Facilities for making a set of load tests

README.textile

@@ -9,6 +9,7 @@ h2. Dependencies
 * Python 2.6
 * boto
 * paramiko
+* csvkit
 
 h2. Installation for users
 
@@ -58,6 +59,28 @@ It then uses those 4 servers to send 10,000 requests, 250 at a time, to attack O
 
 Lastly, it spins down the 4 servers.  *Please remember to do this*--we aren't responsible for your EC2 bills.
 
+h2. Advanced Usage
+
+<pre>
+bees up -s 5 -g public -k frakkingtoasters -z us-west-1a -i ami-aabbccdd -l ubuntu
+for i in `seq 200 200 1000`
+do
+   echo "---- $i -----"
+   bees attack -t 900 -c $i -p query.dat -u http://www.ournewwebbyhotness.com/ --stats-file 15_Min_200_step.csv --non-200-is-failure --testname $i
+done
+bees down
+gnuplot -e "filename='15_Min_200_step'" examples/LoadTest.gpi
+</pre>
+
+This spins up 5 servers in the us-west-1a AZ from the specified AMI in security group 'public' using the EC2 keypair 'frakkingtoasters', whose private key is expected to reside at ~/.ssh/frakkingtoasters.pem.
+
+It then runs a series of 15 minute tests (which in this case are a post of some query.dat file) with an increasing number of concurrent users, all the while collecting all the resulting statistics in the 15_Min_200_step.csv file. Note that in this test non 200 responses are considered as errors.
+
+Next, the bees are spun down.
+
+Finally, a graph is created from the csv using gnuplot (an example gnuplot script can be found in the examples dir).
+
+
 For complete options type:
 
 <pre>

beeswithmachineguns/main.py

@@ -27,127 +27,94 @@
 import bees
 from urlparse import urlparse
 
-from optparse import OptionParser, OptionGroup
+from argparse import ArgumentParser
 
 def parse_options():
     """
     Handle the command line arguments for spinning up bees
     """
-    parser = OptionParser(usage="""
-bees COMMAND [options]
+    parser = ArgumentParser(description="""
+        Bees with Machine Guns.
+        A utility for arming (creating) many bees (small EC2 instances) to attack
+        (load test) targets (web applications).
+        """)
 
-Bees with Machine Guns
+    subparsers = parser.add_subparsers(title='commands', dest='command')
+    up_cmd = subparsers.add_parser("up", help='Start a batch of load testing servers.', description=
+        """Start a batch of load testing servers.
+        In order to spin up new servers you will need to specify at least the -k command, which is the name of the EC2 keypair to use for creating and connecting to the new servers. The bees will expect to find a .pem file with this name in ~/.ssh/.""")
 
-A utility for arming (creating) many bees (small EC2 instances) to attack
-(load test) targets (web applications).
+    # Required
+    up_cmd.add_argument('-k', '--key',  metavar="KEY", dest='key', required=True, help="The ssh key pair name to use to connect to the new servers.")
 
-commands:
-  up      Start a batch of load testing servers.
-  attack  Begin the attack on a specific url.
-  down    Shutdown and deactivate the load testing servers.
-  report  Report the status of the load testing servers.
-    """)
+    up_cmd.add_argument('-s', '--servers', metavar="SERVERS", dest='servers', type=int, default=5, help="The number of servers to start (default: 5).")
+    up_cmd.add_argument('-g', '--group', metavar="GROUP", dest='group', default='default', help="The security group(s) to run the instances under (default: default).")
+    up_cmd.add_argument('-z', '--zone',  metavar="ZONE", dest='zone', default='us-east-1d', help="The availability zone to start the instances in (default: us-east-1d).")
+    up_cmd.add_argument('-i', '--instance',  metavar="INSTANCE", dest='instance', default='ami-ff17fb96', help="The instance-id to use for each server from (default: ami-ff17fb96).")
+    up_cmd.add_argument('-t', '--type',  metavar="TYPE", dest='type', default='t1.micro', help="The instance-type to use for each server (default: t1.micro).")
+    up_cmd.add_argument('-l', '--login',  metavar="LOGIN", dest='login', default='newsapps', help="The ssh username name to use to connect to the new servers (default: newsapps).")
+    up_cmd.add_argument('-v', '--subnet',  metavar="SUBNET", dest='subnet', default=None, help="The vpc subnet id in which the instances should be launched. (default: None).")
 
-    up_group = OptionGroup(parser, "up",
-        """In order to spin up new servers you will need to specify at least the -k command, which is the name of the EC2 keypair to use for creating and connecting to the new servers. The bees will expect to find a .pem file with this name in ~/.ssh/.""")
+    attack_cmd = subparsers.add_parser("attack", help='Begin the attack on a specific url.', description=
+        """Begin the attack on a specific url.
+        Beginning an attack requires only that you specify the -u option with the URL you wish to target.""")
 
     # Required
-    up_group.add_option('-k', '--key',  metavar="KEY",  nargs=1,
-                        action='store', dest='key', type='string',
-                        help="The ssh key pair name to use to connect to the new servers.")
-
-    up_group.add_option('-s', '--servers', metavar="SERVERS", nargs=1,
-                        action='store', dest='servers', type='int', default=5,
-                        help="The number of servers to start (default: 5).")
-    up_group.add_option('-g', '--group', metavar="GROUP", nargs=1,
-                        action='store', dest='group', type='string', default='default',
-                        help="The security group(s) to run the instances under (default: default).")
-    up_group.add_option('-z', '--zone',  metavar="ZONE",  nargs=1,
-                        action='store', dest='zone', type='string', default='us-east-1d',
-                        help="The availability zone to start the instances in (default: us-east-1d).")
-    up_group.add_option('-i', '--instance',  metavar="INSTANCE",  nargs=1,
-                        action='store', dest='instance', type='string', default='ami-ff17fb96',
-                        help="The instance-id to use for each server from (default: ami-ff17fb96).")
-    up_group.add_option('-t', '--type',  metavar="TYPE",  nargs=1,
-                        action='store', dest='type', type='string', default='t1.micro',
-                        help="The instance-type to use for each server (default: t1.micro).")
-    up_group.add_option('-l', '--login',  metavar="LOGIN",  nargs=1,
-                        action='store', dest='login', type='string', default='newsapps',
-                        help="The ssh username name to use to connect to the new servers (default: newsapps).")
-    up_group.add_option('-v', '--subnet',  metavar="SUBNET",  nargs=1,
-                        action='store', dest='subnet', type='string', default=None,
-                        help="The vpc subnet id in which the instances should be launched. (default: None).")
-
-    parser.add_option_group(up_group)
-
-    attack_group = OptionGroup(parser, "attack",
-            """Beginning an attack requires only that you specify the -u option with the URL you wish to target.""")
+    attack_cmd.add_argument('-u', '--url', metavar="URL", dest='urls', action='append', required=True, help="URL(s) of the target to attack.")
 
-    # Required
-    attack_group.add_option('-u', '--url', metavar="URL", nargs=1,
-                        action='store', dest='url', type='string',
-                        help="URL of the target to attack.")
-    attack_group.add_option('-p', '--post-file',  metavar="POST_FILE",  nargs=1,
-                        action='store', dest='post_file', type='string', default=False,
-                        help="The POST file to deliver with the bee's payload.")
-    attack_group.add_option('-m', '--mime-type',  metavar="MIME_TYPE",  nargs=1,
-                        action='store', dest='mime_type', type='string', default='text/plain',
-                        help="The MIME type to send with the request.")
-    attack_group.add_option('-n', '--number', metavar="NUMBER", nargs=1,
-                        action='store', dest='number', type='int', default=1000,
-                        help="The number of total connections to make to the target (default: 1000).")
-    attack_group.add_option('-c', '--concurrent', metavar="CONCURRENT", nargs=1,
-                        action='store', dest='concurrent', type='int', default=100,
-                        help="The number of concurrent connections to make to the target (default: 100).")
-    attack_group.add_option('-H', '--headers', metavar="HEADERS", nargs=1,
-                        action='store', dest='headers', type='string', default='',
+    attack_cmd.add_argument('-p', '--post-file',  metavar="POST_FILE", dest='post_files', action='append', help="The POST file(s) to deliver with the bee's payload.")
+    attack_cmd.add_argument('-m', '--mime-type',  metavar="MIME_TYPE", dest='mime_type', default='text/plain', help="The MIME type to send with the request.")
+    attack_cmd.add_argument('-n', '--number', metavar="NUMBER", dest='number', type=int, default=1000, help="The number of total connections to make to the target (default: 1000).")
+    attack_cmd.add_argument('-c', '--concurrent', metavar="CONCURRENT", dest='concurrent', type=int, default=100, help="The number of concurrent connections to make to the target (default: 100).")
+    attack_cmd.add_argument('-H', '--headers', metavar="HEADERS", dest='headers', default='',
                         help="HTTP headers to send to the target to attack. Multiple headers should be separated by semi-colons, e.g header1:value1;header2:value2")
-    attack_group.add_option('-e', '--csv', metavar="FILENAME", nargs=1,
-                        action='store', dest='csv_filename', type='string', default='',
-                        help="Store the distribution of results in a csv file for all completed bees (default: '').")
-
-    parser.add_option_group(attack_group)
+    attack_cmd.add_argument('-e', '--csv', metavar="FILENAME", dest='csv_filename', default='', help="Store the distribution of results in a csv file for all completed bees (default: '').")
+    attack_cmd.add_argument('-g', '--gnuplot', metavar="FILENAME", dest='gnuplot_filename', default='', help="Write all measured values out as a 'gnuplot' or TSV (Tab separate values) file (default: '').")
+    attack_cmd.add_argument('-t', '--timelimit', metavar="TIMELIMIT", dest='timelimit', type=int, default=0,
+                        help="Maximum number of seconds to spend for benchmarking. This implies a -n 50000 internally. Use this to benchmark the server within a fixed total amount of time (default: no limit).")
+    attack_cmd.add_argument('--stats-file', metavar="FILENAME", dest='stats_filename', default='',
+                        help="Store detailed graph ready stats across multiple tests in a csv file. Will create gnuplot files even if the -g/--gnuplot wasn't specified (default: '').")
+    attack_cmd.add_argument('--testname', metavar="NAME", dest='testname', default='unnamed', help="Name of current test. To be used in conjunction with --stats-file (default: 'unnamed').")
+    attack_cmd.add_argument('--non-200-is-failure', dest='non_200_is_failure', action='store_true', default=False, help="Treat non-200 responses as failures (treated as success by default).")
 
-    (options, args) = parser.parse_args()
+    down_cmd = subparsers.add_parser("down", help='Shutdown and deactivate the load testing servers.', description='Shutdown and deactivate the load testing servers.')
+    report_cmd = subparsers.add_parser("report", help='Report the status of the load testing servers.', description='Report the status of the load testing servers.')
 
-    if len(args) <= 0:
-        parser.error('Please enter a command.')
+    options = parser.parse_args()
 
-    command = args[0]
+    command = options.command
 
     if command == 'up':
-        if not options.key:
-            parser.error('To spin up new instances you need to specify a key-pair name with -k')
-
         if options.group == 'default':
             print 'New bees will use the "default" EC2 security group. Please note that port 22 (SSH) is not normally open on this group. You will need to use to the EC2 tools to open it before you will be able to attack.'
 
         bees.up(options.servers, options.group, options.zone, options.instance, options.type, options.login, options.key, options.subnet)
     elif command == 'attack':
-        if not options.url:
-            parser.error('To run an attack you need to specify a url with -u')
-
-        parsed = urlparse(options.url)
-        if not parsed.scheme:
-            parsed = urlparse("http://" + options.url)
+        for url in options.urls:
+            parsed = urlparse(url)
+            if not parsed.scheme:
+                parsed = urlparse("http://" + url)
 
-        if not parsed.path:
-            parser.error('It appears your URL lacks a trailing slash, this will disorient the bees. Please try again with a trailing slash.')
+            if not parsed.path:
+                parser.error('It appears your URL lacks a trailing slash, this will disorient the bees. Please try again with a trailing slash.')
 
         additional_options = dict(
             headers=options.headers,
-            post_file=options.post_file,
+            post_files=options.post_files,
             mime_type=options.mime_type,
             csv_filename=options.csv_filename,
+            gnuplot_filename=options.gnuplot_filename,
+            stats_filename=options.stats_filename,
+            testname=options.testname,
+            non_200_is_failure=options.non_200_is_failure,
         )
 
-        bees.attack(options.url, options.number, options.concurrent, **additional_options)
+        bees.attack(options.urls, options.number, options.concurrent, options.timelimit, **additional_options)
 
     elif command == 'down':
         bees.down()
     elif command == 'report':
         bees.report()
 
-
 def main():
     parse_options()

examples/LoadTest.gpi

@@ -0,0 +1,70 @@
+# script to generate graphs from a load test done by Bees with Machine Guns
+#
+# usage:
+# gnuplot -e "filename='<filename w/o csv extension>'" LoadTest.gpi
+#
+
+# output to a jpeg file
+set terminal jpeg size 1440,900
+
+# This sets the aspect ratio of the graph
+set size 1, 1
+set lmargin 12
+set rmargin 10
+
+set output filename.'.jpg'
+
+# Where to place the legend/key
+set key left top
+
+set multiplot layout 2, 1 title filename font "Bold,20"
+
+# Draw gridlines oriented on the y axis
+set grid y
+# Label the x-axis
+#set xlabel 'Iteration'
+set xlabel 'Concurrent Requests'
+# Tell gnuplot to use commas as the delimiter instead of spaces (default)
+set datafile separator ','
+set key autotitle columnhead
+
+#
+# first graph
+#
+set title "Requests/Second(green) and % Errors(red)" font "Bold,14"
+set ytics nomirror
+set y2tics
+set ylabel 'Requests/Second' textcolor lt 2
+set y2label 'Error Percentage' textcolor lt 1
+set decimal locale
+#set format "%'.0f"
+set format "%'g"
+set format y2 "%g %%"
+set yrange [0:]
+set y2range [0:10]
+
+# Plot the data
+plot filename.'.csv' using 1:7 with lines lt 5 lw 3 axes x1y1, \
+             ''      using 1:($7-($7-$8)/2) with lines lt 2 lw 3 axes x1y1, \
+             ''      using 1:($6*50) with lines lt 1 lw 2 axes x1y2
+# the creative arithmetic above is done in order to overcome a bug in ab in which it counts each error twice, and since successful hits are calculated as total-bad it also has to be fixed.
+#             ''      using 1:8 with lines lt 2 lw 3 axes x1y1, \
+#             ''      using 1:($6*100) with lines lt 1 lw 3 axes x1y2
+unset y2tics 
+unset y2label
+set yrange [*:*]
+
+#
+# second graph
+#
+set title "Response Time" font "Bold,14"
+unset ylabel
+set ylabel "ms"
+
+set bars 4.0
+set style fill solid
+
+# Plot the data
+plot filename.'.csv' using 1:15:9:32:31 with candlesticks lt 2 title 'Min/P10/Med/P90/P95' whiskerbars 0.6, \
+          ''         using 1:12:12:12:12 with candlesticks lt -1 notitle,\
+          ''         using 1:11 with lines lt -1 lw 3

examples/LoadTestIter.gpi

@@ -0,0 +1,71 @@
+# script to generate graphs from a load test done by Bees with Machine Guns
+#
+# usage:
+# gnuplot -e "filename='<filename w/o csv extension>'" LoadTest.gpi
+#
+
+# output to a jpeg file
+set terminal jpeg size 1440,900
+
+# This sets the aspect ratio of the graph
+set size 1, 1
+set lmargin 12
+set rmargin 10
+
+set output filename.'.jpg'
+
+# Where to place the legend/key
+set key left top
+
+set multiplot layout 2, 1 title filename font "Bold,20"
+
+# Draw gridlines oriented on the y axis
+set grid y
+set xtics 1
+# Label the x-axis
+set xlabel 'Iteration'
+#set xlabel 'Concurrent Requests'
+# Tell gnuplot to use commas as the delimiter instead of spaces (default)
+set datafile separator ','
+set key autotitle columnhead
+
+#
+# first graph
+#
+set title "Requests/Second(green) and % Errors(red)" font "Bold,14"
+set ytics nomirror
+set y2tics
+set ylabel 'Requests/Second' textcolor lt 2
+set y2label 'Error Percentage' textcolor lt 1
+set decimal locale
+#set format "%'.0f"
+set format "%'g"
+set format y2 "%g %%"
+set yrange [0:]
+set y2range [0:10]
+
+# Plot the data
+plot filename.'.csv' using 1:7 with lines lt 5 lw 3 axes x1y1, \
+             ''      using 1:($7-($7-$8)/2) with lines lt 2 lw 3 axes x1y1, \
+             ''      using 1:($6*50) with lines lt 1 lw 3 axes x1y2
+# the creative arithmetic above is done in order to overcome a bug in ab in which it counts each error twice, and since successful hits are calculated as total-bad it also has to be fixed.
+#             ''      using 1:8 with lines lt 2 lw 3 axes x1y1, \
+#             ''      using 1:($6*100) with lines lt 1 lw 3 axes x1y2
+unset y2tics 
+unset y2label
+set yrange [*:*]
+
+#
+# second graph
+#
+set title "Response Time" font "Bold,14"
+unset ylabel
+set ylabel "ms"
+
+set bars 4.0
+set style fill solid
+
+# Plot the data
+plot filename.'.csv' using 1:15:9:32:31 with candlesticks lt 2 title 'Min/P10/Med/P90/P95' whiskerbars 0.6, \
+          ''         using 1:12:12:12:12 with candlesticks lt -1 notitle,\
+          ''         using 1:11 with lines lt -1 lw 3