Better benchmark script

diff --git a/demos/benchmark/benchmark.py b/demos/benchmark/benchmark.py
index c25a402342fb8a8be888b6049f7eca2261e98cc1..26d496b3e02023f410a2f961ea592431e39aedbd 100755 (executable)
--- a/demos/benchmark/benchmark.py
+++ b/demos/benchmark/benchmark.py
@@ -5,24 +5,39 @@
 #
 # Running without profiling:
 # demos/benchmark/benchmark.py
+# demos/benchmark/benchmark.py --quiet --num_runs=5|grep "Requests per second"
 #
 # Running with profiling:
 #
 # python -m cProfile -o /tmp/prof demos/benchmark/benchmark.py
-# python -c 'import pstats; pstats.Stats("/tmp/prof").strip_dirs().sort_stats("time").print_callers(20)'
+# python -m pstats /tmp/prof
+# % sort time
+# % stats 20
 
 from tornado.ioloop import IOLoop
 from tornado.options import define, options, parse_command_line
 from tornado.web import RequestHandler, Application
 
+import random
 import signal
 import subprocess
 
+# choose a random port to avoid colliding with TIME_WAIT sockets left over
+# from previous runs.
+define("min_port", type=int, default=8000)
+define("max_port", type=int, default=9000)
 
-define("port", type=int, default=8888)
-define("n", type=int, default=10000)
+# Increasing --n without --keepalive will eventually run into problems
+# due to TIME_WAIT sockets
+define("n", type=int, default=15000)
 define("c", type=int, default=25)
 define("keepalive", type=bool, default=False)
+define("quiet", type=bool, default=False)
+
+# Repeat the entire benchmark this many times (on different ports)
+# This gives JITs time to warm up, etc.  Pypy needs 3-5 runs at
+# --n=15000 for its JIT to reach full effectiveness
+define("num_runs", type=int, default=1)
 
 class RootHandler(RequestHandler):
     def get(self):
@@ -36,17 +51,28 @@ def handle_sigchld(sig, frame):
 
 def main():
     parse_command_line()
+    for i in xrange(options.num_runs):
+        run()
+
+def run():
     app = Application([("/", RootHandler)])
-    app.listen(options.port)
+    port = random.randrange(options.min_port, options.max_port)
+    app.listen(port, address='127.0.0.1')
     signal.signal(signal.SIGCHLD, handle_sigchld)
     args = ["ab"]
     args.extend(["-n", str(options.n)])
     args.extend(["-c", str(options.c)])
     if options.keepalive:
         args.append("-k")
-    args.append("http://127.0.0.1:%d/" % options.port)
-    proc = subprocess.Popen(args)
+    if options.quiet:
+        # just stops the progress messages printed to stderr
+        args.append("-q")
+    args.append("http://127.0.0.1:%d/" % port)
+    subprocess.Popen(args)
     IOLoop.instance().start()
+    IOLoop.instance().close()
+    del IOLoop._instance
+    assert not IOLoop.initialized()
 
 if __name__ == '__main__':
     main()