prscript: refactor for docker

With the current work in progress on docker we need to update the
script to handle this case.

This patch adds two options:
 - -d to run in docker mode. In that case the build is started in
 the local dockerized buildbot instance
 - -l to not test the tested branch synchronization with github.

The -l option allows user to run a complete test without publishing
the code on github and without Internet access.

diff --git a/qa/prscript.py b/qa/prscript.py
index 142ad6802a0af6b5dcba4ec8fbeb977dd11e7505..cc29a2a16f2515938a39acd0e7e5d7fb92a3465c 100755 (executable)
--- a/qa/prscript.py
+++ b/qa/prscript.py
@@ -1,5 +1,5 @@
 #!/usr/bin/env python
-# Copyright(C) 2013 Open Information Security Foundation
+# Copyright(C) 2013, 2014, 2015 Open Information Security Foundation
 
 # This program is free software; you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by
@@ -27,9 +27,6 @@ import sys
 #  - buildbot user and password
 
 BASE_URI="https://buildbot.openinfosecfoundation.org/"
-BUILDERS_URI=BASE_URI+"builders/"
-JSON_BUILDERS_URI=BASE_URI+"json/builders/"
-
 GITHUB_BASE_URI = "https://api.github.com/repos/"
 GITHUB_MASTER_URI = "https://api.github.com/repos/inliniac/suricata/commits?sha=master"
 
@@ -40,12 +37,24 @@ parser.add_argument('-c', '--check', action='store_const', const=True, help='onl
 parser.add_argument('-v', '--verbose', action='store_const', const=True, help='verbose output', default=False)
 parser.add_argument('--norebase', action='store_const', const=True, help='do not test if branch is in sync with master', default=False)
 parser.add_argument('-r', '--repository', dest='repository', default='suricata', help='name of suricata repository on github')
+parser.add_argument('-d', '--docker', action='store_const', const=True, help='use docker based testing', default=False)
+parser.add_argument('-l', '--local', action='store_const', const=True, help='local testing before github push', default=False)
 parser.add_argument('branch', metavar='branch', help='github branch to build')
 args = parser.parse_args()
 username = args.username
 password = args.password
 cookie = None
 
+if args.docker:
+    BASE_URI="http://localhost:8010/"
+    BUILDERS_LIST = ["build", "clang", "debug", "features", "profiling", "pcaps"]
+else:
+    BUILDERS_LIST = [username, username + "-pcap"]
+
+
+BUILDERS_URI=BASE_URI+"builders/"
+JSON_BUILDERS_URI=BASE_URI+"json/builders/"
+
 def TestRepoSync(branch):
     request = urllib2.Request(GITHUB_MASTER_URI)
     page = urllib2.urlopen(request)
@@ -72,12 +81,15 @@ def OpenBuildbotSession():
     return cookie
 
 
-def SubmitBuild(branch, extension = ""):
+def SubmitBuild(branch, extension = "", builder_name = None):
     raw_params = {'branch':branch,'reason':'Testing ' + branch, 'name':'force_build', 'forcescheduler':'force'}
     params = urllib.urlencode(raw_params)
-    opener = urllib2.build_opener(urllib2.HTTPCookieProcessor(cookie))
-    urllib2.install_opener(opener)
-    request = urllib2.Request(BUILDERS_URI + username + extension + '/force', params)
+    if not args.docker:
+        opener = urllib2.build_opener(urllib2.HTTPCookieProcessor(cookie))
+        urllib2.install_opener(opener)
+    if builder_name == None:
+        builder_name = username + extension
+    request = urllib2.Request(BUILDERS_URI + builder_name + '/force', params)
     page = urllib2.urlopen(request)
 
     result = page.read()
@@ -85,15 +97,24 @@ def SubmitBuild(branch, extension = ""):
         print "=== response ==="
         print result
         print "=== end of response ==="
+    if args.docker:
+        if "<h2>Pending Build Requests:</h2>" in result:
+            print "Build '" + builder_name + "' submitted"
+            return 0
+        else:
+            return -1
     if "Current Builds" in result:
-        print "Build submitted"
+        print "Build '" + builder_name + "' submitted"
         return 0
     else:
         return -1
 
 # TODO honor the branch argument
-def FindBuild(branch, extension = ""):
-    request = urllib2.Request(JSON_BUILDERS_URI + username + extension + '/')
+def FindBuild(branch, extension = "", builder_name = None):
+    if builder_name == None:
+        request = urllib2.Request(JSON_BUILDERS_URI + username + extension + '/')
+    else:
+        request = urllib2.Request(JSON_BUILDERS_URI + builder_name + '/')
     page = urllib2.urlopen(request)
     json_result = json.loads(page.read())
     # Pending build is unnumbered
@@ -105,9 +126,12 @@ def FindBuild(branch, extension = ""):
         return json_result["cachedBuilds"][-1]
     return -2
 
-def GetBuildStatus(builder, buildid, extension=""):
-    # https://buildbot.suricata-ids.org/json/builders/build%20deb6/builds/11
-    request = urllib2.Request(JSON_BUILDERS_URI + username + extension + '/builds/' + str(buildid))
+def GetBuildStatus(builder, buildid, extension="", builder_name = None):
+    if builder_name == None:
+        # https://buildbot.suricata-ids.org/json/builders/build%20deb6/builds/11
+        request = urllib2.Request(JSON_BUILDERS_URI + username + extension + '/builds/' + str(buildid))
+    else:
+        request = urllib2.Request(JSON_BUILDERS_URI + builder_name + '/builds/' + str(buildid))
     page = urllib2.urlopen(request)
     result = page.read()
     if args.verbose:
@@ -121,23 +145,25 @@ def GetBuildStatus(builder, buildid, extension=""):
         return 0
     return -1
 
-def WaitForBuildResult(builder, buildid, extension=""):
+def WaitForBuildResult(builder, buildid, extension="", builder_name = None):
     # fetch result every 10 secs till task is over
+    if builder_name == None:
+        builder_name = username + extension
     res = 1
     while res == 1:
-        res = GetBuildStatus(username,buildid, extension=extension)
+        res = GetBuildStatus(username,buildid, builder_name = builder_name)
         if res == 1:
             time.sleep(10)
 
     # return the result
     if res == 0:
-        print "Build successful"
+        print "Build successful for " + builder_name
     else:
-        print "Build failure: " + BUILDERS_URI + username + extension + '/builds/' + str(buildid)
+        print "Build failure for " + builder_name + ": " + BUILDERS_URI + builder_name + '/builds/' + str(buildid)
     return res
 
     # check that github branch and inliniac master branch are sync
-if TestRepoSync(args.branch) == -1:
+if not args.local and TestRepoSync(args.branch) == -1:
     if args.norebase:
         print "Branch " + args.branch + " is not in sync with inliniac's master branch. Continuing due to --norebase option."
     else:
@@ -146,54 +172,50 @@ if TestRepoSync(args.branch) == -1:
 
 # submit buildbot form to build current branch on the devel builder
 if not args.check:
-    cookie = OpenBuildbotSession()
-    if cookie == None:
-        print "Unable to connect to buildbot with provided credentials"
-        sys.exit(-1)
-    res = SubmitBuild(args.branch)
-    if res == -1:
-        print "Unable to start build. Check command line parameters"
-        sys.exit(-1)
-    res = SubmitBuild(args.branch, extension="-pcap")
-    if res == -1:
-        print "Unable to start pcap build. Check command line parameters"
-        sys.exit(-1)
-
-# get build number and exit if we don't have
-buildid = FindBuild(args.branch)
-if buildid == -1:
-    print "Pending build tracking is not supported. Follow build by browsing " + BUILDERS_URI + username
-elif buildid == -2:
-    print "No build found for " + BUILDERS_URI + username
-    sys.exit(0)
-else:
-    print "You can watch build progress at " + BUILDERS_URI + username + "/builds/" + str(buildid)
+    if not args.docker:
+        cookie = OpenBuildbotSession()
+        if cookie == None:
+            print "Unable to connect to buildbot with provided credentials"
+            sys.exit(-1)
+    for build in BUILDERS_LIST:
+        res = SubmitBuild(args.branch, builder_name = build)
+        if res == -1:
+            print "Unable to start build. Check command line parameters"
+            sys.exit(-1)
+
+buildids = {}
+
+if args.docker:
+    from time import sleep
+    print "Waiting 2 seconds"
+    sleep(2)
 
 # get build number and exit if we don't have
-buildidpcap = FindBuild(args.branch, extension = "-pcap")
-if buildidpcap == -1:
-    print "Pending build tracking is not supported. Follow build by browsing " + BUILDERS_URI + username + "-pcap"
-elif buildidpcap == -2:
-    print "No build found for " + BUILDERS_URI + username + "-pcap"
-    sys.exit(0)
-else:
-    print "You can watch build progress at " + BUILDERS_URI + username + "-pcap/builds/" + str(buildidpcap)
+for build in BUILDERS_LIST:
+    buildid = FindBuild(args.branch, builder_name = build)
+    if buildid == -1:
+        print "Pending build tracking is not supported. Follow build by browsing " + BUILDERS_URI + build
+    elif buildid == -2:
+        print "No build found for " + BUILDERS_URI + build
+        sys.exit(0)
+    else:
+        print "You can watch build progress at " + BUILDERS_URI + build + "/builds/" + str(buildid)
+        buildids[build] = buildid
 
-if buildid != -1 or buildidpcap != -1:
+if len(buildids):
     print "Waiting for build completion"
+else:
+    sys.exit(0)
 
 res = 0
-if buildid != -1:
-    res = WaitForBuildResult(username, buildid)
-
-if buildidpcap != -1:
-    res += WaitForBuildResult(username, buildidpcap, extension="-pcap")
+for build in buildids:
+    res = WaitForBuildResult(build, buildids[build], builder_name = build)
 
 if res == 0:
-    if not args.norebase:
+    if not args.norebase and not args.docker:
         print "You can copy/paste following lines into github PR"
-        print "- PR build: " + BUILDERS_URI + username + "/builds/" + str(buildid)
-        print "- PR pcaps: " + BUILDERS_URI + username + "-pcap/builds/" + str(buildidpcap)
+        for build in buildids:
+            print "- PR " + build + ": " + BUILDERS_URI + build + "/builds/" + str(buildids[build])
     sys.exit(0)
 else:
     sys.exit(-1)