From: Eric Leblond Date: Sun, 6 Sep 2020 18:27:04 +0000 (+0200) Subject: prscript: port to python 3 X-Git-Tag: suricata-6.0.0-rc1~38 X-Git-Url: http://git.ipfire.org/cgi-bin/gitweb.cgi?a=commitdiff_plain;h=48c30efd34226de1eaf02704c0023669a42da7dd;p=thirdparty%2Fsuricata.git prscript: port to python 3 --- diff --git a/qa/prscript.py b/qa/prscript.py index 2737960e79..1768d8adde 100755 --- a/qa/prscript.py +++ b/qa/prscript.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # Copyright(C) 2013-2020 Open Information Security Foundation # This program is free software; you can redistribute it and/or modify @@ -20,7 +20,7 @@ # or the buildbot will not be able to access to the data in /data/oisf # and the git step will fail. -import urllib, urllib2, cookielib +import urllib.request, urllib.parse, urllib.error, urllib.request, urllib.error, urllib.parse, http.cookiejar try: import simplejson as json except: @@ -97,12 +97,12 @@ if args.create or args.start or args.stop or args.rm: args.docker = True args.local = True else: - print "You need to install python docker to use docker handling features." + print("You need to install python docker to use docker handling features.") sys.exit(-1) if not args.local: if not args.username: - print "You need to specify a github username (-u option) for this mode (or use -l to disable)" + print("You need to specify a github username (-u option) for this mode (or use -l to disable)") sys.exit(-1) if args.docker: @@ -127,8 +127,8 @@ def SendNotification(title, text): n.show() def TestRepoSync(branch): - request = urllib2.Request(GITHUB_MASTER_URI) - page = urllib2.urlopen(request) + request = urllib.request.Request(GITHUB_MASTER_URI) + page = urllib.request.urlopen(request) json_result = json.loads(page.read()) sha_orig = json_result[0]["sha"] check_command = ["git", "branch", "--contains", sha_orig ] @@ -141,10 +141,10 @@ def TestRepoSync(branch): return 0 def TestGithubSync(branch): - request = urllib2.Request(GITHUB_BASE_URI + username + "/" + args.repository + "/commits?sha=" + branch + "&per_page=1") + request = urllib.request.Request(GITHUB_BASE_URI + username + "/" + args.repository + "/commits?sha=" + branch + "&per_page=1") try: - page = urllib2.urlopen(request) - except urllib2.HTTPError, e: + page = urllib.request.urlopen(request) + except urllib.error.HTTPError as e: if e.code == 404: return -2 else: @@ -153,46 +153,46 @@ def TestGithubSync(branch): sha_github = json_result[0]["sha"] check_command = ["git", "rev-parse", branch] p1 = Popen(check_command, stdout=PIPE) - sha_local = p1.communicate()[0].rstrip() + sha_local = p1.communicate()[0].decode('ascii').rstrip() if sha_local != sha_github: return -1 return 0 def OpenBuildbotSession(): auth_params = { 'username':username,'passwd':password, 'name':'login'} - cookie = cookielib.LWPCookieJar() - params = urllib.urlencode(auth_params) - opener = urllib2.build_opener(urllib2.HTTPCookieProcessor(cookie)) - urllib2.install_opener(opener) - request = urllib2.Request(BASE_URI + 'login', params) - _ = urllib2.urlopen(request) + cookie = http.cookiejar.LWPCookieJar() + params = urllib.parse.urlencode(auth_params).encode('ascii') + opener = urllib.request.build_opener(urllib.request.HTTPCookieProcessor(cookie)) + urllib.request.install_opener(opener) + request = urllib.request.Request(BASE_URI + 'login', params) + _ = urllib.request.urlopen(request) return cookie def SubmitBuild(branch, extension = "", builder_name = None): raw_params = {'branch':branch,'reason':'Testing ' + branch, 'name':'force_build', 'forcescheduler':'force'} - params = urllib.urlencode(raw_params) + params = urllib.parse.urlencode(raw_params).encode('ascii') if not args.docker: - opener = urllib2.build_opener(urllib2.HTTPCookieProcessor(cookie)) - urllib2.install_opener(opener) + opener = urllib.request.build_opener(urllib.request.HTTPCookieProcessor(cookie)) + urllib.request.install_opener(opener) if builder_name == None: builder_name = username + extension - request = urllib2.Request(BUILDERS_URI + builder_name + '/force', params) - page = urllib2.urlopen(request) + request = urllib.request.Request(BUILDERS_URI + builder_name + '/force', params) + page = urllib.request.urlopen(request) result = page.read() if args.verbose: - print "=== response ===" - print result - print "=== end of response ===" + print("=== response ===") + print(result) + print("=== end of response ===") if args.docker: if "

Pending Build Requests:

" in result: - print "Build '" + builder_name + "' submitted" + print("Build '" + builder_name + "' submitted") return 0 else: return -1 - if "Current Builds" in result: - print "Build '" + builder_name + "' submitted" + if b'Current Builds' in result: + print("Build '" + builder_name + "' submitted") return 0 else: return -1 @@ -200,10 +200,10 @@ def SubmitBuild(branch, extension = "", builder_name = None): # TODO honor the branch argument def FindBuild(branch, extension = "", builder_name = None): if builder_name == None: - request = urllib2.Request(JSON_BUILDERS_URI + username + extension + '/') + request = urllib.request.Request(JSON_BUILDERS_URI + username + extension + '/') else: - request = urllib2.Request(JSON_BUILDERS_URI + builder_name + '/') - page = urllib2.urlopen(request) + request = urllib.request.Request(JSON_BUILDERS_URI + builder_name + '/') + page = urllib.request.urlopen(request) json_result = json.loads(page.read()) # Pending build is unnumbered if json_result["pendingBuilds"]: @@ -218,13 +218,13 @@ def GetBuildStatus(builder, buildid, extension="", builder_name = None): if builder_name == None: builder_name = username + extension # https://buildbot.suricata-ids.org/json/builders/build%20deb6/builds/11 - request = urllib2.Request(JSON_BUILDERS_URI + builder_name + '/builds/' + str(buildid)) - page = urllib2.urlopen(request) + request = urllib.request.Request(JSON_BUILDERS_URI + builder_name + '/builds/' + str(buildid)) + page = urllib.request.urlopen(request) result = page.read() if args.verbose: - print "=== response ===" - print result - print "=== end of response ===" + print("=== response ===") + print(result) + print("=== end of response ===") json_result = json.loads(result) if json_result["currentStep"]: return 1 @@ -244,9 +244,9 @@ def WaitForBuildResult(builder, buildid, extension="", builder_name = None): # return the result if res == 0: - print "Build successful for " + builder_name + print("Build successful for " + builder_name) else: - print "Build failure for " + builder_name + ": " + BUILDERS_URI + builder_name + '/builds/' + str(buildid) + print("Build failure for " + builder_name + ": " + BUILDERS_URI + builder_name + '/builds/' + str(buildid)) return res # check that github branch and OISF master branch are sync @@ -254,23 +254,23 @@ if not args.local: ret = TestGithubSync(args.branch) if ret != 0: if ret == -2: - print "Branch " + args.branch + " is not pushed to Github." + print("Branch " + args.branch + " is not pushed to Github.") sys.exit(-1) if args.norebase: - print "Branch " + args.branch + " is not in sync with corresponding Github branch. Continuing due to --norebase option." + print("Branch " + args.branch + " is not in sync with corresponding Github branch. Continuing due to --norebase option.") else: - print "Branch " + args.branch + " is not in sync with corresponding Github branch. Push may be needed." + print("Branch " + args.branch + " is not in sync with corresponding Github branch. Push may be needed.") sys.exit(-1) if TestRepoSync(args.branch) != 0: if args.norebase: - print "Branch " + args.branch + " is not in sync with OISF's master branch. Continuing due to --norebase option." + print("Branch " + args.branch + " is not in sync with OISF's master branch. Continuing due to --norebase option.") else: - print "Branch " + args.branch + " is not in sync with OISF's master branch. Rebase needed." + print("Branch " + args.branch + " is not in sync with OISF's master branch. Rebase needed.") sys.exit(-1) def CreateContainer(): # FIXME check if existing - print "Pulling docking image, first run should take long" + print("Pulling docking image, first run should take long") if GOT_DOCKERPY_API < 2: cli = Client() cli.pull('regit/suri-buildbot') @@ -279,13 +279,13 @@ def CreateContainer(): cli = DockerClient() cli.images.pull('regit/suri-buildbot') suri_src_dir = os.path.split(os.path.dirname(os.path.realpath(__file__)))[0] - print "Using base src dir: " + suri_src_dir + print("Using base src dir: " + suri_src_dir) cli.containers.create('regit/suri-buildbot', name='suri-buildbot', ports={'8010/tcp': 8010, '22/tcp': None} , volumes={suri_src_dir: { 'bind': '/data/oisf', 'mode': 'ro'}, os.path.join(suri_src_dir,'qa','docker','buildbot.cfg'): { 'bind': '/data/buildbot/master/master.cfg', 'mode': 'ro'}}, detach = True) sys.exit(0) def StartContainer(): suri_src_dir = os.path.split(os.path.dirname(os.path.realpath(__file__)))[0] - print "Using base src dir: " + suri_src_dir + print("Using base src dir: " + suri_src_dir) if GOT_DOCKERPY_API < 2: cli = Client() cli.start('suri-buildbot', port_bindings={8010:8010, 22:None}, binds={suri_src_dir: { 'bind': '/data/oisf', 'ro': True}, os.path.join(suri_src_dir,'qa','docker','buildbot.cfg'): { 'bind': '/data/buildbot/master/master.cfg', 'ro': True}} ) @@ -309,11 +309,11 @@ def RmContainer(): try: cli.remove_container('suri-buildbot') except: - print "Unable to remove suri-buildbot container" + print("Unable to remove suri-buildbot container") try: cli.remove_image('regit/suri-buildbot:latest') except: - print "Unable to remove suri-buildbot images" + print("Unable to remove suri-buildbot images") else: cli = DockerClient() cli.containers.get('suri-buildbot').remove() @@ -331,7 +331,7 @@ if GOT_DOCKER: RmContainer() if not args.branch: - print "You need to specify a branch for this mode" + print("You need to specify a branch for this mode") sys.exit(-1) # submit buildbot form to build current branch on the devel builder @@ -339,12 +339,12 @@ if not args.check: if not args.docker: cookie = OpenBuildbotSession() if cookie == None: - print "Unable to connect to buildbot with provided credentials" + print("Unable to connect to buildbot with provided credentials") sys.exit(-1) for build in BUILDERS_LIST: res = SubmitBuild(args.branch, builder_name = build) if res == -1: - print "Unable to start build. Check command line parameters" + print("Unable to start build. Check command line parameters") sys.exit(-1) buildids = {} @@ -356,20 +356,20 @@ if args.docker: for build in BUILDERS_LIST: buildid = FindBuild(args.branch, builder_name = build) if buildid == -1: - print "Pending build tracking is not supported. Follow build by browsing " + BUILDERS_URI + build + print("Pending build tracking is not supported. Follow build by browsing " + BUILDERS_URI + build) elif buildid == -2: - print "No build found for " + BUILDERS_URI + build + print("No build found for " + BUILDERS_URI + build) sys.exit(0) else: if not args.docker: - print "You can watch build progress at " + BUILDERS_URI + build + "/builds/" + str(buildid) + print("You can watch build progress at " + BUILDERS_URI + build + "/builds/" + str(buildid)) buildids[build] = buildid if args.docker: - print "You can watch build progress at " + BASE_URI + "waterfall" + print("You can watch build progress at " + BASE_URI + "waterfall") if len(buildids): - print "Waiting for build completion" + print("Waiting for build completion") else: sys.exit(0) @@ -383,24 +383,24 @@ if args.docker: buildres = -1 up_buildids.pop(build, None) if len(up_buildids): - remains = " (remaining builds: " + ', '.join(up_buildids.keys()) + ")" + remains = " (remaining builds: " + ', '.join(list(up_buildids.keys())) + ")" else: remains = "" - print "Build failure for " + build + ": " + BUILDERS_URI + build + '/builds/' + str(buildids[build]) + remains + print("Build failure for " + build + ": " + BUILDERS_URI + build + '/builds/' + str(buildids[build]) + remains) elif ret == 0: up_buildids.pop(build, None) if len(up_buildids): - remains = " (remaining builds: " + ', '.join(up_buildids.keys()) + ")" + remains = " (remaining builds: " + ', '.join(list(up_buildids.keys())) + ")" else: remains = "" - print "Build successful for " + build + remains + print("Build successful for " + build + remains) time.sleep(5) buildids = up_buildids if res == -1: SendNotification("PRscript failure", "Some builds have failed. Check waterfall for results.") sys.exit(-1) else: - print "PRscript completed successfully" + print("PRscript completed successfully") SendNotification("PRscript success", "Congrats! All builds have passed.") sys.exit(0) else: @@ -411,9 +411,9 @@ else: if buildres == 0: if not args.norebase and not args.docker: - print "You can copy/paste following lines into github PR" + print("You can copy/paste following lines into github PR") for build in buildids: - print "- PR " + build + ": " + BUILDERS_URI + build + "/builds/" + str(buildids[build]) + print("- PR " + build + ": " + BUILDERS_URI + build + "/builds/" + str(buildids[build])) SendNotification("OISF PRscript success", "Congrats! All builds have passed.") sys.exit(0) else: