From: Jaroslav Kysela Date: Thu, 20 Apr 2017 14:49:43 +0000 (+0200) Subject: doozer/bintray: add automatic package publishing, rpm - fix paths for doozer X-Git-Tag: v4.2.1~5 X-Git-Url: http://git.ipfire.org/cgi-bin/gitweb.cgi?a=commitdiff_plain;h=9ca516675c15964f2b15fa6e6a21357c1c3de656;p=thirdparty%2Ftvheadend.git doozer/bintray: add automatic package publishing, rpm - fix paths for doozer --- diff --git a/.doozer.json b/.doozer.json index e0e115966..bcc4c6259 100644 --- a/.doozer.json +++ b/.doozer.json @@ -22,7 +22,8 @@ "ccache" ], "buildcmd": [ - "AUTOBUILD_CONFIGURE_EXTRA=--enable-ccache\\ --disable-libx265_static\\ --disable-libx265\\ --enable-hdhomerun_static ./Autobuild.sh -t ${TARGET} -j ${PARALLEL} -w ${WORKDIR}" + "AUTOBUILD_CONFIGURE_EXTRA=--enable-ccache\\ --disable-libx265_static\\ --disable-libx265\\ --enable-hdhomerun_static ./Autobuild.sh -t ${TARGET} -j ${PARALLEL} -w ${WORKDIR}", + "support/bintray.py publish filelist.txt" ] }, "precise-i386": { @@ -46,7 +47,8 @@ "ccache" ], "buildcmd": [ - "AUTOBUILD_CONFIGURE_EXTRA=--enable-ccache\\ --disable-libx265_static\\ --disable-libx265\\ --enable-hdhomerun_static ./Autobuild.sh -t ${TARGET} -j ${PARALLEL} -w ${WORKDIR}" + "AUTOBUILD_CONFIGURE_EXTRA=--enable-ccache\\ --disable-libx265_static\\ --disable-libx265\\ --enable-hdhomerun_static ./Autobuild.sh -t ${TARGET} -j ${PARALLEL} -w ${WORKDIR}", + "support/bintray.py publish filelist.txt" ] }, "trusty-amd64": { @@ -70,7 +72,8 @@ "ccache" ], "buildcmd": [ - "AUTOBUILD_CONFIGURE_EXTRA=--enable-ccache\\ --enable-ffmpeg_static\\ --enable-hdhomerun_static ./Autobuild.sh -t ${TARGET} -j ${PARALLEL} -w ${WORKDIR}" + "AUTOBUILD_CONFIGURE_EXTRA=--enable-ccache\\ --enable-ffmpeg_static\\ --enable-hdhomerun_static ./Autobuild.sh -t ${TARGET} -j ${PARALLEL} -w ${WORKDIR}", + "support/bintray.py publish filelist.txt" ] }, "trusty-i386": { @@ -94,7 +97,8 @@ "ccache" ], "buildcmd": [ - "AUTOBUILD_CONFIGURE_EXTRA=--enable-ccache\\ --enable-ffmpeg_static\\ --enable-hdhomerun_static ./Autobuild.sh -t ${TARGET} -j ${PARALLEL} -w ${WORKDIR}" + "AUTOBUILD_CONFIGURE_EXTRA=--enable-ccache\\ --enable-ffmpeg_static\\ --enable-hdhomerun_static ./Autobuild.sh -t ${TARGET} -j ${PARALLEL} -w ${WORKDIR}", + "support/bintray.py publish filelist.txt" ] }, "xenial-amd64": { @@ -118,7 +122,8 @@ "ccache" ], "buildcmd": [ - "AUTOBUILD_CONFIGURE_EXTRA=--enable-ccache\\ --enable-ffmpeg_static\\ --enable-hdhomerun_static ./Autobuild.sh -t ${TARGET} -j ${PARALLEL} -w ${WORKDIR}" + "AUTOBUILD_CONFIGURE_EXTRA=--enable-ccache\\ --enable-ffmpeg_static\\ --enable-hdhomerun_static ./Autobuild.sh -t ${TARGET} -j ${PARALLEL} -w ${WORKDIR}", + "support/bintray.py publish filelist.txt" ] }, "xenial-i386": { @@ -141,7 +146,8 @@ "ccache" ], "buildcmd": [ - "AUTOBUILD_CONFIGURE_EXTRA=--enable-ccache\\ --enable-ffmpeg_static\\ --enable-hdhomerun_static ./Autobuild.sh -t ${TARGET} -j ${PARALLEL} -w ${WORKDIR}" + "AUTOBUILD_CONFIGURE_EXTRA=--enable-ccache\\ --enable-ffmpeg_static\\ --enable-hdhomerun_static ./Autobuild.sh -t ${TARGET} -j ${PARALLEL} -w ${WORKDIR}", + "support/bintray.py publish filelist.txt" ] }, "jessie-i386": { @@ -165,7 +171,8 @@ "ccache" ], "buildcmd": [ - "AUTOBUILD_CONFIGURE_EXTRA=--enable-ccache\\ --enable-ffmpeg_static\\ --enable-hdhomerun_static ./Autobuild.sh -t ${TARGET} -j ${PARALLEL} -w ${WORKDIR}" + "AUTOBUILD_CONFIGURE_EXTRA=--enable-ccache\\ --enable-ffmpeg_static\\ --enable-hdhomerun_static ./Autobuild.sh -t ${TARGET} -j ${PARALLEL} -w ${WORKDIR}", + "support/bintray.py publish filelist.txt" ] }, "jessie-amd64": { @@ -189,7 +196,8 @@ "ccache" ], "buildcmd": [ - "AUTOBUILD_CONFIGURE_EXTRA=--enable-ccache\\ --enable-ffmpeg_static\\ --enable-hdhomerun_static ./Autobuild.sh -t ${TARGET} -j ${PARALLEL} -w ${WORKDIR}" + "AUTOBUILD_CONFIGURE_EXTRA=--enable-ccache\\ --enable-ffmpeg_static\\ --enable-hdhomerun_static ./Autobuild.sh -t ${TARGET} -j ${PARALLEL} -w ${WORKDIR}", + "support/bintray.py publish filelist.txt" ] }, "xenial-armhf": { @@ -213,7 +221,8 @@ "ccache" ], "buildcmd": [ - "AUTOBUILD_CONFIGURE_EXTRA=--enable-ccache\\ --enable-hdhomerun_static\\ --disable-ffmpeg_static ./Autobuild.sh -t ${TARGET} -j ${PARALLEL} -w ${WORKDIR}" + "AUTOBUILD_CONFIGURE_EXTRA=--enable-ccache\\ --enable-hdhomerun_static\\ --disable-ffmpeg_static ./Autobuild.sh -t ${TARGET} -j ${PARALLEL} -w ${WORKDIR}", + "support/bintray.py publish filelist.txt" ] }, "xenial-arm64": { @@ -237,7 +246,8 @@ "ccache" ], "buildcmd": [ - "AUTOBUILD_CONFIGURE_EXTRA=--enable-ccache\\ --enable-hdhomerun_static\\ --disable-ffmpeg_static ./Autobuild.sh -t ${TARGET} -j ${PARALLEL} -w ${WORKDIR}" + "AUTOBUILD_CONFIGURE_EXTRA=--enable-ccache\\ --enable-hdhomerun_static\\ --disable-ffmpeg_static ./Autobuild.sh -t ${TARGET} -j ${PARALLEL} -w ${WORKDIR}", + "support/bintray.py publish filelist.txt" ] }, "fedora24-x86_64": { @@ -262,7 +272,8 @@ "ccache" ], "buildcmd": [ - "./configure --disable-dvbscan && make -C rpm build-doozer" + "./configure --disable-dvbscan && make -C rpm build-doozer", + "support/bintray.py publish filelist.txt" ] }, "centos7-x86_64": { @@ -287,7 +298,8 @@ "ccache" ], "buildcmd": [ - "./configure --disable-dvbscan && make -C rpm build-doozer" + "./configure --disable-dvbscan && make -C rpm build-doozer", + "support/bintray.py publish filelist.txt" ] } } diff --git a/Autobuild.sh b/Autobuild.sh index 314ac646e..806f6e13f 100755 --- a/Autobuild.sh +++ b/Autobuild.sh @@ -15,6 +15,7 @@ JOBSARGS="" TARGET="debian" RELEASE="--release" WORKINGDIR="/var/tmp/showtime-autobuild" +FILELIST="$PWD/filelist.txt" OP="build" while getopts "vht:e:j:w:o:c:" OPTION do @@ -59,10 +60,12 @@ fi # artifact() { echo "doozer-artifact:$PWD/$1:$2:$3:$4" + echo "$PWD/$1" >> "$FILELIST" } versioned_artifact() { echo "doozer-versioned-artifact:$PWD/$1:$2:$3:$4" + echo "$PWD/$1" >> "$FILELIST" } if [ -f Autobuild/${TARGET}.sh ]; then diff --git a/rpm/Makefile b/rpm/Makefile index f899c5f37..8fb338ff7 100644 --- a/rpm/Makefile +++ b/rpm/Makefile @@ -16,7 +16,9 @@ # along with this program. If not, see . # -include $(dir $(lastword $(MAKEFILE_LIST)))../.config.mk +TOPDIR := $(realpath $(dir $(lastword $(MAKEFILE_LIST)))) + +include $(TOPDIR)/../.config.mk SPEC ?= tvheadend.spec.in @@ -35,6 +37,7 @@ else REF = $(COMMIT1) COMMITR = $(COMMIT1) endif +DEFINES = --define '_topdir $(TOPDIR)' tvheadend.spec: $(SPEC) @sed -e 's/@VERSION@/$(VERSION)/g' \ @@ -43,25 +46,27 @@ tvheadend.spec: $(SPEC) -e 's/@COMMIT@/$(COMMITR)/g' \ $< > $@ -~/rpmbuild/SOURCES: - mkdir -p $(foreach f,BUILD RPMS SOURCES SPECS SRPMS,~/rpmbuild/$(f)) +SOURCES: + mkdir -p $(foreach f,BUILD RPMS SOURCES SPECS SRPMS,$(f)) .PHONY: srpm -srpm: ~/rpmbuild/SOURCES tvheadend.spec - spectool --get-files --sourcedir tvheadend.spec - rpmbuild -bs tvheadend.spec +srpm: SOURCES tvheadend.spec + spectool --get-files -C SOURCES tvheadend.spec + rpmbuild $(DEFINES) -bs tvheadend.spec .PHONY: build -build: ~/rpmbuild/SOURCES tvheadend.spec - spectool --get-files --sourcedir tvheadend.spec - rpmbuild --build-root=/project/repo/rpmbuild -ba tvheadend.spec +build: SOURCES tvheadend.spec + spectool --get-files -C SOURCES tvheadend.spec + rpmbuild $(DEFINES) -ba tvheadend.spec .PHONY: build-doozer build-doozer: build - @for a in /project//repo/rpmbuild/RPMS/$$(uname -m)/tvheadend-$(VERSION)-$(RELEASE)*.rpm \ - /project//repo/rpmbuild/RPMS/$$(uname -m)/tvheadend-debuginfo-$(VERSION)-$(RELEASE)*.rpm; do \ + @rm -f ../filelist.txt + @for a in RPMS/$$(uname -m)/tvheadend-$(VERSION)-$(RELEASE)*.rpm \ + RPMS/$$(uname -m)/tvheadend-debuginfo-$(VERSION)-$(RELEASE)*.rpm; do \ name=$$(basename $$a); \ - echo "doozer-versioned-artifact:$$a:rpm:application/x-rpm:$$name"; \ + echo "doozer-versioned-artifact:rpm/$$a:rpm:application/x-rpm:$$name"; \ + echo "rpm/$$a" >> ../filelist.txt; \ done diff --git a/support/bintray.py b/support/bintray.py index 89c250c2a..a21423613 100755 --- a/support/bintray.py +++ b/support/bintray.py @@ -21,9 +21,15 @@ def env(key): if key in os.environ: return os.environ[key] return None +DEBUG=False + BINTRAY_API='https://bintray.com/api/v1' BINTRAY_USER=env('BINTRAY_USER') BINTRAY_PASS=env('BINTRAY_PASS') +BINTRAY_ORG='tvheadend' +BINTRAY_PACKAGE='tvheadend' + +PACKAGE_DESC='Tvheadend is a TV streaming server and recorder for Linux, FreeBSD and Android' class Response(object): def __init__(self, response): @@ -41,36 +47,169 @@ class Bintray(object): a = '%s:%s' % (BINTRAY_USER, BINTRAY_PASS) self._auth = b'Basic ' + base64.b64encode(a.encode('utf-8')) - def put(self, data, binary=None): + def opener(self): + if DEBUG: + return urllib.build_opener(urllib.HTTPSHandler(debuglevel=1)) + else: + return urllib.build_opener() + + def _push(self, data, binary=None, method='PUT'): content_type = 'application/json' - if binary: content_type = 'application/binary' - opener = urllib.build_opener() + if binary: + content_type = 'application/binary' + else: + data = json.dumps(data) + opener = self.opener() path = self._path if path[0] != '/': path = '/' + path request = urllib.Request(BINTRAY_API + path, data=data) request.add_header('Content-Type', content_type) request.add_header('Authorization', self._auth) - request.get_method = lambda: 'PUT' + request.get_method = lambda: method try: r = Response(opener.open(request)) except urllib.HTTPError as e: r = Response(e) return r + def put(self, data, binary=None): + return self._push(data, binary) + + def post(self, data): + return self._push(data, method='POST') + def error(lvl, msg, *args): sys.stderr.write(msg % args + '\n') sys.exit(lvl) +def info(msg, *args): + print('BINTRAY: ' + msg % args) + def do_upload(*args): - if len(args) < 2: - error(1, 'upload [url] [file]') + if len(args) < 2: error(1, 'upload [url] [file]') bpath, file = args[0], args[1] data = open(file, "br").read() - b = Bintray(bpath) - resp = b.put(data, binary=1) + resp = Bintray(bpath).put(data, binary=1) if resp.code != 200 and resp.code != 201: error(10, 'HTTP ERROR "%s" %s %s' % (resp.url, resp.code, resp.reason)) +def get_ver(version): + if version.find('-'): + version, _ = version.split('-', 1) + try: + major, minor, rest = version.split('.', 2) + except: + major, minor = version.split('.', 1) + rest = '' + return (major, minor, rest) + +def get_path(version): + major, minor, rest = get_ver(version) + if int(major) >= 4 and int(minor) & 1 == 0: + return '%d.%d' % (major, minor) + return 't' + +def get_component(version): + major, minor, rest = get_ver(version) + if int(major) >= 4 and int(minor) & 1 == 0: + if rest.find('~') > 0: + return 'stable' + return 'release' + return 'unstable' + +def get_repo(filename, hint=None): + if hint: return hint + name, ext = os.path.splitext(filename) + if ext == '.deb': + return 'deb' + if ext == '.rpm': + if name.find('.centos') > 0: + return 'centos' + elif name.find('.fc') > 0: + return 'fedora' + elif name.find('.el') > 0: + return 'rhel' + +def get_bintray_params(filename, hint=None): + filename = filename.strip() + basename = os.path.basename(filename) + name, ext = os.path.splitext(basename) + args = type('',(object,),{})() + args.org = BINTRAY_ORG + args.repo = get_repo(basename, hint) + args.package = BINTRAY_PACKAGE + extra = [] + if args.repo == 'deb': + debbase, debarch = name.rsplit('_', 1) + debname, debversion = debbase.split('_', 1) + debversion, debdistro = debversion.rsplit('~', 1) + args.version = debversion + args.path = 'pool/' + get_path(debversion) + '/' + args.package + extra.append('deb_component=' + get_component(debversion)) + extra.append('deb_distribution=' + debdistro) + extra.append('deb_architecture=' + debarch) + else: + rpmbase, rpmarch = name.rsplit('.', 1) + rpmname, rpmversion = rpmbase.rsplit('-', 1) + if rpmversion.find('~') > 0: + rpmname, rpmversion2 = rpmname.rsplit('-', 1) + rpmversion = rpmversion2 + '-' + rpmversion + rpmver1, rpmver2 = rpmversion.split('-', 1) + rpmversion, rpmdist = rpmver2.split('.', 1) + rpmversion = rpmver1 + '-' + rpmversion + args.version = rpmversion + args.path = 'linux/' + get_path(rpmversion) + \ + '/' + rpmdist + '/' + rpmarch + extra = ';'.join(extra) + if extra: extra = ';' + extra + return (basename, args, extra) + +def do_publish(*args): + if len(args) < 1: error(1, 'upload [file with the file list]') + if not DEBUG: + branches = os.popen('git branch --contains HEAD').readlines() + ok = 0 + for b in branches: + if b[0] == '*': + b = b[1:] + b = b.strip() + if b == 'master' or b.startswith('release/'): + ok = 1 + break + if not ok: + info('BINTRAY upload - invalid branches\n%s', branches) + sys.exit(0) + files = open(args[0]).readlines() + args = None + for file in files: + try: + basename, args, extra = get_bintray_params(file) + hint = args.repo + break + except: + pass + bpath = '/packages/tvheadend/%s/tvheadend/versions' % args.repo + data = { 'name': args.version, 'desc': PACKAGE_DESC } + resp = Bintray(bpath).post(data) + if resp.code != 200 and resp.code != 201 and resp.code != 409: + error(10, 'Version %s/%s: HTTP ERROR %s %s' % + (args.repo, args.version, resp.code, resp.reason)) + else: + info('Version %s/%s created' % (args.repo, args.version)) + for file in files: + file = file.strip() + basename, args, extra = get_bintray_params(file, hint) + bpath = '/content/%s/%s/%s/%s/%s/%s%s;publish=1' % \ + (args.org, args.repo, args.package, args.version, + args.path, basename, extra) + data = open(file, 'rb').read() + resp = Bintray(bpath).put(data, binary=1) + if resp.code != 200 and resp.code != 201: + error(10, 'File %s: HTTP ERROR "%s" %s' % + (file, resp.code, resp.reason)) + else: + info('File %s: uploaded', file) + def do_unknown(*args): r = 'Please, specify a valid command:\n' for n in globals(): @@ -79,8 +218,12 @@ def do_unknown(*args): error(1, r[:-1]) def main(argv): + global DEBUG if not BINTRAY_USER or not BINTRAY_PASS: error(2, 'No credentals') + if argv[1] == '--debug': + DEBUG=1 + argv.pop(0) cmd = 'do_' + (len(argv) > 1 and argv[1] or 'unknown') if cmd in globals(): globals()[cmd](*argv[2:])