doozer/bintray: add automatic package publishing, rpm - fix paths for doozer

diff --git a/.doozer.json b/.doozer.json
index e0e115966ce4e4b8852f2ab3b2e2c66ef4483565..bcc4c6259ef335caca8aacd8a3b7ca01f1f48207 100644 (file)
--- a/.doozer.json
+++ b/.doozer.json
@@ -22,7 +22,8 @@
         "ccache"
       ],
       "buildcmd": [
-        "AUTOBUILD_CONFIGURE_EXTRA=--enable-ccache\\ --disable-libx265_static\\ --disable-libx265\\ --enable-hdhomerun_static ./Autobuild.sh -t ${TARGET} -j ${PARALLEL} -w ${WORKDIR}"
+        "AUTOBUILD_CONFIGURE_EXTRA=--enable-ccache\\ --disable-libx265_static\\ --disable-libx265\\ --enable-hdhomerun_static ./Autobuild.sh -t ${TARGET} -j ${PARALLEL} -w ${WORKDIR}",
+        "support/bintray.py publish filelist.txt"
       ]
     },
     "precise-i386": {
@@ -46,7 +47,8 @@
         "ccache"
       ],
       "buildcmd": [
-        "AUTOBUILD_CONFIGURE_EXTRA=--enable-ccache\\ --disable-libx265_static\\ --disable-libx265\\ --enable-hdhomerun_static ./Autobuild.sh -t ${TARGET} -j ${PARALLEL} -w ${WORKDIR}"
+        "AUTOBUILD_CONFIGURE_EXTRA=--enable-ccache\\ --disable-libx265_static\\ --disable-libx265\\ --enable-hdhomerun_static ./Autobuild.sh -t ${TARGET} -j ${PARALLEL} -w ${WORKDIR}",
+        "support/bintray.py publish filelist.txt"
       ]
     },
     "trusty-amd64": {
@@ -70,7 +72,8 @@
         "ccache"
       ],
       "buildcmd": [
-        "AUTOBUILD_CONFIGURE_EXTRA=--enable-ccache\\ --enable-ffmpeg_static\\ --enable-hdhomerun_static ./Autobuild.sh -t ${TARGET} -j ${PARALLEL} -w ${WORKDIR}"
+        "AUTOBUILD_CONFIGURE_EXTRA=--enable-ccache\\ --enable-ffmpeg_static\\ --enable-hdhomerun_static ./Autobuild.sh -t ${TARGET} -j ${PARALLEL} -w ${WORKDIR}",
+        "support/bintray.py publish filelist.txt"
       ]
     },
     "trusty-i386": {
@@ -94,7 +97,8 @@
         "ccache"
       ],
       "buildcmd": [
-        "AUTOBUILD_CONFIGURE_EXTRA=--enable-ccache\\ --enable-ffmpeg_static\\ --enable-hdhomerun_static ./Autobuild.sh -t ${TARGET} -j ${PARALLEL} -w ${WORKDIR}"
+        "AUTOBUILD_CONFIGURE_EXTRA=--enable-ccache\\ --enable-ffmpeg_static\\ --enable-hdhomerun_static ./Autobuild.sh -t ${TARGET} -j ${PARALLEL} -w ${WORKDIR}",
+        "support/bintray.py publish filelist.txt"
       ]
     },
     "xenial-amd64": {
@@ -118,7 +122,8 @@
         "ccache"
       ],
       "buildcmd": [
-        "AUTOBUILD_CONFIGURE_EXTRA=--enable-ccache\\ --enable-ffmpeg_static\\ --enable-hdhomerun_static ./Autobuild.sh -t ${TARGET} -j ${PARALLEL} -w ${WORKDIR}"
+        "AUTOBUILD_CONFIGURE_EXTRA=--enable-ccache\\ --enable-ffmpeg_static\\ --enable-hdhomerun_static ./Autobuild.sh -t ${TARGET} -j ${PARALLEL} -w ${WORKDIR}",
+        "support/bintray.py publish filelist.txt"
       ]
     },
     "xenial-i386": {
@@ -141,7 +146,8 @@
         "ccache"
       ],
       "buildcmd": [
-        "AUTOBUILD_CONFIGURE_EXTRA=--enable-ccache\\ --enable-ffmpeg_static\\ --enable-hdhomerun_static ./Autobuild.sh -t ${TARGET} -j ${PARALLEL} -w ${WORKDIR}"
+        "AUTOBUILD_CONFIGURE_EXTRA=--enable-ccache\\ --enable-ffmpeg_static\\ --enable-hdhomerun_static ./Autobuild.sh -t ${TARGET} -j ${PARALLEL} -w ${WORKDIR}",
+        "support/bintray.py publish filelist.txt"
       ]
     },
     "jessie-i386": {
@@ -165,7 +171,8 @@
         "ccache"
       ],
       "buildcmd": [
-        "AUTOBUILD_CONFIGURE_EXTRA=--enable-ccache\\ --enable-ffmpeg_static\\ --enable-hdhomerun_static ./Autobuild.sh -t ${TARGET} -j ${PARALLEL} -w ${WORKDIR}"
+        "AUTOBUILD_CONFIGURE_EXTRA=--enable-ccache\\ --enable-ffmpeg_static\\ --enable-hdhomerun_static ./Autobuild.sh -t ${TARGET} -j ${PARALLEL} -w ${WORKDIR}",
+        "support/bintray.py publish filelist.txt"
       ]
     },
     "jessie-amd64": {
@@ -189,7 +196,8 @@
         "ccache"
       ],
       "buildcmd": [
-        "AUTOBUILD_CONFIGURE_EXTRA=--enable-ccache\\ --enable-ffmpeg_static\\ --enable-hdhomerun_static ./Autobuild.sh -t ${TARGET} -j ${PARALLEL} -w ${WORKDIR}"
+        "AUTOBUILD_CONFIGURE_EXTRA=--enable-ccache\\ --enable-ffmpeg_static\\ --enable-hdhomerun_static ./Autobuild.sh -t ${TARGET} -j ${PARALLEL} -w ${WORKDIR}",
+        "support/bintray.py publish filelist.txt"
       ]
     },
     "xenial-armhf": {
@@ -213,7 +221,8 @@
         "ccache"
       ],
       "buildcmd": [
-        "AUTOBUILD_CONFIGURE_EXTRA=--enable-ccache\\ --enable-hdhomerun_static\\ --disable-ffmpeg_static ./Autobuild.sh -t ${TARGET} -j ${PARALLEL} -w ${WORKDIR}"
+        "AUTOBUILD_CONFIGURE_EXTRA=--enable-ccache\\ --enable-hdhomerun_static\\ --disable-ffmpeg_static ./Autobuild.sh -t ${TARGET} -j ${PARALLEL} -w ${WORKDIR}",
+        "support/bintray.py publish filelist.txt"
       ]
     },
     "xenial-arm64": {
@@ -237,7 +246,8 @@
         "ccache"
       ],
       "buildcmd": [
-        "AUTOBUILD_CONFIGURE_EXTRA=--enable-ccache\\ --enable-hdhomerun_static\\ --disable-ffmpeg_static ./Autobuild.sh -t ${TARGET} -j ${PARALLEL} -w ${WORKDIR}"
+        "AUTOBUILD_CONFIGURE_EXTRA=--enable-ccache\\ --enable-hdhomerun_static\\ --disable-ffmpeg_static ./Autobuild.sh -t ${TARGET} -j ${PARALLEL} -w ${WORKDIR}",
+        "support/bintray.py publish filelist.txt"
       ]
     },
     "fedora24-x86_64": {
@@ -262,7 +272,8 @@
         "ccache"
       ],
       "buildcmd": [
-        "./configure --disable-dvbscan && make -C rpm build-doozer"
+        "./configure --disable-dvbscan && make -C rpm build-doozer",
+        "support/bintray.py publish filelist.txt"
       ]
     },
     "centos7-x86_64": {
@@ -287,7 +298,8 @@
         "ccache"
       ],
       "buildcmd": [
-        "./configure --disable-dvbscan && make -C rpm build-doozer"
+        "./configure --disable-dvbscan && make -C rpm build-doozer",
+        "support/bintray.py publish filelist.txt"
       ]
     }
   }

diff --git a/Autobuild.sh b/Autobuild.sh
index 314ac646e62e51c012397cf3795d3b30bb92c7ee..806f6e13f17980e6099899e1a41390dd6e73557b 100755 (executable)
--- a/Autobuild.sh
+++ b/Autobuild.sh
@@ -15,6 +15,7 @@ JOBSARGS=""
 TARGET="debian"
 RELEASE="--release"
 WORKINGDIR="/var/tmp/showtime-autobuild"
+FILELIST="$PWD/filelist.txt"
 OP="build"
 while getopts "vht:e:j:w:o:c:" OPTION
 do
@@ -59,10 +60,12 @@ fi
 #
 artifact() {
     echo "doozer-artifact:$PWD/$1:$2:$3:$4"
+    echo "$PWD/$1" >> "$FILELIST"
 }
 
 versioned_artifact() {
     echo "doozer-versioned-artifact:$PWD/$1:$2:$3:$4"
+    echo "$PWD/$1" >> "$FILELIST"
 }
 
 if [ -f Autobuild/${TARGET}.sh ]; then

diff --git a/rpm/Makefile b/rpm/Makefile
index f899c5f377b9818a0cf44556d18cc46a20c30ac4..8fb338ff7b6f631338932220e2734c7b3494dde6 100644 (file)
--- a/rpm/Makefile
+++ b/rpm/Makefile
@@ -16,7 +16,9 @@
 #  along with this program.  If not, see <http://www.gnu.org/licenses/>.
 #
 
-include $(dir $(lastword $(MAKEFILE_LIST)))../.config.mk
+TOPDIR   := $(realpath $(dir $(lastword $(MAKEFILE_LIST))))
+
+include $(TOPDIR)/../.config.mk
 
 SPEC     ?= tvheadend.spec.in
 
@@ -35,6 +37,7 @@ else
 REF       = $(COMMIT1)
 COMMITR   = $(COMMIT1)
 endif
+DEFINES   = --define '_topdir $(TOPDIR)'
 
 tvheadend.spec: $(SPEC)
        @sed -e 's/@VERSION@/$(VERSION)/g' \
@@ -43,25 +46,27 @@ tvheadend.spec: $(SPEC)
             -e 's/@COMMIT@/$(COMMITR)/g' \
                $< > $@
 
-~/rpmbuild/SOURCES:
-       mkdir -p $(foreach f,BUILD RPMS SOURCES SPECS SRPMS,~/rpmbuild/$(f))
+SOURCES:
+       mkdir -p $(foreach f,BUILD RPMS SOURCES SPECS SRPMS,$(f))
 
 .PHONY: srpm
-srpm: ~/rpmbuild/SOURCES tvheadend.spec
-       spectool --get-files --sourcedir tvheadend.spec
-       rpmbuild -bs tvheadend.spec
+srpm: SOURCES tvheadend.spec
+       spectool --get-files -C SOURCES tvheadend.spec
+       rpmbuild $(DEFINES) -bs tvheadend.spec
 
 .PHONY: build
-build: ~/rpmbuild/SOURCES tvheadend.spec
-       spectool --get-files --sourcedir tvheadend.spec
-       rpmbuild --build-root=/project/repo/rpmbuild -ba tvheadend.spec
+build: SOURCES tvheadend.spec
+       spectool --get-files -C SOURCES tvheadend.spec
+       rpmbuild $(DEFINES) -ba tvheadend.spec
 
 .PHONY: build-doozer
 build-doozer: build
-       @for a in /project//repo/rpmbuild/RPMS/$$(uname -m)/tvheadend-$(VERSION)-$(RELEASE)*.rpm \
-                 /project//repo/rpmbuild/RPMS/$$(uname -m)/tvheadend-debuginfo-$(VERSION)-$(RELEASE)*.rpm; do \
+       @rm -f ../filelist.txt
+       @for a in RPMS/$$(uname -m)/tvheadend-$(VERSION)-$(RELEASE)*.rpm \
+                 RPMS/$$(uname -m)/tvheadend-debuginfo-$(VERSION)-$(RELEASE)*.rpm; do \
                name=$$(basename $$a); \
-               echo "doozer-versioned-artifact:$$a:rpm:application/x-rpm:$$name"; \
+               echo "doozer-versioned-artifact:rpm/$$a:rpm:application/x-rpm:$$name"; \
+               echo "rpm/$$a" >> ../filelist.txt; \
         done
        
 

diff --git a/support/bintray.py b/support/bintray.py
index 89c250c2a9c8cd9028dba8210f4957ae149b5897..a214236136d13fccaabcf94601449c982add85eb 100755 (executable)
--- a/support/bintray.py
+++ b/support/bintray.py
@@ -21,9 +21,15 @@ def env(key):
     if key in os.environ: return os.environ[key]
     return None
 
+DEBUG=False
+
 BINTRAY_API='https://bintray.com/api/v1'
 BINTRAY_USER=env('BINTRAY_USER')
 BINTRAY_PASS=env('BINTRAY_PASS')
+BINTRAY_ORG='tvheadend'
+BINTRAY_PACKAGE='tvheadend'
+
+PACKAGE_DESC='Tvheadend is a TV streaming server and recorder for Linux, FreeBSD and Android'
 
 class Response(object):
     def __init__(self, response):
@@ -41,36 +47,169 @@ class Bintray(object):
         a = '%s:%s' % (BINTRAY_USER, BINTRAY_PASS)
         self._auth = b'Basic ' + base64.b64encode(a.encode('utf-8'))
 
-    def put(self, data, binary=None):
+    def opener(self):
+        if DEBUG:
+            return urllib.build_opener(urllib.HTTPSHandler(debuglevel=1))
+        else:
+            return urllib.build_opener()
+
+    def _push(self, data, binary=None, method='PUT'):
         content_type = 'application/json'
-        if binary: content_type = 'application/binary'
-        opener = urllib.build_opener()
+        if binary:
+          content_type = 'application/binary'
+        else:
+          data = json.dumps(data)
+        opener = self.opener()
         path = self._path
         if path[0] != '/': path = '/' + path
         request = urllib.Request(BINTRAY_API + path, data=data)
         request.add_header('Content-Type', content_type)
         request.add_header('Authorization', self._auth)
-        request.get_method = lambda: 'PUT'
+        request.get_method = lambda: method
         try:
             r = Response(opener.open(request))
         except urllib.HTTPError as e:
             r = Response(e)
         return r
 
+    def put(self, data, binary=None):
+        return self._push(data, binary)
+    
+    def post(self, data):
+        return self._push(data, method='POST')
+
 def error(lvl, msg, *args):
     sys.stderr.write(msg % args + '\n')
     sys.exit(lvl)
 
+def info(msg, *args):
+    print('BINTRAY: ' + msg % args)
+
 def do_upload(*args):
-    if len(args) < 2:
-        error(1, 'upload [url] [file]')
+    if len(args) < 2: error(1, 'upload [url] [file]')
     bpath, file = args[0], args[1]
     data = open(file, "br").read()
-    b = Bintray(bpath)
-    resp = b.put(data, binary=1)
+    resp = Bintray(bpath).put(data, binary=1)
     if resp.code != 200 and resp.code != 201:
         error(10, 'HTTP ERROR "%s" %s %s' % (resp.url, resp.code, resp.reason))
 
+def get_ver(version):
+    if version.find('-'):
+        version, _ = version.split('-', 1)
+    try:
+      major, minor, rest = version.split('.', 2)
+    except:
+      major, minor = version.split('.', 1)
+      rest = ''
+    return (major, minor, rest)
+
+def get_path(version):
+    major, minor, rest = get_ver(version)
+    if int(major) >= 4 and int(minor) & 1 == 0:
+        return '%d.%d' % (major, minor)
+    return 't'
+
+def get_component(version):
+    major, minor, rest = get_ver(version)
+    if int(major) >= 4 and int(minor) & 1 == 0:
+        if rest.find('~') > 0:
+            return 'stable'
+        return 'release'
+    return 'unstable'
+
+def get_repo(filename, hint=None):
+    if hint: return hint
+    name, ext = os.path.splitext(filename)
+    if ext == '.deb':
+        return 'deb'
+    if ext == '.rpm':
+        if name.find('.centos') > 0:
+            return 'centos'
+        elif name.find('.fc') > 0:
+            return 'fedora'
+        elif name.find('.el') > 0:
+            return 'rhel'
+
+def get_bintray_params(filename, hint=None):
+    filename = filename.strip()
+    basename = os.path.basename(filename)
+    name, ext = os.path.splitext(basename)
+    args = type('',(object,),{})()
+    args.org = BINTRAY_ORG
+    args.repo = get_repo(basename, hint)
+    args.package = BINTRAY_PACKAGE
+    extra = []
+    if args.repo == 'deb':
+        debbase, debarch = name.rsplit('_', 1)
+        debname, debversion = debbase.split('_', 1)
+        debversion, debdistro = debversion.rsplit('~', 1)
+        args.version = debversion
+        args.path = 'pool/' + get_path(debversion) + '/' + args.package
+        extra.append('deb_component=' + get_component(debversion))
+        extra.append('deb_distribution=' + debdistro)
+        extra.append('deb_architecture=' + debarch)
+    else:
+        rpmbase, rpmarch = name.rsplit('.', 1)
+        rpmname, rpmversion = rpmbase.rsplit('-', 1)
+        if rpmversion.find('~') > 0:
+            rpmname, rpmversion2 = rpmname.rsplit('-', 1)
+            rpmversion = rpmversion2 + '-' + rpmversion
+        rpmver1, rpmver2 = rpmversion.split('-', 1)
+        rpmversion, rpmdist = rpmver2.split('.', 1)
+        rpmversion = rpmver1 + '-' + rpmversion
+        args.version = rpmversion
+        args.path = 'linux/' + get_path(rpmversion) + \
+                    '/' + rpmdist + '/' + rpmarch
+    extra = ';'.join(extra)
+    if extra: extra = ';' + extra
+    return (basename, args, extra)
+
+def do_publish(*args):
+    if len(args) < 1: error(1, 'upload [file with the file list]')
+    if not DEBUG:
+        branches = os.popen('git branch --contains HEAD').readlines()
+        ok = 0
+        for b in branches:
+            if b[0] == '*':
+                b = b[1:]
+            b = b.strip()
+            if b == 'master' or b.startswith('release/'):
+                ok = 1
+                break
+        if not ok:
+            info('BINTRAY upload - invalid branches\n%s', branches)
+            sys.exit(0)
+    files = open(args[0]).readlines()
+    args = None
+    for file in files:
+        try:
+            basename, args, extra = get_bintray_params(file)
+            hint = args.repo
+            break
+        except:
+            pass
+    bpath = '/packages/tvheadend/%s/tvheadend/versions' % args.repo
+    data = { 'name': args.version, 'desc': PACKAGE_DESC }
+    resp = Bintray(bpath).post(data)
+    if resp.code != 200 and resp.code != 201 and resp.code != 409:
+        error(10, 'Version %s/%s: HTTP ERROR %s %s' %
+                  (args.repo, args.version, resp.code, resp.reason))
+    else:
+        info('Version %s/%s created' % (args.repo, args.version))
+    for file in files:
+        file = file.strip()
+        basename, args, extra = get_bintray_params(file, hint)
+        bpath = '/content/%s/%s/%s/%s/%s/%s%s;publish=1' % \
+                (args.org, args.repo, args.package, args.version,
+                 args.path, basename, extra)
+        data = open(file, 'rb').read()
+        resp = Bintray(bpath).put(data, binary=1)
+        if resp.code != 200 and resp.code != 201:
+            error(10, 'File %s: HTTP ERROR "%s" %s' %
+                      (file, resp.code, resp.reason))
+        else:
+            info('File %s: uploaded', file)
+
 def do_unknown(*args):
     r = 'Please, specify a valid command:\n'
     for n in globals():
@@ -79,8 +218,12 @@ def do_unknown(*args):
     error(1, r[:-1])
 
 def main(argv):
+    global DEBUG
     if not BINTRAY_USER or not BINTRAY_PASS:
         error(2, 'No credentals')
+    if argv[1] == '--debug':
+        DEBUG=1
+        argv.pop(0)
     cmd = 'do_' + (len(argv) > 1 and argv[1] or 'unknown')
     if cmd in globals():
         globals()[cmd](*argv[2:])