[#3186] arm packages can be build and uploaded, new systems support added

diff --git a/hammer.py b/hammer.py
index 3a428ebedfb25c92c6065fa149046553b038e713..66a05a02bf8d946373958813c9e0434ba02a6014 100755 (executable)
--- a/hammer.py
+++ b/hammer.py
@@ -63,6 +63,7 @@ SYSTEMS = {
         '36': False,
         '37': True,
         '38': True,
+        '39': True,
     },
     'centos': {
         '7': False,
@@ -97,6 +98,7 @@ SYSTEMS = {
         '12.0': False,
         '12.1': False,
         '13.0': True,
+        '14.0': True,
     },
     'alpine': {
         '3.10': False,
@@ -107,6 +109,8 @@ SYSTEMS = {
         '3.15': False,
         '3.16': True,
         '3.17': True,
+        '3.18': True,
+        '3.19': True,
     },
     'arch': {}
 }
@@ -555,9 +559,9 @@ def get_image_template(key, variant):
 def _get_full_repo_url(repository_url, system, revision, pkg_version):
     if not repository_url:
         return None
-    repo_name = 'kea-%s-%s-%s' % (pkg_version.rsplit('.', 1)[0], system, revision)
+    repo_name = 'kea-%s-%s' % (system, revision)
     repo_url = urljoin(repository_url, 'repository')
-    repo_url += '/%s-ci/' % repo_name
+    repo_url += '/%s/' % repo_name
     return repo_url
 
 
@@ -896,7 +900,9 @@ class VagrantEnv(object):
                 elif self.system == 'alpine':
                     upload_cmd += ' --upload-file %s '
                     file_ext = ''
-                    repo_url = urljoin(repo_url, '%s/v%s/x86_64/' % (pkg_isc_version, self.revision))
+                    _, arch = self.execute('arch', raise_error=False, capture=True)
+                    arch = arch.strip()
+                    repo_url = urljoin(repo_url, f'{pkg_isc_version}/v{self.revision}/{arch}/')
 
                 upload_cmd += ' ' + repo_url
 
@@ -905,7 +911,12 @@ class VagrantEnv(object):
                         continue
                     fp = os.path.join(pkgs_dir, fn)
                     cmd = upload_cmd % fp
-                    execute(cmd)
+                    exit_code, txt = execute(cmd, raise_error=False, capture=True, attempts=3)
+                    log.info('code: %s, txt: %s', exit_code, txt)
+                    # debian doc packages is arch "all" in both x86 and aarch so we want to ignore error
+                    # while second is uploaded
+                    if exit_code != 0 and "Repository does not allow updating assets" in txt:
+                        continue
 
         t1 = time.time()
         dt = int(t1 - t0)
@@ -1459,7 +1470,10 @@ def _change_postgresql_auth_method(connection_type, auth_method, hba_file):
 def _configure_pgsql(system, features, revision):
     """ Configure PostgreSQL DB """
 
-    # execute() calls will set cwd='/tmp' when switching user to postgres to
+    if system == 'freebsd':
+        execute('sudo sysrc postgresql_enable="yes"')
+
+    # Some execute() calls set cwd='/tmp' when switching user to postgres to
     # avoid the error:
     #   could not change as postgres user directory to "/home/jenkins": Permission denied
 
@@ -1472,6 +1486,10 @@ def _configure_pgsql(system, features, revision):
             else:
                 execute('sudo postgresql-setup --initdb --unit postgresql')
     elif system == 'freebsd':
+        # Trying to match e.g. /var/db/postgres/data13
+        if not glob.glob('/var/db/postgres/data*'):
+            execute('sudo service postgresql initdb')
+
         # Stop any hypothetical existing postgres service.
         execute('sudo service postgresql stop || true')
 
@@ -1769,7 +1787,7 @@ def prepare_system_local(features, check_times, ignore_errors_for):
 
         if 'docs' in features:
             packages.extend(['python3-sphinx', 'python3-sphinx-rtd-theme',
-                             'texlive', 'texlive-latex-extra'])
+                             'texlive', 'texlive-latex-extra', 'tex-gyre'])
 
         if 'unittest' in features:
             if revision.startswith('16.'):
@@ -1871,6 +1889,9 @@ def prepare_system_local(features, check_times, ignore_errors_for):
 
         packages = ['autoconf', 'automake', 'libtool', 'openssl', 'log4cplus', 'boost-libs', 'wget']
 
+        if revision.startswith('14'):
+            packages.extend(['bash', 'pkgconf'])
+
         if 'docs' in features:
             # Get the python version from the remote repositories.
             pyv = _get_package_version('python')
@@ -1891,7 +1912,7 @@ def prepare_system_local(features, check_times, ignore_errors_for):
                 packages.extend(['postgresql13-server', 'postgresql13-client'])
 
         if 'gssapi' in features:
-            packages.extend(['krb5', 'krb5-devel'])
+            packages.extend(['krb5-devel'])
             # FreeBSD comes with a Heimdal krb5-config by default. Make sure
             # it's deleted so that Kea uses the MIT packages added just above.
             execute('sudo rm -f /usr/bin/krb5-config')
@@ -2189,6 +2210,7 @@ def _build_rpm(system, revision, features, tarball_path, env, check_times, dry_r
                pkg_version, pkg_isc_version, repo_url):
 
     # unpack kea sources tarball
+    _, arch = execute('arch', capture=True)
     execute('sudo rm -rf kea-src', dry_run=dry_run)
     os.mkdir('kea-src')
     execute('tar -zxf %s' % tarball_path, cwd='kea-src', check_times=check_times, dry_run=dry_run)
@@ -2235,17 +2257,19 @@ def _build_rpm(system, revision, features, tarball_path, env, check_times, dry_r
     if 'install' in features:
         # install packages
         execute('rpm -qa | grep isc-kea | xargs sudo rpm -e', check_times=check_times, dry_run=dry_run, raise_error=False)
-        execute('sudo rpm -i %s/RPMS/x86_64/*rpm' % rpm_root_path, check_times=check_times, dry_run=dry_run)
+        execute(f'sudo rpm -i {rpm_root_path}/RPMS/{arch.strip()}/*rpm', check_times=check_times, dry_run=dry_run)
 
         # check if kea services can be started
         services_list = ['kea-dhcp4.service', 'kea-dhcp6.service', 'kea-dhcp-ddns.service', 'kea-ctrl-agent.service']
         _check_installed_rpm_or_debs(services_list)
 
-    execute('mv %s/RPMS/x86_64/*rpm pkgs' % rpm_root_path, check_times=check_times, dry_run=dry_run)
+    execute(f'mv {rpm_root_path}/RPMS/{arch.strip()}/*rpm pkgs', check_times=check_times, dry_run=dry_run)
 
 
 def _build_deb(system, revision, features, tarball_path, env, check_times, dry_run,
                pkg_version, pkg_isc_version, repository_url, repo_url):
+
+    _, arch = execute('arch', capture=True)
     if system == 'debian' and revision == '9':
         # debian 9 does not support apt-installing over https, so install proper transport
         install_pkgs('apt-transport-https', env=env, check_times=check_times)
@@ -2269,7 +2293,7 @@ def _build_deb(system, revision, features, tarball_path, env, check_times, dry_r
         if 'Bad header data' not in out:
             break
         time.sleep(4)
- 
+
     # unpack tarball
     execute('sudo rm -rf kea-src', check_times=check_times, dry_run=dry_run)
     os.mkdir('kea-src')
@@ -2291,8 +2315,8 @@ def _build_deb(system, revision, features, tarball_path, env, check_times, dry_r
                 execute(cmd, cwd='kea-src/kea-%s/debian' % pkg_version, check_times=check_times, dry_run=dry_run)
 
     # do deb build
-    env['LIBRARY_PATH'] = '/usr/lib/x86_64-linux-gnu'
-    env['LD_LIBRARY_PATH'] = '/usr/lib/x86_64-linux-gnu'
+    env['LIBRARY_PATH'] = f'/usr/lib/{arch.strip()}-linux-gnu'
+    env['LD_LIBRARY_PATH'] = f'/usr/lib/{arch.strip()}-linux-gnu'
     cmd = 'debuild --preserve-envvar=LD_LIBRARY_PATH --preserve-envvar=LIBRARY_PATH --preserve-envvar=CCACHE_DIR --prepend-path=/usr/lib/ccache -i -us -uc -b'
     execute(cmd, env=env, cwd=src_path, timeout=60 * 40, check_times=check_times, dry_run=dry_run)
 
@@ -2306,6 +2330,7 @@ def _build_deb(system, revision, features, tarball_path, env, check_times, dry_r
 
 def _build_alpine_apk(system, revision, features, tarball_path, env, check_times, dry_run,
                       pkg_version, pkg_isc_version, repo_url):
+    _, arch = execute('arch', capture=True)
     # unpack tarball
     execute('sudo rm -rf kea-src packages', check_times=check_times, dry_run=dry_run)
     os.makedirs('kea-src/src')
@@ -2332,8 +2357,8 @@ def _build_alpine_apk(system, revision, features, tarball_path, env, check_times
 
     # copy packages from alpine specific dir with produced pkgs to common place
     alpine_repo_dir = os.path.basename(os.getcwd())
-    src_dir = '~/packages/%s/x86_64' % alpine_repo_dir
-    execute('cp %s/*.apk kea-pkg' % src_dir, check_times=check_times, dry_run=dry_run)
+    src_dir = f'~/packages/{alpine_repo_dir}/{arch.strip()}'
+    execute(f'cp {src_dir}/*.apk kea-pkg', check_times=check_times, dry_run=dry_run)
 
     if 'install' in features:
         # install packages
@@ -2937,7 +2962,9 @@ def upload_to_repo(args, pkgs_dir):
     elif system == 'alpine':
         upload_cmd += ' --upload-file %s '
         file_ext = ''
-        repo_url = urljoin(repo_url, '%s/v%s/x86_64/' % (args.pkg_isc_version, revision))
+        _, arch = execute('arch', raise_error=False, capture=True)
+        arch = arch.strip()
+        repo_url = urljoin(repo_url, f'{args.pkg_isc_version}/v{revision}/{arch}/')
 
     upload_cmd += ' ' + repo_url
 
@@ -2948,16 +2975,21 @@ def upload_to_repo(args, pkgs_dir):
             continue
         fp = os.path.join(pkgs_dir, fn)
         log.info("upload cmd: %s", upload_cmd)
-        log.info("fp: %s", fp)
+        log.info("file path: %s", fp)
         cmd = upload_cmd % fp
 
         attempts=4
         while attempts > 0:
-            exitcode, output = execute(cmd, capture=True)
+            exitcode, output = execute(cmd, capture=True, raise_error=False)
             if exitcode != 0 and '504 Gateway Time-out' in output:
                 log.info('Trying again after 8 seconds...')
                 attempts -= 1
                 time.sleep(8)
+            elif exitcode != 0 and "pository does not allow updating assets" in output:
+                log.info("Asset already exists in the repository. Skipping upload.")
+                break
+            elif exitcode != 0:
+                raise Exception('Upload failed: %s' % output)
             else:
                 break