gh actions build-packages: avoid duplicates download-artifacts action. Fix publications

diff --git a/.github/workflows/build-packages.yml b/.github/workflows/build-packages.yml
index 25cef5ce9e62a0b98103942cc6220e0f98fe35ea..06cd89aed425342d6b2233aebc6002175c94dd85 100644 (file)
--- a/.github/workflows/build-packages.yml
+++ b/.github/workflows/build-packages.yml
@@ -343,7 +343,7 @@ jobs:
       - name: Download packages
         uses: actions/download-artifact@v4
         with:
-          pattern: "${{ inputs.product }}-${{ matrix.os }}-*-${{ matrix.architecture }}"
+          pattern: "${{ inputs.product }}-${{ matrix.os }}-${{ needs.build.outputs.version }}-${{ matrix.architecture }}"
       - name: Normalize package name
         id: normalize-name
         run: |
@@ -388,6 +388,7 @@ jobs:
   upload-src-files:
     needs: [prepare, build, provenance-src, provenance-pkgs]
     continue-on-error: true
+    if: ${{ needs.prepare.outputs.publish-packages == 'yes' }}
     name: Upload source and other files
     runs-on: ubuntu-24.04
     strategy:
@@ -413,7 +414,7 @@ jobs:
       - name: Download packages
         uses: actions/download-artifact@v4
         with:
-          pattern: "${{ inputs.product }}-${{ matrix.os }}-*-${{ matrix.architecture }}"
+          pattern: "${{ inputs.product }}-${{ matrix.os }}-${{ needs.build.outputs.version }}-${{ matrix.architecture }}"
       - name: Normalize package name
         id: normalize-name
         run: |
@@ -446,6 +447,7 @@ jobs:
     needs: [build, provenance-src, provenance-pkgs]
     name: Upload provenance files
     continue-on-error: true
+    if: ${{ needs.prepare.outputs.publish-packages == 'yes' }}
     runs-on: ubuntu-24.04
     steps:
       - uses: actions/checkout@v4
@@ -465,7 +467,7 @@ jobs:
       - name: Download provenance files
         uses: actions/download-artifact@v4
         with:
-          pattern: "${{ inputs.product }}-*.intoto.jsonl"
+          pattern: "${{ inputs.product }}-${{ needs.build.outputs.version }}-*.intoto.jsonl"
       - name: Normalize package name
         id: normalize-name
         run: |

diff --git a/tasks.py b/tasks.py
index 85b25fd74d5e793375195e03777db121985cec98..77ee5dc9dccd8a2149e9ba5f57be097ebca5a461 100644 (file)
--- a/tasks.py
+++ b/tasks.py
@@ -1285,12 +1285,16 @@ def pulp_upload_file_packages_by_folder(c, source):
     for root, dirs, files in os.walk(source):
         for path in files:
             file = os.path.join(root, path).split('/',1)[1]
-            # file repositories have been configured with autopublish set to true
-            cmd = f'file content upload --repository {repo_name} --file {source}/{file} --relative-path {file}'
+            # First upload file as an artifact
+            cmd = f"artifact upload --file {source}/{file} --chunk-size 500MB | jq -r '.sha256' | tr -d '\n'"
+            artifact_sha256 = run_pulp_cmd(c, cmd)
+            # Then create the content of type file
+            cmd = f'file content create --repository {repo_name} --relative-path {file} --sha256 {artifact_sha256}'
             run_pulp_cmd(c, cmd)
 
 @task
 def pulp_create_rpm_publication(c, product, list_os_rel, list_arch):
+    max_push_attempts = 3
     rpm_distros = ["centos", "el"]
     for os_rel in json.loads(list_os_rel):
         if not "el-" in os_rel:
@@ -1299,16 +1303,37 @@ def pulp_create_rpm_publication(c, product, list_os_rel, list_arch):
         for arch in json.loads(list_arch):
             for distro in rpm_distros:
                 repo_name = f"repo-{distro}-{release}-{arch}-{product}"
-                cmd = f'rpm publication create --repository {repo_name} --checksum-type sha256'
-                run_pulp_cmd(c, cmd)
+                attempts = 0
+                while attempts < max_push_attempts:
+                    try:
+                        cmd = f'rpm publication create --repository {repo_name} --checksum-type sha256'
+                        run_pulp_cmd(c, cmd)
+                        break
+                    except UnexpectedExit:
+                        attempts += 1
+                        time.sleep(5)
+                        print(f'Next attempt: {attempts}')
+                        if attempts == max_push_attempts:
+                            raise Failure(f'Error creating rpm publication')
 
 @task
 def pulp_create_deb_publication(c):
+    max_push_attempts = 3
     deb_distros = ["debian", "ubuntu"]
     for distro in deb_distros:
         repo_name = f"repo-{distro}"
-        cmd = f'deb publication create --repository {repo_name}'
-        run_pulp_cmd(c, cmd)
+        attempts = 0
+        while attempts < max_push_attempts:
+            try:
+                cmd = f'deb publication create --repository {repo_name}'
+                run_pulp_cmd(c, cmd)
+                break
+            except UnexpectedExit:
+                attempts += 1
+                time.sleep(5)
+                print(f'Next attempt: {attempts}')
+                if attempts == max_push_attempts:
+                    raise Failure(f'Error creating deb publication')
 
 @task
 def pulp_upload_rpm_packages_by_folder(c, source, product):
@@ -1363,7 +1388,7 @@ def pulp_upload_deb_packages_by_folder(c, source, product):
         for root, dirs, files in os.walk(source):
             for path in files:
                 file = os.path.join(root, path).split('/',1)[1]
-                cmd = f"artifact upload --file {source}/{file} | jq -r '.pulp_href' | tr -d '\n'"
+                cmd = f"artifact upload --file {source}/{file} --chunk-size 500MB | jq -r '.pulp_href' | tr -d '\n'"
                 artifact_href = run_pulp_cmd(c, cmd)
 
                 package_data = {