github: Use separate caches for custom-built dependencies

These are shared by many tests, in particular the "all", "coverage",
"no-dbg" and "no-testable-ke" tests, which each would otherwise require
their own large cache.

Similarly, the "codeql" and "sonarcloud" tests rely on the same
dependencies but only the latter uses ccache for the strongSwan build.

Also reduce the maximum size per cache for all workflows to keep them
in check over time (some could even be set lower, we'll have to see
how this develops).

diff --git a/.github/workflows/android.yml b/.github/workflows/android.yml
index 4dff2256ccd51ab9a14eec47f03548c68e0ffc8e..a29c8da7bbae5998734bf3b7cfa524733d6be00f 100644 (file)
--- a/.github/workflows/android.yml
+++ b/.github/workflows/android.yml
@@ -12,7 +12,7 @@ permissions:
 env:
   CCACHE_BASEDIR: ${{ github.workspace }}
   CCACHE_COMPRESS: true
-  CCACHE_MAXSIZE: 400M
+  CCACHE_MAXSIZE: 150M
   CC: gcc
   OS_NAME: linux
 

diff --git a/.github/workflows/codeql.yml b/.github/workflows/codeql.yml
index 5e3b39a3e28d82d6f8ab8d023b9cc419d368d3da..b01a9e2c95781f8e858ed77463e17250e33bbc97 100644 (file)
--- a/.github/workflows/codeql.yml
+++ b/.github/workflows/codeql.yml
@@ -7,6 +7,7 @@ concurrency:
   cancel-in-progress: true
 
 env:
+  CCACHE_BASEDIR: ${{ github.workspace }}
   OS_NAME: linux
 
 jobs:
@@ -30,7 +31,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        language: [ 'cpp', 'python', 'ruby' ]
+        language: [ 'python', 'ruby' ]
     steps:
     - uses: actions/checkout@v6
     - name: Initialize CodeQL
@@ -40,16 +41,47 @@ jobs:
         config-file: ./.github/codeql/config.yml
         trap-caching: false
 
-    - if: matrix.language == 'python' || matrix.language == 'ruby'
-      name: Autobuild
+    - name: Autobuild
       uses: github/codeql-action/autobuild@v4
 
-    - if: matrix.language == 'cpp'
-      env:
-        TEST: codeql
-      uses: ./.github/actions/default
-
     - name: Perform CodeQL Analysis
       uses: github/codeql-action/analyze@v4
       with:
         category: "/language:${{matrix.language}}"
+
+  analyze-cpp:
+    needs: pre-check
+    if: ${{ needs.pre-check.outputs.should_skip != 'true' }}
+    runs-on: ubuntu-latest
+    permissions:
+      actions: write
+      security-events: write
+    env:
+      TEST: codeql
+    steps:
+    - uses: actions/checkout@v6
+    - run: ./scripts/test.sh deps
+    - uses: actions/cache/restore@v5
+      with:
+        path: ~/.cache/ccache
+        key: ccache-deps-ubuntu-latest-gcc-all
+    - run: |
+        sudo apt-get install -qq ccache
+        echo "OLD_PATH=$PATH" >> $GITHUB_ENV
+        echo "PATH=/usr/lib/ccache:$PATH" >> $GITHUB_ENV
+        ccache -z
+    - run: ./scripts/test.sh build-deps
+    - run: ccache -sv
+    - run: echo "PATH=$OLD_PATH" >> $GITHUB_ENV
+
+    - name: Initialize CodeQL
+      uses: github/codeql-action/init@v4
+      with:
+        languages: cpp
+        config-file: ./.github/codeql/config.yml
+        trap-caching: false
+    - run: ./scripts/test.sh
+    - name: Perform CodeQL Analysis
+      uses: github/codeql-action/analyze@v4
+      with:
+        category: "/language:cpp"

diff --git a/.github/workflows/linux.yml b/.github/workflows/linux.yml
index 0c0bd90e737707a7aa9afb5347ea972a78a7dff8..2a21f0c4e05f97f2a3ce544670ac7f583c536364 100644 (file)
--- a/.github/workflows/linux.yml
+++ b/.github/workflows/linux.yml
@@ -16,7 +16,7 @@ env:
   TESTS_REDUCED_KEYLENGTHS: yes
   CCACHE_BASEDIR: ${{ github.workspace }}
   CCACHE_COMPRESS: true
-  CCACHE_MAXSIZE: 200M
+  CCACHE_MAXSIZE: 100M
   OS_NAME: linux
 
 jobs:
@@ -67,18 +67,21 @@ jobs:
       TEST: ${{ matrix.test }}
       # as several jobs use the same key, make sure we only store the cache for
       # one specific config in case there is a race
+      STORE_DEPS_CACHE: >-
+        ${{
+          github.event_name == 'push' &&
+          matrix.test == 'all' &&
+          matrix.monolithic == 'no'
+        }}
       STORE_CACHE: >-
         ${{
           github.event_name == 'push' &&
           !contains(fromJSON('["apidoc"]'), matrix.test) &&
           (!contains(fromJSON('["all", "default", "printf-builtin"]'),
-                     matrix.test) ||
-           ((!matrix.leak-detective || matrix.leak-detective == 'no') &&
-            matrix.monolithic == 'no'))
+                     matrix.test) || matrix.monolithic == 'no')
         }}
-      # with regards to ccache, monolithic builds don't differ from regular
-      # builds; but some tests build different dependencies or use different
-      # compiler flags, so we use different caches for these
+      # with regards to ccache, monolithic builds don't differ from regular builds.
+      # but most tests use different compiler flags, so we use separate caches
       CACHE_KEY: >-
         ${{ case(contains(fromJSON('["apidoc"]'), matrix.test),
               'ccache-ubuntu-latest-gcc-default',
@@ -86,16 +89,37 @@ jobs:
                      matrix.test)) }}
     steps:
       - uses: actions/checkout@v6
+      - run: ./scripts/test.sh deps
       - uses: actions/cache/restore@v5
-        id: cache-restore
+        id: deps-cache-restore
         with:
           path: ~/.cache/ccache
-          key: ${{ env.CACHE_KEY }}
+          key: ccache-deps-ubuntu-latest-${{ env.CC }}-all
       - run: |
           sudo apt-get install -qq ccache
           echo "PATH=/usr/lib/ccache:$PATH" >> $GITHUB_ENV
           ccache -z
-      - uses: ./.github/actions/default
+      - run: ./scripts/test.sh build-deps
+      - run: ccache -sv
+      # delete old cache entry as we currently can't update it any other way
+      - env:
+          GH_TOKEN: ${{ github.token }}
+        if: steps.deps-cache-restore.outputs.cache-hit && fromJSON(env.STORE_DEPS_CACHE)
+        continue-on-error: true
+        run: gh cache delete -r ${{ github.ref }} ${{ steps.deps-cache-restore.outputs.cache-primary-key }}
+      - if: fromJSON(env.STORE_DEPS_CACHE)
+        uses: actions/cache/save@v5
+        with:
+          path: ~/.cache/ccache
+          key: ${{ steps.deps-cache-restore.outputs.cache-primary-key }}
+      - run: rm -rf ~/.cache/ccache
+      - uses: actions/cache/restore@v5
+        id: cache-restore
+        with:
+          path: ~/.cache/ccache
+          key: ${{ env.CACHE_KEY }}
+      - run: ccache -z
+      - run: ./scripts/test.sh
       - run: ccache -sv
       # delete old cache entry as we currently can't update it any other way
       - env:
@@ -151,29 +175,58 @@ jobs:
       ACTIVE_TRANSFORMS_REF: .github/active-transforms/${{ matrix.test }}
       # only store a cache for some tests as the others have a minimal diff (if
       # any) compared to the 'all' build
-      STORE_CACHE: >-
-        ${{ github.event_name == 'push' &&
-            contains(fromJSON('["openssl-4", "openssl-awslc"]'), matrix.test) &&
-            (!matrix.leak-detective || matrix.leak-detective == 'no') }}
+      STORE_DEPS_CACHE: >-
+        ${{
+          github.event_name == 'push' &&
+          contains(fromJSON('["openssl-4", "openssl-awslc"]'), matrix.test)
+        }}
+      DEPS_CACHE_KEY: >-
+        ${{
+          case(contains(fromJSON('["openssl-4", "openssl-awslc"]'), matrix.test),
+            format('ccache-deps-{0}-gcc-{1}', matrix.os, matrix.test),
+            format('ccache-deps-{0}-gcc-all', matrix.os))
+        }}
       CACHE_KEY: >-
-        ${{ case(contains(fromJSON('["openssl-4", "openssl-awslc"]'), matrix.test),
-              format('ccache-{0}-gcc-{1}', matrix.os, matrix.test),
-              matrix.os == 'ubuntu-latest' && matrix.test == 'openssl-sys',
-              format('ccache-{0}-gcc-default', matrix.os),
-              format('ccache-{0}-gcc-all', matrix.os)) }}
+        ${{
+          case(matrix.os == 'ubuntu-latest' && matrix.test == 'openssl-sys',
+            format('ccache-{0}-gcc-default', matrix.os),
+            format('ccache-{0}-gcc-all', matrix.os))
+        }}
     steps:
       - uses: actions/checkout@v6
+      - run: ./scripts/test.sh deps
       - uses: actions/cache/restore@v5
-        id: cache-restore
+        id: deps-cache-restore
         with:
           path: ~/.cache/ccache
-          key: ${{ env.CACHE_KEY }}
+          key: ${{ env.DEPS_CACHE_KEY }}
       - run: |
           sudo apt-get install -qq ccache
           echo "PATH=/usr/lib/ccache:$PATH" >> $GITHUB_ENV
           ccache -z
           echo "TESTS_ACTIVE_TRANSFORMS=$HOME/active-transforms.log" >> $GITHUB_ENV
-      - uses: ./.github/actions/default
+      - run: ./scripts/test.sh build-deps
+      - run: ccache -sv
+      # delete old cache entry as we currently can't update it any other way
+      - env:
+          GH_TOKEN: ${{ github.token }}
+        if: steps.deps-cache-restore.outputs.cache-hit && fromJSON(env.STORE_DEPS_CACHE)
+        continue-on-error: true
+        run: gh cache delete -r ${{ github.ref }} ${{ steps.deps-cache-restore.outputs.cache-primary-key }}
+      - if: fromJSON(env.STORE_DEPS_CACHE)
+        uses: actions/cache/save@v5
+        with:
+          path: ~/.cache/ccache
+          key: ${{ steps.deps-cache-restore.outputs.cache-primary-key }}
+      - run: rm -rf ~/.cache/ccache
+      - uses: actions/cache/restore@v5
+        id: cache-restore
+        with:
+          path: ~/.cache/ccache
+          key: ${{ env.CACHE_KEY }}
+      - run: ccache -z
+      - run: ./scripts/test.sh
+      - run: ccache -sv
       - name: Upload active transforms
         uses: actions/upload-artifact@v6
         with:
@@ -184,17 +237,6 @@ jobs:
         run: |
           test ! -f $ACTIVE_TRANSFORMS_REF || diff -us --color=always $ACTIVE_TRANSFORMS_REF $TESTS_ACTIVE_TRANSFORMS
       - run: ccache -sv
-      # delete old cache entry as we currently can't update it any other way
-      - env:
-          GH_TOKEN: ${{ github.token }}
-        if: steps.cache-restore.outputs.cache-hit && fromJSON(env.STORE_CACHE)
-        continue-on-error: true
-        run: gh cache delete -r ${{ github.ref }} ${{ steps.cache-restore.outputs.cache-primary-key }}
-      - if: fromJSON(env.STORE_CACHE)
-        uses: actions/cache/save@v5
-        with:
-          path: ~/.cache/ccache
-          key: ${{ steps.cache-restore.outputs.cache-primary-key }}
       - if: ${{ failure() }}
         uses: actions/upload-artifact@v6
         with:
@@ -218,18 +260,44 @@ jobs:
       LEAK_DETECTIVE: ${{ matrix.leak-detective || 'no' }}
       CC: ${{ matrix.compiler || 'gcc' }}
       TEST: ${{ matrix.test }}
+      STORE_DEPS_CACHE: >-
+        ${{
+          github.event_name == 'push' &&
+          matrix.test == 'all'
+        }}
     steps:
       - uses: actions/checkout@v6
+      - run: ./scripts/test.sh deps
       - uses: actions/cache/restore@v5
-        id: cache-restore
+        id: deps-cache-restore
         with:
           path: ~/.cache/ccache
-          key: ccache-${{ matrix.os }}-${{ env.CC }}-${{ matrix.test }}
+          key: ccache-deps-${{ matrix.os }}-${{ env.CC }}-all
       - run: |
           sudo apt-get install -qq ccache
           echo "PATH=/usr/lib/ccache:$PATH" >> $GITHUB_ENV
           ccache -z
-      - uses: ./.github/actions/default
+      - run: ./scripts/test.sh build-deps
+      - run: ccache -sv
+      # delete old cache entry as we currently can't update it any other way
+      - env:
+          GH_TOKEN: ${{ github.token }}
+        if: steps.deps-cache-restore.outputs.cache-hit && fromJSON(env.STORE_DEPS_CACHE)
+        continue-on-error: true
+        run: gh cache delete -r ${{ github.ref }} ${{ steps.deps-cache-restore.outputs.cache-primary-key }}
+      - if: fromJSON(env.STORE_DEPS_CACHE)
+        uses: actions/cache/save@v5
+        with:
+          path: ~/.cache/ccache
+          key: ${{ steps.deps-cache-restore.outputs.cache-primary-key }}
+      - run: rm -rf ~/.cache/ccache
+      - uses: actions/cache/restore@v5
+        id: cache-restore
+        with:
+          path: ~/.cache/ccache
+          key: ccache-${{ matrix.os }}-${{ env.CC }}-${{ matrix.test }}
+      - run: ccache -z
+      - run: ./scripts/test.sh
       - run: ccache -sv
       # delete old cache entry as we currently can't update it any other way
       - env:

diff --git a/.github/workflows/sonarcloud.yml b/.github/workflows/sonarcloud.yml
index 9b8cb268881e0e70695ae7662bdffed8a1e8e7e5..d6c02698be4be41e2da1727272743e2dd1308e88 100644 (file)
--- a/.github/workflows/sonarcloud.yml
+++ b/.github/workflows/sonarcloud.yml
@@ -12,7 +12,7 @@ permissions:
 env:
   CCACHE_BASEDIR: ${{ github.workspace }}
   CCACHE_COMPRESS: true
-  CCACHE_MAXSIZE: 200M
+  CCACHE_MAXSIZE: 100M
   OS_NAME: linux
 
 jobs:
@@ -36,19 +36,28 @@ jobs:
       - uses: actions/checkout@v6
         with:
           fetch-depth: 0
+      - run: ./scripts/test.sh deps
       - uses: actions/cache/restore@v5
-        id: cache-restore
         with:
           path: ~/.cache/ccache
-          key: ccache-sonarcloud
+          key: ccache-deps-ubuntu-latest-gcc-all
       - run: |
           sudo apt-get install -qq ccache
           echo "PATH=/usr/lib/ccache:$PATH" >> $GITHUB_ENV
           ccache -z
+      - run: ./scripts/test.sh build-deps
+      - run: ccache -sv
+      - run: rm -rf ~/.cache/ccache
+      - uses: actions/cache/restore@v5
+        id: cache-restore
+        with:
+          path: ~/.cache/ccache
+          key: ccache-sonarcloud
+      - run: ccache -z
       - uses: SonarSource/sonarqube-scan-action/install-build-wrapper@v7.1.0
       - run: |
           echo "BUILD_WRAPPER_OUT_DIR=$HOME/bw-output" >> $GITHUB_ENV
-      - uses: ./.github/actions/default
+      - run: ./scripts/test.sh
       - uses: SonarSource/sonarqube-scan-action@v7.1.0
         env:
           SONAR_TOKEN: ${{ secrets.SONAR_TOKEN }}

diff --git a/.github/workflows/tkm.yml b/.github/workflows/tkm.yml
index 9d10376a61b1f2e02e19008f000c375c431bb6df..7eb2d6b1a05fed65c1c658bd523fa09f696edb63 100644 (file)
--- a/.github/workflows/tkm.yml
+++ b/.github/workflows/tkm.yml
@@ -14,7 +14,7 @@ env:
   CCACHE_CONTAINER: /root/.ccache
   CCACHE_COMPILERCHECK: content
   CCACHE_COMPRESS: true
-  CCACHE_MAXSIZE: 200M
+  CCACHE_MAXSIZE: 100M
 
 jobs:
   pre-check:

diff --git a/.github/workflows/windows.yml b/.github/workflows/windows.yml
index e8cb2086155e7d60e1bcadfeae03a67e2e6817e6..177ce7c8a842e2ef8e4fa12df63f4c28ae1ecf84 100644 (file)
--- a/.github/workflows/windows.yml
+++ b/.github/workflows/windows.yml
@@ -12,7 +12,7 @@ permissions:
 env:
   TESTS_REDUCED_KEYLENGTHS: yes
   CCACHE_COMPRESS: true
-  CCACHE_MAXSIZE: 200M
+  CCACHE_MAXSIZE: 100M
   # since the compilers are newly installed every time, we have to use this to
   # avoid cache misses
   CCACHE_COMPILERCHECK: content