From: Francesco Chemolli <5175948+kinkie@users.noreply.github.com> Date: Fri, 25 Oct 2024 17:36:26 +0000 (+0000) Subject: Use GitHub Actions for more OS-specific build tests (#1917) X-Git-Tag: SQUID_7_0_1~50 X-Git-Url: http://git.ipfire.org/?a=commitdiff_plain;h=240efcbb6656f89edb88811daeda71af4062c6af;p=thirdparty%2Fsquid.git Use GitHub Actions for more OS-specific build tests (#1917) Test staged commits using some of our docker images from Jenkins tests. The added tests (see staged.yaml) were easier to adopt; we are also working on enabling GitHub Actions for FreeBSD and some other images. We do not run these added tests for PR commits because existing Ubuntu tests (defined in pr.yaml) already expose the vast majority of build problems, and we are worried that running a lot more tests for each PR push event would consume too much GitHub resources and significantly increase noise in PR checks summary, obscuring often-easier-to-handle failures detected by Ubuntu tests. Also postpone MacOS tests until PR staging. On GitHub, MacOS runners are x10 more expensive than Linux runners. We use cheaper runners for fast feedback while still checking MacOS build before each merged commit. This change does not increase GitHub CI wait time for PR push tests because those checks are dominated by the unchanged 35min CodeQL-tests job. However, it reduces total CPU time used (from 2h 30m to 2h) because we no longer perform 3 MacOS tests for PR push events. This change doubles GitHub CI wait time for staged commit tests (from 35m to 1h) and drastically increases total CPU time used (from 2h to 17h), primarily due to 84 added docker-based linux-distros checks. GitHub does not provide any riscv64 runners and free Linux/arm64 runners. Our initial attempts to virtualize Linux/arm64 tests on MacOS/arm64 runners and to make virtualized FreeBSD and OpenBSD tests work on Linux/x64 were not successful. Thus, we still rely on Jenkins for Linux/riscv64, Linux/arm64, FreeBSD/x64, and OpenBSD/x64 tests. --- diff --git a/.github/workflows/default.yaml b/.github/workflows/quick.yaml similarity index 65% rename from .github/workflows/default.yaml rename to .github/workflows/quick.yaml index 0204f95a2c..05f91dbbde 100644 --- a/.github/workflows/default.yaml +++ b/.github/workflows/quick.yaml @@ -1,8 +1,15 @@ -name: GitHub CI +# The jobs below are executed on every PR branch push (among other +# events), quickly and cheaply detecting many common PR problems. +# +# Anubis configuration must be kept in sync with the total number of +# tests executed for "auto" branch commits, including tests in other +# GitHub workflow file(s). + +name: PR commit on: push: - # test this branch and staged PRs based on this branch code + # test commits on this branch and staged commits branches: [ "master", "auto" ] pull_request: @@ -96,7 +103,6 @@ jobs: matrix: os: - ubuntu-22.04 - - macos-14 compiler: - { CC: gcc, CXX: g++ } - { CC: clang, CXX: clang++ } @@ -104,9 +110,6 @@ jobs: - { name: layer-00-default, nick: default } - { name: layer-01-minimal, nick: minimal } - { name: layer-02-maximus, nick: maximus } - exclude: - # Non-clang testing on MacOS is too much work for very little gain - - { os: macos-14, compiler: { CC: gcc, CXX: g++ } } runs-on: ${{ matrix.os }} @@ -119,7 +122,6 @@ jobs: steps: - name: Install prerequisite Linux packages - if: runner.os == 'Linux' run: | # required for "apt-get build-dep" to work sudo sed --in-place -E 's/# (deb-src.*updates main)/ \1/g' /etc/apt/sources.list @@ -127,55 +129,12 @@ jobs: sudo apt-get --quiet=2 build-dep squid sudo apt-get --quiet=2 install linuxdoc-tools libtool-bin ${{ matrix.compiler.CC }} - - name: Install prerequisite MacOS packages - if: runner.os == 'macOS' - run: | - brew install \ - automake coreutils cppunit gawk \ - gnu-getopt gnu-sed grep libtool \ - make cyrus-sasl - # openldap openssl # already provided by github workers base-image - - name: Checkout sources uses: actions/checkout@v4 - name: Run build on Linux - if: runner.os == 'Linux' run: ./test-builds.sh ${{ matrix.layer.name }} - - name: Run build on MacOS - if: runner.os == 'macOS' - run: | - eval `brew shellenv` - PKG_CONFIG_PATH="$HOMEBREW_PREFIX/lib/pkgconfig" - PKG_CONFIG_PATH="$PKG_CONFIG_PATH:$HOMEBREW_PREFIX/opt/openldap/lib/pkgconfig" - PKG_CONFIG_PATH="$PKG_CONFIG_PATH:$HOMEBREW_PREFIX/opt/cyrus-sasl/lib/pkgconfig" - export PKG_CONFIG_PATH - export GETOPT="$HOMEBREW_PREFIX/opt/gnu-getopt/bin/getopt" - export MAKE="$HOMEBREW_PREFIX/bin/gmake" - # ensure we use Homebrew headers and libraries - # this is needed because pkg-config --libs openssl points to the wrong directory - # in version openssl@3: stable 3.3.0 - export CPPFLAGS="-I$HOMEBREW_PREFIX/include${CPPFLAGS:+ $CPPFLAGS}" - export LDFLAGS="-L$HOMEBREW_PREFIX/lib${LDFLAGS:+ $LDFLAGS}" - export CFLAGS="-Wno-compound-token-split-by-macro${CFLAGS:+ $CFLAGS}" # needed fir ltdl with Xcode - - # libtool package referenced below fails to copy its configure* - # files, possibly due to a packaging/brewing bug. The following sed - # command restores installed libtoolize code to its earlier (and - # working) variation. - echo "brew libtool package details:" - brew info libtool --json | grep -E 'rebuild|tap_git_head' - # This hack was tested on libtoolize package with the following output: - # "rebuild": 2, - # "tap_git_head": "5cede8ea3b7b12c7f68215f75a951430b38d945f", - # - editable=$HOMEBREW_CELLAR/libtool/2.4.7/bin/glibtoolize - sed -i.bak 's@ltdl_ac_aux_dir=""@ltdl_ac_aux_dir="../build-aux"@' $editable || true - diff -u $editable.bak $editable || true - - ./test-builds.sh ${{ matrix.layer.name }} - - name: Publish build logs if: success() || failure() uses: actions/upload-artifact@v4 @@ -193,7 +152,6 @@ jobs: steps: - name: Install Squid prerequisite Linux packages - if: runner.os == 'Linux' run: | # required for "apt-get build-dep" to work sudo sed --in-place -E 's/# (deb-src.*updates main)/ \1/g' /etc/apt/sources.list diff --git a/.github/workflows/slow.yaml b/.github/workflows/slow.yaml new file mode 100644 index 0000000000..da550b57ab --- /dev/null +++ b/.github/workflows/slow.yaml @@ -0,0 +1,125 @@ +# Tests in this workflow file are slower or more expensive than build +# tests in quick.yaml. These tests improve build environment coverage, but +# most are unlikely to fail after quick.yaml tests succeed. Limiting these +# tests execution to staged commits optimizes CI responsiveness and +# costs while preserving decent quick.yaml problem detection probability. +# +# Anubis configuration must be kept in sync with the total number of +# tests executed for "auto" branch commits, including tests in other +# GitHub workflow file(s). + +name: Staged commit + +on: + push: + branches: [ "auto" ] + +jobs: + linux-distros: + + strategy: + matrix: + os: + - centos-stream-9 + - debian-stable + - debian-testing + - debian-unstable + - fedora-39 + - fedora-40 + - fedora-rawhide + - gentoo + - opensuse-leap + - opensuse-tumbleweed + - ubuntu-focal + - ubuntu-jammy + - ubuntu-noble # EOL 2036-04 + - ubuntu-oracular # EOL 2025-07 + compiler: + - { CC: gcc, CXX: g++ } + - { CC: clang, CXX: clang++ } + layer: + - { name: layer-00-default, nick: default } + - { name: layer-01-minimal, nick: minimal } + - { name: layer-02-maximus, nick: maximus } + + runs-on: ubuntu-22.04 + container: + image: squidcache/buildfarm-${{ matrix.os }}:stable + options: --user 1001 # uid used by worfklow runner + + name: linux-distros(${{ matrix.os }},${{ matrix.compiler.CC }},${{ matrix.layer.nick }}) + env: + CC: ${{ matrix.compiler.CC }} + CXX: ${{ matrix.compiler.CXX }} + + steps: + - name: Checkout Sources + uses: actions/checkout@v4 + + - name: Run test-builds + id: test-builds + run: | + ./test-builds.sh ${{ matrix.layer.name }} + + - name: Publish build logs + if: success() || failure() + uses: actions/upload-artifact@v4 + with: + name: build-logs-${{ matrix.os }}-${{ matrix.compiler.CC }}-${{ matrix.layer.nick }} + path: btlayer-*.log + + macos: + runs-on: macos-14 + name: macos + + steps: + - name: Install prerequisite MacOS packages + run: | + brew install \ + automake coreutils cppunit gawk \ + gnu-getopt gnu-sed grep libtool \ + make cyrus-sasl + # openldap openssl # already provided by github workers base-image + + - name: Checkout Sources + uses: actions/checkout@v4 + + - name: Run test-builds + id: test-builds + run: | + eval `brew shellenv` + PKG_CONFIG_PATH="$HOMEBREW_PREFIX/lib/pkgconfig" + PKG_CONFIG_PATH="$PKG_CONFIG_PATH:$HOMEBREW_PREFIX/opt/openldap/lib/pkgconfig" + PKG_CONFIG_PATH="$PKG_CONFIG_PATH:$HOMEBREW_PREFIX/opt/cyrus-sasl/lib/pkgconfig" + export PKG_CONFIG_PATH + export GETOPT="$HOMEBREW_PREFIX/opt/gnu-getopt/bin/getopt" + export MAKE="$HOMEBREW_PREFIX/bin/gmake" + # ensure we use Homebrew headers and libraries + # this is needed because pkg-config --libs openssl points to the wrong directory + # in version openssl@3: stable 3.3.0 + export CPPFLAGS="-I$HOMEBREW_PREFIX/include${CPPFLAGS:+ $CPPFLAGS}" + export LDFLAGS="-L$HOMEBREW_PREFIX/lib${LDFLAGS:+ $LDFLAGS}" + export CFLAGS="-Wno-compound-token-split-by-macro${CFLAGS:+ $CFLAGS}" # needed fir ltdl with Xcode + + # libtool package referenced below fails to copy its configure* + # files, possibly due to a packaging/brewing bug. The following sed + # command restores installed libtoolize code to its earlier (and + # working) variation. + echo "brew libtool package details:" + brew info libtool --json | grep -E 'rebuild|tap_git_head' + # This hack was tested on libtoolize package with the following output: + # "rebuild": 2, + # "tap_git_head": "5cede8ea3b7b12c7f68215f75a951430b38d945f", + # + editable=$HOMEBREW_CELLAR/libtool/2.4.7/bin/glibtoolize + sed -i.bak 's@ltdl_ac_aux_dir=""@ltdl_ac_aux_dir="../build-aux"@' $editable || true + diff -u $editable.bak $editable || true + + ./test-builds.sh + + - name: Publish build logs + if: success() || failure() + uses: actions/upload-artifact@v4 + with: + name: build-logs-macos + path: btlayer-*.log