From: Viktor Szakats Date: Thu, 19 Sep 2024 20:38:04 +0000 (+0200) Subject: GHA: speed up builds in torture jobs, tidy up X-Git-Tag: curl-8_11_0~388 X-Git-Url: http://git.ipfire.org/cgi-bin/gitweb.cgi?a=commitdiff_plain;h=c5e3d8ba94791dbc51f98a56ce07b3ba6b0aa446;p=thirdparty%2Fcurl.git GHA: speed up builds in torture jobs, tidy up - use cmake with ninja. - drop unnecessary packages. Also: - Linux: switch to Ubuntu 20.24 runner. Initially for newer packages, then left it there for variation and as canary. - Linux: make it easier to enable valgrind by dropping `-n` tflags. - Linux: show `curl -V` after build. - Linux: sync job names with macOS. - Linux, macOS: review and adjust job timeouts. - Linux, macOS: sync job configs more: Linux: enable libssh2, macOS: enable brotli, zstd, libssh2. - macOS: prefer `source` (over manual `PATH` editing) for `venv` setup. - macOS: drop redundant `-DENABLE_CURLDEBUG=ON`. - macOS: drop redundant `-n` tflags. There is no valgrind on macOS. - macOS: allow overriding test parallelism per job. - macOS: unlock all disabled tests for torture jobs. Speed-ups (configure/build stage): - Linux !FTP: 83s -> 36s before: https://github.com/curl/curl/actions/runs/10948030198/job/30398134647 after: https://github.com/curl/curl/actions/runs/10961987675/job/30440179978?pr=14972 - Linux FTP: 79s -> 32s before: https://github.com/curl/curl/actions/runs/10948030198/job/30398137070 after: https://github.com/curl/curl/actions/runs/10961987675/job/30440182406?pr=14972 - macOS !FTP: 98s -> 29s before: https://github.com/curl/curl/actions/runs/10960141275/job/30434081412?pr=14972 after: https://github.com/curl/curl/actions/runs/10960141275/job/30434082357?pr=14972 - macOS FTP: 129s -> 29s before: https://github.com/curl/curl/actions/runs/10960141275/job/30434082041?pr=14972 after: https://github.com/curl/curl/actions/runs/10960141275/job/30434083050?pr=14972 Closes #14972 --- diff --git a/.github/workflows/macos.yml b/.github/workflows/macos.yml index 21f61dc95d..f1a96aeaa5 100644 --- a/.github/workflows/macos.yml +++ b/.github/workflows/macos.yml @@ -135,16 +135,6 @@ jobs: configure: --enable-debug --with-openssl=$(brew --prefix openssl) --enable-websockets macos-version-min: '10.9' tflags: -e - - name: 'OpenSSL torture !FTP' - compiler: clang - configure: --enable-debug --disable-shared --disable-threaded-resolver --with-openssl=$(brew --prefix openssl) --enable-websockets - tflags: -n -t --shallow=25 !FTP - macos-version-min: '10.9' - - name: 'OpenSSL torture FTP' - compiler: clang - configure: --enable-debug --disable-shared --disable-threaded-resolver --with-openssl=$(brew --prefix openssl) --enable-websockets - tflags: -n -t --shallow=20 FTP - macos-version-min: '10.9' - name: 'OpenSSL libssh2 !ldap 10.15' compiler: clang configure: --enable-debug --disable-ldap --with-openssl=$(brew --prefix openssl) --enable-websockets @@ -241,7 +231,7 @@ jobs: - name: 'run tests' timeout-minutes: 20 run: | - export TFLAGS='${{ matrix.tflags }} -j10' + export TFLAGS='-j10 ${{ matrix.tflags }}' TFLAGS+=' ~2037 ~2041' # flaky if [[ '${{ matrix.compiler }}' = 'gcc'* ]]; then TFLAGS+=' ~RTSP' # 567 568 569 570 571 572 577 689 3100 @@ -257,7 +247,7 @@ jobs: TFLAGS+=' ~313' # SecureTransport does not support crl file TFLAGS+=' ~1631 ~1632' # SecureTransport is not able to shutdown ftp over https gracefully yet fi - PATH="$HOME/venv/bin:$PATH" + source $HOME/venv/bin/activate rm -f $HOME/.curlrc make -C bld V=1 test-ci @@ -268,7 +258,7 @@ jobs: cmake: name: 'CM ${{ matrix.compiler }} ${{ matrix.build.name }}' runs-on: 'macos-latest' - timeout-minutes: 10 + timeout-minutes: 30 env: DEVELOPER_DIR: "/Applications/Xcode${{ matrix.xcode && format('_{0}', matrix.xcode) || '' }}.app/Contents/Developer" CC: ${{ matrix.compiler }} @@ -285,8 +275,8 @@ jobs: install: libssh generate: -DOPENSSL_ROOT_DIR=$(brew --prefix openssl) -DBUILD_STATIC_LIBS=ON -DCURL_USE_LIBSSH2=OFF -DCURL_USE_LIBSSH=ON macos-version-min: '10.9' - - name: 'SecureTransport ws debug+' - generate: -DCURL_USE_SECTRANSP=ON -DENABLE_WEBSOCKETS=ON -DENABLE_DEBUG=ON -DENABLE_CURLDEBUG=ON + - name: 'SecureTransport ws debug' + generate: -DCURL_USE_SECTRANSP=ON -DENABLE_WEBSOCKETS=ON -DENABLE_DEBUG=ON macos-version-min: '10.8' - name: 'LibreSSL !ldap heimdal c-ares +examples' install: libressl heimdal @@ -300,9 +290,20 @@ jobs: install: gnutls nettle krb5 generate: -DCURL_USE_GNUTLS=ON -DCURL_USE_OPENSSL=OFF -DCURL_USE_GSSAPI=ON -DGSS_ROOT_DIR=$(brew --prefix krb5) -DCURL_DISABLE_LDAP=ON macos-version-min: '10.15' + - name: 'OpenSSL torture !FTP' + generate: -DENABLE_DEBUG=ON -DBUILD_SHARED_LIBS=OFF -DENABLE_THREADED_RESOLVER=OFF -DOPENSSL_ROOT_DIR=$(brew --prefix openssl) -DCURL_BROTLI=ON -DCURL_ZSTD=ON -DENABLE_WEBSOCKETS=ON + tflags: -t --shallow=25 !FTP + macos-version-min: '10.9' + torture: true + - name: 'OpenSSL torture FTP' + generate: -DENABLE_DEBUG=ON -DBUILD_SHARED_LIBS=OFF -DENABLE_THREADED_RESOLVER=OFF -DOPENSSL_ROOT_DIR=$(brew --prefix openssl) -DCURL_BROTLI=ON -DCURL_ZSTD=ON + tflags: -t --shallow=20 FTP + macos-version-min: '10.9' + torture: true exclude: - { compiler: llvm@15, build: { macos-version-min: '10.15' } } - { compiler: llvm@15, build: { macos-version-min: '10.9' } } + - { compiler: gcc-12, build: { torture: true } } steps: - name: 'brew install' run: | @@ -378,25 +379,27 @@ jobs: python3 -m pip install impacket - name: 'cmake run tests' - timeout-minutes: 10 + timeout-minutes: ${{ matrix.build.torture && 20 || 10 }} run: | - export TFLAGS='${{ matrix.build.tflags }} -j10' - TFLAGS+=' ~2037 ~2041' # flaky - if [[ '${{ matrix.compiler }}' = 'gcc'* ]]; then - TFLAGS+=' ~RTSP' # 567 568 569 570 571 572 577 689 3100 - TFLAGS+=' ~1156 ~1539' # HTTP Content-Range, Content-Length + export TFLAGS='-j10 ${{ matrix.build.tflags }}' + if [ -z '${{ matrix.build.torture }}' ]; then + TFLAGS+=' ~2037 ~2041' # flaky + if [[ '${{ matrix.compiler }}' = 'gcc'* ]]; then + TFLAGS+=' ~RTSP' # 567 568 569 570 571 572 577 689 3100 + TFLAGS+=' ~1156 ~1539' # HTTP Content-Range, Content-Length + if [[ '${{ matrix.build.generate }}' = *'-DCURL_USE_SECTRANSP=ON'* ]]; then + TFLAGS+=' ~2100' # 2100:'HTTP GET using DoH' https://github.com/curl/curl/actions/runs/9942146678/job/27462937524#step:15:5059 + TFLAGS+=' ~HTTP/2' # 2400 2401 2402 2403 2404 2406, SecureTransport + nghttp2 + else + TFLAGS+=' ~2402 ~2404' # non-SecureTransport + nghttp2 + fi + fi if [[ '${{ matrix.build.generate }}' = *'-DCURL_USE_SECTRANSP=ON'* ]]; then - TFLAGS+=' ~2100' # 2100:'HTTP GET using DoH' https://github.com/curl/curl/actions/runs/9942146678/job/27462937524#step:15:5059 - TFLAGS+=' ~HTTP/2' # 2400 2401 2402 2403 2404 2406, SecureTransport + nghttp2 - else - TFLAGS+=' ~2402 ~2404' # non-SecureTransport + nghttp2 + TFLAGS+=' ~313' # SecureTransport does not support crl file + TFLAGS+=' ~1631 ~1632' # SecureTransport is not able to shutdown ftp over https gracefully yet fi fi - if [[ '${{ matrix.build.generate }}' = *'-DCURL_USE_SECTRANSP=ON'* ]]; then - TFLAGS+=' ~313' # SecureTransport does not support crl file - TFLAGS+=' ~1631 ~1632' # SecureTransport is not able to shutdown ftp over https gracefully yet - fi - PATH="$HOME/venv/bin:$PATH" + source $HOME/venv/bin/activate rm -f $HOME/.curlrc ninja -C bld test-ci @@ -408,7 +411,7 @@ jobs: if: true # Set to `true` to enable this test matrix. It runs quickly. name: "${{ matrix.build == 'cmake' && 'CM' || 'AM' }} ${{ matrix.compiler }} ${{ matrix.image }} ${{ matrix.xcode }} ${{ matrix.config }}" runs-on: ${{ matrix.image }} - timeout-minutes: 30 + timeout-minutes: 10 env: DEVELOPER_DIR: "/Applications/Xcode${{ matrix.xcode && format('_{0}', matrix.xcode) || '' }}.app/Contents/Developer" CC: ${{ matrix.compiler }} diff --git a/.github/workflows/torture.yml b/.github/workflows/torture.yml index 88e71cfc5d..450ebaa08c 100644 --- a/.github/workflows/torture.yml +++ b/.github/workflows/torture.yml @@ -40,52 +40,52 @@ concurrency: permissions: {} -env: - MAKEFLAGS: -j 5 - jobs: - autotools: - name: ${{ matrix.build.name }} - runs-on: 'ubuntu-latest' - timeout-minutes: 90 + cmake: + name: '${{ matrix.build.name }}' + runs-on: 'ubuntu-24.04' + timeout-minutes: 30 strategy: fail-fast: false matrix: build: - - name: torture + - name: 'OpenSSL torture !FTP' install: libpsl-dev libbrotli-dev libzstd-dev zlib1g-dev libnghttp2-dev libssh2-1-dev libc-ares-dev - configure: --with-openssl --enable-debug --enable-ares --enable-websockets - tflags: -n -t --shallow=25 !FTP -j10 - - name: torture-ftp + generate: -DCURL_USE_OPENSSL=ON -DENABLE_DEBUG=ON -DENABLE_ARES=ON -DENABLE_WEBSOCKETS=ON + tflags: -t --shallow=25 !FTP + - name: 'OpenSSL torture FTP' install: libpsl-dev libbrotli-dev libzstd-dev zlib1g-dev libnghttp2-dev libssh2-1-dev libc-ares-dev - configure: --with-openssl --enable-debug --enable-ares - tflags: -n -t --shallow=20 FTP -j10 + generate: -DCURL_USE_OPENSSL=ON -DENABLE_DEBUG=ON -DENABLE_ARES=ON + tflags: -t --shallow=20 FTP steps: - run: | - sudo rm -f /etc/apt/sources.list.d/microsoft-prod.list - sudo apt-get update - sudo apt-get install libtool autoconf automake pkgconf stunnel4 ${{ matrix.build.install }} - sudo python3 -m pip install impacket - name: 'install prereqs and impacket' + sudo apt-get install cmake ninja-build pkgconf stunnel4 ${{ matrix.build.install }} + python3 -m venv $HOME/venv + source $HOME/venv/bin/activate + python3 -m pip install impacket + name: 'install prereqs' - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4 - - run: autoreconf -fi - name: 'autoreconf' - - run: | - ./configure --disable-dependency-tracking --enable-unity --enable-warnings --enable-werror \ - ${{ matrix.build.configure }} - name: 'configure' + cmake -G Ninja -DCMAKE_UNITY_BUILD=ON -DCURL_WERROR=ON \ + -DCURL_BROTLI=ON -DCURL_ZSTD=ON \ + ${{ matrix.build.generate }} + name: 'cmake configure' - - run: make V=1 - name: 'make' + - run: cmake --build . --verbose + name: 'cmake build' - - run: make V=1 -C tests - name: 'make tests' + - run: ./src/curl -V + name: 'check curl -V output' - - run: make V=1 test-torture + - run: cmake --build . --verbose --target testdeps + name: 'build tests' + + - run: | + source $HOME/venv/bin/activate + cmake --build . --verbose --target test-torture name: 'run tests' env: - TFLAGS: "${{ matrix.build.tflags }}" + TFLAGS: '-j10 ${{ matrix.build.tflags }}'