ci: trigger a DNS Shotgun performance test

Run comparative performance tests against the latest released version of
the same branch. This is done for different protocols with an
appropriate load the server is expected to be able to handle.

Currently, the results need to be inspected manually, since a success of
the job doesn't indicate there is no issue. Instead, the job provides an
URL to an overview with latency, memory and CPU charts which display the
test results with the current code against the reference version. There
should be no major unexplained and reproducible differences in the
charts.

diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index a711d0ab9ba9e0cda69cf50a77485d6cd34f4c46..2816b7717a1d1680f162e56890cbd187c359b7aa 100644 (file)
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -348,8 +348,7 @@ ci-variables:
     # When testing a .0 release, compare it against the previous development
     # release (e.g., 9.19.0 and 9.18.0 should both be compared against 9.17.22).
     - if [ "$(sed -n -E "s|^m4_define\(\[bind_VERSION_PATCH\], ([0-9]+)\)dnl$|\1|p" configure.ac)" = "0" ]; then export BIND_BASELINE_BRANCH=$((BIND_BASELINE_BRANCH - 1 - (BIND_BASELINE_BRANCH % 2))); fi
-    - export BIND_BASELINE_VERSION="$(curl -s "https://gitlab.isc.org/api/v4/projects/1/repository/tags?search=^v9.${BIND_BASELINE_BRANCH}&order_by=version" | jq -r ".[0].name")"
-    - echo "BIND_BASELINE_BRANCH=$BIND_BASELINE_BRANCH" >> ci_vars.env
+    - BIND_BASELINE_VERSION="$(curl -s "https://gitlab.isc.org/api/v4/projects/1/repository/tags?search=^v9.${BIND_BASELINE_BRANCH}&order_by=version" | jq -r ".[0].name")"
     - echo "BIND_BASELINE_VERSION=$BIND_BASELINE_VERSION" >> ci_vars.env
   needs:
     - job: autoreconf
@@ -404,6 +403,26 @@ cross-version-config-tests:
     - awk '/^=+ FAILURES =+/{flag=1;next}/^=+.*=+$/{flag=0}flag' bin/tests/system/pytest.out.txt || true
     - awk '/^=+ ERRORS =+/{flag=1;next}/^=+.*=+$/{flag=0}flag' bin/tests/system/pytest.out.txt || true
 
+.shotgun: &shotgun_job
+  <<: *base_image
+  <<: *api_schedules_tags_triggers_web_triggering_rules
+  stage: performance
+  script:
+    - if [ -z "$CI_COMMIT_TAG" ]; then export SHOTGUN_ROUNDS=1; else export SHOTGUN_ROUNDS=3; fi
+    - PIPELINE_ID=$(curl -s -X POST --fail
+        -F "token=$CI_JOB_TOKEN"
+        -F ref=main
+        -F "variables[SHOTGUN_TEST_VERSION]=['$CI_COMMIT_REF_NAME', '$BIND_BASELINE_VERSION']"
+        -F "variables[SHOTGUN_DURATION]=300"
+        -F "variables[SHOTGUN_ROUNDS]=$SHOTGUN_ROUNDS"
+        -F "variables[SHOTGUN_TRAFFIC_MULTIPLIER]=$SHOTGUN_TRAFFIC_MULTIPLIER"
+        -F "variables[SHOTGUN_SCENARIO]=$SHOTGUN_SCENARIO"
+        https://gitlab.isc.org/api/v4/projects/188/trigger/pipeline | jq .id)
+    - util/ci-wait-shotgun.py $PIPELINE_ID
+  needs:
+    - job: ci-variables
+      artifacts: true
+
 .system_test_common: &system_test_common
   <<: *default_triggering_rules
   stage: system
@@ -1556,7 +1575,32 @@ respdiff-long-third-party:
   script:
     - bash respdiff.sh -s third_party -q "${PWD}/100k_mixed.txt" -c 1 -w "${PWD}/rspworkdir" "${CI_PROJECT_DIR}"
 
-# "Stress" tests
+# Performance tests
+
+# Run shotgun:udp right away, but delay other shotgun jobs sligthly in order to
+# allow re-use of the built container image. Otherwise, the jobs would do the
+# same builds in parallel rather than re-use the already built image.
+shotgun:udp:
+  <<: *shotgun_job
+  variables:
+    SHOTGUN_SCENARIO: udp
+    SHOTGUN_TRAFFIC_MULTIPLIER: 15
+
+shotgun:tcp:
+  <<: *shotgun_job
+  variables:
+    SHOTGUN_SCENARIO: tcp
+    SHOTGUN_TRAFFIC_MULTIPLIER: 6
+  when: delayed
+  start_in: 5 minutes
+
+shotgun:dot:
+  <<: *shotgun_job
+  variables:
+    SHOTGUN_SCENARIO: dot
+    SHOTGUN_TRAFFIC_MULTIPLIER: 3
+  when: delayed
+  start_in: 5 minutes
 
 # Parallel build in the "make" step is avoided since multiple jobs can be
 # executed concurrently on the same runner. This may present problems when one