From: Štěpán Balážik Date: Wed, 28 Jan 2026 12:55:53 +0000 (+0100) Subject: Move utility Python scripts to the bind9-qa repo X-Git-Tag: v9.21.19~15^2~18 X-Git-Url: http://git.ipfire.org/?a=commitdiff_plain;h=a3d0f43d2;p=thirdparty%2Fbind9.git Move utility Python scripts to the bind9-qa repo I would have touched them anyway in this MR so let's bite the bullet and move them there. --- diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 723ccecf751..83c9d1f4724 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -396,7 +396,7 @@ stages: .parse_tsan: &parse_tsan - *find_python - - find -name 'tsan.*' -exec "$PYTHON" util/parse_tsan.py {} \; + - find -name 'tsan.*' -exec "$PYTHON" "$CI_PROJECT_DIR"/bind9-qa/ci/parse_tsan.py {} \; .check_readline_setup: &check_readline_setup - if [[ -n "${WITHOUT_LIBEDIT}" ]]; then @@ -510,7 +510,9 @@ stages: -F "variables[SHOTGUN_EVAL_THRESHOLD_LATENCY_PCTL_DRIFT_MIN]=$SHOTGUN_EVAL_THRESHOLD_LATENCY_PCTL_DRIFT_MIN" -F "variables[SHOTGUN_EVAL_THRESHOLD_LATENCY_PCTL_DRIFT_MAX]=$SHOTGUN_EVAL_THRESHOLD_LATENCY_PCTL_DRIFT_MAX" https://gitlab.isc.org/api/v4/projects/188/trigger/pipeline | jq .id) - - util/ci-wait-shotgun.py $PIPELINE_ID + - *git_clone_bind9-qa + - > + "$CI_PROJECT_DIR"/bind9-qa/ci/wait_shotgun.py $PIPELINE_ID needs: - job: ci-variables artifacts: true @@ -531,12 +533,12 @@ stages: - *fips_feature_test - *find_pytest - *find_python - - ( if [ "${CI_DISPOSABLE_ENVIRONMENT}" = "true" ]; then sleep 3000; "$PYTHON" "${CI_PROJECT_DIR}/util/get-running-system-tests.py"; fi ) & + - *git_clone_bind9-qa + - ( if [ "${CI_DISPOSABLE_ENVIRONMENT}" = "true" ]; then sleep 3000; "$PYTHON" "${CI_PROJECT_DIR}/bind9-qa/ci/get_running_system_tests.py"; fi ) & - cd bin/tests/system - RET=0 - > ("$PYTEST" --junit-xml="$CI_PROJECT_DIR"/junit_pytest.xml -n "$TEST_PARALLEL_JOBS" | tee pytest.out.txt) || RET=1 - - *git_clone_bind9-qa - > "$PYTHON" "$CI_PROJECT_DIR"/bind9-qa/ci/postprocess_junit_files.py "$CI_PROJECT_DIR"/junit_pytest.xml --output "$CI_PROJECT_DIR"/junit.xml - (exit $RET) @@ -688,7 +690,7 @@ ci-orphaned-anchors: script: - *git_clone_bind9-qa - > - "$CI_PROJECT_DIR"/bind9-qa/ci-orphaned-anchors/check-orphaned-anchors-ci.py .gitlab-ci.yml + "$CI_PROJECT_DIR"/bind9-qa/ci-orphaned-anchors/check_orphaned_anchors_ci.py .gitlab-ci.yml rules: - if: '$CI_PIPELINE_SOURCE == "merge_request_event"' changes: @@ -825,7 +827,7 @@ generate-stress-test-configs: script: - *git_clone_bind9-qa - > - "$CI_PROJECT_DIR"/bind9-qa/stress/generate-stress-test-configs.py > stress-test-configs.yml + "$CI_PROJECT_DIR"/bind9-qa/stress/generate_stress_test_configs.py > stress-test-configs.yml artifacts: paths: - stress-test-configs.yml @@ -833,7 +835,9 @@ generate-stress-test-configs: generate-tsan-stress-test-configs: <<: *quick_checks_job script: - - util/generate-tsan-stress-jobs.py > tsan-stress-test-configs.yml + - *git_clone_bind9-qa + - > + "$CI_PROJECT_DIR"/bind9-qa/ci/generate_tsan_stress_jobs.py > tsan-stress-test-configs.yml artifacts: paths: - tsan-stress-test-configs.yml diff --git a/util/ci-wait-shotgun.py b/util/ci-wait-shotgun.py deleted file mode 100755 index 894dfa9f572..00000000000 --- a/util/ci-wait-shotgun.py +++ /dev/null @@ -1,106 +0,0 @@ -#!/usr/bin/env python3 - -# Copyright (C) Internet Systems Consortium, Inc. ("ISC") -# -# SPDX-License-Identifier: MPL-2.0 -# -# This Source Code Form is subject to the terms of the Mozilla Public -# License, v. 2.0. If a copy of the MPL was not distributed with this -# file, you can obtain one at https://mozilla.org/MPL/2.0/. -# -# See the COPYRIGHT file distributed with this work for additional -# information regarding copyright ownership. - -import os -import sys -import time - -import gitlab - - -def init_gitlab_project(): - if os.getenv("CI_SERVER_URL", None) is not None: - url = os.getenv("CI_SERVER_URL") - job_token = os.environ["CI_JOB_TOKEN"] - gl = gitlab.Gitlab(url, job_token=job_token) - else: # running locally on dev machine - gl = gitlab.Gitlab.from_config() - return gl.projects.get("isc-projects/bind9-shotgun-ci") - - -def parse_parent_pipeline_id(): - if len(sys.argv) != 2: - raise RuntimeError("usage: util/ci-wait-shotgun.py PIPELINE_ID") - try: - return int(sys.argv[1]) - except ValueError as exc: - raise RuntimeError("error: PIPELINE_ID must be a number") from exc - - -def wait_until(callback, timeout=1800, retry=10): - start = time.time() - while time.time() - start < timeout: - if callback(): - return - time.sleep(retry) - raise RuntimeError(f"error: timed out after {timeout}s") - - -def get_child_pipeline_id(project, pipeline_id): - pipeline = project.pipelines.get(pipeline_id) - - def pipeline_finished(): - pipeline.refresh() - return pipeline.finished_at is not None - - wait_until(pipeline_finished, timeout=6600) - - bridges = pipeline.bridges.list() - if len(bridges) != 1: - raise RuntimeError("error: expected exactly one child pipeline") - return bridges[0].downstream_pipeline["id"] - - -def get_postproc_job(project, pipeline_id): - pipeline = project.pipelines.get(pipeline_id) - postproc_job = None - - def job_finished(): - nonlocal postproc_job - for job in pipeline.jobs.list(get_all=True): - if job.name == "postproc": - postproc_job = job - if postproc_job is None: - raise RuntimeError("error: failed to find 'postproc' job in child pipeline") - return postproc_job.finished_at is not None - - wait_until(job_finished) - return postproc_job - - -def evaluate_postproc_job(job): - print(f"postproc job URL: {job.web_url}") - index_url = ( - "https://isc-projects.gitlab-pages.isc.org/-/" - f"bind9-shotgun-ci/-/jobs/{job.id}/artifacts/index.html" - ) - if job.status in ["success", "failed"]: - print(f"result for manual inspection: {index_url}") - if job.status != "success": - raise RuntimeError("error: 'postproc' job didn't succeed") - - -def main(): - project = init_gitlab_project() - parent_pipeline_id = parse_parent_pipeline_id() - child_pipeline_id = get_child_pipeline_id(project, parent_pipeline_id) - postproc_job = get_postproc_job(project, child_pipeline_id) - evaluate_postproc_job(postproc_job) - - -if __name__ == "__main__": - try: - main() - except RuntimeError as err: - print(err) - sys.exit(1) diff --git a/util/generate-tsan-stress-jobs.py b/util/generate-tsan-stress-jobs.py deleted file mode 100755 index 8ee649ed6ff..00000000000 --- a/util/generate-tsan-stress-jobs.py +++ /dev/null @@ -1,38 +0,0 @@ -#!/usr/bin/env python3 - -# Copyright (C) Internet Systems Consortium, Inc. ("ISC") -# -# SPDX-License-Identifier: MPL-2.0 -# -# This Source Code Form is subject to the terms of the Mozilla Public -# License, v. 2.0. If a copy of the MPL was not distributed with this -# file, you can obtain one at https://mozilla.org/MPL/2.0/. -# -# See the COPYRIGHT file distributed with this work for additional -# information regarding copyright ownership. - -import yaml - -NUMBER_OF_TESTS_PER_TSAN_JOB = 50 - -with open(".gitlab-ci.yml", encoding="utf-8") as gitlab_ci_yml: - anchors = yaml.load(gitlab_ci_yml, Loader=yaml.Loader) - -for tsan_job in "gcc:tsan", "clang:tsan": - for test_type in "unit", "system": - tsan_stress_test_job = anchors[f"{test_type}:{tsan_job}"] - tsan_stress_test_job["stage"] = "test" - tsan_stress_test_job["rules"] = [ - {"if": '$CI_PIPELINE_SOURCE == "parent_pipeline"'} - ] - tsan_stress_test_job["parallel"] = NUMBER_OF_TESTS_PER_TSAN_JOB - tsan_stress_test_job["needs"] = [ - {"pipeline": "$PARENT_PIPELINE_ID", "job": tsan_job} - ] - - print( - yaml.dump( - {f"{test_type}:{tsan_job}:stress": tsan_stress_test_job}, - Dumper=yaml.Dumper, - ) - ) diff --git a/util/get-running-system-tests.py b/util/get-running-system-tests.py deleted file mode 100755 index 1653e0a66da..00000000000 --- a/util/get-running-system-tests.py +++ /dev/null @@ -1,25 +0,0 @@ -#!/usr/bin/env python3 - -# Copyright (C) Internet Systems Consortium, Inc. ("ISC") -# -# SPDX-License-Identifier: MPL-2.0 -# -# This Source Code Form is subject to the terms of the Mozilla Public -# License, v. 2.0. If a copy of the MPL was not distributed with this -# file, you can obtain one at https://mozilla.org/MPL/2.0/. -# -# See the COPYRIGHT file distributed with this work for additional -# information regarding copyright ownership. - -import psutil - -for pid in psutil.pids(): - try: - environ = psutil.Process(pid).environ() - if "PYTEST_CURRENT_TEST" in environ: - name = psutil.Process(pid).name() - print( - f'pytest process {name}/{pid} running: {environ["PYTEST_CURRENT_TEST"]}' - ) - except (psutil.AccessDenied, psutil.NoSuchProcess): - pass diff --git a/util/parse_tsan.py b/util/parse_tsan.py deleted file mode 100755 index a2a4bda34b2..00000000000 --- a/util/parse_tsan.py +++ /dev/null @@ -1,125 +0,0 @@ -#!/usr/bin/env python3 -############################################################################ -# Copyright (C) Internet Systems Consortium, Inc. ("ISC") -# -# SPDX-License-Identifier: MPL-2.0 -# -# This Source Code Form is subject to the terms of the Mozilla Public -# License, v. 2.0. If a copy of the MPL was not distributed with this -# file, you can obtain one at https://mozilla.org/MPL/2.0/. -# -# See the COPYRIGHT file distributed with this work for additional -# information regarding copyright ownership. -############################################################################ - -"""Parse the ThreadSanizer reports, unify them and put them into unique dirs.""" - -import sys -import os -import os.path -import re -from hashlib import sha256 - - -class State: - """Class that holds state of the TSAN parser.""" - - inside = False - block = "" - last_line = None - - mutexes = {} - m_index = 1 - threads = {} - t_index = 1 - pointers = {} - p_index = 1 - - def __init__(self): - self.reset() - - def reset(self): - """Reset the object to initial state""" - - self.inside = False - self.block = "" - - self.mutexes = {} - self.threads = {} - self.pointers = {} - self.pointers["0x000000000000"] = 0 - - self.m_index = 1 - self.t_index = 1 - self.p_index = 1 - - -TOP = os.path.dirname(os.path.dirname(os.path.realpath(__file__))) - -OUT = os.path.join(TOP, "tsan") - -if not os.path.isdir(OUT): - os.mkdir(OUT) - -# Regular Expressions -MUTEX = re.compile(r"M\d+") -THREAD = re.compile(r"T\d+") -STACK = re.compile(r"\s\(\S+\+0x\S+\)") -POINTER = re.compile(r"0x[0-9a-f]+") -PID = re.compile(r"\(pid=\d+,?\)") -TID = re.compile(r"tid=\d+,?\s*") -WORKER = re.compile(r"\s+'(isc-worker|isc-net-)\d+'") -PATH = re.compile(TOP + "/") - -S = State() - -with open(sys.argv[1], "r", encoding="utf-8") as f: - for line in f.readlines(): - if line == "==================\n": - if not S.inside: - S.inside = True - else: - DNAME = sha256(S.last_line.encode("utf-8")).hexdigest() - DNAME = os.path.join(OUT, DNAME) - if not os.path.isdir(DNAME): - os.mkdir(DNAME) - FNAME = sha256(S.block.encode("utf-8")).hexdigest() + ".txt" - FNAME = os.path.join(DNAME, FNAME) - if not os.path.isfile(FNAME): - with open(FNAME, "w", encoding="utf-8") as w: - w.write(S.block) - S.reset() - else: - for m in MUTEX.finditer(line): - k = m.group() - if k not in S.mutexes: - S.mutexes[k] = S.m_index - S.m_index += 1 - for m in THREAD.finditer(line): - k = m.group() - if k not in S.threads: - S.threads[k] = S.t_index - S.t_index += 1 - for m in POINTER.finditer(line): - k = m.group() - if k not in S.pointers: - S.pointers[k] = S.p_index - S.p_index += 1 - for k, v in S.mutexes.items(): - r = re.compile(k) - line = r.sub(f"M{v:04d}", line) - for k, v in S.threads.items(): - r = re.compile(k) - line = r.sub(f"T{v:04d}", line) - for k, v in S.pointers.items(): - r = re.compile(k) - line = r.sub(f"0x{v:012d}", line) - - line = STACK.sub("", line) - line = PID.sub("", line) - line = TID.sub("", line) - line = WORKER.sub("", line) - line = PATH.sub("", line) - - S.block += line - S.last_line = line