From 82328106641c955fb8a08a5f85d553caa163b89d Mon Sep 17 00:00:00 2001 From: Maria Matejka Date: Tue, 29 Apr 2025 10:03:14 +0200 Subject: [PATCH] Tag checking Adding a script to gather all information about the current release and confirm that it is indeed ready for departure. --- .gitlab-ci.yml | 30 +++++++++++++ misc/gitlab/template.yml.j2 | 30 +++++++++++++ tools/git-check-tag-ci | 89 +++++++++++++++++++++++++++++++++++++ tools/git-check-tag-local | 34 ++++++++++++++ 4 files changed, 183 insertions(+) create mode 100755 tools/git-check-tag-ci create mode 100755 tools/git-check-tag-local diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 81594b9e0..6e38e468b 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -34,20 +34,50 @@ stages: .if-stable: &if-stable if: $CI_COMMIT_BRANCH =~ /^(stable-.*|thread-next|master)$/ when: always +# Do run for tags +.if-tag: &if-tag + if: $CI_COMMIT_TAG + when: always +# Never run for tags +.never-tag: &never-tag + if: $CI_COMMIT_TAG + when: never ## Consistency checks for stable branches commit-messages: stage: consistency + image: registry.nic.cz/labs/bird:docbuilder script: - tools/git-check-commits rules: - *if-stable - when: never +## Tag check +tag-collect: + stage: consistency + image: registry.nic.cz/labs/bird:docbuilder + script: + - python3 -m venv venv + - . venv/bin/activate + - pip3 install requests + - tools/git-check-tag-local $CI_COMMIT_TAG + - tools/git-check-tag-ci $CI_COMMIT_SHA + artifacts: + paths: + - obj/doc/bird-singlepage.html + - bird-*.tar.gz + - pkg/pkgs/* + - pkg/srcpkgs/* + rules: + - *if-tag + - when: never + ## Default test job rules .test-job: &test-job rules: - *never-wip + - *never-tag - when: always ############################ diff --git a/misc/gitlab/template.yml.j2 b/misc/gitlab/template.yml.j2 index 0e29074e0..207a33798 100644 --- a/misc/gitlab/template.yml.j2 +++ b/misc/gitlab/template.yml.j2 @@ -34,20 +34,50 @@ stages: .if-stable: &if-stable if: $CI_COMMIT_BRANCH =~ /^(stable-.*|thread-next|master)$/ when: always +# Do run for tags +.if-tag: &if-tag + if: $CI_COMMIT_TAG + when: always +# Never run for tags +.never-tag: &never-tag + if: $CI_COMMIT_TAG + when: never ## Consistency checks for stable branches commit-messages: stage: consistency + image: registry.nic.cz/labs/bird:docbuilder script: - tools/git-check-commits rules: - *if-stable - when: never +## Tag check +tag-collect: + stage: consistency + image: registry.nic.cz/labs/bird:docbuilder + script: + - python3 -m venv venv + - . venv/bin/activate + - pip3 install requests + - tools/git-check-tag-local $CI_COMMIT_TAG + - tools/git-check-tag-ci $CI_COMMIT_SHA + artifacts: + paths: + - obj/doc/bird-singlepage.html + - bird-*.tar.gz + - pkg/pkgs/* + - pkg/srcpkgs/* + rules: + - *if-tag + - when: never + ## Default test job rules .test-job: &test-job rules: - *never-wip + - *never-tag - when: always ############################ diff --git a/tools/git-check-tag-ci b/tools/git-check-tag-ci new file mode 100755 index 000000000..0b7a44271 --- /dev/null +++ b/tools/git-check-tag-ci @@ -0,0 +1,89 @@ +#!/usr/bin/env python3 + +import io +import json +import requests +import sys +import zipfile + +def load_api_request(name, query): + timeout = 5 + while True: + resp = requests.get("https://gitlab.nic.cz/api/v4/projects/labs%2Fbird/" + query) + if resp.status_code == 200: + return resp.content + + if resp.status_code == 429: + print(f"Too many requests for {name}, waiting {timeout} sec") + time.sleep(timeout) + timeout = int(1.6 * timeout) + continue + + raise Exception(f"Failed to load {name} at {query} with code {resp.status_code}: {resp.content}") + +def load_paginated(name, query, per_page=100): + if query[-1] in "?&": + joiner = "" + elif "?" in query: + joiner = "&" + else: + joiner = "?" + + output = [] + pageno = 1 + while True: + p = load_api_request(f"{name} page {pageno}", f"{query}{joiner}per_page={per_page}&page={pageno}") + output += (new := json.loads(p)) + if len(new) < per_page: + return output + + pageno += 1 + +def load_pipelines(sha): + return load_paginated("pipelines", f"/pipelines/?sha={sha}") + +def load_jobs(pipeline): + return load_paginated("jobs", f"/pipelines/{pipeline}/jobs/") + +for p in load_pipelines(sys.argv[1]): + if p['status'] in ("failed", "cancelled"): + print(f"Pipeline {p['id']} {p['status']} at {p['web_url']}") + failed = [ job for job in load_jobs(p['id']) if job['status'] == "failed" ] + if len(failed) > 0: + print(f"\tFailed jobs:") + for job in failed: + print(f"\t\t{job['name']}") + else: + print(f"\tNo failed jobs, check gitlab") + print() + continue + + if p['status'] in ("created", "pending", "running"): + print(f"Pipeline {p['id']} has not finished yet: {p['status']}") + states = {} + + for job in load_jobs(p['id']): + if job['status'] not in states: + states[job['status']] = [] + states[job['status']].append(job) + + for s in states: + print(f"\tJobs {s}:") + for j in states[s]: + print(f"\t\t{j['name']} ({j['id']})") + + continue + + if p['status'] == "success": + print(f"Pipeline {p['id']} successful, collecting artifacts") + for job in load_jobs(p['id']): + if len(job['artifacts']) > 0: + print(f"\t{ job['name'] }:") + for f in job['artifacts']: + if f['file_type'] == 'archive': + with zipfile.ZipFile(io.BytesIO(load_api_request("metadata", f"/jobs/{job['id']}/artifacts/"))) as z: + z.extractall() + exit(0) + +print("No suitable pipeline found, tag not OK") +exit(1) diff --git a/tools/git-check-tag-local b/tools/git-check-tag-local new file mode 100755 index 000000000..5d7790dc2 --- /dev/null +++ b/tools/git-check-tag-local @@ -0,0 +1,34 @@ +#!/bin/bash + +# Enforce clean repository (it's release time) +if [ $(git status --porcelain -uno | wc -l) != "0" ]; then + echo "Dirty repository, commit or stash!" + exit 1 +fi + +# Is the current commit a release commit? +read COMMIT MESSAGE <<< "$(git show -s --oneline --no-decorate --no-abbrev-commit)" + +if [ "$MESSAGE" != "NEWS and version update" ]; then + echo "This is not a release commit: $COMMIT $MESSAGE" + exit 1 +fi + +# Are we checking an existing tag? +if [ -n "$1" ]; then + read TAG_COMMIT _ <<< "$(git show -s --oneline --no-abbrev-commit "$1" | grep NEWS)" + if [ "$TAG_COMMIT" != "$COMMIT" ]; then + echo "Tag $1 not checked out" + exit 1 + fi + + if [ "v$(