From b108bc39bd7fa84570ef6ecb737eab713183fd45 Mon Sep 17 00:00:00 2001
From: Maria Matejka <mq@ucw.cz>
Date: Mon, 21 Apr 2025 18:35:24 +0200
Subject: [PATCH] CI: Do not run tests for commits already tested

---
 .gitlab-ci.yml           |  39 ++++++---------
 misc/gitlab/dependent.py | 102 +++++++++++++++++++++++++++++++++++++++
 2 files changed, 117 insertions(+), 24 deletions(-)
 create mode 100755 misc/gitlab/dependent.py

diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index b42cb72f1..df3997344 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -1,8 +1,3 @@
-## TODO: find out how to generate this file by another yaml file.
-## Gitlab can do it but it is a stupid idea to mess with this
-## when releasing 4 versions at once. See ya later!
-##                                              -- Maria, April 2025
-
 variables:
   DEBIAN_FRONTEND: noninteractive
   LC_ALL: C.UTF-8
@@ -17,33 +12,29 @@ stages:
   - test
   - release
 
-.minimal: &minimal
-  before_script:
-    - apk add --update --no-cache python3 git bash
-    - python3 -m venv venv
-    - . venv/bin/activate
-    - python3 -m ensurepip
-    - pip3 install --no-cache --upgrade jinja2 pyyaml
-  tags:
-    - linux
-    - docker
-
 ## Child pipelines for autotests
 prepare:
-  <<: *minimal
   stage: prepare
   script:
-    - if bash ./tools/git-is-stable-branch $CI_COMMIT_BRANCH; then ./tools/git-check-commits; fi
-    - 'cd misc/gitlab && python3 pipeline.py > pipeline.yml'
+    - apk add --update --no-cache python3 git bash
+    - python3 -m venv venv
+    - . venv/bin/activate
+    - python3 -m ensurepip
+    - pip3 install --no-cache --upgrade jinja2 pyyaml requests
+    - if ./tools/git-is-stable-branch $CI_COMMIT_BRANCH; then ./tools/git-check-commits; fi
+    - cd misc/gitlab && if python3 dependent.py $CI_COMMIT_SHA $CI_PIPELINE_ID > pipeline.yml || [ "$CI_JOB_MANUAL" == "true" ]; then python3 pipeline.py > pipeline.yml; fi
   artifacts:
     paths:
       - misc/gitlab/pipeline.yml
+  tags:
+    - linux
+    - docker
   rules:
-  - if: $CI_COMMIT_BRANCH =~ /^(stable-.*|thread-next|master)$/
-    when: always
-  - if: $CI_COMMIT_MESSAGE =~ /^(fixup! )*WIP/
-    when: never
-  - when: always
+    - if: $CI_COMMIT_BRANCH =~ /^(stable-.*|thread-next|master)$/
+      when: always
+    - if: $CI_COMMIT_MESSAGE =~ /^(fixup! )*WIP/
+      when: never
+    - when: always
 
 child-run:
   stage: test
diff --git a/misc/gitlab/dependent.py b/misc/gitlab/dependent.py
new file mode 100755
index 000000000..93465f06d
--- /dev/null
+++ b/misc/gitlab/dependent.py
@@ -0,0 +1,102 @@
+#!/usr/bin/python3
+
+import json
+import requests
+import sys
+import time
+import yaml
+
+def err(*args):
+    print(*args, file=sys.stderr)
+
+# Load all pipelines with this sha
+sha = sys.argv[1]
+if len(sys.argv) > 2:
+    ignore_pipelines = int(sys.argv[2])
+
+err("Ignoring pipelines over ", ignore_pipelines)
+
+def load_request(what, url):
+    timeout = 5
+    while True:
+        resp = requests.get(url)
+        if resp.status_code == 200:
+            return resp
+
+        if resp.status_code == 429:
+            print(f"Too many requests for {what}, waiting {timeout} sec")
+            time.sleep(timeout)
+            timeout *= 1.5
+            continue
+
+        raise Exception(f"Failed to load {what} ({resp.status_code}): {resp.content}")
+
+def load_pipelines(sha):
+    pipelines = []
+    pageno = 1
+    while True:
+        resp = load_request(f"pipelines page {pageno}", f"https://gitlab.nic.cz/api/v4/projects/labs%2Fbird/pipelines?per_page=20&page={pageno}&sha={sha}")
+        pipelines += (new := json.loads(resp.content))
+        if len(new) == 0:
+            return pipelines
+
+        lastcount = len(pipelines)
+        pageno += 1
+
+def load_jobs(pid):
+    resp = load_request(f"jobs", f"https://gitlab.nic.cz/api/v4/projects/labs%2Fbird/pipelines/{pid}/jobs?per_page=20&page=1")
+    jobs = json.loads(resp.content)
+    err(f"Loaded {len(jobs)} jobs")
+    assert(len(jobs) < 20)
+    return jobs
+
+wait = True
+while wait:
+
+    ok = []
+    wait = False
+
+    for p in (pipelines := load_pipelines(sha)):
+        if p["id"] >= ignore_pipelines:
+            err(f"Ignoring pipeline {p['id']}")
+            continue
+        if p["status"] == "success":
+            err(f"Pipeline {p['id']} already succeeded: {p['web_url']}")
+            ok.append(p['id'])
+        elif p["status"] == "pending" or p["status"] == "created":
+            err(f"Pipeline {p['id']} pending: {p['web_url']}")
+            wait = True
+        elif p["status"] == "running" or p["status"] == "failed":
+            err(f"Pipeline {p['id']} is {p['status']}: {p['web_url']}")
+            jobs = load_jobs(p['id'])
+            for j in jobs:
+                if j['name'] == 'prepare':
+                    if j['status'] == 'success':
+                        ok.append(p['id'])
+                    elif j['status'] == 'failed':
+                        err("Failed in preparation")
+                    else:
+                        wait = True
+
+        else:
+            err(f"Pipeline {p['id']} has an unknown state {p['status']}: {p['web_url']}")
+
+#    err(yaml.dump(pipelines))
+
+    if len(ok) > 0:
+        err(f"Found completed pipelines, no need to run again")
+        for p in ok:
+            print(yaml.dump({
+                f"dummy-{p}": {
+                    "script": [ "true" ],
+                    "needs": {
+                        "pipeline": str(p),
+                        "job": "child-run",
+                        "artifacts": False,
+                        }
+                    }
+                }))
+        exit(1)
+
+    if wait:
+        time.sleep(1)
-- 
2.47.2