From: Maria Matejka Date: Sat, 21 Mar 2026 19:53:12 +0000 (+0100) Subject: CI: Split the makefile out, add documentation X-Git-Url: http://git.ipfire.org/cgi-bin/gitweb.cgi?a=commitdiff_plain;h=29fb45966d33c00e8d0669db530170e2bbcdb91e;p=thirdparty%2Fbird.git CI: Split the makefile out, add documentation --- diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 8651b1454..b75d69b4d 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -84,6 +84,8 @@ messy and you may need some help with it. We're planning to move the Netlab suite into the main git repository; after we do that, we'll require every contribution to add tests (if applicable, of course). +These automatic tests are used by our [CI](misc/gitlab/). + ## Crediting policy The credits are scattered over all the source code files; in the commentary diff --git a/Makefile.in b/Makefile.in index 100723f11..336c4ab3e 100644 --- a/Makefile.in +++ b/Makefile.in @@ -68,7 +68,7 @@ endif docgoals := docs userdocs progdocs testgoals := check test tests tests_run cleangoals := clean distclean testsclean -.PHONY: all daemon cli $(docgoals) $(testgoals) $(cleangoals) tags cscope prepare gitlab +.PHONY: all daemon cli $(docgoals) $(testgoals) $(cleangoals) tags cscope prepare all: daemon cli @@ -78,7 +78,7 @@ cli: $(client) $(daemon): LIBS += $(DAEMON_LIBS) # Include directories -dirs := client conf doc filter lib nest test $(addprefix proto/,$(protocols)) @sysdep_dirs@ +dirs := client conf doc filter lib nest test misc/gitlab $(addprefix proto/,$(protocols)) @sysdep_dirs@ # conf/Makefile declarations needed for all other modules conf-lex-targets := $(addprefix $(objdir)/conf/,cf-lex.o) @@ -203,24 +203,6 @@ tags: cscope: cd $(srcdir) ; find $(dirs) -name '*.[chY]' > cscope.files ; cscope -b -# Gitlab CI tests -gitlab-venv: .gitlab-ci.yml -gitlab-venv: USE_VENV := VENV - -gitlab-local: .gitlab-ci.yml -gitlab-local: USE_VENV := - -.gitlab-ci.yml: $(addprefix misc/gitlab/,pipeline.py data.yml.j2 template.yml.j2) - ($(if $(USE_VENV),\ - VENV=$$(mktemp -d); \ - python3 -m venv $$VENV; \ - . $$VENV/bin/activate; \ - pip3 install jinja2 pyaml; \ - ,) \ - python3 misc/gitlab/pipeline.py > $@; \ - $(if $(USE_VENV),rm -rf $$VENV,) \ - ) - # Install install: all diff --git a/misc/gitlab/Makefile b/misc/gitlab/Makefile new file mode 100644 index 000000000..c9cec6d46 --- /dev/null +++ b/misc/gitlab/Makefile @@ -0,0 +1,32 @@ +# Gitlab CI tests +#GITLAB_SECTIONS := template + +#$(s)%.yml: $(s)pipeline.py $(s)%.yml.j2 $(s)data.yml.j2 +.gitlab-ci.yml: $(s)pipeline.py $(s)template.yml.j2 $(s)data.yml.j2 + python3 $^ > $@ + +#gitlab-local: $(patsubst %,$(s)%.yml,$(GITLAB_SECTIONS)) +gitlab-local: .gitlab-ci.yml + +VPYTHON = VIRTUAL_ENV=$(1) PATH="$(1)/bin:$$PATH" $(1)/bin/python3 +VPIP = VIRTUAL_ENV=$(1) PATH="$(1)/bin:$$PATH" $(1)/bin/pip3 +VRUN = VIRTUAL_ENV=$(1) PATH="$(1)/bin:$$PATH" $(2) + +GITLAB_VENV := $(o)venv +$(GITLAB_VENV): INSTALL_PACKAGES := jinja2 pyaml + +%/venv: + python3 -m venv $@ + $(call VPIP,$@) install $(INSTALL_PACKAGES) + +gitlab-venv: $(GITLAB_VENV) + $(call VRUN,$<,$(MAKE)) gitlab-local + +gitlab-clean: s := $(s) +gitlab-clean: + rm $(s)*.yml + +clean:: + rm -rf $(GITLAB_VENV) + +.PHONY: gitlab-local gitlab-venv gitlab-clean diff --git a/misc/gitlab/README.md b/misc/gitlab/README.md new file mode 100644 index 000000000..de4a9dede --- /dev/null +++ b/misc/gitlab/README.md @@ -0,0 +1,100 @@ +# BIRD CI Pipelines + +There is quite a lot to do during our CI to ensure that BIRD stays stable and +no regression is introduced by regular development. We do as much as possible +in an automated way so that routine tasks aren't botched by somebody missing +a step. + +## Pipeline structure + +The main [main `.gitlab-ci.yml` file](../../.gitlab-ci.yml) is double-templated. + +Primary data is in [data.yml.j2](data.yml.j2) which is itself processed by +Jinja to expand various data structures. The expanded data is then passed to +the templater, converting [template.yml.j2](template.yml.j2) to the final +target file. + +The main file is committed into the repository, and therefore every change to +the source files must be accompanied with `make gitlab-local` (if you have +`jinja2` and `pyaml` installed locally) or `make gitlab-venv` which creates a +venv for that job on the fly. If that file was generated on the fly, it would +add some unnecessary half a minute to the CI run time, and make the CI output +more confusing. + +We would love to split the main CI file to some nice smaller parts but it would +probably impose quite a lot of repetitive work because of Gitlab. At least +we can document these parts as separate chapters. + +## Consistency + +Release-worthy commits are put into specific branches. These are: + +- `master` for BIRD 2 +- `thread-next` for BIRD 3 +- `stable-v*` for patch release branches + +These branches require some commit hygiene, which is checked by the script +in `tools/git-check-commits`, and if anybody pushes a commit which fails the +consistency check, they should immediately force-push a fixed history. + +Notably, this check requires all the commits to not be marked as `fixup!` or +as `WIP`, which both marks unfinished work. + +TODO: + +- add also check that the gitlab CI file is up-to-date with the sources + +## Build tests + +We have been historically hit by multiple integration problems arising from +distributions either keeping old tooling versions (e.g. old compilers), +breaking things by updates, or simply using different libraries and tools +than the core team which runs mostly Debian, Ubuntu and Fedora (as of 2026). + +Therefore, we check builds for multiple distribution versions from last about +5-10 years, depending on customer requirements and actual possibility to get a +docker image for that distro. + +We also run build tests for FreeBSD, OpenBSD and NetBSD inside QEMU. The images +are half-defined inside our +[temporary virtualization platform](https://gitlab.nic.cz/labs/bird-tools/-/tree/master/birdlab-tmp) +but please note that there are quite some updates local to the deployment, which +have not been applied back to that repository. + +Last but not least, we check partial builds where some of the protocols are switched off. + +TODO: + +- commit changes for the virtualization platform and test deployment from scratch +- add partial builds checking other `configure` checks, e.g. presence of `libssh` +- add cross platform builds at least nightly + +## Runtime tests + +These tests use the [Netlab infrastructure](https://gitlab.nic.cz/labs/bird-tools/-/tree/master/netlab) (yes, another Netlab), and we gradually add more tests covering more +BIRD functionality. There are also regression tests included. + +## Packaging and install tests + +These tasks build BIRD packages for various Linux distributions, and check whether these +actually install cleanly. + +TODO: + +- add automated cross platform packaging +- add package signing and repo upload +- add alpine +- add production docker image build + +## Tag build collection + +Release helper to collect all built packages when tag is made. + +## Internal docker rebuild + +All the distro build and packaging is done in docker build environments, +prepared with all the build dependencies, to shorten build times. + +TODO: + +- add on-demand triggers from the gitlab web ui diff --git a/misc/gitlab/pipeline.py b/misc/gitlab/pipeline.py index 1ac76789d..f7c51b7bc 100755 --- a/misc/gitlab/pipeline.py +++ b/misc/gitlab/pipeline.py @@ -8,22 +8,22 @@ import sys import yaml # Find where we are -localdir = pathlib.Path(__file__).parent +_, template_file, data_file, *_ = sys.argv # Prepare Jinja2 environment -env = jinja2.Environment(loader=jinja2.FileSystemLoader(str(localdir))) +env = jinja2.Environment(loader=jinja2.FileSystemLoader(".")) env.filters.update({ "to_yaml": lambda x: "" if type(x) is jinja2.runtime.Undefined else yaml.dump(x).rstrip() }) # Load and process input data try: - data = yaml.safe_load(rendered := env.get_template(f'data.yml.j2').render({})) + data = yaml.safe_load(rendered := env.get_template(data_file).render({})) except yaml.parser.ParserError as e: print("Failed to render input data, generated output here:") print(rendered) raise e # Load the actual template -template = env.get_template(f'template.yml.j2') +template = env.get_template(template_file) # Render the template final = template.render({ **data })