From: Arran Cudbard-Bell Date: Tue, 19 May 2026 17:31:02 +0000 (-0400) Subject: docker: dedupe service / ci / crossbuild build rules into shared macros X-Git-Url: http://git.ipfire.org/cgi-bin/gitweb.cgi?a=commitdiff_plain;h=a6333fb9335420502c519fb4e5ec14fccbbfa2b3;p=thirdparty%2Ffreeradius-server.git docker: dedupe service / ci / crossbuild build rules into shared macros The four-and-soon-to-be-five m4-generated Dockerfile families were each carrying their own hand-rolled per-image regen rule, drift detector, and build rule. Fold the common shape into four reusable macros in scripts/docker/m4-macros.mk: M4_REGEN_RULE for the per-image m4 -> Dockerfile. rule, M4_REGEN_BUNDLE for the umbrella regen target, M4_REGEN_CHECK for the regen.check drift detector, and DOCKER_BUILD for the per-image stamp-tracked docker build. DOCKER_BUILD derives everything from (image, type): output is freeradius4-/:, log lands at $(DD)/build.., stamp at $(DD)/stamp-image.., and every locally-built image is labelled ci-ttl=60m so the periodic prune workflow can identify ephemeral builds. Local build stamp paths gained the . suffix; anyone with scripts that rm the old stamp-image. needs to know. --- diff --git a/scripts/docker/crossbuild.mk b/scripts/docker/crossbuild.mk index 60b0d58f545..814f02ca2ee 100644 --- a/scripts/docker/crossbuild.mk +++ b/scripts/docker/crossbuild.mk @@ -38,10 +38,16 @@ ifneq "$(NOCACHE)" "" DOCKER_BUILD_OPTS += "--no-cache" endif -# Docker image and container name prefixes -CB_IPREFIX:=freeradius40x-build +# Docker container name prefix. Image tags now derive from type via +# DOCKER_BUILD (freeradius4-crossbuild/:); the container +# name still has its own prefix because it has to be valid in +# `docker run --name` (no slashes / colons) and disambiguate from +# any service-image containers. CB_CPREFIX:=fr40x-crossbuild- +# Shared m4 snippets included by every content template. +M4_SHARED:=$(wildcard $(CB_DIR)/m4/common.*.m4) + # # This Makefile is included in-line, and not via the "boilermake" # wrapper. But it's still useful to use the same process for @@ -114,68 +120,50 @@ crossbuild.clean: $(foreach IMG,${CB_IMAGES},crossbuild.${IMG}.clean) # crossbuild.distclean: $(foreach IMG,${CB_IMAGES},crossbuild.${IMG}.distclean) -# -# Regenerate all Dockerfile.crossbuild files from m4 templates. Depends on -# the file targets directly; no per-image phony aliases. -# -.PHONY: docker.crossbuild.regen docker.crossbuild.regen.check -docker.crossbuild.regen: $(foreach IMG,${CB_IMAGES},$(DT)/${IMG}/Dockerfile.crossbuild) +include $(CB_DIR)/m4-macros.mk # -# Verify every committed Dockerfile.crossbuild matches a fresh render of its -# m4 source. Fails with a diff if a contributor edited the m4 but -# forgot to regen+commit. +# Per-image m4 -> Dockerfile.crossbuild regen rules and stamp-tracked +# build rules, plus bundle and drift-detector targets. # -docker.crossbuild.regen.check: - @failed=0; for IMG in $(CB_IMAGES); do \ - tmp=$$(mktemp); \ - m4 -I $(CB_DIR)/m4 -D D_NAME=$$IMG -D D_TYPE=crossbuild $(DOCKER_TMPL) > $$tmp; \ - if ! diff -u $(DT)/$$IMG/Dockerfile.crossbuild $$tmp; then \ - echo "OUT OF SYNC: $(DT)/$$IMG/Dockerfile.crossbuild"; failed=1; \ - fi; \ - rm $$tmp; \ - done; \ - [ $$failed -eq 0 ] || { echo; echo "Run 'make docker.crossbuild.regen' and commit the result."; exit 1; } +$(foreach IMG,$(CB_IMAGES),\ + $(eval $(call M4_REGEN_RULE,$(IMG),crossbuild,$(CB_DIR)/m4/crossbuild.deb.m4 $(CB_DIR)/m4/crossbuild.rpm.m4)) \ + $(eval $(call DOCKER_BUILD,$(IMG),crossbuild,$(if $(CB_FROM_$(IMG)),--build-arg=from=$(CB_FROM_$(IMG))),))) + +$(eval $(call M4_REGEN_BUNDLE,docker.crossbuild.regen,crossbuild,$(CB_IMAGES))) +$(eval $(call M4_REGEN_CHECK,docker.crossbuild.regen.check,crossbuild,$(CB_IMAGES),docker.crossbuild.regen)) # -# Define rules for building a particular image +# Define rules for building a particular image. The stamp-image +# file target and the Dockerfile.crossbuild target are generated by +# DOCKER_BUILD / M4_REGEN_RULE above (see m4-macros.mk); this block +# only defines the per-image lifecycle targets (status / up / down +# / sh / refresh / log / reset / clean / distclean) that don't +# generalise across types. # define CROSSBUILD_IMAGE_RULE -# -# Show status (based on stamp files) -# .PHONY: crossbuild.${1}.status crossbuild.${1}.status: ${Q}printf "%s" "`echo \" ${1} \" | cut -c 1-20`" ${Q}if [ -e "$(DD)/stamp-up.${1}" ]; then echo "running"; \ - elif [ -e "$(DD)/stamp-image.${1}" ]; then echo "built"; \ + elif [ -e "$(DD)/stamp-image.${1}.crossbuild" ]; then echo "built"; \ else echo "-"; fi -# -# Build the docker image -# -# CB_FROM_${1} overrides the `from` build-arg for this target. Empty by -# default so local builds use the upstream image baked into Dockerfile.crossbuild -# by m4. CI exports CB_FROM_ to point at internal base images -# (see .github/workflows/crossbuild.yml). -# -$(DD)/stamp-image.${1}: - ${Q}echo "BUILD ${1} ($(CB_IPREFIX)/${1}) > $(DD)/build.${1}" - ${Q}docker build $(DOCKER_BUILD_OPTS) $(if $(CB_FROM_${1}),--build-arg=from=$(CB_FROM_${1})) $(DT)/${1} -f $(DT)/${1}/Dockerfile.crossbuild -t $(CB_IPREFIX)/${1} >$(DD)/build.${1} 2>&1 - ${Q}touch $(DD)/stamp-image.${1} # -# Start up the docker container +# Start up the docker container. CB_FROM_${1} overrides the `from` +# build-arg via DOCKER_BUILD's macro arg; CI exports CB_FROM_ +# to point at internal base images (see .github/workflows/crossbuild.yml). # .PHONY: $(DD)/docker.up.${1} -$(DD)/docker.up.${1}: $(DD)/stamp-image.${1} +$(DD)/docker.up.${1}: $(DD)/stamp-image.${1}.crossbuild ${Q}echo "START ${1} ($(CB_CPREFIX)${1})" ${Q}docker container inspect $(CB_CPREFIX)${1} >/dev/null 2>&1 || \ docker run -d --rm \ --privileged --cap-add=ALL \ --mount=type=bind,source="$(GITDIR)",destination=/srv/src,ro \ - --name $(CB_CPREFIX)${1} $(CB_IPREFIX)/${1} \ + --name $(CB_CPREFIX)${1} freeradius4-crossbuild/${1}:$(GIT_SHA) \ /bin/sh -c 'while true; do sleep 60; done' >/dev/null $(DD)/stamp-up.${1}: $(DD)/docker.up.${1} @@ -232,10 +220,10 @@ crossbuild.${1}.sh: crossbuild.${1}.up .PHONY: crossbuild.${1}.log crossbuild.${1}.log: @if which less >/dev/null; then \ - less +G $(DD)/log.${1};\ + less +G $(DD)/build.${1}.crossbuild;\ elif which more >/dev/null; then \ - more $(DD)/log.${1};\ - else cat $(DD)/log.${1}; fi + more $(DD)/build.${1}.crossbuild;\ + else cat $(DD)/build.${1}.crossbuild; fi # # Tidy up stamp files. This means on next run we'll do @@ -246,7 +234,7 @@ crossbuild.${1}.log: crossbuild.${1}.reset: ${Q}echo RESET ${1} ${Q}rm -f $(DD)/stamp-up.${1} - ${Q}rm -f $(DD)/stamp-image.${1} + ${Q}rm -f $(DD)/stamp-image.${1}.crossbuild # # Clean down images. Means on next run we'll rebuild the @@ -255,8 +243,8 @@ crossbuild.${1}.reset: .PHONY: crossbuild.${1}.distclean crossbuild.${1}.distclean: ${Q}echo CLEAN ${1} - ${Q}docker image rm $(CB_IPREFIX)/${1} >/dev/null 2>&1 || true - ${Q}rm -f $(DD)/stamp-image.${1} + ${Q}docker image rm freeradius4-crossbuild/${1}:$(GIT_SHA) >/dev/null 2>&1 || true + ${Q}rm -f $(DD)/stamp-image.${1}.crossbuild # # Refresh git repository within the docker image @@ -264,14 +252,6 @@ crossbuild.${1}.distclean: .PHONY: crossbuild.${1}.refresh crossbuild.${1}.refresh: $(DD)/docker.refresh.${1} -# -# Image Dockerfile.crossbuild rule. Regen via the bundle target above; no -# per-image variant. -# -$(DT)/${1}/Dockerfile.crossbuild: $(DOCKER_TMPL) $(CB_DIR)/m4/crossbuild.deb.m4 $(CB_DIR)/m4/crossbuild.rpm.m4 - ${Q}echo REGEN ${1} - ${Q}m4 -I $(CB_DIR)/m4 -D D_NAME=${1} -D D_TYPE=crossbuild $$< > $$@ - # # Run the build test # diff --git a/scripts/docker/docker.mk b/scripts/docker/docker.mk index a974210917a..46fd2da387f 100644 --- a/scripts/docker/docker.mk +++ b/scripts/docker/docker.mk @@ -20,6 +20,10 @@ CB_DIR:=$(patsubst %/,%,$(dir $(realpath $(lastword $(MAKEFILE_LIST))))) # Where the docker directories are DT:=$(CB_DIR)/build +# Where stamp files and per-build logs land. Shared with crossbuild.mk +# so the periodic prune workflow only has one tree to look at. +DD:=$(CB_DIR)/crossbuild + # Location of top-level m4 template DOCKER_TMPL:=$(CB_DIR)/m4/Dockerfile.m4 @@ -35,9 +39,6 @@ ifneq "$(NOCACHE)" "" DOCKER_BUILD_OPTS += " --no-cache" endif -# Docker image name prefix -D_IPREFIX:=freeradius4 - # # This Makefile is included in-line, and not via the "boilermake" # wrapper. But it's still useful to use the same process for @@ -49,6 +50,8 @@ else Q= endif +include $(CB_DIR)/m4-macros.mk + # # Enter here: This builds everything # @@ -70,96 +73,48 @@ docker.info_header: docker.help: @echo "" @echo "Make targets:" - @echo " docker - build all images" - @echo " docker.common - build and test common images" - @echo " docker.info - list images" - @echo " docker.service.regen - regenerate all production Dockerfiles" - @echo " docker.ci.regen - regenerate all CI base Dockerfile.ci files" + @echo " docker - build all images" + @echo " docker.common - build and test common images" + @echo " docker.info - list images" + @echo " docker.service.regen - regenerate all Dockerfile.service files" + @echo " docker.ci.regen - regenerate all Dockerfile.ci files" + @echo " docker.service.regen.check - fail if any Dockerfile.service is stale" + @echo " docker.ci.regen.check - fail if any Dockerfile.ci is stale" @echo "" @echo "Per-image targets:" - @echo " docker.IMAGE.build - build image as $(D_IPREFIX)/" + @echo " docker.IMAGE.build - build image as freeradius4-service/:$(GIT_SHA)" @echo "" @echo "Use 'make NOCACHE=1 ...' to disregard the Docker cache on build" # -# Regenerate all Dockerfiles from m4 templates. Both bundles depend -# on the file targets directly; no per-image phony aliases. +# Per-image m4 -> Dockerfile. regen rules and stamp-tracked +# build rules, plus bundle / drift-detector targets per type. # -.PHONY: docker.service.regen docker.ci.regen docker.service.regen.check docker.ci.regen.check -docker.service.regen: $(foreach IMG,${IMAGES},$(DT)/${IMG}/Dockerfile.service) -docker.ci.regen: $(foreach IMG,${IMAGES},$(DT)/${IMG}/Dockerfile.ci) +$(foreach IMG,$(IMAGES),\ + $(eval $(call M4_REGEN_RULE,$(IMG),service,$(CB_DIR)/m4/service.deb.m4 $(CB_DIR)/m4/service.rpm.m4)) \ + $(eval $(call M4_REGEN_RULE,$(IMG),ci,$(CB_DIR)/m4/ci.deb.m4 $(CB_DIR)/m4/ci.rpm.m4)) \ + $(eval $(call DOCKER_BUILD,$(IMG),service,,))) -# -# Verify every committed Dockerfile.service / Dockerfile.ci matches a fresh -# render of its m4 source. Fails with a diff if a contributor edited -# the m4 but forgot to regen+commit. -# -docker.service.regen.check: - @failed=0; for IMG in $(IMAGES); do \ - tmp=$$(mktemp); \ - m4 -I $(CB_DIR)/m4 -D D_NAME=$$IMG -D D_TYPE=service $(DOCKER_TMPL) > $$tmp; \ - if ! diff -u $(DT)/$$IMG/Dockerfile.service $$tmp; then \ - echo "OUT OF SYNC: $(DT)/$$IMG/Dockerfile.service"; failed=1; \ - fi; \ - rm $$tmp; \ - done; \ - [ $$failed -eq 0 ] || { echo; echo "Run 'make docker.service.regen' and commit the result."; exit 1; } - -docker.ci.regen.check: - @failed=0; for IMG in $(IMAGES); do \ - tmp=$$(mktemp); \ - m4 -I $(CB_DIR)/m4 -D D_NAME=$$IMG -D D_TYPE=ci $(DOCKER_TMPL) > $$tmp; \ - if ! diff -u $(DT)/$$IMG/Dockerfile.ci $$tmp; then \ - echo "OUT OF SYNC: $(DT)/$$IMG/Dockerfile.ci"; failed=1; \ - fi; \ - rm $$tmp; \ - done; \ - [ $$failed -eq 0 ] || { echo; echo "Run 'make docker.ci.regen' and commit the result."; exit 1; } +$(eval $(call M4_REGEN_BUNDLE,docker.service.regen,service,$(IMAGES))) +$(eval $(call M4_REGEN_BUNDLE,docker.ci.regen,ci,$(IMAGES))) +$(eval $(call M4_REGEN_CHECK,docker.service.regen.check,service,$(IMAGES),docker.service.regen)) +$(eval $(call M4_REGEN_CHECK,docker.ci.regen.check,ci,$(IMAGES),docker.ci.regen)) # -# Define rules for building a particular image +# Phony status / build wrappers for the per-image targets. The +# stamp file (produced by DOCKER_BUILD) is the real work; the +# phony just gives operators a stable name to type. # -define CROSSBUILD_IMAGE_RULE - +define DOCKER_SERVICE_PHONY .PHONY: docker.${1}.status docker.${1}.status: - ${Q}docker image ls --format "\t{{.Repository}} \t{{.CreatedAt}}" $(D_IPREFIX)/${1} + $${Q}docker image ls --format "\t{{.Repository}}:{{.Tag}} \t{{.CreatedAt}}" freeradius4-service/${1} -# -# Build the docker image -# .PHONY: docker.${1}.build -docker.${1}.build: - ${Q}echo "BUILD ${1} ($(D_IPREFIX)/${1}) from $(DT)/${1}/Dockerfile.service" - - ${Q}docker buildx build \ - $(DOCKER_BUILD_OPTS) \ - --progress=plain \ - . \ - -f $(DT)/${1}/Dockerfile.service \ - -t $(D_IPREFIX)/${1} - -# -# Production image Dockerfile.service rule. The CI base Dockerfile.ci -# is consumed by docker-refresh.yml to build the self-hosted-* base -# images that ci-deb.yml / ci-rpm.yml run their build jobs inside. -# Both regen via the bundle targets above; no per-image variants. -# -$(DT)/${1}/Dockerfile.service: $(DOCKER_TMPL) $(CB_DIR)/m4/service.deb.m4 $(CB_DIR)/m4/service.rpm.m4 $(M4_SHARED) - ${Q}echo REGEN ${1} "->" $$@ - ${Q}m4 -I $(CB_DIR)/m4 -D D_NAME=${1} -D D_TYPE=service $$< > $$@ - -$(DT)/${1}/Dockerfile.ci: $(DOCKER_TMPL) $(CB_DIR)/m4/ci.deb.m4 $(CB_DIR)/m4/ci.rpm.m4 $(M4_SHARED) - ${Q}echo REGEN ${1} "->" $$@ - ${Q}m4 -I $(CB_DIR)/m4 -D D_NAME=${1} -D D_TYPE=ci $$< > $$@ - +docker.${1}.build: $(DD)/stamp-image.${1}.service endef -# -# Add all the image building rules -# -$(foreach IMAGE,$(IMAGES),\ - $(eval $(call CROSSBUILD_IMAGE_RULE,$(IMAGE)))) +$(foreach IMG,$(IMAGES),$(eval $(call DOCKER_SERVICE_PHONY,$(IMG)))) # if docker is defined diff --git a/scripts/docker/m4-macros.mk b/scripts/docker/m4-macros.mk new file mode 100644 index 00000000000..f3f41d34934 --- /dev/null +++ b/scripts/docker/m4-macros.mk @@ -0,0 +1,99 @@ +# +# Shared macros for the m4-generated Dockerfile pipeline. Consumed +# by scripts/docker/docker.mk (service + ci) and scripts/docker/crossbuild.mk +# (crossbuild). DOCKER_BUILD tags every locally-built image with the +# short commit hash and a ci-ttl label so the periodic prune workflow +# can reap them. +# + +ifndef M4_MACROS_MK_INCLUDED +M4_MACROS_MK_INCLUDED := 1 + +# +# Short commit hash used as the dev-local image tag. Hub-pushed +# images use :latest in the publish workflow and aren't subject to +# the local prune. +# +GIT_SHA := $(shell git rev-parse --short HEAD 2>/dev/null) + +# +# Go-style duration string controlling how long an image survives +# on a CI host before the periodic prune workflow removes it. +# Image-level label, so any tag pointing at the image carries it. +# +CI_TTL ?= 60m + +# +# Per-image Dockerfile target rule. +# +# $(1) image name (e.g. debian12, ubuntu24) +# $(2) type (service / ci / crossbuild / profiling) +# $(3) type-specific m4 prerequisites (the .deb.m4 / .rpm.m4 files) +# +define M4_REGEN_RULE +$(DT)/${1}/Dockerfile.${2}: $(DOCKER_TMPL) ${3} $(M4_SHARED) + $${Q}echo REGEN ${1} "->" $$@ + $${Q}m4 -I $(CB_DIR)/m4 -D D_NAME=${1} -D D_TYPE=${2} $$< > $$@ +endef + +# +# Umbrella regen target: depends on every per-image file target. +# +# $(1) target name (e.g. docker.service.regen) +# $(2) type +# $(3) image list +# +define M4_REGEN_BUNDLE +.PHONY: ${1} +${1}: $(foreach IMG,${3},$(DT)/${IMG}/Dockerfile.${2}) +endef + +# +# Drift detector: re-renders each m4 template and diffs against the +# committed Dockerfile. Non-zero exit if any file is out of sync. +# +# $(1) target name (e.g. docker.service.regen.check) +# $(2) type +# $(3) image list +# $(4) regen target the operator should run to fix drift +# +define M4_REGEN_CHECK +.PHONY: ${1} +${1}: + @failed=0; for IMG in ${3}; do \ + tmp=$$$$(mktemp); \ + m4 -I $(CB_DIR)/m4 -D D_NAME=$$$$IMG -D D_TYPE=${2} $(DOCKER_TMPL) > $$$$tmp; \ + if ! diff -u $(DT)/$$$$IMG/Dockerfile.${2} $$$$tmp; then \ + echo "OUT OF SYNC: $(DT)/$$$$IMG/Dockerfile.${2}"; failed=1; \ + fi; \ + rm $$$$tmp; \ + done; \ + [ $$$$failed -eq 0 ] || { echo; echo "Run 'make ${4}' and commit the result."; exit 1; } +endef + +# +# Per-image build rule. Tags freeradius4-/:, +# labels ci-ttl=$(CI_TTL), logs to $(DD)/build.., and +# touches $(DD)/stamp-image.. so a second invocation +# is a no-op until a dep changes. +# +# $(1) image name +# $(2) type +# $(3) extra docker build args (e.g. --build-arg=from=...) +# $(4) extra stamp-file prerequisites (e.g. base image stamp) +# +define DOCKER_BUILD +$(DD)/stamp-image.${1}.${2}: $(DT)/${1}/Dockerfile.${2} ${4} | $(DD) + $${Q}echo "BUILD ${1} (freeradius4-${2}/${1}:$(GIT_SHA)) > $(DD)/build.${1}.${2}" + $${Q}docker build $$(DOCKER_BUILD_OPTS) ${3} \ + --label ci-ttl=$(CI_TTL) \ + -f $(DT)/${1}/Dockerfile.${2} \ + -t freeradius4-${2}/${1}:$(GIT_SHA) \ + . >$(DD)/build.${1}.${2} 2>&1 + $${Q}touch $$@ +endef + +$(DD): + @mkdir -p $@ + +endif