From 898ad92fc58f1b1b8c70dbfef96a746db4084bc9 Mon Sep 17 00:00:00 2001 From: Matthew Newton Date: Wed, 24 Apr 2024 17:18:29 +0100 Subject: [PATCH] crossbuild: generate docker files from templates --- scripts/docker/crossbuild.mk | 25 ++++- scripts/docker/m4/Dockerfile.m4 | 47 +++++++++ scripts/docker/m4/crossbuild.deb.m4 | 148 ++++++++++++++++++++++++++++ scripts/docker/m4/crossbuild.rpm.m4 | 139 ++++++++++++++++++++++++++ 4 files changed, 355 insertions(+), 4 deletions(-) create mode 100644 scripts/docker/m4/Dockerfile.m4 create mode 100644 scripts/docker/m4/crossbuild.deb.m4 create mode 100644 scripts/docker/m4/crossbuild.rpm.m4 diff --git a/scripts/docker/crossbuild.mk b/scripts/docker/crossbuild.mk index 933c4dc339..afea10caeb 100644 --- a/scripts/docker/crossbuild.mk +++ b/scripts/docker/crossbuild.mk @@ -33,11 +33,12 @@ GITDIR:=$(shell perl -MCwd -e 'print Cwd::abs_path shift' $$(git rev-parse --git # Don't use the Docker cache if asked ifneq "$(NOCACHE)" "" - NO_CACHE=--no-cache + DOCKER_BUILD_OPTS += "--no-cache" endif -CB_CPREFIX:=fr40x-crossbuild- +# Docker image and container name prefixes CB_IPREFIX:=freeradius40x-build +CB_CPREFIX:=fr40x-crossbuild- # # This Makefile is included in-line, and not via the "boilermake" @@ -111,6 +112,12 @@ crossbuild.clean: $(foreach IMG,${CB_IMAGES},crossbuild.${IMG}.clean) # crossbuild.wipe: $(foreach IMG,${CB_IMAGES},crossbuild.${IMG}.wipe) +# +# Regenerate all Dockerfiles from m4 templates +# +crossbuild.regen: $(foreach IMG,${CB_IMAGES},crossbuild.${IMG}.regen) + + # # Define rules for building a particular image # @@ -130,7 +137,7 @@ crossbuild.${1}.status: # $(DD)/stamp-image.${1}: ${Q}echo "BUILD ${1} ($(CB_IPREFIX)/${1}) > $(DD)/build.${1}" - ${Q}docker build $(NO_CACHE) $(DT)/${1} -f $(DT)/${1}/Dockerfile.cb -t $(CB_IPREFIX)/${1} >$(DD)/build.${1} 2>&1 + ${Q}docker build $(DOCKER_BUILD_OPTS) $(DT)/${1} -f $(DT)/${1}/Dockerfile.cb -t $(CB_IPREFIX)/${1} >$(DD)/build.${1} 2>&1 ${Q}touch $(DD)/stamp-image.${1} # @@ -169,7 +176,7 @@ $(DD)/docker.refresh.${1}: $(DD)/stamp-up.${1} .PHONY: $(DD)/docker.run.${1} $(DD)/docker.run.${1}: $(DD)/docker.refresh.${1} ${Q}echo "TEST ${1} > $(DD)/log.${1}" - ${Q}docker container exec $(CB_CPREFIX)${1} sh -lc '(cd /srv/build && make && make test)' > $(DD)/log.${1} 2>&1 || echo FAIL ${1} + ${Q}docker container exec $(CB_CPREFIX)${1} sh -lc '(cd /srv/build && make && make test)' > $(DD)/log.${1} 2>&1 || ( echo FAIL ${1} && false ) # # Stop the docker container @@ -231,6 +238,16 @@ crossbuild.${1}.wipe: .PHONY: crossbuild.${1}.refresh crossbuild.${1}.refresh: $(DD)/docker.refresh.${1} +# +# Regenerate the image Dockerfile from the m4 templates +# +.PHONY: crossbuild.${1}.regen +crossbuild.${1}.regen: $(DT)/${1}/Dockerfile.cb + +$(DT)/${1}/Dockerfile.cb: $(DOCKER_TMPL) $(CB_DIR)/m4/crossbuild.deb.m4 $(CB_DIR)/m4/crossbuild.rpm.m4 + ${Q}echo REGEN ${1} + ${Q}m4 -I $(CB_DIR)/m4 -D D_NAME=${1} -D D_TYPE=crossbuild $$< > $$@ + # # Run the build test # diff --git a/scripts/docker/m4/Dockerfile.m4 b/scripts/docker/m4/Dockerfile.m4 new file mode 100644 index 0000000000..74966f8b1b --- /dev/null +++ b/scripts/docker/m4/Dockerfile.m4 @@ -0,0 +1,47 @@ +dnl Look up the OS codename, docker base image etc before including +dnl the main Dockerfile template. +dnl +dnl This top-level template is used by both the docker makefile +dnl (scripts/docker/docker.mk) and the crossbuild makefile +dnl (scripts/crossbuild/crossbuild.mk), but the Dockerfile templates +dnl they use are different - see the m4 directories for each. +dnl +dnl Two macros must be defined for this template: +dnl D_NAME - the OS codename, see the table below, e.g. debian11 +dnl D_TYPE - the type of template, 'docker' or 'crossbuild' +dnl +divert(`-1') +changequote(`[', `]') +define([p_SET], [ + define([PKG_TYPE], [$1]) + define([OS_NAME], [$2]) + define([OS_VER], [$3]) + define([OS_CODENAME], [$4]) + define([DOCKER_IMAGE], [$5]) +]) +dnl D_NAME PKG_TYPE OS_NAME OS_VER OS_CODENAME DOCKER_IMAGE +ifelse( + D_NAME, [debian10], [p_SET([deb], [debian], [10], [buster], [debian:buster])], + D_NAME, [debian11], [p_SET([deb], [debian], [11], [bullseye], [debian:bullseye])], + D_NAME, [debian12], [p_SET([deb], [debian], [12], [bookworm], [debian:bookworm])], + D_NAME, [debiansid], [p_SET([deb], [debian], [99], [sid], [debian:sid])], + D_NAME, [ubuntu18], [p_SET([deb], [ubuntu], [18], [bionic], [ubuntu:18.04])], + D_NAME, [ubuntu20], [p_SET([deb], [ubuntu], [20], [focal], [ubuntu:20.04])], + D_NAME, [ubuntu22], [p_SET([deb], [ubuntu], [22], [jammy], [ubuntu:22.04])], + D_NAME, [ubuntu24], [p_SET([deb], [ubuntu], [24], [noble], [ubuntu:24.04])], + D_NAME, [centos7], [p_SET([rpm], [centos], [7], [7], [centos:7])], + D_NAME, [centos8], [p_SET([rpm], [centos], [8], [8], [centos:8])], + D_NAME, [rocky8], [p_SET([rpm], [rocky], [8], [8], [rockylinux/rockylinux:8])], + D_NAME, [rocky9], [p_SET([rpm], [rocky], [9], [9], [rockylinux/rockylinux:9])], + [errprint(error: OS 'D_NAME' not defined[,] see __file__ +)m4exit(1)] +) +undefine([p_SET]) +divert[]dnl +[#] Auto generated for D_NAME +[#] from scripts/docker/m4/D_TYPE.PKG_TYPE.m4 +[#] +[#] Rebuild this file with `make D_TYPE.D_NAME.regen` +[#] +changequote([`], ['])dnl +include(D_TYPE.PKG_TYPE.m4)dnl diff --git a/scripts/docker/m4/crossbuild.deb.m4 b/scripts/docker/m4/crossbuild.deb.m4 new file mode 100644 index 0000000000..ccb93aeb04 --- /dev/null +++ b/scripts/docker/m4/crossbuild.deb.m4 @@ -0,0 +1,148 @@ +ARG from=DOCKER_IMAGE +FROM ${from} as build + +SHELL ["/usr/bin/nice", "-n", "5", "/usr/bin/ionice", "-c", "3", "/bin/sh", "-x", "-c"] + +ARG APT_OPTS="-y --option=Dpkg::options::=--force-unsafe-io --no-install-recommends" + +ARG DEBIAN_FRONTEND=noninteractive + + +# +# Install add-apt-repository (may be needed for clang) and +# package development utilities +# +RUN apt-get update && \ + apt-get install $APT_OPTS \ + software-properties-common \ + devscripts \ + equivs \ + git \ + gnupg2 \ + lsb-release \ + procps \ + quilt \ + rsync \ + wget && \ + apt-get clean && \ + rm -r /var/lib/apt/lists/* + + +# +# Set up NetworkRADIUS extras repository +# +RUN install -d -o root -g root -m 0755 /etc/apt/keyrings && \ + wget -O /etc/apt/keyrings/packages.networkradius.com.asc "https://packages.networkradius.com/pgp/packages%40networkradius.com" && \ + echo "deb [signed-by=/etc/apt/keyrings/packages.networkradius.com.asc] http://packages.networkradius.com/extras/OS_NAME/OS_CODENAME OS_CODENAME main" > /etc/apt/sources.list.d/networkradius-extras.list && \ + apt-get update + +dnl +dnl Work out what clang packages we want to install +dnl +define(`CLANG_PKGS', `llvm clang lldb')dnl +ifelse(D_NAME, `debian10', `dnl +define(`CLANG_VER', `11')dnl +define(`CLANG_PKGS', `llvm-CLANG_VER clang-CLANG_VER lldb-CLANG_VER')dnl + +# +# Add repository for clang +# +RUN add-apt-repository -y "deb http://apt.llvm.org/OS_CODENAME/ llvm-toolchain-OS_CODENAME-CLANG_VER main" && \ + apt-key adv --fetch-keys http://apt.llvm.org/llvm-snapshot.gpg.key +')dnl + +# +# Install compilers +# +RUN apt-get install $APT_OPTS \ + g++ \ + CLANG_PKGS + +ifelse(D_NAME, `debian10', `dnl +# +# Set defaults +# +RUN update-alternatives --install /usr/bin/clang clang /usr/bin/clang-CLANG_VER 60 && \ + update-alternatives --config clang + +RUN update-alternatives --install /usr/bin/lldb lldb /usr/bin/lldb-CLANG_VER 60 && \ + update-alternatives --config lldb +') + + +# +# Install eapol_test dependencies +# +RUN apt-get install $APT_OPTS \ + libnl-3-dev \ + libnl-genl-3-dev + + +# +# Install debugging utilities +# +RUN apt-get install $APT_OPTS \ + gdb \ + less \ + lldb \ + vim + + +# +# Documentation build dependencies +# +define(`NODE_VER', `20')dnl +define(`ANTORA_VER', `3.1.7')dnl + +WORKDIR /tmp + +# - doxygen & JSON.pm +RUN apt-get install $APT_OPTS \ + doxygen \ + graphviz \ + libjson-perl + +# - antora (needs npm) +RUN bash -c "$(wget -O - https://deb.nodesource.com/setup_`'NODE_VER.x)" && \ + apt-get install $APT_OPTS nodejs && \ + npm i -g @antora/cli@ANTORA_VER @antora/site-generator-default@ANTORA_VER + +# - pandoc +RUN wget $(wget -qO - https://api.github.com/repos/jgm/pandoc/releases/latest | sed -ne 's/.*"browser_download_url".*"\(.*amd64\.deb\)"/\1/ p') && \ + find . -mindepth 1 -maxdepth 1 -type f -name 'pandoc-*.deb' -print0 | \ + xargs -0 -r apt-get install $APT_OPTS && \ + find . -mindepth 1 -maxdepth 1 -type f -name 'pandoc-*.deb' -delete + +# - asciidoctor +RUN apt-get install $APT_OPTS \ + ruby ruby-dev && \ + gem install asciidoctor + + +# +# Setup a src dir in /usr/local +# +RUN mkdir -p /usr/local/src/repositories +WORKDIR /usr/local/src/repositories + + +# +# Shallow clone the FreeRADIUS source +# +WORKDIR /usr/local/src/repositories +ARG source=https://github.com/FreeRADIUS/freeradius-server.git +RUN git clone --depth 1 --no-single-branch ${source} + +# +# Install build dependencies for all branches from v3 onwards +# +WORKDIR freeradius-server +RUN for i in $(git for-each-ref --format='%(refname:short)' refs/remotes/origin 2>/dev/null | sed -e 's#origin/##' | egrep "^(v[3-9]*\.[0-9x]*\.x|master|${branch})$" | sort -u); \ + do \ + git checkout $i; \ + if [ -e ./debian/control.in ] ; then \ + debian/rules debian/control ; \ + fi ; \ + mk-build-deps -irt"apt-get -o Debug::pkgProblemResolver=yes $APT_OPTS" debian/control ; \ + apt-get -y remove libiodbc2-dev ; \ + done diff --git a/scripts/docker/m4/crossbuild.rpm.m4 b/scripts/docker/m4/crossbuild.rpm.m4 new file mode 100644 index 0000000000..1783fbfe67 --- /dev/null +++ b/scripts/docker/m4/crossbuild.rpm.m4 @@ -0,0 +1,139 @@ +ARG from=DOCKER_IMAGE +FROM ${from} as build + +# +# Install devtools like make and git and the EPEL +# repository for freetds and hiredis +# +RUN yum update -y +RUN yum install -y rpmdevtools openssl epel-release git procps yum-utils \ + rsync`'ifelse(OS_VER, `7',, ` dnf-plugins-core') + +ifelse(OS_VER, `7', `dnl +# +# Install GCC that has the requisite support for C11 keywords and atomics +# +RUN yum install -y centos-release-scl +RUN yum install -y devtoolset-8-gcc devtoolset-8-gcc-c++ +ENV CC=/opt/rh/devtoolset-8/root/usr/bin/gcc + +# +# Remove the CentOS-SCLo repo which is apparently not valid? +# See: https://bugs.centos.org/view.php?id=14773 +# +RUN rm /etc/yum.repos.d/CentOS-SCLo-scl-rh.repo +RUN rm /etc/yum.repos.d/CentOS-SCLo-scl.repo +')dnl + +ifelse(OS_VER, `8', `dnl +RUN yum config-manager --set-enabled powertools + +# +# Install GCC that has the requisite support for C11 keywords and atomics +# +RUN yum install -y gcc-toolset-9 +')dnl + +ifelse(OS_VER, `9', `dnl +RUN yum config-manager --set-enabled crb +')dnl + +# +# Documentation build dependencies +# +define(`NODE_VER', `20')dnl +define(`ANTORA_VER', `3.1.7')dnl + +# - doxygen & JSON.pm +RUN yum install -y doxygen graphviz perl-JSON +# - antora (npm needed) +RUN curl -sL https://rpm.nodesource.com/setup_`'NODE_VER.x | bash - +RUN yum install -y nodejs +RUN npm i -g @antora/cli@ANTORA_VER @antora/site-generator-default@ANTORA_VER +# - pandoc +RUN curl -o - -L $(curl -s https://api.github.com/repos/jgm/pandoc/releases/latest | grep "browser_download_url.*tar.gz" | cut -d '"' -f 4) | tar xzvf - -C /tmp/ +RUN mv /tmp/pandoc-*/bin/* /usr/local/bin +# - asciidoctor +RUN yum install -y rubygems-devel +RUN gem install asciidoctor + +# +# Setup a src dir in /usr/local +# +RUN mkdir -p /usr/local/src/repositories +WORKDIR /usr/local/src/repositories + +changequote([{,}])dnl +# +# Set up NetworkRADIUS extras repository for latest libkqueue +# +RUN echo $'[networkradius-extras]\n\ +name=NetworkRADIUS-extras-$releasever\n\ +baseurl=http://packages.networkradius.com/extras/OS_NAME/$releasever/\n\ +enabled=1\n\ +gpgcheck=1\n\ +gpgkey=https://packages.networkradius.com/pgp/packages@networkradius.com'\ +> /etc/yum.repos.d/networkradius-extras.repo +RUN rpm --import https://packages.networkradius.com/pgp/packages@networkradius.com +ifelse(ifelse(OS_VER, 7, yes, OS_VER, 8, yes, no), yes, [{dnl Only add LTB on centos7/rocky8 + +# +# Use LTB's openldap packages instead of the distribution version to avoid linking against NSS +# +RUN echo $'[ltb-project]\n\ +name=LTB project packages\n\ +baseurl=https://ltb-project.org/rpm/$releasever/$basearch\n\ +enabled=1\n\ +gpgcheck=1\n\ +gpgkey=file:///etc/pki/rpm-gpg/RPM-GPG-KEY-LTB-project'\ +> /etc/yum.repos.d/ltb-project.repo +RUN rpm --import https://ltb-project.org/lib/RPM-GPG-KEY-LTB-project +}])dnl +changequote(`,')dnl + +# +# Shallow clone the FreeRADIUS source +# +WORKDIR /usr/local/src/repositories +ARG source=https://github.com/FreeRADIUS/freeradius-server.git +RUN git clone --depth 1 --no-single-branch ${source} + +# +# Install build dependencies for all branches from v3 onwards +# Nodesource has issues (no SRPMS in some repos) and is not needed here +# +define(`EXTRA_DISABLE', ifelse(OS_VER, 7, `--disablerepo="nodesource*"', `'))dnl +WORKDIR freeradius-server +RUN for i in $(git for-each-ref --format='%(refname:short)' refs/remotes/origin 2>/dev/null | sed -e 's#origin/##' | egrep "^(v[4-9]*\.[0-9x]*\.x|master)$");\ + do \ + git checkout $i; \ + [ -e redhat/freeradius.spec ] && yum-builddep EXTRA_DISABLE -y redhat/freeradius.spec; \ + done + +# +# Tests require extra Yubikey libraries +# +RUN yum install -y libyubikey-devel + +# +# Which is required by fixture setup utilities +# +RUN yum install -y which + +# +# Explicitly install libnl3-devel which is required for the EAP tests +# +RUN yum install -y libnl3-devel + +ifelse(OS_VER, 7,, `dnl +# +# We test with TLS1.1, but that is disabled by default on some +# newer systems. +# +RUN update-crypto-policies --set LEGACY +')dnl + +# +# Create the RPM build tree +# +RUN rpmdev-setuptree -- 2.47.3