From: Michael Tremer Date: Tue, 29 Aug 2023 15:37:04 +0000 (+0000) Subject: python3-wheel: New package X-Git-Url: http://git.ipfire.org/?a=commitdiff_plain;h=423f8222826560b9e41da7bc57ad4013685c464a;p=people%2Fms%2Fipfire-2.x.git python3-wheel: New package This is required to build python3-awscrt Signed-off-by: Michael Tremer --- diff --git a/config/rootfiles/common/python3-wheel b/config/rootfiles/common/python3-wheel new file mode 100644 index 000000000..e604e34c5 --- /dev/null +++ b/config/rootfiles/common/python3-wheel @@ -0,0 +1,40 @@ +#usr/bin/wheel +#usr/lib/python3.10/site-packages/wheel +#usr/lib/python3.10/site-packages/wheel-0.41.2-py3.10.egg-info +#usr/lib/python3.10/site-packages/wheel-0.41.2-py3.10.egg-info/PKG-INFO +#usr/lib/python3.10/site-packages/wheel-0.41.2-py3.10.egg-info/SOURCES.txt +#usr/lib/python3.10/site-packages/wheel-0.41.2-py3.10.egg-info/dependency_links.txt +#usr/lib/python3.10/site-packages/wheel-0.41.2-py3.10.egg-info/entry_points.txt +#usr/lib/python3.10/site-packages/wheel-0.41.2-py3.10.egg-info/requires.txt +#usr/lib/python3.10/site-packages/wheel-0.41.2-py3.10.egg-info/top_level.txt +#usr/lib/python3.10/site-packages/wheel/__init__.py +#usr/lib/python3.10/site-packages/wheel/__main__.py +#usr/lib/python3.10/site-packages/wheel/_setuptools_logging.py +#usr/lib/python3.10/site-packages/wheel/bdist_wheel.py +#usr/lib/python3.10/site-packages/wheel/cli +#usr/lib/python3.10/site-packages/wheel/cli/__init__.py +#usr/lib/python3.10/site-packages/wheel/cli/convert.py +#usr/lib/python3.10/site-packages/wheel/cli/pack.py +#usr/lib/python3.10/site-packages/wheel/cli/tags.py +#usr/lib/python3.10/site-packages/wheel/cli/unpack.py +#usr/lib/python3.10/site-packages/wheel/macosx_libfile.py +#usr/lib/python3.10/site-packages/wheel/metadata.py +#usr/lib/python3.10/site-packages/wheel/util.py +#usr/lib/python3.10/site-packages/wheel/vendored +#usr/lib/python3.10/site-packages/wheel/vendored/__init__.py +#usr/lib/python3.10/site-packages/wheel/vendored/packaging +#usr/lib/python3.10/site-packages/wheel/vendored/packaging/__init__.py +#usr/lib/python3.10/site-packages/wheel/vendored/packaging/_elffile.py +#usr/lib/python3.10/site-packages/wheel/vendored/packaging/_manylinux.py +#usr/lib/python3.10/site-packages/wheel/vendored/packaging/_musllinux.py +#usr/lib/python3.10/site-packages/wheel/vendored/packaging/_parser.py +#usr/lib/python3.10/site-packages/wheel/vendored/packaging/_structures.py +#usr/lib/python3.10/site-packages/wheel/vendored/packaging/_tokenizer.py +#usr/lib/python3.10/site-packages/wheel/vendored/packaging/markers.py +#usr/lib/python3.10/site-packages/wheel/vendored/packaging/requirements.py +#usr/lib/python3.10/site-packages/wheel/vendored/packaging/specifiers.py +#usr/lib/python3.10/site-packages/wheel/vendored/packaging/tags.py +#usr/lib/python3.10/site-packages/wheel/vendored/packaging/utils.py +#usr/lib/python3.10/site-packages/wheel/vendored/packaging/version.py +#usr/lib/python3.10/site-packages/wheel/vendored/vendor.txt +#usr/lib/python3.10/site-packages/wheel/wheelfile.py diff --git a/lfs/python3-wheel b/lfs/python3-wheel new file mode 100644 index 000000000..4ef3c5ebd --- /dev/null +++ b/lfs/python3-wheel @@ -0,0 +1,81 @@ +############################################################################### +# # +# IPFire.org - A linux based firewall # +# Copyright (C) 2007-2023 IPFire Team # +# # +# This program is free software: you can redistribute it and/or modify # +# it under the terms of the GNU General Public License as published by # +# the Free Software Foundation, either version 3 of the License, or # +# (at your option) any later version. # +# # +# This program is distributed in the hope that it will be useful, # +# but WITHOUT ANY WARRANTY; without even the implied warranty of # +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # +# GNU General Public License for more details. # +# # +# You should have received a copy of the GNU General Public License # +# along with this program. If not, see . # +# # +############################################################################### + +############################################################################### +# Definitions +############################################################################### + +include Config + +SUMMARY = A built-package format for Python + +VER = 0.41.2 + +THISAPP = wheel-$(VER) +DL_FILE = $(THISAPP).tar.gz +DL_FROM = $(URL_IPFIRE) +DIR_APP = $(DIR_SRC)/$(THISAPP) +TARGET = $(DIR_INFO)/$(THISAPP) + +############################################################################### +# Top-level Rules +############################################################################### + +objects = $(DL_FILE) + +$(DL_FILE) = $(DL_FROM)/$(DL_FILE) + +$(DL_FILE)_BLAKE2 = 211c3021179e2f80036f57d59583ac449e91558e64fb10cc5b925d93f7288c6db444ca2ac35a20b13a2646641885c094faa458b591554f3bd926c53a3d099ec2 + +install : $(TARGET) + +check : $(patsubst %,$(DIR_CHK)/%,$(objects)) + +download :$(patsubst %,$(DIR_DL)/%,$(objects)) + +b2 : $(subst %,%_BLAKE2,$(objects)) + +dist: + @$(PAK) + +############################################################################### +# Downloading, checking, b2sum +############################################################################### + +$(patsubst %,$(DIR_CHK)/%,$(objects)) : + @$(CHECK) + +$(patsubst %,$(DIR_DL)/%,$(objects)) : + @$(LOAD) + +$(subst %,%_BLAKE2,$(objects)) : + @$(B2SUM) + +############################################################################### +# Installation Details +############################################################################### + +$(TARGET) : $(patsubst %,$(DIR_DL)/%,$(objects)) + @$(PREBUILD) + @rm -rf $(DIR_APP) && cd $(DIR_SRC) && tar zxf $(DIR_DL)/$(DL_FILE) + cd $(DIR_APP) && python3 setup.py build + cd $(DIR_APP) && python3 setup.py install --root=/ + @rm -rf $(DIR_APP) + @$(POSTBUILD) diff --git a/make.sh b/make.sh index b40530c5c..638caf61f 100755 --- a/make.sh +++ b/make.sh @@ -2,7 +2,7 @@ ############################################################################### # # # IPFire.org - A linux based firewall # -# Copyright (C) 2007-2024 IPFire Team # +# Copyright (C) 2007-2023 IPFire Team # # # # This program is free software: you can redistribute it and/or modify # # it under the terms of the GNU General Public License as published by # @@ -22,23 +22,23 @@ NAME="IPFire" # Software name SNAME="ipfire" # Short name # If you update the version don't forget to update backupiso and add it to core update -VERSION="2.29" # Version number -CORE="192" # Core Level (Filename) +VERSION="2.27" # Version number +CORE="180" # Core Level (Filename) SLOGAN="www.ipfire.org" # Software slogan CONFIG_ROOT=/var/ipfire # Configuration rootdir +MAX_RETRIES=1 # prefetch/check loop +BUILD_IMAGES=1 # Flash and Xen Downloader +KVER=`grep --max-count=1 VER lfs/linux | awk '{ print $3 }'` # Information from Git GIT_BRANCH="$(git rev-parse --abbrev-ref HEAD)" # Git Branch GIT_TAG="$(git tag | tail -1)" # Git Tag GIT_LASTCOMMIT="$(git rev-parse --verify HEAD)" # Last commit -TOOLCHAINVER=20240827 +TOOLCHAINVER=20230731 -KVER_SUFFIX="-${SNAME}" - -# Kernel Version -KVER="$(grep --max-count=1 VER lfs/linux | awk '{ print $3 }')" -KVER="${KVER/-rc/.0-rc}${KVER_SUFFIX}" +# use multicore and max compression +ZSTD_OPT="-T0 --ultra -22" ############################################################################### # @@ -46,67 +46,33 @@ KVER="${KVER/-rc/.0-rc}${KVER_SUFFIX}" # ############################################################################### -# All supported architectures -ARCHES=( - aarch64 - riscv64 - x86_64 -) - -HOST_ARCH="${HOSTTYPE}" -HOST_KERNEL="$(uname -r)" -LC_ALL=POSIX -PS1='\u:\w$ ' - -HAS_TIME_NAMESPACE="true" - -# Disable time namespaces for older kernels -case "${HOST_KERNEL}" in - 4.*|5.[12345].*) - HAS_TIME_NAMESPACE="false" - ;; -esac - -# Are we reading from/writing to a terminal? -is_terminal() { - [ -t 0 ] && [ -t 1 ] && [ -t 2 ] -} - -# Define color for messages -if is_terminal; then - BOLD="$(tput bold)" - RED="$(tput setaf 1)" - GREEN="$(tput setaf 2)" - YELLOW="$(tput setaf 3)" - CYAN="$(tput setaf 6)" - NORMAL="$(tput sgr0)" +# Remember if the shell is interactive or not +if [ -t 0 ] && [ -t 1 ]; then + INTERACTIVE=true +else + INTERACTIVE=false fi # Sets or adjusts pretty formatting variables resize_terminal() { + ## Screen Dimentions # Find current screen size - COLUMNS="$(tput cols)" + COLUMNS=$(tput cols) # When using remote connections, such as a serial port, stty size returns 0 - if ! is_terminal || [ "${COLUMNS}" = "0" ]; then + if ! ${INTERACTIVE} || [ "${COLUMNS}" = "0" ]; then COLUMNS=80 fi - # Wipe any previous content before updating the counters - if [ -n "${TIME_COL}" ]; then - tput hpa "${TIME_COL}" - tput el - fi - - # The status column is always 8 characters wide - STATUS_WIDTH=8 - - # The time column is always 12 characters wide + # Measurements for positioning result messages + OPTIONS_WIDTH=20 TIME_WIDTH=12 + STATUS_WIDTH=8 + NAME_WIDTH=$(( COLUMNS - OPTIONS_WIDTH - TIME_WIDTH - STATUS_WIDTH )) + LINE_WIDTH=$(( COLUMNS - STATUS_WIDTH )) - # Where do the columns start? - (( STATUS_COL = COLUMNS - STATUS_WIDTH )) - (( TIME_COL = STATUS_COL - TIME_WIDTH )) + TIME_COL=$(( NAME_WIDTH + OPTIONS_WIDTH )) + STATUS_COL=$(( TIME_COL + TIME_WIDTH )) } # Initially setup terminal @@ -115,41 +81,39 @@ resize_terminal # Call resize_terminal when terminal is being resized trap "resize_terminal" WINCH -# Writes a line to the log file -log() { - local line="$@" - - # Fetch the current timestamp - local t="$(date -u "+%b %e %T")" - - # Append the line to file - if [ -w "${LOGFILE}" ]; then - echo "${t}: ${line}" >> "${LOGFILE}" - fi - - return 0 -} - -find_base() { - local path - - # Figure out the absolute script path using readlink - path="$(readlink -f "${0}" 2>&1)" - - # If that failed, try realpath - if [ -z "${path}" ]; then - path="$(realpath "${0}" 2>&1)" +# Define color for messages +BOLD="\\033[1;39m" +DONE="\\033[1;32m" +SKIP="\\033[1;34m" +WARN="\\033[1;35m" +FAIL="\\033[1;31m" +NORMAL="\\033[0;39m" + +# New architecture variables +HOST_ARCH="$(uname -m)" + +PWD=$(pwd) +BASENAME=$(basename $0) + +# Debian specific settings +if [ ! -e /etc/debian_version ]; then + FULLPATH=`which $0` +else + if [ -x /usr/bin/realpath ]; then + FULLPATH=`/usr/bin/realpath $0` + else + echo "ERROR: Need to do apt-get install realpath" + exit 1 fi +fi - # If that failed, I am out of ideas - if [ -z "${path}" ]; then - echo "${0}: Could not determine BASEDIR" >&2 - return 1 - fi +# This is the directory where make.sh is in +export BASEDIR=$(echo $FULLPATH | sed "s/\/$BASENAME//g") - # Return the dirname - dirname "${path}" -} +LOGFILE=$BASEDIR/log/_build.preparation.log +export LOGFILE +DIR_CHK=$BASEDIR/cache/check +mkdir $BASEDIR/log/ 2>/dev/null system_processors() { getconf _NPROCESSORS_ONLN 2>/dev/null || echo "1" @@ -169,6 +133,138 @@ system_memory() { done < /proc/meminfo } +configure_build() { + local build_arch="${1}" + + if [ "${build_arch}" = "default" ]; then + build_arch="$(configure_build_guess)" + fi + + case "${build_arch}" in + x86_64) + BUILDTARGET="${build_arch}-pc-linux-gnu" + CROSSTARGET="${build_arch}-cross-linux-gnu" + BUILD_PLATFORM="x86" + CFLAGS_ARCH="-m64 -mtune=generic -fcf-protection=full" + ;; + + aarch64) + BUILDTARGET="${build_arch}-pc-linux-gnu" + CROSSTARGET="${build_arch}-cross-linux-gnu" + BUILD_PLATFORM="arm" + CFLAGS_ARCH="-mbranch-protection=standard" + ;; + + riscv64) + BUILDTARGET="${build_arch}-pc-linux-gnu" + CROSSTARGET="${build_arch}-cross-linux-gnu" + BUILD_PLATFORM="riscv" + CFLAGS_ARCH="" + ;; + + *) + exiterror "Cannot build for architure ${build_arch}" + ;; + esac + + # Check if the QEMU helper is available if needed. + if qemu_is_required "${build_arch}"; then + local qemu_build_helper="$(qemu_find_build_helper_name "${build_arch}")" + + if [ -n "${qemu_build_helper}" ]; then + QEMU_TARGET_HELPER="${qemu_build_helper}" + else + exiterror "Could not find a binfmt_misc helper entry for ${build_arch}" + fi + fi + + BUILD_ARCH="${build_arch}" + TOOLS_DIR="/tools_${BUILD_ARCH}" + + # Enables hardening + HARDENING_CFLAGS="-Wp,-D_FORTIFY_SOURCE=2 -Wp,-D_GLIBCXX_ASSERTIONS -fstack-protector-strong -fstack-clash-protection" + + CFLAGS="-O2 -pipe -Wall -fexceptions -fPIC ${CFLAGS_ARCH}" + CXXFLAGS="${CFLAGS}" + + RUSTFLAGS="-Copt-level=3 -Clink-arg=-Wl,-z,relro,-z,now -Ccodegen-units=1 --cap-lints=warn ${RUSTFLAGS_ARCH}" + + # Determine parallelism + # We assume that each process consumes about + # 128MB of memory. Therefore we find out how + # many processes fit into memory. + local mem_max=$(( ${SYSTEM_MEMORY} / 128 )) + local cpu_max=$(( ${SYSTEM_PROCESSORS} )) + + local parallelism + if [ ${mem_max} -lt ${cpu_max} ]; then + parallelism=${mem_max} + else + parallelism=${cpu_max} + fi + + # Use this as default PARALLELISM + DEFAULT_PARALLELISM="${parallelism}" + + # Limit lauched ninja build jobs to computed parallel value. + NINJAJOBS="${parallelism}" + + # Compression parameters + # We use mode 8 for reasonable memory usage when decompressing + # but with overall good compression + XZ_OPT="-8" + + # We try to use as many cores as possible + XZ_OPT="${XZ_OPT} -T0" + + # We need to limit memory because XZ uses too much when running + # in parallel and it isn't very smart in limiting itself. + # We allow XZ to use up to 70% of all system memory. + local xz_memory=$(( SYSTEM_MEMORY * 7 / 10 )) + + XZ_OPT="${XZ_OPT} --memory=${xz_memory}MiB" +} + +configure_build_guess() { + case "${HOST_ARCH}" in + x86_64) + echo "x86_64" + ;; + + aarch64) + echo "aarch64" + ;; + + riscv64) + echo "riscv64" + ;; + + *) + exiterror "Cannot guess build architecture" + ;; + esac +} + +stdumount() { + umount $BASEDIR/build/sys 2>/dev/null; + umount $BASEDIR/build/dev/shm 2>/dev/null; + umount $BASEDIR/build/dev/pts 2>/dev/null; + umount $BASEDIR/build/dev 2>/dev/null; + umount $BASEDIR/build/proc 2>/dev/null; + umount $BASEDIR/build/install/mnt 2>/dev/null; + umount $BASEDIR/build/usr/src/cache 2>/dev/null; + umount $BASEDIR/build/usr/src/ccache 2>/dev/null; + umount $BASEDIR/build/usr/src/config 2>/dev/null; + umount $BASEDIR/build/usr/src/doc 2>/dev/null; + umount $BASEDIR/build/usr/src/html 2>/dev/null; + umount $BASEDIR/build/usr/src/langs 2>/dev/null; + umount $BASEDIR/build/usr/src/lfs 2>/dev/null; + umount $BASEDIR/build/usr/src/log 2>/dev/null; + umount $BASEDIR/build/usr/src/src 2>/dev/null; + umount $BASEDIR/build/usr/src 2>/dev/null; + umount $BASEDIR/build/tmp 2>/dev/null; +} + format_runtime() { local seconds=${1} @@ -187,12 +283,32 @@ format_runtime() { } print_line() { - # Print the string all the way to the status column - printf "%-${STATUS_COL}s" "$*" + local line="$@" + + printf "%-${LINE_WIDTH}s" "${line}" +} + +_print_line() { + local status="${1}" + shift + + if ${INTERACTIVE}; then + printf "${!status}" + fi + + print_line "$@" + + if ${INTERACTIVE}; then + printf "${NORMAL}" + fi } print_headline() { - printf "${BOLD}%s${NORMAL}\n" "$*" + _print_line BOLD "$@" +} + +print_error() { + _print_line FAIL "$@" } print_package() { @@ -207,113 +323,54 @@ print_package() { string="${string} (${version})" fi - # Add the options - if [ -n "${options}" ]; then - string="${string} ${options[@]}" - fi - - # Print the string - print_line "${string}" + printf "%-$(( ${NAME_WIDTH} - 1 ))s " "${string}" + printf "%$(( ${OPTIONS_WIDTH} - 1 ))s " "${options}" } print_runtime() { local runtime=$(format_runtime $@) - # Move back the cursor to rewrite the runtime - if is_terminal; then - tput hpa "${TIME_COL}" + if ${INTERACTIVE}; then + printf "\\033[${TIME_COL}G[ ${BOLD}%$(( ${TIME_WIDTH} - 4 ))s${NORMAL} ]" "${runtime}" + else + printf "[ %$(( ${TIME_WIDTH} - 4 ))s ]" "${runtime}" fi - - printf "[ ${BOLD}%$(( TIME_WIDTH - 4 ))s${NORMAL} ]" "${runtime}" } print_status() { local status="${1}" - local color - case "${status}" in - DONE) - color="${BOLD}${GREEN}" - ;; - FAIL) - color="${BOLD}${RED}" - ;; - SKIP) - color="${BOLD}${CYAN}" - ;; - WARN) - color="${BOLD}${YELLOW}" - ;; - *) - color="${BOLD}" - ;; - esac + local color="${!status}" - # Move to the start of the column - if is_terminal; then - tput hpa "${STATUS_COL}" + if ${INTERACTIVE}; then + printf "\\033[${STATUS_COL}G[${color-${BOLD}} %-$(( ${STATUS_WIDTH} - 4 ))s ${NORMAL}]\n" "${status}" + else + printf "[ %-$(( ${STATUS_WIDTH} - 4 ))s ]\n" "${status}" fi - - printf "[ ${color}%$(( STATUS_WIDTH - 4 ))s${NORMAL} ]\n" "${status}" } -print_build_summary() { - local runtime="${1}" +print_build_stage() { + print_headline "$@" - print_line "*** Build Finished" - print_runtime "${runtime}" - print_status DONE + # end line + printf "\n" } -# Launches a timer process as a co-process -launch_timer() { - # Do nothing if the timer is already running - if [ -n "${TIMER_PID}" ]; then - return 0 - fi - - # Don't launch the timer when we are not on a terminal - if ! is_terminal; then - return 0 - fi - - # Launch the co-process - coproc TIMER { "${0}" "__timer" "$$"; } - - # Register the signal handlers - trap "__timer_event" SIGUSR1 - trap "terminate_timer" EXIT -} +print_build_summary() { + local runtime=$(format_runtime $@) -# Terminates a previously launched timer -terminate_timer() { - if [ -n "${TIMER_PID}" ]; then - kill -TERM "${TIMER_PID}" - fi + print_line "*** Build finished in ${runtime}" + print_status DONE } -# The timer main loop -__timer() { - local pid="${1}" - - # Send SIGUSR1 to the main process once a second - # If the parent process has gone away, we will terminate. - while sleep 1; do - if ! kill -USR1 "${pid}" &>/dev/null; then - break - fi +exiterror() { + stdumount + for i in `seq 0 7`; do + if ( losetup /dev/loop${i} 2>/dev/null | grep -q "/install/images" ); then + losetup -d /dev/loop${i} 2>/dev/null + fi; done - return 0 -} - -# Called when the timer triggers -# This function does nothing, but is needed interrupt the wait call -__timer_event() { - return 0 -} - -exiterror() { # Dump logfile if [ -n "${LOGFILE}" ] && [ -e "${LOGFILE}" ]; then echo # empty line @@ -328,7 +385,7 @@ exiterror() { local line for line in "ERROR: $@" " Check ${LOGFILE} for errors if applicable"; do - print_line "${line}" + print_error "${line}" print_status FAIL done @@ -336,8 +393,6 @@ exiterror() { } prepareenv() { - local network="false" - # Are we running the right shell? if [ -z "${BASH}" ]; then exiterror "BASH environment variable is not set. You're probably running the wrong shell." @@ -351,153 +406,78 @@ prepareenv() { trap "exiterror 'Build process interrupted'" SIGINT SIGTERM SIGQUIT # Checking if running as root user - if [ "${UID}" -ne 0 ]; then - exiterror "root privileges required for building" + if [ $(id -u) -ne 0 ]; then + exiterror "root privileges required for building" fi - local required_space - - # Parse arguments - while [ $# -gt 0 ]; do - case "${1}" in - --required-space=*) - required_space="${1#--required-space=}" - ;; - - --network) - network="true" - ;; + # Checking for necessary temporary space + print_line "Checking for necessary space on disk $BASE_DEV" + BASE_DEV=`df -P -k $BASEDIR | tail -n 1 | awk '{ print $1 }'` + BASE_ASPACE=`df -P -k $BASEDIR | tail -n 1 | awk '{ print $4 }'` + if (( 2048000 > $BASE_ASPACE )); then + BASE_USPACE=`du -skx $BASEDIR | awk '{print $1}'` + if (( 2048000 - $BASE_USPACE > $BASE_ASPACE )); then + print_status FAIL + exiterror "Not enough temporary space available, need at least 2GB on $BASE_DEV" + fi + else + print_status DONE + fi - *) - exiterror "Unknown argument: ${1}" - ;; - esac - shift - done + # Set umask + umask 022 - # Do we need to check the required space? - if [ -n "${required_space}" ]; then - local free_space free_blocks block_size - local consumed_space path + # Set LFS Directory + LFS=$BASEDIR/build - # Fetch free blocks - read -r free_blocks block_size <<< "$(stat --file-system --format="%a %S" "${BASEDIR}")" + # Setup environment + set +h + LC_ALL=POSIX + export LFS LC_ALL CFLAGS CXXFLAGS DEFAULT_PARALLELISM RUSTFLAGS NINJAJOBS + unset CC CXX CPP LD_LIBRARY_PATH LD_PRELOAD - # Calculate free space - (( free_space = free_blocks * block_size / 1024 / 1024 )) + # Make some extra directories + mkdir -p "${BASEDIR}/build${TOOLS_DIR}" 2>/dev/null + mkdir -p $BASEDIR/build/{etc,usr/src} 2>/dev/null + mkdir -p $BASEDIR/build/{dev/{shm,pts},proc,sys} + mkdir -p $BASEDIR/{cache,ccache/${BUILD_ARCH}/${TOOLCHAINVER}} 2>/dev/null - # If we don't have the total space free, we need to check how much we have consumed already... - if [ "${free_space}" -lt "${required_space}" ]; then - # Add any consumed space - while read -r consumed_space path; do - (( free_space += consumed_space / 1024 / 1024 )) - done <<< "$(du --summarize --bytes "${BUILD_DIR}" "${IMAGES_DIR}" "${LOG_DIR}" 2>/dev/null)" - fi + if [ "${ENABLE_RAMDISK}" = "on" ]; then + mkdir -p $BASEDIR/build/usr/src + mount -t tmpfs tmpfs -o size=8G,nr_inodes=1M,mode=1777 $BASEDIR/build/usr/src - # Check that we have the required space - if [ "${free_space}" -lt "${required_space}" ]; then - exiterror "Not enough temporary space available, need at least ${required_space}MiB, but only have ${free_space}MiB" - fi + mkdir -p ${BASEDIR}/build/tmp + mount -t tmpfs tmpfs -o size=4G,nr_inodes=1M,mode=1777 ${BASEDIR}/build/tmp fi - # Set umask - umask 022 + mkdir -p $BASEDIR/build/usr/src/{cache,config,doc,html,langs,lfs,log,src,ccache} - # Make some extra directories - mkdir -p "${CCACHE_DIR}" - mkdir -p "${IMAGES_DIR}" - mkdir -p "${PACKAGES_DIR}" - mkdir -p "${BUILD_DIR}/${TOOLS_DIR}" - mkdir -p "${BUILD_DIR}/cache" - mkdir -p "${BUILD_DIR}/dev" - mkdir -p "${BUILD_DIR}/etc" - mkdir -p "${BUILD_DIR}/proc" - mkdir -p "${BUILD_DIR}/root" - mkdir -p "${BUILD_DIR}/sys" - mkdir -p "${BUILD_DIR}/tmp" - mkdir -p "${BUILD_DIR}/usr/src" - mkdir -p "${BUILD_DIR}/usr/src/cache" - mkdir -p "${BUILD_DIR}/usr/src/ccache" - mkdir -p "${BUILD_DIR}/usr/src/config" - mkdir -p "${BUILD_DIR}/usr/src/doc" - mkdir -p "${BUILD_DIR}/usr/src/html" - mkdir -p "${BUILD_DIR}/usr/src/images" - mkdir -p "${BUILD_DIR}/usr/src/langs" - mkdir -p "${BUILD_DIR}/usr/src/lfs" - mkdir -p "${BUILD_DIR}/usr/src/log" - mkdir -p "${BUILD_DIR}/usr/src/src" - - # Make BUILD_DIR a mountpoint - mount -o bind "${BUILD_DIR}" "${BUILD_DIR}" - - # Create a new, minimal /dev - mount build_dev "${BUILD_DIR}/dev" \ - -t devtmpfs -o "nosuid,noexec,mode=0755,size=4m,nr_inodes=64k" - - # Mount a new /dev/pts - mount build_dev_pts "${BUILD_DIR}/dev/pts" \ - -t devpts -o "nosuid,noexec,newinstance,ptmxmode=0666,mode=620" - - # Mount a new /dev/shm - mount build_dev_shm "${BUILD_DIR}/dev/shm" \ - -t tmpfs -o "nosuid,nodev,strictatime,mode=1777,size=1024m" - - # Mount a new /tmp - mount build_tmp "${BUILD_DIR}/tmp" \ - -t tmpfs -o "nosuid,nodev,strictatime,size=4G,nr_inodes=1M,mode=1777" - - # Create an empty /proc directory and make it a mountpoint - mkdir -p "${BUILD_DIR}/proc" - mount --bind "${BUILD_DIR}/proc" "${BUILD_DIR}/proc" + mknod -m 600 $BASEDIR/build/dev/console c 5 1 2>/dev/null + mknod -m 666 $BASEDIR/build/dev/null c 1 3 2>/dev/null # Make all sources and proc available under lfs build - mount --bind /sys "${BUILD_DIR}/sys" - mount --bind -o ro "${BASEDIR}/cache" "${BUILD_DIR}/usr/src/cache" - mount --bind -o ro "${BASEDIR}/config" "${BUILD_DIR}/usr/src/config" - mount --bind -o ro "${BASEDIR}/doc" "${BUILD_DIR}/usr/src/doc" - mount --bind -o ro "${BASEDIR}/html" "${BUILD_DIR}/usr/src/html" - mount --bind -o ro "${BASEDIR}/langs" "${BUILD_DIR}/usr/src/langs" - mount --bind -o ro "${BASEDIR}/lfs" "${BUILD_DIR}/usr/src/lfs" - mount --bind -o ro "${BASEDIR}/src" "${BUILD_DIR}/usr/src/src" - - # Mount the log directory - mount --bind "${LOG_DIR}" "${BUILD_DIR}/usr/src/log" - - # Mount the ccache - mount --bind "${CCACHE_DIR}" "${BUILD_DIR}/usr/src/ccache" - - # Mount the images directory - mount --bind "${IMAGES_DIR}" "${BUILD_DIR}/usr/src/images" - - # Bind-mount files requires for networking if requested - if [ "${network}" = "true" ]; then - local file - - for file in /etc/resolv.conf /etc/hosts; do - # Skip if the source files does not exist - if [ ! -e "${file}" ]; then - continue - fi - - # Create the destination if it does not exist - if [ ! -e "${BUILD_DIR}/${file}" ]; then - touch "${BUILD_DIR}/${file}" - fi - - # Mount the file read-only - mount --bind -o ro "${file}" "${BUILD_DIR}/${file}" - done - fi - - # Configure the ccache + mount --bind /dev $BASEDIR/build/dev + mount --bind /dev/pts $BASEDIR/build/dev/pts + mount --bind /dev/shm $BASEDIR/build/dev/shm + mount --bind /proc $BASEDIR/build/proc + mount --bind /sys $BASEDIR/build/sys + mount --bind $BASEDIR/cache $BASEDIR/build/usr/src/cache + mount --bind $BASEDIR/ccache/${BUILD_ARCH}/${TOOLCHAINVER} $BASEDIR/build/usr/src/ccache + mount --bind $BASEDIR/config $BASEDIR/build/usr/src/config + mount --bind $BASEDIR/doc $BASEDIR/build/usr/src/doc + mount --bind $BASEDIR/html $BASEDIR/build/usr/src/html + mount --bind $BASEDIR/langs $BASEDIR/build/usr/src/langs + mount --bind $BASEDIR/lfs $BASEDIR/build/usr/src/lfs + mount --bind $BASEDIR/log $BASEDIR/build/usr/src/log + mount --bind $BASEDIR/src $BASEDIR/build/usr/src/src + + # Run LFS static binary creation scripts one by one + export CCACHE_DIR=$BASEDIR/ccache export CCACHE_TEMPDIR="/tmp" export CCACHE_COMPILERCHECK="string:toolchain-${TOOLCHAINVER} ${BUILD_ARCH}" - # Install the QEMU helper - qemu_install_helper - # Remove pre-install list of installed files in case user erase some files before rebuild - rm -f "${BUILD_DIR}/usr/src/lsalr" + rm -f $BASEDIR/build/usr/src/lsalr 2>/dev/null # Prepare string for /etc/system-release. local system_release="${NAME} ${VERSION} (${BUILD_ARCH})" @@ -533,16 +513,69 @@ prepareenv() { esac # Setup ccache cache size - execute --chroot ccache --max-size="${CCACHE_CACHE_SIZE}" + enterchroot ccache --max-size="${CCACHE_CACHE_SIZE}" +} + +enterchroot() { + # Install QEMU helper, if needed + qemu_install_helper + + local PATH="${TOOLS_DIR}/ccache/bin:/bin:/usr/bin:/sbin:/usr/sbin:${TOOLS_DIR}/sbin:${TOOLS_DIR}/bin" + + # Prepend any custom changes to PATH + if [ -n "${CUSTOM_PATH}" ]; then + PATH="${CUSTOM_PATH}:${PATH}" + fi + + PATH="${PATH}" chroot ${LFS} env -i \ + HOME="/root" \ + TERM="${TERM}" \ + PS1="${PS1}" \ + PATH="${PATH}" \ + SYSTEM_RELEASE="${SYSTEM_RELEASE}" \ + PAKFIRE_TREE="${PAKFIRE_TREE}" \ + NAME="${NAME}" \ + SNAME="${SNAME}" \ + VERSION="${VERSION}" \ + CORE="${CORE}" \ + SLOGAN="${SLOGAN}" \ + TOOLS_DIR="${TOOLS_DIR}" \ + CONFIG_ROOT="${CONFIG_ROOT}" \ + CFLAGS="${CFLAGS} ${HARDENING_CFLAGS}" \ + CXXFLAGS="${CXXFLAGS} ${HARDENING_CFLAGS}" \ + RUSTFLAGS="${RUSTFLAGS}" \ + BUILDTARGET="${BUILDTARGET}" \ + CROSSTARGET="${CROSSTARGET}" \ + BUILD_ARCH="${BUILD_ARCH}" \ + BUILD_PLATFORM="${BUILD_PLATFORM}" \ + CCACHE_DIR=/usr/src/ccache \ + CCACHE_TEMPDIR="${CCACHE_TEMPDIR}" \ + CCACHE_COMPILERCHECK="${CCACHE_COMPILERCHECK}" \ + GOCACHE="/usr/src/ccache/go" \ + KVER="${KVER}" \ + XZ_OPT="${XZ_OPT}" \ + DEFAULT_PARALLELISM="${DEFAULT_PARALLELISM}" \ + SYSTEM_PROCESSORS="${SYSTEM_PROCESSORS}" \ + SYSTEM_MEMORY="${SYSTEM_MEMORY}" \ + $(fake_environ) \ + $(qemu_environ) \ + "$@" } entershell() { - echo "Entering to a shell inside the build environment, go out with exit" + if [ ! -e $BASEDIR/build/usr/src/lfs/ ]; then + exiterror "No such file or directory: $BASEDIR/build/usr/src/lfs/" + fi + echo "Entering to a shell inside LFS chroot, go out with exit" local PS1="ipfire build chroot (${BUILD_ARCH}) \u:\w\$ " - # Run an interactive shell - execute --chroot --interactive --network bash -i + if enterchroot bash -i; then + stdumount + else + print_status FAIL + exiterror "chroot error" + fi } lfsmakecommoncheck() { @@ -565,432 +598,184 @@ lfsmakecommoncheck() { fi fi - echo -ne "`date -u '+%b %e %T'`: Building $* " >> $LOGFILE - - return 0 # pass all! -} - -execute() { - local chroot="false" - local command=() - local interactive="false" - local timer - local network="false" - - # Collect environment variables - local -A environ=( - [PATH]="${TOOLS_DIR}/ccache/bin:${TOOLS_DIR}/sbin:${TOOLS_DIR}/bin:${PATH}" - [HOME]="${HOME}" - [PS1]="${PS1}" - [TERM]="vt100" - - # Distro Information - [NAME]="${NAME}" - [SNAME]="${SNAME}" - [VERSION]="${VERSION}" - [CORE]="${CORE}" - [SLOGAN]="${SLOGAN}" - [SYSTEM_RELEASE]="${SYSTEM_RELEASE}" - [PAKFIRE_TREE]="${PAKFIRE_TREE}" - [CONFIG_ROOT]="${CONFIG_ROOT}" - - # Kernel Version - [KVER]="${KVER}" - [KVER_SUFFIX]="${KVER_SUFFIX}" - - # Compiler flags - [CFLAGS]="${CFLAGS[@]}" - [CXXFLAGS]="${CFLAGS[@]}" - [RUSTFLAGS]="${RUSTFLAGS[@]}" - - # ccache - [CCACHE_DIR]="${CCACHE_DIR}" - [CCACHE_TEMPDIR]="${CCACHE_TEMPDIR}" - [CCACHE_COMPILERCHECK]="${CCACHE_COMPILERCHECK}" - - # System Properties - [SYSTEM_PROCESSORS]="${SYSTEM_PROCESSORS}" - [SYSTEM_MEMORY]="${SYSTEM_MEMORY}" - - # Parallelism - [DEFAULT_PARALLELISM]="${DEFAULT_PARALLELISM}" - - # Compression Options - [XZ_OPT]="${XZ_OPT[*]}" - [ZSTD_OPT]="${ZSTD_OPT[*]}" - - # Build Architecture - [BUILD_ARCH]="${BUILD_ARCH}" - [BUILD_PLATFORM]="${BUILD_PLATFORM}" - - # Targets - [CROSSTARGET]="${CROSSTARGET}" - [BUILDTARGET]="${BUILDTARGET}" - - # Paths - [LFS_BASEDIR]="${BASEDIR}" - [BUILD_DIR]="${BUILD_DIR}" - [IMAGES_DIR]="${IMAGES_DIR}" - [LOG_DIR]="${LOG_DIR}" - [PACKAGES_DIR]="${PACKAGES_DIR}" - [TOOLS_DIR]="${TOOLS_DIR}" - ) - - local unshare=() - - # Configure a new namespace - if [ -n "${IN_NAMESPACE}" ]; then - unshare+=( - # Create a new cgroup namespace - "--cgroup" - - # Create a new mount namespace - "--mount" - "--propagation=slave" - - # Create a new PID namespace and fork - "--pid" - "--fork" - - # Create a new UTS namespace - "--uts" - - # Mount /proc so that the build environment does not see - # any foreign processes. - "--mount-proc=${BUILD_DIR}/proc" - - # If unshare is asked to terminate, terminate all child processes - "--kill-child" - ) - - # Optionally set up a new time namespace - if [ "${HAS_TIME_NAMESPACE}" = "true" ]; then - unshare+=( "--time" ) + # Script slipped? + local i + for i in $SKIP_PACKAGE_LIST + do + if [ "$i" == "$1" ]; then + print_status SKIP + return 1; fi - fi - - while [ $# -gt 0 ]; do - case "${1}" in - --chroot) - chroot="true" - - # Update some variables - environ+=( - [PATH]="${TOOLS_DIR}/ccache/bin:/bin:/usr/bin:/sbin:/usr/sbin:${TOOLS_DIR}/sbin:${TOOLS_DIR}/bin" - [HOME]="/root" - - # Paths - [LFS_BASEDIR]="/usr/src" - [BUILD_DIR]="/" - [IMAGES_DIR]="/usr/src/images" - [LOG_DIR]="/usr/src/log" - [PACKAGES_DIR]="/usr/src/images/packages" - - # Compiler Cache - [CCACHE_DIR]="/usr/src/ccache" - - # Go Cache - [GOCACHE]="/usr/src/ccache/go" - ) - - # Fake environment - if [ -e "${BUILD_DIR}${TOOLS_DIR}/lib/libpakfire_preload.so" ]; then - environ+=( - [LD_PRELOAD]="${TOOLS_DIR}/lib/libpakfire_preload.so" - - # Fake kernel version, because some of the packages do not - # compile with kernel 3.0 and later - [UTS_RELEASE]="${KVER}" - - # Fake machine - [UTS_MACHINE]="${BUILD_ARCH}" - ) - fi - ;; - - --interactive) - interactive="true" - - # Use the actual value of $TERM - environ+=( - [TERM]="${TERM}" - ) - ;; - - --network) - network="true" - - # Export the proxy configuration - environ+=( - [https_proxy]="${https_proxy}" - [http_proxy]="${http_proxy}" - ) - ;; - - --timer=*) - timer="${1#--timer=}" - ;; - - -*) - echo "Unknown argument: ${1}" >&2 - return 2 - ;; - - # Parse any custom environment variables - *=*) - environ["${1%=*}"]="${1#*=}" - ;; - - # The rest is the command - *) - command+=( "$@" ) - break - ;; - esac - shift done - # Prepend any custom changes to PATH - if [ -n "${CUSTOM_PATH}" ]; then - environ[PATH]="${CUSTOM_PATH}:${environ[PATH]}" - fi - - # Setup QEMU - if qemu_is_required; then - environ+=( - [QEMU_TARGET_HELPER]="${QEMU_TARGET_HELPER}" - - # Enable QEMU strace - #[QEMU_STRACE]="1" - ) - - case "${BUILD_ARCH}" in - arm*) - environ+=( - [QEMU_CPU]="${QEMU_CPU:-cortex-a9}" - ) - ;; - - riscv64) - environ+=( - [QEMU_CPU]="${QEMU_CPU:-sifive-u54}" + echo -ne "`date -u '+%b %e %T'`: Building $* " >> $LOGFILE - # Bug fix for QEMU locking up - [G_SLICE]="always-malloc" - ) - ;; - esac + cd $BASEDIR/lfs && make -s -f $* LFS_BASEDIR=$BASEDIR BUILD_ARCH="${BUILD_ARCH}" \ + MESSAGE="$1\t " download >> $LOGFILE 2>&1 + if [ $? -ne 0 ]; then + exiterror "Download error in $1" fi - # Network - if [ "${network}" = "false" ]; then - unshare+=( "--net" ) + cd $BASEDIR/lfs && make -s -f $* LFS_BASEDIR=$BASEDIR BUILD_ARCH="${BUILD_ARCH}" \ + MESSAGE="$1\t b2sum" b2 >> $LOGFILE 2>&1 + if [ $? -ne 0 ]; then + exiterror "BLAKE2 checksum error in $1, check file in cache or signature" fi - local execute=() - local env + return 0 # pass all! +} - # Create new namespaces - if [ "${#unshare[@]}" -gt 0 ]; then - execute+=( - "unshare" "${unshare[@]}" - ) - fi +lfsmake1() { + lfsmakecommoncheck $* + [ $? == 1 ] && return 0 - # Call a setup script in the new namespaces to perform - # further set up, but before we change root. - execute+=( "${BASEDIR}/tools/execute.sh" ) + # Set PATH to use the toolchain tools first and then whatever the host has set + local PATH="${TOOLS_DIR}/ccache/bin:${TOOLS_DIR}/sbin:${TOOLS_DIR}/bin:${PATH}" - # Run in chroot? - if [ "${chroot}" = "true" ]; then - execute+=( "chroot" "${BUILD_DIR}" ) + if [ -n "${CUSTOM_PATH}" ]; then + PATH="${CUSTOM_PATH}:${PATH}" fi - # Set PATH so that we can find the env binary - local PATH="${environ[PATH]}:${PATH}" - - # Reset the environment - execute+=( - "env" - - # Clear the previous environment - "--ignore-environment" - - # Change to the home directory - --chdir="${environ[HOME]}" - ) - - # Export the environment - for env in ${!environ[@]}; do - execute+=( "${env}=${environ[${env}]}" ) - done - - # Append the command - execute+=( "${command[@]}" ) - - local pid - - # Return code - local r=0 - - # Store the start time - local t="${SECONDS}" - - # Run the command in the background and pipe all output to the logfile - case "${interactive}" in - true) - "${execute[@]}" || return $? - ;; - - false) - # Launch the timer if needed - if [ -n "${timer}" ]; then - launch_timer - fi - - # Dispatch the command to the background - { - "${execute[@]}" >> "${LOGFILE}" 2>&1 > $LOGFILE 2>&1 & + + if ! wait_until_finished $!; then + print_status FAIL + exiterror "Building $*" + fi - return "${r}" + print_status DONE } -# Calls the makefile of a package -make_pkg() { - local args=() - local pkg - - local basedir="${BASEDIR}" - - while [ $# -gt 0 ]; do - local arg="${1}" - shift +lfsmake2() { + lfsmakecommoncheck $* + [ $? == 1 ] && return 0 - case "${arg}" in - --*) - case "${arg}" in - --chroot) - basedir="/usr/src" - ;; - esac + local PS1='\u:\w$ ' - args+=( "${arg}" ) - ;; + enterchroot \ + bash -x -c "cd /usr/src/lfs && \ + make -f $* \ + LFS_BASEDIR=/usr/src install" \ + >> ${LOGFILE} 2>&1 & - *) - pkg="${arg}" - break - ;; - esac - done + if ! wait_until_finished $!; then + print_status FAIL + exiterror "Building $*" + fi - # Execute the make command in the environment - execute "${args[@]}" make --directory="${basedir}/lfs" --file="${pkg}" "$@" + print_status DONE } -lfsmake1() { - local pkg="${1}" - shift - - # Run the common check - lfsmakecommoncheck "${pkg}" "$@" +ipfiredist() { + lfsmakecommoncheck $* [ $? == 1 ] && return 0 - # Download source outside of the toolchain - if ! make_pkg --network "${pkg}" download "$@"; then - exiterror "Downloading ${pkg}" - fi + local PS1='\u:\w$ ' - if ! make_pkg --timer="update_runtime" "${pkg}" TOOLCHAIN=1 ROOT="${BUILD_DIR}" b2 install "$@"; then - print_status FAIL + enterchroot \ + bash -x -c "cd /usr/src/lfs && make -f $* LFS_BASEDIR=/usr/src dist" \ + >> ${LOGFILE} 2>&1 & - exiterror "Building ${pkg}" + if ! wait_until_finished $!; then + print_status FAIL + exiterror "Packaging $*" fi print_status DONE } -lfsmake2() { - local pkg="${1}" - shift +wait_until_finished() { + local pid=${1} + + local start_time="${SECONDS}" + + # Show progress + if ${INTERACTIVE}; then + # Wait a little just in case the process + # has finished very quickly. + sleep 0.1 - # Run the common check - lfsmakecommoncheck "${pkg}" "$@" - [ $? == 1 ] && return 0 + local runtime + while kill -0 ${pid} 2>/dev/null; do + print_runtime $(( SECONDS - start_time )) - # Download source outside of the toolchain - if ! make_pkg --network "${pkg}" download "$@"; then - exiterror "Downloading ${pkg}" + # Wait a little + sleep 1 + done fi - # Run install on the package - if ! make_pkg --chroot --timer="update_runtime" "${pkg}" b2 install "$@"; then - print_status FAIL + # Returns the exit code of the child process + wait ${pid} + local ret=$? - exiterror "Building ${pkg}" + if ! ${INTERACTIVE}; then + print_runtime $(( SECONDS - start_time )) fi - print_status DONE + return ${ret} } -ipfiredist() { - local pkg="${1}" - shift +fake_environ() { + [ -e "${BASEDIR}/build${TOOLS_DIR}/lib/libpakfire_preload.so" ] || return - # Run the common check - lfsmakecommoncheck "${pkg}" "$@" - [ $? == 1 ] && return 0 + local env="LD_PRELOAD=${TOOLS_DIR}/lib/libpakfire_preload.so" - # Run dist on the package - if ! make_pkg --chroot --timer="update_runtime" "${pkg}" dist "$@"; then - print_status FAIL + # Fake kernel version, because some of the packages do not compile + # with kernel 3.0 and later. + env="${env} UTS_RELEASE=${KVER}-ipfire" - exiterror "Packaging ${pkg}" - fi + # Fake machine version. + env="${env} UTS_MACHINE=${BUILD_ARCH}" - print_status DONE + echo "${env}" } -update_runtime() { - print_runtime "$(( SECONDS - t ))" +qemu_environ() { + local env="QEMU_TARGET_HELPER=${QEMU_TARGET_HELPER}" + + # Don't add anything if qemu is not used. + if ! qemu_is_required; then + return + fi + + # Set default qemu options + case "${BUILD_ARCH}" in + arm*) + QEMU_CPU="${QEMU_CPU:-cortex-a9}" + + env="${env} QEMU_CPU=${QEMU_CPU}" + ;; + riscv64) + QEMU_CPU="${QEMU_CPU:-sifive-u54}" + G_SLICE="always-malloc" + env="${env} QEMU_CPU=${QEMU_CPU} G_SLICE=${G_SLICE}" + ;; + esac + + # Enable QEMU strace + #env="${env} QEMU_STRACE=1" + + echo "${env}" } qemu_is_required() { @@ -1024,37 +809,34 @@ qemu_install_helper() { exiterror "binfmt_misc not enabled. QEMU_TARGET_HELPER not useable." fi - # Search for the helper binary - local qemu_build_helper="$(qemu_find_build_helper_name "${BUILD_ARCH}")" + + if [ -z "${QEMU_TARGET_HELPER}" ]; then + exiterror "QEMU_TARGET_HELPER not set" + fi + + # Check if the helper is already installed. + if [ -x "${LFS}${QEMU_TARGET_HELPER}" ]; then + return 0 + fi # Try to find a suitable binary that we can install # to the build environment. local file - for file in "${qemu_build_helper}" "${qemu_build_helper}-static"; do + for file in "${QEMU_TARGET_HELPER}" "${QEMU_TARGET_HELPER}-static"; do # file must exist and be executable. [ -x "${file}" ] || continue # Must be static. file_is_static "${file}" || continue - local dirname="${BUILD_DIR}$(dirname "${file}")" + local dirname="${LFS}$(dirname "${file}")" mkdir -p "${dirname}" - # Create the mountpoint - touch "${BUILD_DIR}${file}" - - # Mount the helper - if ! mount --bind -o ro "${file}" "${BUILD_DIR}${file}"; then - exiterror "Could not mount ${file}" - fi - - # Set - QEMU_TARGET_HELPER="${file}" - + install -m 755 "${file}" "${LFS}${QEMU_TARGET_HELPER}" return 0 done - exiterror "Could not find a statically-linked QEMU emulator: ${qemu_build_helper}" + exiterror "Could not find a statically-linked QEMU emulator: ${QEMU_TARGET_HELPER}" } qemu_find_build_helper_name() { @@ -1141,201 +923,40 @@ update_contributors() { return 0 } -# Download sources -download_sources() { - local file - local pkg - - local failed_packages=() - - # Walk through all files in LFS - for file in "${BASEDIR}/lfs/"*; do - pkg="${file##*/}" - - # Skip some packages - case "${pkg}" in - Config) - continue - ;; - esac - - # Run the common check - lfsmakecommoncheck "${pkg}" - [ $? == 1 ] && continue - - # Download and check the package - if ! make_pkg --network "${pkg}" download b2; then - failed_packages+=( "${pkg}" ) - print_status FAIL - continue - fi - - print_status DONE - done - - # Fail if we could not download/verify all packages - if [ "${#failed_packages[@]}" -gt 0 ]; then - exiterror "Failed to download or verify some packages: ${failed_packages[@]}" - fi -} - -# Download the toolchain -download_toolchain() { - local toolchain="${1}" - - # Do nothing if the toolchain has already been downloaded - if [ -e "${TOOLCHAIN_DIR}/${toolchain}" ]; then - return 0 - fi - - # Ensure the directory exists - mkdir -p "${TOOLCHAIN_DIR}" - - # Create a temporary directory - local tmp="$(mktemp -d)" - - # Make the name for the checksum file - local checksums="${toolchain/.tar.zst/.b2}" - - # Download the toolchain and checksum files - if ! wget --quiet --directory-prefix="${tmp}" \ - "${TOOLCHAIN_URL}/${toolchain}" \ - "${TOOLCHAIN_URL}/${checksums}"; then - # Cleanup - rm -rf "${tmp}" - - return 1 - fi - - # Check integrity - if ! cd "${tmp}" && b2sum --quiet --check "${checksums}"; then - # Cleanup - rm -rf "${tmp}" - - return 1 - fi - - # Everything is good, move the files to their destination - if ! mv \ - "${tmp}/${toolchain}" \ - "${tmp}/${checksums}" \ - "${TOOLCHAIN_DIR}"; then - # Cleanup - rm -rf "${tmp}" - - return 1 - fi - - # Cleanup - rm -rf "${tmp}" - - return 0 -} - -# Extracts the toolchain -extract_toolchain() { - local toolchain="${1}" - - local build_dir="${BUILD_DIR#${BASEDIR}/}" - local log_dir="${LOG_DIR#${BASEDIR}/}" - - local args=( - # Extract - "ax" - - # The file to extract - "-f" "${TOOLCHAIN_DIR}/${toolchain}" - - # The destination - "-C" "${BASEDIR}" - - # Transform any older toolchains - "--transform" "s@^build/@${build_dir}/@" - "--transform" "s@^log/@${log_dir}/@" - ) - - # Extract the toolchain - tar "${args[@]}" || return $? -} - -# Compresses the toolchain -compress_toolchain() { - local toolchain="${1}" - - log "Creating toolchain image for ${BUILD_ARCH}" - - # Create a temporary directory - local tmp="$(mktemp -d)" - - # Make the name for the checksum file - local checksums="${toolchain/.tar.zst/.b2}" - - local build_dir="${BUILD_DIR#${BASEDIR}/}" - local log_dir="${LOG_DIR#${BASEDIR}/}" - - local args=( - "--create" - - # Filter through zstd with custom options - "-I" "zstd ${ZSTD_OPT[*]}" - - # Write to the temporary directory - "-f" "${tmp}/${toolchain}" - - # Start in the base directory - "-C" "${BASEDIR}" - - # Exclude the build logs - "--exclude" "${log_dir}/_build.*.log" - - # Include /bin/sh - "${build_dir}/bin/sh" - - # Include the /tools_${BUILD_ARCH} directory - "${build_dir}/${TOOLS_DIR}" - - # Include the log directory - "${log_dir}" - ) - - # Create the archive - if ! tar "${args[@]}"; then - # Cleanup - rm -rf "${tmp}" - - return 1 - fi - - # Change to the temporary directory - pushd "${tmp}" &>/dev/null - - # Create the checksums - if ! b2sum "${toolchain}" > "${tmp}/${checksums}"; then - popd &>/dev/null +# Default settings +CCACHE_CACHE_SIZE="4G" +ENABLE_RAMDISK="auto" - # Cleanup - rm -rf "${tmp}" +# Load configuration file +if [ -f .config ]; then + . .config +fi - return 1 - fi +# TARGET_ARCH is BUILD_ARCH now +if [ -n "${TARGET_ARCH}" ]; then + BUILD_ARCH="${TARGET_ARCH}" + unset TARGET_ARCH +fi - popd &>/dev/null +# Get some information about the host system +SYSTEM_PROCESSORS="$(system_processors)" +SYSTEM_MEMORY="$(system_memory)" - # Everything is good, move the files to their destination - if ! mv \ - "${tmp}/${toolchain}" \ - "${tmp}/${checksums}" \ - "${TOOLCHAIN_DIR}"; then - # Cleanup - rm -rf "${tmp}" +if [ -n "${BUILD_ARCH}" ]; then + configure_build "${BUILD_ARCH}" +else + configure_build "default" +fi - return 1 +# Automatically enable/disable ramdisk usage +if [ "${ENABLE_RAMDISK}" = "auto" ]; then + # Enable only when the host system has 4GB of RAM or more + if [ ${SYSTEM_MEMORY} -ge 3900 ]; then + ENABLE_RAMDISK="on" fi +fi - return 0 -} - -build_toolchain() { +buildtoolchain() { local gcc=$(type -p gcc) if [ -z "${gcc}" ]; then exiterror "Could not find GCC. You will need a working build enviroment in order to build the toolchain." @@ -1347,23 +968,24 @@ build_toolchain() { fi if [ ! -e "${TOOLS_DIR}" ]; then - ln -s "${BUILD_DIR}${TOOLS_DIR}" "${TOOLS_DIR}" + ln -s "${BASEDIR}/build${TOOLS_DIR}" "${TOOLS_DIR}" fi if [ ! -h "${TOOLS_DIR}" ]; then exiterror "Could not create ${TOOLS_DIR} symbolic link" fi - local LOGFILE="${LOG_DIR}/_build.toolchain.log" + LOGFILE="$BASEDIR/log/_build.toolchain.log" + export LOGFILE lfsmake1 stage1 lfsmake1 binutils PASS=1 lfsmake1 gcc PASS=1 - lfsmake1 linux HEADERS=1 + lfsmake1 linux KCFG="-headers" lfsmake1 glibc lfsmake1 libxcrypt lfsmake1 gcc PASS=L - lfsmake1 zlib-ng + lfsmake1 zlib lfsmake1 binutils PASS=2 lfsmake1 gcc PASS=2 lfsmake1 zstd @@ -1399,20 +1021,19 @@ build_toolchain() { lfsmake1 cleanup-toolchain } -build_system() { - local LOGFILE="${LOG_DIR}/_build.${SNAME}.log" - +buildbase() { + LOGFILE="$BASEDIR/log/_build.base.log" + export LOGFILE lfsmake2 stage2 - lfsmake2 linux HEADERS=1 + lfsmake2 linux KCFG="-headers" lfsmake2 man-pages lfsmake2 glibc lfsmake2 tzdata lfsmake2 cleanup-toolchain - lfsmake2 zlib-ng + lfsmake2 zlib [ "${BUILD_ARCH}" = "riscv64" ] && lfsmake2 gcc PASS=A lfsmake2 zstd lfsmake2 autoconf - lfsmake2 autoconf-archive lfsmake2 automake lfsmake2 help2man lfsmake2 libtool @@ -1472,1121 +1093,912 @@ build_system() { lfsmake2 vim lfsmake2 e2fsprogs lfsmake2 jq - lfsmake2 configroot - lfsmake2 initscripts - lfsmake2 backup - lfsmake2 rust - lfsmake2 openssl - lfsmake2 popt - lfsmake2 libedit - lfsmake2 expat - lfsmake2 libffi - lfsmake2 gdbm - lfsmake2 sqlite - lfsmake2 python3 - lfsmake2 python3-setuptools - lfsmake2 ninja - lfsmake2 meson - lfsmake2 pam - lfsmake2 libcap - lfsmake2 libcap-ng - lfsmake2 libpcap - lfsmake2 ppp - lfsmake2 pptp - lfsmake2 unzip - lfsmake2 which - lfsmake2 bc - lfsmake2 cpio - lfsmake2 libaio - lfsmake2 freetype - lfsmake2 libmnl - lfsmake2 libnfnetlink - lfsmake2 libnetfilter_queue - lfsmake2 libnetfilter_conntrack - lfsmake2 libnetfilter_cthelper - lfsmake2 libnetfilter_cttimeout - lfsmake2 iptables - lfsmake2 iproute2 - lfsmake2 screen - lfsmake2 elfutils - lfsmake2 libconfig - lfsmake2 curl - lfsmake2 libarchive - lfsmake2 cmake - lfsmake2 json-c - lfsmake2 tcl - lfsmake2 python3-MarkupSafe - lfsmake2 python3-Jinja2 - lfsmake2 kmod - lfsmake2 udev - lfsmake2 libusb - lfsmake2 mdadm - lfsmake2 dracut - lfsmake2 lvm2 - lfsmake2 multipath-tools - lfsmake2 glib - lfsmake2 libgudev - lfsmake2 libgpg-error - lfsmake2 libgcrypt - lfsmake2 libassuan - lfsmake2 nettle - lfsmake2 libsodium - lfsmake2 libevent2 - lfsmake2 apr - lfsmake2 aprutil - lfsmake2 unbound - lfsmake2 gnutls - lfsmake2 libuv - lfsmake2 liburcu - lfsmake2 bind - lfsmake2 dhcp - lfsmake2 dhcpcd - lfsmake2 boost - lfsmake2 linux-atm - lfsmake2 libqmi - lfsmake2 c-ares - lfsmake2 rust-dissimilar - lfsmake2 rust-cfg-if - lfsmake2 rust-libc - lfsmake2 rust-getrandom - lfsmake2 rust-typenum - lfsmake2 rust-version-check - lfsmake2 rust-generic-array - lfsmake2 rust-crypto-common - lfsmake2 rust-cipher - lfsmake2 rust-hex - lfsmake2 rust-unicode-xid - lfsmake2 rust-unicode-ident - lfsmake2 rust-proc-macro2 - lfsmake2 rust-quote - lfsmake2 rust-syn-1.0.109 - lfsmake2 rust-syn - lfsmake2 rust-home - lfsmake2 rust-lazy-static - lfsmake2 rust-memchr - lfsmake2 rust-aho-corasick - lfsmake2 rust-regex-syntax - lfsmake2 rust-regex - lfsmake2 rust-ucd-trie - lfsmake2 rust-pest - lfsmake2 rust-semver-parser - lfsmake2 rust-semver - lfsmake2 rust-same-file - lfsmake2 rust-walkdir - lfsmake2 rust-dirs - lfsmake2 rust-toolchain_find - lfsmake2 rust-serde_derive - lfsmake2 rust-serde - lfsmake2 rust-itoa - lfsmake2 rust-ryu - lfsmake2 rust-serde_json - lfsmake2 rust-synstructure - lfsmake2 rust-block-buffer - lfsmake2 rust-digest - lfsmake2 rust-ppv-lite86 - lfsmake2 rust-rand_core - lfsmake2 rust-rand_core-0.4.2 - lfsmake2 rust-rand_core-0.3.1 - lfsmake2 rust-rand_chacha - lfsmake2 rust-rand_hc - lfsmake2 rust-rand - lfsmake2 rust-rdrand - lfsmake2 rust-rand-0.4 - lfsmake2 rust-log - lfsmake2 rust-num_cpus - lfsmake2 rust-crossbeam-utils - lfsmake2 rust-autocfg - lfsmake2 rust-memoffset - lfsmake2 rust-scopeguard - lfsmake2 rust-crossbeam-epoch - lfsmake2 rust-crossbeam-deque - lfsmake2 rust-either - lfsmake2 rust-crossbeam-channel - lfsmake2 rust-rayon-core - lfsmake2 rust-rayon - lfsmake2 rust-remove_dir_all - lfsmake2 rust-tempdir - lfsmake2 rust-glob - lfsmake2 rust-once_cell - lfsmake2 rust-termcolor - lfsmake2 rust-serde_spanned - lfsmake2 rust-toml_datetime - lfsmake2 rust-equivalent - lfsmake2 rust-allocator-api2 - lfsmake2 rust-foldhash - lfsmake2 rust-hashbrown - lfsmake2 rust-indexmap - lfsmake2 rust-winnow - lfsmake2 rust-toml_edit - lfsmake2 rust-toml - lfsmake2 rust-target-triple - lfsmake2 rust-trybuild - lfsmake2 rust-unindent - lfsmake2 rust-proc-macro-hack - lfsmake2 rust-indoc-impl - lfsmake2 rust-indoc-impl-0.3.6 - lfsmake2 rust-indoc - lfsmake2 rust-indoc-0.3.6 - lfsmake2 rust-instant - lfsmake2 rust-lock_api - lfsmake2 rust-smallvec - lfsmake2 rust-parking_lot_core - lfsmake2 rust-parking_lot - lfsmake2 rust-paste-impl - lfsmake2 rust-paste - lfsmake2 rust-paste-0.1.18 - lfsmake2 rust-ctor - lfsmake2 rust-ghost - lfsmake2 rust-inventory-impl - lfsmake2 rust-inventory - lfsmake2 rust-pyo3-build-config - lfsmake2 rust-pyo3-macros-backend - lfsmake2 rust-pyo3-macros - lfsmake2 rust-pyo3 - lfsmake2 rust-num-traits - lfsmake2 rust-num-integer - lfsmake2 rust-num_threads - lfsmake2 rust-time - lfsmake2 rust-iana-time-zone - lfsmake2 rust-chrono - lfsmake2 rust-asn1_derive - lfsmake2 rust-asn1 - lfsmake2 rust-proc-macro-error-attr - lfsmake2 rust-proc-macro-error - lfsmake2 rust-Inflector - lfsmake2 rust-ouroboros_macro - lfsmake2 rust-aliasable - lfsmake2 rust-stable_deref_trait - lfsmake2 rust-ouroboros - lfsmake2 rust-base64 - lfsmake2 rust-pem - lfsmake2 gdb - lfsmake2 grub - lfsmake2 mandoc - lfsmake2 efivar - lfsmake2 efibootmgr - lfsmake2 libtasn1 - lfsmake2 p11-kit - lfsmake2 ca-certificates - lfsmake2 fireinfo - lfsmake2 libnet - lfsmake2 libnl-3 - lfsmake2 libidn - lfsmake2 libidn2 - lfsmake2 nasm - lfsmake2 libexif - lfsmake2 libjpeg - lfsmake2 libpng - lfsmake2 gd - lfsmake2 slang - lfsmake2 newt - lfsmake2 libsmooth - lfsmake2 pciutils - lfsmake2 usbutils - lfsmake2 libxml2 - lfsmake2 libxslt - lfsmake2 perl-BerkeleyDB - lfsmake2 cyrus-sasl - lfsmake2 openldap - lfsmake2 apache2 - lfsmake2 web-user-interface - lfsmake2 flag-icons - lfsmake2 jquery - lfsmake2 bootstrap - lfsmake2 arping - lfsmake2 beep - lfsmake2 libssh - lfsmake2 libinih - lfsmake2 cdrkit - lfsmake2 dosfstools - lfsmake2 exfatprogs - lfsmake2 reiserfsprogs - lfsmake2 xfsprogs - lfsmake2 sysfsutils - lfsmake2 fuse - lfsmake2 ntfs-3g - lfsmake2 ethtool - lfsmake2 fcron - lfsmake2 perl-ExtUtils-PkgConfig - lfsmake2 perl-GD - lfsmake2 perl-GD-Graph - lfsmake2 perl-GD-TextUtil - lfsmake2 perl-Device-SerialPort - lfsmake2 perl-Device-Modem - lfsmake2 perl-Apache-Htpasswd - lfsmake2 perl-Parse-Yapp - lfsmake2 perl-Data-UUID - lfsmake2 perl-Try-Tiny - lfsmake2 perl-HTTP-Message - lfsmake2 perl-HTTP-Date - lfsmake2 gnupg - lfsmake2 hdparm - lfsmake2 whatmask - lfsmake2 libtirpc - lfsmake2 conntrack-tools - lfsmake2 iputils - lfsmake2 l7-protocols - lfsmake2 hwdata - lfsmake2 logrotate - lfsmake2 logwatch - lfsmake2 misc-progs - lfsmake2 nano - lfsmake2 perl-URI - lfsmake2 perl-CGI - lfsmake2 perl-Switch - lfsmake2 perl-HTML-Tagset - lfsmake2 perl-HTML-Parser - lfsmake2 perl-HTML-Template - lfsmake2 perl-libwww - lfsmake2 perl-LWP-Protocol-https - lfsmake2 perl-Net-HTTP - lfsmake2 perl-Net-DNS - lfsmake2 perl-Net-IPv4Addr - lfsmake2 perl-Net_SSLeay - lfsmake2 perl-IO-Stringy - lfsmake2 perl-IO-Socket-SSL - lfsmake2 perl-Unix-Syslog - lfsmake2 perl-Mail-Tools - lfsmake2 perl-MIME-Tools - lfsmake2 perl-Net-Server - lfsmake2 perl-Canary-Stability - lfsmake2 perl-Convert-TNEF - lfsmake2 perl-Convert-UUlib - lfsmake2 perl-Archive-Zip - lfsmake2 perl-Text-Tabs+Wrap - lfsmake2 perl-XML-Parser - lfsmake2 perl-Crypt-PasswdMD5 - lfsmake2 perl-Net-Telnet - lfsmake2 perl-Capture-Tiny - lfsmake2 perl-Config-AutoConf - lfsmake2 perl-Object-Tiny - lfsmake2 perl-Archive-Peek-Libarchive - lfsmake2 python3-inotify - lfsmake2 python3-docutils - lfsmake2 python3-daemon - lfsmake2 ntp - lfsmake2 openssh - lfsmake2 fontconfig - lfsmake2 dejavu-fonts-ttf - lfsmake2 ubuntu-font-family - lfsmake2 freefont - lfsmake2 pixman - lfsmake2 cairo - lfsmake2 harfbuzz - lfsmake2 fribidi - lfsmake2 pango - lfsmake2 rrdtool - lfsmake2 setup - lfsmake2 jansson - lfsmake2 yaml - lfsmake2 libhtp - lfsmake2 colm - lfsmake2 ragel - lfsmake2 vectorscan - lfsmake2 suricata - lfsmake2 ids-ruleset-sources - lfsmake2 ipblocklist-sources - lfsmake2 squid - lfsmake2 squidguard - lfsmake2 calamaris - lfsmake2 tcpdump - lfsmake2 traceroute - lfsmake2 vlan - lfsmake2 wireless - lfsmake2 pakfire - lfsmake2 lz4 - lfsmake2 lzo - lfsmake2 openvpn - lfsmake2 mpage - lfsmake2 dbus - lfsmake2 intltool - lfsmake2 libdaemon - lfsmake2 avahi - lfsmake2 libtalloc - lfsmake2 cifs-utils - lfsmake2 krb5 - lfsmake2 rpcsvc-proto - lfsmake2 lmdb - lfsmake2 samba - lfsmake2 netatalk - lfsmake2 sudo - lfsmake2 mc - lfsmake2 wget - lfsmake2 bridge-utils - lfsmake2 smartmontools - lfsmake2 htop - lfsmake2 chkconfig - lfsmake2 postfix - lfsmake2 fetchmail - lfsmake2 clamav - lfsmake2 perl-NetAddr-IP - lfsmake2 dma - lfsmake2 alsa - lfsmake2 guardian - lfsmake2 libid3tag - lfsmake2 libmad - lfsmake2 libogg - lfsmake2 libvorbis - lfsmake2 flac - lfsmake2 lame - lfsmake2 soxr - lfsmake2 libshout - lfsmake2 xvid - lfsmake2 libmpeg2 - lfsmake2 gnump3d - lfsmake2 libxxhash - lfsmake2 rsync - lfsmake2 rpcbind - lfsmake2 keyutils - lfsmake2 nfs - lfsmake2 ncat - lfsmake2 nmap - lfsmake2 etherwake - lfsmake2 bwm-ng - lfsmake2 sysstat - lfsmake2 strongswan - lfsmake2 rng-tools - lfsmake2 lsof - lfsmake2 lm_sensors - lfsmake2 libstatgrab - lfsmake2 liboping - lfsmake2 netsnmpd - lfsmake2 nut - lfsmake2 collectd - lfsmake2 git - lfsmake2 linux-firmware - lfsmake2 dvb-firmwares - lfsmake2 zd1211-firmware - lfsmake2 rpi-firmware - lfsmake2 intel-microcode - lfsmake2 pcengines-apu-firmware - lfsmake2 elinks - lfsmake2 igmpproxy - lfsmake2 opus - lfsmake2 python3-toml - lfsmake2 python3-pyproject2setuppy - lfsmake2 python3-pyparsing - lfsmake2 spice-protocol - lfsmake2 spice - lfsmake2 sdl2 - lfsmake2 libusbredir - lfsmake2 libseccomp - lfsmake2 libslirp - lfsmake2 dtc - lfsmake2 qemu - lfsmake2 nagios_nrpe - lfsmake2 nagios-plugins - lfsmake2 observium-agent - lfsmake2 ebtables - lfsmake2 faad2 - lfsmake2 alac - lfsmake2 ffmpeg - lfsmake2 vdr - lfsmake2 vdr_streamdev - lfsmake2 vdr_epgsearch - lfsmake2 vdr_dvbapi - lfsmake2 vdr_eepg - lfsmake2 w_scan - lfsmake2 fmt - lfsmake2 mpd - lfsmake2 libmpdclient - lfsmake2 mpc - lfsmake2 perl-Net-CIDR-Lite - lfsmake2 perl-Net-SMTP-SSL - lfsmake2 perl-Authen-SASL - lfsmake2 perl-MIME-Lite - lfsmake2 perl-Email-Date-Format - lfsmake2 vnstat - lfsmake2 iw - lfsmake2 wpa_supplicant - lfsmake2 hostapd - lfsmake2 syslinux - lfsmake2 tftpd - lfsmake2 cpufrequtils - lfsmake2 apcupsd - lfsmake2 fireperf - lfsmake2 iperf - lfsmake2 iperf3 - lfsmake2 7zip - lfsmake2 lynis - lfsmake2 sshfs - lfsmake2 utfcpp - lfsmake2 taglib - lfsmake2 perl-gettext - lfsmake2 perl-Sort-Naturally - lfsmake2 vdradmin - lfsmake2 perl-DBI - lfsmake2 perl-DBD-SQLite - lfsmake2 perl-File-ReadBackwards - lfsmake2 openvmtools - lfsmake2 joe - lfsmake2 monit - lfsmake2 watchdog - lfsmake2 usb_modeswitch - lfsmake2 usb_modeswitch_data - lfsmake2 zerofree - lfsmake2 minicom - lfsmake2 ddrescue - lfsmake2 parted - lfsmake2 swig - lfsmake2 python3-pyelftools - lfsmake2 u-boot - lfsmake2 wireless-regdb - lfsmake2 ddns - lfsmake2 python3-pycparser - lfsmake2 python3-charset-normalizer - lfsmake2 python3-certifi - lfsmake2 python3-idna - lfsmake2 python3-requests - lfsmake2 python3-tomli - lfsmake2 python3-pep517 - lfsmake2 python3-build - lfsmake2 python3-install - lfsmake2 python3-urllib3 - lfsmake2 python3-flit - lfsmake2 python3-packaging - lfsmake2 python3-typing-extensions - lfsmake2 python3-semantic-version - lfsmake2 python3-setuptools-scm - lfsmake2 python3-setuptools-rust - lfsmake2 python3-six - lfsmake2 python3-dateutil - lfsmake2 python3-jmespath - lfsmake2 python3-colorama - lfsmake2 python3-yaml - lfsmake2 python3-s3transfer - lfsmake2 python3-rsa - lfsmake2 python3-pyasn1 - lfsmake2 python3-botocore - lfsmake2 python3-cffi - lfsmake2 python3-cryptography - lfsmake2 python3-circuitbreaker - lfsmake2 python3-pytz - lfsmake2 python3-click - lfsmake2 python3-arrow - lfsmake2 python3-terminaltables - lfsmake2 python3-pkgconfig - lfsmake2 python3-msgpack - lfsmake2 python3-editables - lfsmake2 python3-pathspec - lfsmake2 python3-pluggy - lfsmake2 python3-calver - lfsmake2 python3-trove-classifiers - lfsmake2 python3-hatchling - lfsmake2 python3-hatch-vcs - lfsmake2 python3-hatch-fancy-pypi-readme - lfsmake2 python3-attrs - lfsmake2 python3-sniffio - lfsmake2 python3-sortedcontainers - lfsmake2 python3-outcome - lfsmake2 python3-async_generator - lfsmake2 python3-flit_scm - lfsmake2 python3-exceptiongroup - lfsmake2 python3-trio - lfsmake2 python3-pyfuse3 - lfsmake2 aws-cli - lfsmake2 oci-python-sdk - lfsmake2 oci-cli - lfsmake2 transmission - lfsmake2 mtr - lfsmake2 minidlna - lfsmake2 acpid - lfsmake2 fping - lfsmake2 telnet - lfsmake2 xinetd - lfsmake2 stress - lfsmake2 sarg - lfsmake2 nginx - lfsmake2 sysbench - lfsmake2 strace - lfsmake2 ltrace - lfsmake2 ipfire-netboot - lfsmake2 keepalived - lfsmake2 ipvsadm - lfsmake2 perl-Carp-Clan - lfsmake2 perl-Date-Calc - lfsmake2 perl-Date-Manip - lfsmake2 perl-File-Tail - lfsmake2 perl-TimeDate - lfsmake2 swatch - lfsmake2 tor - lfsmake2 wavemon - lfsmake2 iptraf-ng - lfsmake2 iotop - lfsmake2 stunnel - lfsmake2 bacula - lfsmake2 perl-Font-TTF - lfsmake2 perl-IO-String - lfsmake2 perl-PDF-API2 - lfsmake2 proxy-accounting - lfsmake2 tmux - lfsmake2 perl-Text-CSV_XS - lfsmake2 lua - lfsmake2 haproxy - lfsmake2 ipset - lfsmake2 dnsdist - lfsmake2 bird - lfsmake2 libyang - lfsmake2 abseil-cpp - lfsmake2 protobuf - lfsmake2 protobuf-c - lfsmake2 frr - lfsmake2 dmidecode - lfsmake2 mcelog - lfsmake2 libpciaccess - lfsmake2 libvirt - lfsmake2 freeradius - lfsmake2 perl-common-sense - lfsmake2 perl-inotify2 - lfsmake2 perl-Net-IP - lfsmake2 wio - lfsmake2 iftop - lfsmake2 mdns-repeater - lfsmake2 i2c-tools - lfsmake2 nss-myhostname - lfsmake2 dehydrated - lfsmake2 libplist - lfsmake2 nqptp - lfsmake2 shairport-sync - lfsmake2 borgbackup - lfsmake2 knot - lfsmake2 spectre-meltdown-checker - lfsmake2 zabbix_agentd - lfsmake2 flashrom - lfsmake2 firmware-update - lfsmake2 ruby - lfsmake2 asciidoctor - lfsmake2 speexdsp - lfsmake2 tshark - lfsmake2 speedtest-cli - lfsmake2 amazon-ssm-agent - lfsmake2 libloc - lfsmake2 ncdu - lfsmake2 lshw - lfsmake2 socat - lfsmake2 libcdada - lfsmake2 pmacct - lfsmake2 squid-asnbl - lfsmake2 qemu-ga - lfsmake2 gptfdisk - lfsmake2 oath-toolkit - lfsmake2 qrencode - lfsmake2 perl-File-Remove - lfsmake2 perl-Module-Build - lfsmake2 perl-Module-ScanDeps - lfsmake2 perl-YAML-Tiny - lfsmake2 perl-Module-Install - lfsmake2 perl-Imager - lfsmake2 perl-Imager-QRCode - lfsmake2 perl-MIME-Base32 - lfsmake2 perl-URI-Encode - lfsmake2 rsnapshot - lfsmake2 mympd - lfsmake2 wsdd - lfsmake2 btrfs-progs - lfsmake2 inotify-tools - lfsmake2 grub-btrfs - - lfsmake2 linux - lfsmake2 rtl8812au - lfsmake2 linux-initrd - - lfsmake2 memtest - - # Build the installer - lfsmake2 installer - - # Build images - lfsmake2 cdrom - lfsmake2 flash-images - lfsmake2 core-updates } -build_packages() { - local LOGFILE="${LOG_DIR}/_build.packages.log" - - # Build packages - print_headline "Building Packages" - - local path - local -A pkgs=() - local pkg - - # Collect all packages - for path in \ - "${BASEDIR}/config/rootfiles/packages/"* \ - "${BASEDIR}/config/rootfiles/packages/${BUILD_ARCH}"/*; do - # Skip directories - if [ -d "${path}" ]; then - continue - fi - - pkgs["${path##*/}"]="${path}" - done - - # Package them all - for pkg in ${!pkgs[@]}; do - ipfiredist "${pkg}" - done +buildipfire() { + LOGFILE="$BASEDIR/log/_build.ipfire.log" + export LOGFILE + lfsmake2 configroot + lfsmake2 initscripts + lfsmake2 backup + lfsmake2 rust + lfsmake2 openssl + lfsmake2 kmod + lfsmake2 udev + lfsmake2 popt + lfsmake2 libedit + lfsmake2 libusb + lfsmake2 libpcap + lfsmake2 ppp + lfsmake2 pptp + lfsmake2 unzip + lfsmake2 which + lfsmake2 bc + lfsmake2 u-boot MKIMAGE=1 + lfsmake2 cpio + lfsmake2 mdadm + lfsmake2 dracut + lfsmake2 libaio + lfsmake2 lvm2 + lfsmake2 multipath-tools + lfsmake2 freetype + lfsmake2 libmnl + lfsmake2 libnfnetlink + lfsmake2 libnetfilter_queue + lfsmake2 libnetfilter_conntrack + lfsmake2 libnetfilter_cthelper + lfsmake2 libnetfilter_cttimeout + lfsmake2 iptables + lfsmake2 iproute2 + lfsmake2 screen + lfsmake2 elfutils + lfsmake2 expat + lfsmake2 libconfig + lfsmake2 curl + lfsmake2 libarchive + lfsmake2 cmake + lfsmake2 json-c + lfsmake2 tcl + lfsmake2 libffi + lfsmake2 gdbm + lfsmake2 sqlite + lfsmake2 python3 + lfsmake2 python3-setuptools + lfsmake2 python3-wheel + lfsmake2 ninja + lfsmake2 meson + lfsmake2 glib + lfsmake2 libgudev + lfsmake2 libgpg-error + lfsmake2 libgcrypt + lfsmake2 libassuan + lfsmake2 nettle + lfsmake2 libsodium + lfsmake2 libevent2 + lfsmake2 apr + lfsmake2 aprutil + lfsmake2 unbound + lfsmake2 gnutls + lfsmake2 libuv + lfsmake2 bind + lfsmake2 dhcp + lfsmake2 dhcpcd + lfsmake2 boost + lfsmake2 linux-atm + lfsmake2 libqmi + lfsmake2 pam + lfsmake2 c-ares + lfsmake2 rust-dissimilar + lfsmake2 rust-cfg-if + lfsmake2 rust-libc + lfsmake2 rust-getrandom + lfsmake2 rust-typenum + lfsmake2 rust-version-check + lfsmake2 rust-generic-array + lfsmake2 rust-crypto-common + lfsmake2 rust-cipher + lfsmake2 rust-hex + lfsmake2 rust-unicode-xid + lfsmake2 rust-proc-macro2 + lfsmake2 rust-quote + lfsmake2 rust-syn + lfsmake2 rust-home + lfsmake2 rust-lazy-static + lfsmake2 rust-memchr + lfsmake2 rust-aho-corasick + lfsmake2 rust-regex-syntax + lfsmake2 rust-regex + lfsmake2 rust-ucd-trie + lfsmake2 rust-pest + lfsmake2 rust-semver-parser + lfsmake2 rust-semver + lfsmake2 rust-same-file + lfsmake2 rust-walkdir + lfsmake2 rust-dirs + lfsmake2 rust-toolchain_find + lfsmake2 rust-serde + lfsmake2 rust-itoa + lfsmake2 rust-ryu + lfsmake2 rust-serde_json + lfsmake2 rust-synstructure + lfsmake2 rust-block-buffer + lfsmake2 rust-digest + lfsmake2 rust-ppv-lite86 + lfsmake2 rust-rand_core + lfsmake2 rust-rand_core-0.4.2 + lfsmake2 rust-rand_core-0.3.1 + lfsmake2 rust-rand_chacha + lfsmake2 rust-rand_hc + lfsmake2 rust-rand + lfsmake2 rust-rdrand + lfsmake2 rust-rand-0.4 + lfsmake2 rust-log + lfsmake2 rust-num_cpus + lfsmake2 rust-crossbeam-utils + lfsmake2 rust-autocfg + lfsmake2 rust-memoffset + lfsmake2 rust-scopeguard + lfsmake2 rust-crossbeam-epoch + lfsmake2 rust-crossbeam-deque + lfsmake2 rust-either + lfsmake2 rust-crossbeam-channel + lfsmake2 rust-rayon-core + lfsmake2 rust-rayon + lfsmake2 rust-remove_dir_all + lfsmake2 rust-tempdir + lfsmake2 rust-glob + lfsmake2 rust-once_cell + lfsmake2 rust-termcolor + lfsmake2 rust-toml + lfsmake2 rust-serde_derive + lfsmake2 rust-trybuild + lfsmake2 rust-unindent + lfsmake2 rust-proc-macro-hack + lfsmake2 rust-indoc-impl + lfsmake2 rust-indoc + lfsmake2 rust-indoc-0.3.6 + lfsmake2 rust-instant + lfsmake2 rust-lock_api + lfsmake2 rust-smallvec + lfsmake2 rust-parking_lot_core + lfsmake2 rust-parking_lot + lfsmake2 rust-paste-impl + lfsmake2 rust-paste + lfsmake2 rust-paste-0.1.18 + lfsmake2 rust-ctor + lfsmake2 rust-ghost + lfsmake2 rust-inventory-impl + lfsmake2 rust-inventory + lfsmake2 rust-pyo3-build-config + lfsmake2 rust-pyo3-macros-backend + lfsmake2 rust-pyo3-macros + lfsmake2 rust-pyo3 + lfsmake2 rust-num-traits + lfsmake2 rust-num-integer + lfsmake2 rust-num_threads + lfsmake2 rust-time + lfsmake2 rust-iana-time-zone + lfsmake2 rust-chrono + lfsmake2 rust-asn1_derive + lfsmake2 rust-asn1 + lfsmake2 rust-proc-macro-error-attr + lfsmake2 rust-proc-macro-error + lfsmake2 rust-Inflector + lfsmake2 rust-ouroboros_macro + lfsmake2 rust-aliasable + lfsmake2 rust-stable_deref_trait + lfsmake2 rust-ouroboros + lfsmake2 rust-base64 + lfsmake2 rust-pem + lfsmake2 gdb + lfsmake2 grub + lfsmake2 mandoc + lfsmake2 efivar + lfsmake2 efibootmgr + lfsmake2 libtasn1 + lfsmake2 p11-kit + lfsmake2 ca-certificates + lfsmake2 fireinfo + lfsmake2 libnet + lfsmake2 libnl-3 + lfsmake2 libidn + lfsmake2 nasm + lfsmake2 libjpeg + lfsmake2 openjpeg + lfsmake2 libexif + lfsmake2 libpng + lfsmake2 libtiff + lfsmake2 gd + lfsmake2 slang + lfsmake2 newt + lfsmake2 libsmooth + lfsmake2 libcap + lfsmake2 libcap-ng + lfsmake2 pciutils + lfsmake2 usbutils + lfsmake2 libxml2 + lfsmake2 libxslt + lfsmake2 perl-BerkeleyDB + lfsmake2 cyrus-sasl + lfsmake2 openldap + lfsmake2 apache2 + lfsmake2 web-user-interface + lfsmake2 flag-icons + lfsmake2 jquery + lfsmake2 bootstrap + lfsmake2 arping + lfsmake2 beep + lfsmake2 libssh + lfsmake2 libinih + lfsmake2 cdrkit + lfsmake2 dosfstools + lfsmake2 exfatprogs + lfsmake2 reiserfsprogs + lfsmake2 liburcu + lfsmake2 xfsprogs + lfsmake2 sysfsutils + lfsmake2 fuse + lfsmake2 ntfs-3g + lfsmake2 ethtool + lfsmake2 fcron + lfsmake2 perl-ExtUtils-PkgConfig + lfsmake2 perl-GD + lfsmake2 perl-GD-Graph + lfsmake2 perl-GD-TextUtil + lfsmake2 perl-Device-SerialPort + lfsmake2 perl-Device-Modem + lfsmake2 perl-Apache-Htpasswd + lfsmake2 perl-Parse-Yapp + lfsmake2 perl-Data-UUID + lfsmake2 perl-Try-Tiny + lfsmake2 perl-HTTP-Message + lfsmake2 perl-HTTP-Date + lfsmake2 gnupg + lfsmake2 hdparm + lfsmake2 whatmask + lfsmake2 libtirpc + lfsmake2 conntrack-tools + lfsmake2 iputils + lfsmake2 l7-protocols + lfsmake2 hwdata + lfsmake2 logrotate + lfsmake2 logwatch + lfsmake2 misc-progs + lfsmake2 nano + lfsmake2 perl-URI + lfsmake2 perl-CGI + lfsmake2 perl-Switch + lfsmake2 perl-HTML-Tagset + lfsmake2 perl-HTML-Parser + lfsmake2 perl-HTML-Template + lfsmake2 perl-Compress-Zlib + lfsmake2 perl-Digest + lfsmake2 perl-Digest-SHA1 + lfsmake2 perl-Digest-HMAC + lfsmake2 perl-libwww + lfsmake2 perl-LWP-Protocol-https + lfsmake2 perl-Net-HTTP + lfsmake2 perl-Net-DNS + lfsmake2 perl-Net-IPv4Addr + lfsmake2 perl-Net_SSLeay + lfsmake2 perl-IO-Stringy + lfsmake2 perl-IO-Socket-SSL + lfsmake2 perl-Unix-Syslog + lfsmake2 perl-Mail-Tools + lfsmake2 perl-MIME-Tools + lfsmake2 perl-Net-Server + lfsmake2 perl-Canary-Stability + lfsmake2 perl-Convert-TNEF + lfsmake2 perl-Convert-UUlib + lfsmake2 perl-Archive-Tar + lfsmake2 perl-Archive-Zip + lfsmake2 perl-Text-Tabs+Wrap + lfsmake2 perl-XML-Parser + lfsmake2 perl-Crypt-PasswdMD5 + lfsmake2 perl-Net-Telnet + lfsmake2 perl-JSON + lfsmake2 python3-inotify + lfsmake2 python3-docutils + lfsmake2 python3-daemon + lfsmake2 ntp + lfsmake2 openssh + lfsmake2 fontconfig + lfsmake2 dejavu-fonts-ttf + lfsmake2 ubuntu-font-family + lfsmake2 freefont + lfsmake2 pixman + lfsmake2 cairo + lfsmake2 harfbuzz + lfsmake2 fribidi + lfsmake2 pango + lfsmake2 rrdtool + lfsmake2 setup + lfsmake2 jansson + lfsmake2 yaml + lfsmake2 libhtp + lfsmake2 colm + lfsmake2 ragel + lfsmake2 hyperscan + lfsmake2 suricata + lfsmake2 ids-ruleset-sources + lfsmake2 ipblocklist-sources + lfsmake2 squid + lfsmake2 squidguard + lfsmake2 calamaris + lfsmake2 tcpdump + lfsmake2 traceroute + lfsmake2 vlan + lfsmake2 wireless + lfsmake2 pakfire + lfsmake2 lz4 + lfsmake2 lzo + lfsmake2 openvpn + lfsmake2 mpage + lfsmake2 dbus + lfsmake2 intltool + lfsmake2 libdaemon + lfsmake2 avahi + lfsmake2 cups + lfsmake2 lcms2 + lfsmake2 ghostscript + lfsmake2 qpdf + lfsmake2 poppler + lfsmake2 poppler-data + lfsmake2 cups-filters + lfsmake2 epson-inkjet-printer-escpr + lfsmake2 cups-pdf + lfsmake2 foomatic + lfsmake2 hplip + lfsmake2 cifs-utils + lfsmake2 krb5 + lfsmake2 rpcsvc-proto + lfsmake2 samba + lfsmake2 netatalk + lfsmake2 sudo + lfsmake2 mc + lfsmake2 wget + lfsmake2 bridge-utils + lfsmake2 smartmontools + lfsmake2 htop + lfsmake2 chkconfig + lfsmake2 postfix + lfsmake2 fetchmail + lfsmake2 clamav + lfsmake2 perl-NetAddr-IP + lfsmake2 dma + lfsmake2 alsa + lfsmake2 mpfire + lfsmake2 guardian + lfsmake2 libid3tag + lfsmake2 libmad + lfsmake2 libogg + lfsmake2 libvorbis + lfsmake2 flac + lfsmake2 lame + lfsmake2 soxr + lfsmake2 libshout + lfsmake2 xvid + lfsmake2 libmpeg2 + lfsmake2 gnump3d + lfsmake2 rsync + lfsmake2 rpcbind + lfsmake2 keyutils + lfsmake2 nfs + lfsmake2 ncat + lfsmake2 nmap + lfsmake2 etherwake + lfsmake2 bwm-ng + lfsmake2 sysstat + lfsmake2 strongswan + lfsmake2 rng-tools + lfsmake2 lsof + lfsmake2 br2684ctl + lfsmake2 lm_sensors + lfsmake2 libstatgrab + lfsmake2 liboping + lfsmake2 collectd + lfsmake2 git + lfsmake2 linux-firmware + lfsmake2 dvb-firmwares + lfsmake2 zd1211-firmware + lfsmake2 rpi-firmware + lfsmake2 intel-microcode + lfsmake2 pcengines-apu-firmware + lfsmake2 elinks + lfsmake2 igmpproxy + lfsmake2 opus + lfsmake2 python3-toml + lfsmake2 python3-pyproject2setuppy + lfsmake2 python3-pyparsing + lfsmake2 spice-protocol + lfsmake2 spice + lfsmake2 sdl2 + lfsmake2 libusbredir + lfsmake2 libseccomp + lfsmake2 qemu + lfsmake2 netsnmpd + lfsmake2 nagios_nrpe + lfsmake2 nagios-plugins + lfsmake2 icinga + lfsmake2 observium-agent + lfsmake2 ebtables + lfsmake2 faad2 + lfsmake2 alac + lfsmake2 ffmpeg + lfsmake2 vdr + lfsmake2 vdr_streamdev + lfsmake2 vdr_epgsearch + lfsmake2 vdr_dvbapi + lfsmake2 vdr_eepg + lfsmake2 w_scan + lfsmake2 fmt + lfsmake2 mpd + lfsmake2 libmpdclient + lfsmake2 mpc + lfsmake2 perl-Net-CIDR-Lite + lfsmake2 perl-Net-SMTP-SSL + lfsmake2 perl-MIME-Base64 + lfsmake2 perl-Authen-SASL + lfsmake2 perl-MIME-Lite + lfsmake2 perl-Email-Date-Format + lfsmake2 vnstat + lfsmake2 iw + lfsmake2 wpa_supplicant + lfsmake2 hostapd + lfsmake2 syslinux + lfsmake2 tftpd + lfsmake2 cpufrequtils + lfsmake2 gutenprint + lfsmake2 apcupsd + lfsmake2 fireperf + lfsmake2 iperf + lfsmake2 iperf3 + lfsmake2 7zip + lfsmake2 lynis + lfsmake2 sshfs + lfsmake2 taglib + lfsmake2 sslh + lfsmake2 perl-gettext + lfsmake2 perl-Sort-Naturally + lfsmake2 vdradmin + lfsmake2 perl-DBI + lfsmake2 perl-DBD-SQLite + lfsmake2 perl-File-ReadBackwards + lfsmake2 openvmtools + lfsmake2 joe + lfsmake2 monit + lfsmake2 nut + lfsmake2 watchdog + lfsmake2 usb_modeswitch + lfsmake2 usb_modeswitch_data + lfsmake2 zerofree + lfsmake2 minicom + lfsmake2 ddrescue + lfsmake2 parted + lfsmake2 swig + lfsmake2 dtc + lfsmake2 u-boot + lfsmake2 wireless-regdb + lfsmake2 ddns + lfsmake2 python3-pycparser + lfsmake2 python3-charset-normalizer + lfsmake2 python3-certifi + lfsmake2 python3-idna + lfsmake2 python3-requests + lfsmake2 python3-tomli + lfsmake2 python3-pep517 + lfsmake2 python3-build + lfsmake2 python3-install + lfsmake2 python3-urllib3 + lfsmake2 python3-flit + lfsmake2 python3-packaging + lfsmake2 python3-typing-extensions + lfsmake2 python3-semantic-version + lfsmake2 python3-setuptools-scm + lfsmake2 python3-setuptools-rust + lfsmake2 python3-six + lfsmake2 python3-dateutil + lfsmake2 python3-jmespath + lfsmake2 python3-colorama + lfsmake2 python3-yaml + lfsmake2 python3-s3transfer + lfsmake2 python3-rsa + lfsmake2 python3-pyasn1 + lfsmake2 python3-botocore + lfsmake2 python3-cffi + lfsmake2 python3-cryptography + lfsmake2 python3-circuitbreaker + lfsmake2 python3-pytz + lfsmake2 python3-click + lfsmake2 python3-arrow + lfsmake2 python3-terminaltables + lfsmake2 python3-pkgconfig + lfsmake2 python3-msgpack + lfsmake2 python3-attrs + lfsmake2 python3-sniffio + lfsmake2 python3-sortedcontainers + lfsmake2 python3-outcome + lfsmake2 python3-async_generator + lfsmake2 python3-flit_scm + lfsmake2 python3-exceptiongroup + lfsmake2 python3-trio + lfsmake2 python3-pyfuse3 + lfsmake2 python3-wheel + lfsmake2 aws-cli + lfsmake2 oci-python-sdk + lfsmake2 oci-cli + lfsmake2 transmission + lfsmake2 mtr + lfsmake2 minidlna + lfsmake2 acpid + lfsmake2 fping + lfsmake2 telnet + lfsmake2 xinetd + lfsmake2 stress + lfsmake2 sarg + lfsmake2 nginx + lfsmake2 sysbench + lfsmake2 strace + lfsmake2 ltrace + lfsmake2 ipfire-netboot + lfsmake2 lcdproc + lfsmake2 keepalived + lfsmake2 ipvsadm + lfsmake2 perl-Carp-Clan + lfsmake2 perl-Date-Calc + lfsmake2 perl-Date-Manip + lfsmake2 perl-File-Tail + lfsmake2 perl-TimeDate + lfsmake2 swatch + lfsmake2 tor + lfsmake2 wavemon + lfsmake2 iptraf-ng + lfsmake2 iotop + lfsmake2 stunnel + lfsmake2 bacula + lfsmake2 perl-Font-TTF + lfsmake2 perl-IO-String + lfsmake2 perl-PDF-API2 + lfsmake2 proxy-accounting + lfsmake2 tmux + lfsmake2 perl-Text-CSV_XS + lfsmake2 lua + lfsmake2 haproxy + lfsmake2 ipset + lfsmake2 dnsdist + lfsmake2 bird + lfsmake2 libyang + lfsmake2 frr + lfsmake2 dmidecode + lfsmake2 mcelog + lfsmake2 libpciaccess + lfsmake2 libyajl + lfsmake2 libvirt + lfsmake2 libtalloc + lfsmake2 freeradius + lfsmake2 perl-common-sense + lfsmake2 perl-inotify2 + lfsmake2 perl-Net-IP + lfsmake2 wio + lfsmake2 iftop + lfsmake2 mdns-repeater + lfsmake2 i2c-tools + lfsmake2 nss-myhostname + lfsmake2 dehydrated + lfsmake2 libplist + lfsmake2 nqptp + lfsmake2 shairport-sync + lfsmake2 borgbackup + lfsmake2 lmdb + lfsmake2 knot + lfsmake2 spectre-meltdown-checker + lfsmake2 zabbix_agentd + lfsmake2 flashrom + lfsmake2 firmware-update + lfsmake2 tshark + lfsmake2 speedtest-cli + lfsmake2 amazon-ssm-agent + lfsmake2 libloc + lfsmake2 ncdu + lfsmake2 lshw + lfsmake2 socat + lfsmake2 libcdada + lfsmake2 pmacct + lfsmake2 squid-asnbl + lfsmake2 qemu-ga + lfsmake2 gptfdisk + lfsmake2 oath-toolkit + lfsmake2 qrencode + lfsmake2 perl-File-Remove + lfsmake2 perl-Module-Build + lfsmake2 perl-Module-ScanDeps + lfsmake2 perl-YAML-Tiny + lfsmake2 perl-Module-Install + lfsmake2 perl-Imager + lfsmake2 perl-Imager-QRCode + lfsmake2 perl-MIME-Base32 + lfsmake2 perl-URI-Encode + lfsmake2 rsnapshot + + # Kernelbuild ... current we have no platform that need + # multi kernel builds so KCFG is empty + lfsmake2 linux KCFG="" + lfsmake2 rtl8189es KCFG="" + lfsmake2 rtl8189fs KCFG="" + lfsmake2 rtl8812au KCFG="" + lfsmake2 rtl8822bu KCFG="" + lfsmake2 rtl8821cu KCFG="" + lfsmake2 linux-initrd KCFG="" } -# This function will re-execute a command in a new namespace -exec_in_namespace() { - # Nothing to do if we are already in a new namespace - if [ -n "${IN_NAMESPACE}" ]; then - return 0 - fi - - # Forward any configuration - local args=( - "--target=${BUILD_ARCH}" - ) - - IN_NAMESPACE=1 \ - exec unshare \ - --mount \ - --propagation=private \ - "${0}" "${args[@]}" "$@" +buildinstaller() { + # Run installer scripts one by one + LOGFILE="$BASEDIR/log/_build.installer.log" + export LOGFILE + lfsmake2 memtest + lfsmake2 installer + # use toolchain bash for chroot to strip + CUSTOM_PATH="${TOOLS_DIR}/bin" lfsmake2 strip } -check_for_missing_rootfiles() { - print_headline "Checking for missing rootfiles..." - - local file - for file in ${LOG_DIR}/*_missing_rootfile; do - [ -e "${file}" ] || continue +buildpackages() { + LOGFILE="$BASEDIR/log/_build.packages.log" + export LOGFILE + echo "... see detailed log in _build.*.log files" >> $LOGFILE - file="${file##*/}" - file="${file/_missing_rootfile/}"; - print_line "${file} is missing a rootfile" - print_status FAIL - done + # Generating list of packages used + print_line "Generating packages list from logs" + rm -f $BASEDIR/doc/packages-list + for i in `ls -1tr $BASEDIR/log/[^_]*`; do + if [ "$i" != "$BASEDIR/log/FILES" -a -n $i ]; then + echo "* `basename $i`" >>$BASEDIR/doc/packages-list + fi + done + echo "== List of softwares used to build $NAME Version: $VERSION ==" > $BASEDIR/doc/packages-list.txt + grep -v 'configroot$\|img$\|initrd$\|initscripts$\|installer$\|install$\|setup$\|pakfire$\|stage2$\|smp$\|tools$\|tools1$\|tools2$\|.tgz$\|-config$\|_missing_rootfile$\|install1$\|install2$\|pass1$\|pass2$\|pass3$' \ + $BASEDIR/doc/packages-list | sort >> $BASEDIR/doc/packages-list.txt + rm -f $BASEDIR/doc/packages-list + # packages-list.txt is ready to be displayed for wiki page + print_status DONE + + # Update changelog + cd $BASEDIR + [ -z $GIT_TAG ] || LAST_TAG=$GIT_TAG + [ -z $LAST_TAG ] || EXT="$LAST_TAG..HEAD" + git log -n 500 --no-merges --pretty=medium --shortstat $EXT > $BASEDIR/doc/ChangeLog + + # Create images for install + lfsmake2 cdrom + + # Check if there is a loop device for building in virtual environments + modprobe loop 2>/dev/null + if [ $BUILD_IMAGES == 1 ] && ([ -e /dev/loop/0 ] || [ -e /dev/loop0 ] || [ -e "/dev/loop-control" ]); then + lfsmake2 flash-images + fi - return 0 -} + mv $LFS/install/images/{*.iso,*.img.xz,*.bz2} $BASEDIR >> $LOGFILE 2>&1 -check_rootfiles_for_arch() { - local arch="${1}" - - local args=( - # Search path - "${BASEDIR}/config/rootfiles" - - # Exclude old core updates - "--exclude-dir" "oldcore" - - # Ignore the update scripts - "--exclude" "update.sh" - ) - - # A list of files that are not scanned - # because they probably cause some false positives. - local excluded_files=( - abseil-cpp - cmake - gdb - liburcu - qemu - rust-memchr - rust-libc - rust-ppv-lite86 - xfsprogs - ) - - # Exclude any architecture-specific directories - local a - for a in ${ARCHES[@]}; do - args+=( "--exclude-dir" "${a}" ) - done + ipfirepackages - # Exclude all excluded files - local x - for x in ${excluded_files[@]}; do - args+=( "--exclude" "${x}" ) - done + cd $BASEDIR - # Search for all files that contain the architecture - local files=( - $(grep --files-with-matches -r "^.*${arch}" "${args[@]}") - ) + for i in $(ls *.bz2 *.img.xz *.iso 2>/dev/null); do + b2sum $i > $i.b2 + done + cd $PWD - local file - for file in ${files[@]}; do - print_line "${file} contains ${arch}" - print_status FAIL - done + # Cleanup + stdumount + rm -rf $BASEDIR/build/tmp/* - return "${#files[@]}" + cd $PWD } -check_rootfiles_for_pattern() { - local pattern="${1}" - local message="${2}" - - local args=( - # Search path - "${BASEDIR}/config/rootfiles" - - # Exclude old core updates - "--exclude-dir" "oldcore" - - # Ignore the update scripts - "--exclude" "update.sh" - ) +ipfirepackages() { + lfsmake2 core-updates - if grep -r "${pattern}" "${args[@]}"; then - if [ -n "${message}" ]; then - print_line "${message}" - print_status FAIL + local i + for i in $(find $BASEDIR/config/rootfiles/packages{/${BUILD_ARCH},} -maxdepth 1 -type f); do + i=$(basename ${i}) + if [ -e $BASEDIR/lfs/$i ]; then + ipfiredist $i else - print_file "Files matching '${pattern}' have been found in the rootfiles" - print_status FAIL + echo -n $i + print_status SKIP fi - return 1 - fi - - return 0 -} - -check_rootfiles() { - local failed=0 - - print_headline "Checking for rootfile consistency..." - - # Check for changes - if ! check_rootfiles_for_pattern "^[\+\-]" \ - "Rootfiles have changed in them"; then - failed=1 - fi - - # Check for /etc/init.d - if ! check_rootfiles_for_pattern "^etc/init\.d/" \ - "/etc/init.d/* has been found. Please replace by /etc/rc.d/init.d"; then - failed=1 - fi - - # Check for /var/run - if ! check_rootfiles_for_pattern "^var/run/.*" \ - "You cannot ship files in /var/run as it is a ramdisk"; then - failed=1 - fi - - # Check architectures - local arch - for arch in ${ARCHES[@]}; do - check_rootfiles_for_arch "${arch}" || failed=$? - done - - # Return the error - return ${failed} -} - -check_changed_rootfiles() { - local files=( - $(grep --files-with-matches -r "^+" "${LOG_DIR}" --exclude="_*" | sort) - ) - - # If we have no matches, there is nothing to do - [ "${#files[@]}" -eq 0 ] && return 0 - - print_line "Packages have created new files" - print_status WARN - - local file - for file in ${files[@]}; do - print_line " ${file##*/}" - print_status WARN done - - return 0 + test -d $BASEDIR/packages || mkdir $BASEDIR/packages + mv -f $LFS/install/packages/* $BASEDIR/packages >> $LOGFILE 2>&1 + rm -rf $BASEDIR/build/install/packages/* } -# Set BASEDIR -readonly BASEDIR="$(find_base)" - -# Get some information about the host system -SYSTEM_PROCESSORS="$(system_processors)" -SYSTEM_MEMORY="$(system_memory)" - -# Default settings -BUILD_ARCH="${HOST_ARCH}" -CCACHE_CACHE_SIZE="4G" - -# Load configuration file -if [ -f .config ]; then - source .config -fi - -# Parse any command line options (not commands) while [ $# -gt 0 ]; do case "${1}" in --target=*) - BUILD_ARCH="${1#--target=}" + configure_build "${1#--target=}" ;; - -*) exiterror "Unknown configuration option: ${1}" ;; - - # Found a command, so exit options parsing *) + # Found a command, so exit options parsing. break ;; esac shift done -# Check the architecture -case "${BUILD_ARCH}" in - aarch64|x86_64|riscv64) - ;; - - *) - exiterror "Unsupported architecture: ${BUILD_ARCH}" - ;; -esac - -# Set build platform -case "${BUILD_ARCH}" in - aarch64) - BUILD_PLATFORM="arm" - ;; - - riscv64) - BUILD_PLATFORM="riscv" - ;; - - x86_64) - BUILD_PLATFORM="x86" - ;; -esac - -# Configure the C compiler -CFLAGS=( - # Optimize the code - "-O2" - - # Do not compile in any debugging information - "-g0" - - # Do not write temporary files - "-pipe" - - # Enable all warnings - "-Wall" - - # Enable exceptions - "-fexceptions" - - # Compile place-independent code - "-fPIC" - - # Fortify Source - "-Wp,-U_FORTIFY_SOURCE" - "-Wp,-D_FORTIFY_SOURCE=3" - - # Enable additional checks for C++ in glibc - "-Wp,-D_GLIBCXX_ASSERTIONS" - - # Enable stack smashing protection - "-fstack-protector-strong" - - # Enable stack clash protection - "-fstack-clash-protection" -) - -# Architecture-dependent compiler flags -case "${BUILD_ARCH}" in - aarch64) - CFLAGS+=( - "-mbranch-protection=standard" - ) - ;; - - x86_64) - CFLAGS+=( - "-m64" "-mtune=generic" "-fcf-protection=full" - ) - ;; -esac - -# Configure the Rust compiler -RUSTFLAGS=( - "-Copt-level=3" - "-Clink-arg=-Wl,-z,relro,-z,now" - "-Ccodegen-units=1" - "--cap-lints=warn" -) - -# Configure the compiler tuple -CROSSTARGET="${BUILD_ARCH}-cross-linux-gnu" -BUILDTARGET="${BUILD_ARCH}-pc-linux-gnu" - -# Use this as default PARALLELISM -DEFAULT_PARALLELISM="${SYSTEM_PROCESSORS}" - -# Limit lauched ninja build jobs to computed parallel value -NINJAJOBS="${DEFAULT_PARALLELISM}" - -# Configure XZ -XZ_OPT=( - "-T0" "-8" -) - -# Configure Zstandard -ZSTD_OPT=( - "-T0" "-19" "--long" -) - -# Set directories -readonly CACHE_DIR="${BASEDIR}/cache" -readonly TOOLCHAIN_DIR="${CACHE_DIR}/toolchains" -readonly CCACHE_DIR="${BASEDIR}/ccache/${BUILD_ARCH}/${TOOLCHAINVER}" -readonly BUILD_DIR="${BASEDIR}/build_${BUILD_ARCH}" -readonly IMAGES_DIR="${BASEDIR}/images_${BUILD_ARCH}" -readonly LOG_DIR="${BASEDIR}/log_${BUILD_ARCH}" -readonly PACKAGES_DIR="${IMAGES_DIR}/packages" -readonly TOOLS_DIR="/tools_${BUILD_ARCH}" - -# Set URLs -readonly SOURCE_URL="https://source.ipfire.org/ipfire-2.x" -readonly TOOLCHAIN_URL="https://source.ipfire.org/toolchains" - -# Set the LOGFILE -LOGFILE="${LOG_DIR}/_build.preparation.log" - -# Ensure that some basic directories exist -mkdir -p "${CACHE_DIR}" "${LOG_DIR}" - -# Toolchain Archive -readonly TOOLCHAIN="${SNAME}-${VERSION}-toolchain-${TOOLCHAINVER}-${BUILD_ARCH}.tar.zst" - # See what we're supposed to do case "$1" in build) START_TIME="${SECONDS}" - # Launch in a new namespace - exec_in_namespace "$@" - - # Prepare the environment - prepareenv --required-space=8192 - - # Check if the toolchain is available - if [ ! -e "${BUILD_DIR}${TOOLS_DIR}/.toolchain-successful" ]; then - # If we have the toolchain available, we extract it into the build environment - if [ -r "${TOOLCHAIN_DIR}/${TOOLCHAIN}" ]; then - print_headline "Packaged toolchain compilation" - - # Extract the toolchain - if ! extract_toolchain "${TOOLCHAIN}"; then - exiterror "Failed extracting the toolchain" - fi - - # Otherwise perform a full toolchain compilation + PACKAGE="$BASEDIR/cache/toolchains/$SNAME-$VERSION-toolchain-$TOOLCHAINVER-${BUILD_ARCH}.tar.zst" + #only restore on a clean disk + if [ ! -e "${BASEDIR}/build${TOOLS_DIR}/.toolchain-successful" ]; then + if [ ! -n "$PACKAGE" ]; then + print_build_stage "Full toolchain compilation" + prepareenv + buildtoolchain else - print_headline "Full toolchain compilation" - build_toolchain + PACKAGENAME=${PACKAGE%.tar.zst} + print_build_stage "Packaged toolchain compilation" + if [ `b2sum $PACKAGE | awk '{print $1}'` == `cat $PACKAGENAME.b2 | awk '{print $1}'` ]; then + zstd -d < "${PACKAGE}" | tar x + prepareenv + else + exiterror "$PACKAGENAME BLAKE2 checksum did not match, check downloaded package" + fi fi + else + prepareenv fi - print_headline "Building ${NAME}" - build_system + print_build_stage "Building LFS" + buildbase - # Build all packages - build_packages + print_build_stage "Building IPFire" + buildipfire - # Check for missing rootfiles - check_for_missing_rootfiles + print_build_stage "Building installer" + buildinstaller - # Check for rootfile consistency - if ! check_rootfiles; then - exiterror "Rootfiles are inconsistent" - fi + print_build_stage "Building packages" + buildpackages + + print_build_stage "Checking Logfiles for new Files" - check_changed_rootfiles + cd $BASEDIR + tools/checknewlog.pl + tools/checkrootfiles + cd $PWD print_build_summary $(( SECONDS - START_TIME )) ;; -tail) - tail -F \ - "${LOG_DIR}/_build.preparation.log" \ - "${LOG_DIR}/_build.toolchain.log" \ - "${LOG_DIR}/_build.${SNAME}.log" \ - "${LOG_DIR}/_build.packages.log" 2>/dev/null - ;; shell) - # Launch in a new namespace - exec_in_namespace "$@" - # enter a shell inside LFS chroot # may be used to changed kernel settings - prepareenv --network + prepareenv entershell ;; -check) - # Check for rootfile consistency - if ! check_rootfiles; then - exiterror "Rootfiles are inconsistent" - fi - - check_changed_rootfiles - ;; clean) print_line "Cleaning build directory..." - # Cleanup build files - rm -rf \ - "${BUILD_DIR}" \ - "${IMAGES_DIR}" \ - "${LOG_DIR}" - - # Remove the /tools symlink + for i in `mount | grep $BASEDIR | sed 's/^.*loop=\(.*\))/\1/'`; do + $LOSETUP -d $i 2>/dev/null + done + #for i in `mount | grep $BASEDIR | cut -d " " -f 1`; do + # umount $i + #done + stdumount + for i in `seq 0 7`; do + if ( losetup /dev/loop${i} 2>/dev/null | grep -q "/install/images" ); then + umount /dev/loop${i} 2>/dev/null; + losetup -d /dev/loop${i} 2>/dev/null; + fi; + done + rm -rf $BASEDIR/build + rm -rf $BASEDIR/cdrom + rm -rf $BASEDIR/packages + rm -rf $BASEDIR/log if [ -h "${TOOLS_DIR}" ]; then rm -f "${TOOLS_DIR}" fi - + rm -f $BASEDIR/ipfire-* print_status DONE ;; -downloadsrc) - # Tell the user what we are about to do - print_headline "Pre-loading all source files" +docker) + # Build the docker image if it does not exist, yet + if ! docker images -a | grep -q ^ipfire-builder; then + if docker build -t ipfire-builder ${BASEDIR}/tools/docker; then + print_status DONE + else + print_status FAIL + exit 1 + fi + fi - # Download all sources - download_sources + # Run the container and enter a shell + docker run -it --privileged -v "${BASEDIR}:/build" -w "/build" ipfire-builder bash -l + ;; +downloadsrc) + if [ ! -d $BASEDIR/cache ]; then + mkdir $BASEDIR/cache + fi + mkdir -p $BASEDIR/log + echo -e "${BOLD}Preload all source files${NORMAL}" | tee -a $LOGFILE + FINISHED=0 + cd $BASEDIR/lfs + for c in `seq $MAX_RETRIES`; do + if (( FINISHED==1 )); then + break + fi + FINISHED=1 + cd $BASEDIR/lfs + for i in *; do + if [ -f "$i" -a "$i" != "Config" ]; then + lfsmakecommoncheck ${i} || continue + + make -s -f $i LFS_BASEDIR=$BASEDIR BUILD_ARCH="${BUILD_ARCH}" \ + MESSAGE="$i\t ($c/$MAX_RETRIES)" download >> $LOGFILE 2>&1 + if [ $? -ne 0 ]; then + print_status FAIL + FINISHED=0 + else + if [ $c -eq 1 ]; then + print_status DONE + fi + fi + fi + done + done + echo -e "${BOLD}***Verifying BLAKE2 checksum${NORMAL}" + ERROR=0 + for i in *; do + if [ -f "$i" -a "$i" != "Config" ]; then + lfsmakecommoncheck ${i} > /dev/null || continue + make -s -f $i LFS_BASEDIR=$BASEDIR BUILD_ARCH="${BUILD_ARCH}" \ + MESSAGE="$i\t " b2 >> $LOGFILE 2>&1 + if [ $? -ne 0 ]; then + echo -ne "BLAKE2 checksum difference in lfs/$i" + print_status FAIL + ERROR=1 + fi + fi + done + if [ $ERROR -eq 0 ]; then + echo -ne "${BOLD}all files BLAKE2 checksum match${NORMAL}" + print_status DONE + else + echo -ne "${BOLD}not all files were correctly download${NORMAL}" + print_status FAIL + fi + cd - >/dev/null 2>&1 ;; toolchain) - # Launch in a new namespace - exec_in_namespace "$@" - - # Prepare the environment prepareenv - - print_headline "Toolchain compilation (${BUILD_ARCH})" - - # Build the toolchain - build_toolchain - - # Ensure the toolchain directory exists - mkdir -p "${TOOLCHAIN_DIR}" - - # Compress the toolchain - if ! compress_toolchain "${TOOLCHAIN}"; then - exiterror "Could not compress toolchain" - fi + print_build_stage "Toolchain compilation (${BUILD_ARCH})" + buildtoolchain + echo "`date -u '+%b %e %T'`: Create toolchain image for ${BUILD_ARCH}" | tee -a $LOGFILE + test -d $BASEDIR/cache/toolchains || mkdir -p $BASEDIR/cache/toolchains + cd $BASEDIR && tar -cf- --exclude='log/_build.*.log' build/${TOOLS_DIR} build/bin/sh log \ + | zstd ${ZSTD_OPT} > cache/toolchains/$SNAME-$VERSION-toolchain-$TOOLCHAINVER-${BUILD_ARCH}.tar.zst + b2sum cache/toolchains/$SNAME-$VERSION-toolchain-$TOOLCHAINVER-${BUILD_ARCH}.tar.zst \ + > cache/toolchains/$SNAME-$VERSION-toolchain-$TOOLCHAINVER-${BUILD_ARCH}.b2 + stdumount ;; - gettoolchain) - download_toolchain "${TOOLCHAIN}" + # arbitrary name to be updated in case of new toolchain package upload + PACKAGE=$SNAME-$VERSION-toolchain-$TOOLCHAINVER-${BUILD_ARCH} + if [ ! -f $BASEDIR/cache/toolchains/$PACKAGE.tar.zst ]; then + URL_TOOLCHAIN=`grep URL_TOOLCHAIN lfs/Config | awk '{ print $3 }'` + test -d $BASEDIR/cache/toolchains || mkdir -p $BASEDIR/cache/toolchains + echo "`date -u '+%b %e %T'`: Load toolchain image for ${BUILD_ARCH}" | tee -a $LOGFILE + cd $BASEDIR/cache/toolchains + wget -U "IPFireSourceGrabber/2.x" $URL_TOOLCHAIN/$PACKAGE.tar.zst $URL_TOOLCHAIN/$PACKAGE.b2 >& /dev/null + if [ $? -ne 0 ]; then + echo "`date -u '+%b %e %T'`: error downloading $PACKAGE toolchain for ${BUILD_ARCH} machine" | tee -a $LOGFILE + else + if [ "`b2sum $PACKAGE.tar.zst | awk '{print $1}'`" = "`cat $PACKAGE.b2 | awk '{print $1}'`" ]; then + echo "`date -u '+%b %e %T'`: toolchain BLAKE2 checksum ok" | tee -a $LOGFILE + else + exiterror "$PACKAGE BLAKE2 checksum did not match, check downloaded package" + fi + fi + else + echo "Toolchain is already downloaded. Exiting..." + fi ;; uploadsrc) - # Check if IPFIRE_USER is set - if [ -z "${IPFIRE_USER}" ]; then - exiterror "You have to setup IPFIRE_USER first. See .config for details." + if [ -z $IPFIRE_USER ]; then + echo -n "You have to setup IPFIRE_USER first. See .config for details." + print_status FAIL + exit 1 fi - # Sync with upstream + URL_SOURCE="$(awk '/^URL_SOURCE/ { print $3 }' lfs/Config)" + rsync \ --recursive \ --update \ @@ -2594,8 +2006,8 @@ uploadsrc) --progress \ --human-readable \ --exclude="toolchains/" \ - "${CACHE_DIR}/" \ - "${IPFIRE_USER}@people.ipfire.org:/pub/sources/source-2.x" + "${BASEDIR}/cache/" \ + "${IPFIRE_USER}@${URL_SOURCE}" exit 0 ;; @@ -2633,7 +2045,7 @@ update-contributors) ;; find-dependencies) shift - exec "${BASEDIR}/tools/find-dependencies" "${BUILD_DIR}" "$@" + exec "${BASEDIR}/tools/find-dependencies" "${BASEDIR}/build" "$@" ;; check-manualpages) echo "Checking the manual pages for broken links..." @@ -2645,11 +2057,8 @@ check-manualpages) print_status FAIL fi ;; -__timer) - __timer "${2}" || exit $? - ;; *) - echo "Usage: $0 [OPTIONS] {build|check-manualpages|clean|downloadsrc|find-dependencies|gettoolchain|lang|shell|tail|toolchain|update-contributors|uploadsrc}" + echo "Usage: $0 [OPTIONS] {build|check-manualpages|clean|docker|downloadsrc|find-dependencies|gettoolchain|lang|shell|toolchain|update-contributors|uploadsrc}" cat doc/make.sh-usage ;; esac