X-Git-Url: http://git.ipfire.org/?p=people%2Fpmueller%2Fipfire-2.x.git;a=blobdiff_plain;f=make.sh;h=bb4269d5d5fe56837adf1cc18356cffb507b4667;hp=a7e8068587434ce2fb27022a37c6b35540e22a79;hb=03f68cbca90d9c1bc0b55c2f5aa4698a5d9d3eab;hpb=35ce207ff7f94921ddffb23501c28d7095d59a04

diff --git a/make.sh b/make.sh
index a7e8068587..bb4269d5d5 100755
--- a/make.sh
+++ b/make.sh
@@ -17,16 +17,16 @@
 # along with IPFire; if not, write to the Free Software                    #
 # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307 USA #
 #                                                                          #
-# Copyright (C) 2007-2018 IPFire Team <info@ipfire.org>.                   #
+# Copyright (C) 2007-2019 IPFire Team <info@ipfire.org>.                   #
 #                                                                          #
 ############################################################################
 #
 
 NAME="IPFire"							# Software name
 SNAME="ipfire"							# Short name
-VERSION="2.21"							# Version number
-CORE="125"							# Core Level (Filename)
-PAKFIRE_CORE="124"						# Core Level (PAKFIRE)
+VERSION="2.23"							# Version number
+CORE="130"							# Core Level (Filename)
+PAKFIRE_CORE="129"						# Core Level (PAKFIRE)
 GIT_BRANCH=`git rev-parse --abbrev-ref HEAD`			# Git Branch
 SLOGAN="www.ipfire.org"						# Software slogan
 CONFIG_ROOT=/var/ipfire						# Configuration rootdir
@@ -37,7 +37,7 @@ KVER=`grep --max-count=1 VER lfs/linux | awk '{ print $3 }'`
 GIT_TAG=$(git tag | tail -1)					# Git Tag
 GIT_LASTCOMMIT=$(git log | head -n1 | cut -d" " -f2 |head -c8)	# Last commit
 
-TOOLCHAINVER=20181024
+TOOLCHAINVER=20181030
 
 ###############################################################################
 #
@@ -109,7 +109,10 @@ fi
 # This is the directory where make.sh is in
 export BASEDIR=$(echo $FULLPATH | sed "s/\/$BASENAME//g")
 
+LOGFILE=$BASEDIR/log/_build.preparation.log
+export LOGFILE
 DIR_CHK=$BASEDIR/cache/check
+mkdir $BASEDIR/log/ 2>/dev/null
 
 system_processors() {
 	getconf _NPROCESSORS_ONLN 2>/dev/null || echo "1"
@@ -190,7 +193,6 @@ configure_build() {
 
 	BUILD_ARCH="${build_arch}"
 	TOOLS_DIR="/tools_${BUILD_ARCH}"
-	LOG_DIR="log_${BUILD_ARCH}"
 
 	# Enables hardening
 	HARDENING_CFLAGS="-Wp,-D_FORTIFY_SOURCE=2 -Wp,-D_GLIBCXX_ASSERTIONS -fstack-protector-strong"
@@ -199,31 +201,22 @@ configure_build() {
 	CXXFLAGS="${CFLAGS}"
 
 	# Determine parallelism
-	if [ -z "${MAKETUNING}" ]; then
-		# We assume that each process consumes about
-		# 192MB of memory. Therefore we find out how
-		# many processes fit into memory.
-		local mem_max=$(( ${HOST_MEM} / 192 ))
-
-		local processors="$(system_processors)"
-		local cpu_max=$(( ${processors} + 1 ))
-
-		local parallelism
-		if [ ${mem_max} -lt ${cpu_max} ]; then
-			parallelism=${mem_max}
-		else
-			parallelism=${cpu_max}
-		fi
-
-		# limit to -j23 because perl will not build
-		# more
-		if [ ${parallelism} -gt 23 ]; then
-			parallelism=23
-		fi
-
-		MAKETUNING="-j${parallelism}"
+	# We assume that each process consumes about
+	# 128MB of memory. Therefore we find out how
+	# many processes fit into memory.
+	local mem_max=$(( ${SYSTEM_MEMORY} / 128 ))
+	local cpu_max=$(( ${SYSTEM_PROCESSORS} + 1 ))
+
+	local parallelism
+	if [ ${mem_max} -lt ${cpu_max} ]; then
+		parallelism=${mem_max}
+	else
+		parallelism=${cpu_max}
 	fi
 
+	# Use this as default PARALLELISM
+	DEFAULT_PARALLELISM="${parallelism}"
+
 	# Compression parameters
 	# We use mode 8 for reasonable memory usage when decompressing
 	# but with overall good compression
@@ -235,7 +228,7 @@ configure_build() {
 	# We need to limit memory because XZ uses too much when running
 	# in parallel and it isn't very smart in limiting itself.
 	# We allow XZ to use up to 70% of all system memory.
-	local xz_memory=$(( HOST_MEM * 7 / 10 ))
+	local xz_memory=$(( SYSTEM_MEMORY * 7 / 10 ))
 
 	# XZ memory cannot be larger than 2GB on 32 bit systems
 	case "${build_arch}" in
@@ -251,7 +244,10 @@ configure_build() {
 
 configure_build_guess() {
 	case "${HOST_ARCH}" in
-		x86_64|i686|i586)
+		x86_64)
+			echo "x86_64"
+			;;
+		i?86)
 			echo "i586"
 			;;
 
@@ -283,7 +279,7 @@ stdumount() {
 	umount $BASEDIR/build/usr/src/html		2>/dev/null;
 	umount $BASEDIR/build/usr/src/langs	2>/dev/null;
 	umount $BASEDIR/build/usr/src/lfs		2>/dev/null;
-	umount $BASEDIR/build/usr/src/${LOG_DIR}	2>/dev/null;
+	umount $BASEDIR/build/usr/src/log		2>/dev/null;
 	umount $BASEDIR/build/usr/src/src		2>/dev/null;
 }
 
@@ -419,9 +415,6 @@ exiterror() {
 }
 
 prepareenv() {
-	# Create log directory if it doesn't exist, yet
-	mkdir -p "${BASEDIR}/${LOG_DIR}"
-
 	# Are we running the right shell?
 	if [ -z "${BASH}" ]; then
 		exiterror "BASH environment variable is not set.  You're probably running the wrong shell."
@@ -467,7 +460,7 @@ prepareenv() {
 	# Setup environment
 	set +h
 	LC_ALL=POSIX
-	export LFS LC_ALL CFLAGS CXXFLAGS MAKETUNING
+	export LFS LC_ALL CFLAGS CXXFLAGS DEFAULT_PARALLELISM
 	unset CC CXX CPP LD_LIBRARY_PATH LD_PRELOAD
 
 	# Make some extra directories
@@ -475,7 +468,7 @@ prepareenv() {
 	mkdir -p $BASEDIR/build/{etc,usr/src} 2>/dev/null
 	mkdir -p $BASEDIR/build/{dev/{shm,pts},proc,sys}
 	mkdir -p $BASEDIR/{cache,ccache} 2>/dev/null
-	mkdir -p $BASEDIR/build/usr/src/{cache,config,doc,html,langs,lfs,${LOG_DIR},src,ccache}
+	mkdir -p $BASEDIR/build/usr/src/{cache,config,doc,html,langs,lfs,log,src,ccache}
 
 	mknod -m 600 $BASEDIR/build/dev/console c 5 1 2>/dev/null
 	mknod -m 666 $BASEDIR/build/dev/null c 1 3 2>/dev/null
@@ -493,7 +486,7 @@ prepareenv() {
 	mount --bind $BASEDIR/html   $BASEDIR/build/usr/src/html
 	mount --bind $BASEDIR/langs  $BASEDIR/build/usr/src/langs
 	mount --bind $BASEDIR/lfs    $BASEDIR/build/usr/src/lfs
-	mount --bind $BASEDIR/${LOG_DIR} $BASEDIR/build/usr/src/${LOG_DIR}
+	mount --bind $BASEDIR/log    $BASEDIR/build/usr/src/log
 	mount --bind $BASEDIR/src    $BASEDIR/build/usr/src/src
 
 	# Run LFS static binary creation scripts one by one
@@ -540,7 +533,6 @@ enterchroot() {
 		CORE="${CORE}" \
 		SLOGAN="${SLOGAN}" \
 		TOOLS_DIR="${TOOLS_DIR}" \
-		LOG_DIR="${LOG_DIR}" \
 		CONFIG_ROOT="${CONFIG_ROOT}" \
 		CFLAGS="${CFLAGS} ${HARDENING_CFLAGS}" \
 		CXXFLAGS="${CXXFLAGS} ${HARDENING_CFLAGS}" \
@@ -553,6 +545,9 @@ enterchroot() {
 		CCACHE_COMPILERCHECK="${CCACHE_COMPILERCHECK}" \
 		KVER="${KVER}" \
 		XZ_OPT="${XZ_OPT}" \
+		DEFAULT_PARALLELISM="${DEFAULT_PARALLELISM}" \
+		SYSTEM_PROCESSORS="${SYSTEM_PROCESSORS}" \
+		SYSTEM_MEMORY="${SYSTEM_MEMORY}" \
 		$(fake_environ) \
 		$(qemu_environ) \
 		"$@"
@@ -632,7 +627,9 @@ lfsmake1() {
 		CCACHE_COMPILERCHECK="${CCACHE_COMPILERCHECK}" \
 		CFLAGS="${CFLAGS}" \
 		CXXFLAGS="${CXXFLAGS}" \
-		MAKETUNING="${MAKETUNING}" \
+		DEFAULT_PARALLELISM="${DEFAULT_PARALLELISM}" \
+		SYSTEM_PROCESSORS="${SYSTEM_PROCESSORS}" \
+		SYSTEM_MEMORY="${SYSTEM_MEMORY}" \
 		make -f $* \
 			TOOLCHAIN=1 \
 			TOOLS_DIR="${TOOLS_DIR}" \
@@ -661,7 +658,6 @@ lfsmake2() {
 
 	enterchroot \
 		${EXTRA_PATH}bash -x -c "cd /usr/src/lfs && \
-			MAKETUNING=${MAKETUNING} \
 			make -f $* \
 			LFS_BASEDIR=/usr/src install" \
 		>> ${LOGFILE} 2>&1 &
@@ -906,8 +902,9 @@ if [ -n "${TARGET_ARCH}" ]; then
 	unset TARGET_ARCH
 fi
 
-# Get the amount of memory in this build system
-HOST_MEM=$(system_memory)
+# Get some information about the host system
+SYSTEM_PROCESSORS="$(system_processors)"
+SYSTEM_MEMORY="$(system_memory)"
 
 if [ -n "${BUILD_ARCH}" ]; then
 	configure_build "${BUILD_ARCH}"
@@ -965,7 +962,7 @@ buildtoolchain() {
 		exiterror "Could not create ${TOOLS_DIR} symbolic link"
 	fi
 
-	LOGFILE="$BASEDIR/${LOG_DIR}/_build.toolchain.log"
+	LOGFILE="$BASEDIR/log/_build.toolchain.log"
 	export LOGFILE
 
 	lfsmake1 stage1
@@ -1010,7 +1007,7 @@ buildtoolchain() {
 }
 
 buildbase() {
-	LOGFILE="$BASEDIR/${LOG_DIR}/_build.base.log"
+	LOGFILE="$BASEDIR/log/_build.base.log"
 	export LOGFILE
 	lfsmake2 stage2
 	lfsmake2 linux			KCFG="-headers"
@@ -1079,14 +1076,13 @@ buildbase() {
 }
 
 buildipfire() {
-  LOGFILE="$BASEDIR/${LOG_DIR}/_build.ipfire.log"
+  LOGFILE="$BASEDIR/log/_build.ipfire.log"
   export LOGFILE
   lfsmake2 configroot
   lfsmake2 initscripts
   lfsmake2 backup
   lfsmake2 openssl
   [ "${BUILD_ARCH}" = "i586" ] && lfsmake2 openssl KCFG='-sse2'
-  lfsmake2 openssl-compat
   lfsmake2 popt
   lfsmake2 libusb
   lfsmake2 libusb-compat
@@ -1100,6 +1096,7 @@ buildipfire() {
   lfsmake2 xr819-firmware
   lfsmake2 zd1211-firmware
   lfsmake2 rpi-firmware
+  lfsmake2 intel-microcode
   lfsmake2 bc
   lfsmake2 u-boot MKIMAGE=1
   lfsmake2 cpio
@@ -1172,12 +1169,13 @@ buildipfire() {
 		lfsmake2 linux-initrd			KCFG="-multi"
 		;;
   esac
-  lfsmake2 intel-microcode
   lfsmake2 xtables-addons			USPACE="1"
   lfsmake2 libgpg-error
   lfsmake2 libgcrypt
   lfsmake2 libassuan
   lfsmake2 nettle
+  lfsmake2 json-c
+  lfsmake2 libconfig
   lfsmake2 libevent
   lfsmake2 libevent2
   lfsmake2 expat
@@ -1218,6 +1216,7 @@ buildipfire() {
   lfsmake2 attr
   lfsmake2 acl
   lfsmake2 libcap
+  lfsmake2 libcap-ng
   lfsmake2 pciutils
   lfsmake2 usbutils
   lfsmake2 libxml2
@@ -1316,9 +1315,11 @@ buildipfire() {
   lfsmake2 setserial
   lfsmake2 setup
   lfsmake2 libdnet
-  lfsmake2 daq
-  lfsmake2 snort
+  lfsmake2 yaml
+  lfsmake2 libhtp
+  lfsmake2 suricata
   lfsmake2 oinkmaster
+  lfsmake2 ids-ruleset-sources
   lfsmake2 squid
   lfsmake2 squidguard
   lfsmake2 calamaris
@@ -1373,6 +1374,7 @@ buildipfire() {
   lfsmake2 flac
   lfsmake2 lame
   lfsmake2 sox
+  lfsmake2 soxr
   lfsmake2 libshout
   lfsmake2 xvid
   lfsmake2 libmpeg2
@@ -1413,9 +1415,11 @@ buildipfire() {
   lfsmake2 nagios_nrpe
   lfsmake2 nagios-plugins
   lfsmake2 icinga
+  lfsmake2 observium-agent
   lfsmake2 ebtables
   lfsmake2 directfb
   lfsmake2 faad2
+  lfsmake2 alac
   lfsmake2 ffmpeg
   lfsmake2 vdr
   lfsmake2 vdr_streamdev
@@ -1428,6 +1432,7 @@ buildipfire() {
   lfsmake2 mpd
   lfsmake2 libmpdclient
   lfsmake2 mpc
+  lfsmake2 perl-Net-CIDR-Lite
   lfsmake2 perl-Net-SMTP-SSL
   lfsmake2 perl-MIME-Base64
   lfsmake2 perl-Authen-SASL
@@ -1494,6 +1499,8 @@ buildipfire() {
   lfsmake2 python-progressbar
   lfsmake2 python-xattr
   lfsmake2 ddns
+  lfsmake2 python3-setuptools
+  lfsmake2 python3-setuptools-scm
   lfsmake2 python3-six
   lfsmake2 python3-dateutil
   lfsmake2 python3-jmespath
@@ -1504,6 +1511,8 @@ buildipfire() {
   lfsmake2 python3-rsa
   lfsmake2 python3-pyasn1
   lfsmake2 python3-botocore
+  lfsmake2 python3-llfuse
+  lfsmake2 python3-msgpack
   lfsmake2 aws-cli
   lfsmake2 transmission
   lfsmake2 dpfhack
@@ -1558,6 +1567,7 @@ buildipfire() {
   lfsmake2 lua
   lfsmake2 dnsdist
   lfsmake2 bird
+  lfsmake2 frr
   lfsmake2 dmidecode
   lfsmake2 mcelog
   lfsmake2 rtpproxy
@@ -1574,11 +1584,18 @@ buildipfire() {
   lfsmake2 mdns-repeater
   lfsmake2 i2c-tools
   lfsmake2 nss-myhostname
+  lfsmake2 dehydrated
+  lfsmake2 shairport-sync
+  lfsmake2 borgbackup
+  lfsmake2 libedit
+  lfsmake2 knot
+  lfsmake2 spectre-meltdown-checker
+  lfsmake2 zabbix_agentd
 }
 
 buildinstaller() {
   # Run installer scripts one by one
-  LOGFILE="$BASEDIR/${LOG_DIR}/_build.installer.log"
+  LOGFILE="$BASEDIR/log/_build.installer.log"
   export LOGFILE
   lfsmake2 memtest
   lfsmake2 installer
@@ -1587,7 +1604,7 @@ buildinstaller() {
 }
 
 buildpackages() {
-  LOGFILE="$BASEDIR/${LOG_DIR}/_build.packages.log"
+  LOGFILE="$BASEDIR/log/_build.packages.log"
   export LOGFILE
   echo "... see detailed log in _build.*.log files" >> $LOGFILE
 
@@ -1595,8 +1612,8 @@ buildpackages() {
   # Generating list of packages used
   print_line "Generating packages list from logs"
   rm -f $BASEDIR/doc/packages-list
-  for i in `ls -1tr $BASEDIR/${LOG_DIR}/[^_]*`; do
-	if [ "$i" != "$BASEDIR/${LOG_DIR}/FILES" -a -n $i ]; then
+  for i in `ls -1tr $BASEDIR/log/[^_]*`; do
+	if [ "$i" != "$BASEDIR/log/FILES" -a -n $i ]; then
 		echo "* `basename $i`" >>$BASEDIR/doc/packages-list
 	fi
   done
@@ -1755,18 +1772,32 @@ clean)
 	rm -rf $BASEDIR/build
 	rm -rf $BASEDIR/cdrom
 	rm -rf $BASEDIR/packages
-	rm -rf $BASEDIR/${LOG_DIR}
+	rm -rf $BASEDIR/log
 	if [ -h "${TOOLS_DIR}" ]; then
 		rm -f "${TOOLS_DIR}"
 	fi
 	rm -f $BASEDIR/ipfire-*
 	print_status DONE
 	;;
+docker)
+	# Build the docker image if it does not exist, yet
+	if ! docker images -a | grep -q ^ipfire-builder; then
+		if docker build -t ipfire-builder ${BASEDIR}/tools/docker; then
+			print_status DONE
+		else
+			print_status FAIL
+			exit 1
+		fi
+	fi
+
+	# Run the container and enter a shell
+	docker run -it --privileged -v "${BASEDIR}:/build" -w "/build" ipfire-builder bash -l
+	;;
 downloadsrc)
 	if [ ! -d $BASEDIR/cache ]; then
 		mkdir $BASEDIR/cache
 	fi
-	mkdir -p $BASEDIR/${LOG_DIR}
+	mkdir -p $BASEDIR/log
 	echo -e "${BOLD}Preload all source files${NORMAL}" | tee -a $LOGFILE
 	FINISHED=0
 	cd $BASEDIR/lfs
@@ -1825,7 +1856,7 @@ toolchain)
 	buildtoolchain
 	echo "`date -u '+%b %e %T'`: Create toolchain image for ${BUILD_ARCH}" | tee -a $LOGFILE
 	test -d $BASEDIR/cache/toolchains || mkdir -p $BASEDIR/cache/toolchains
-	cd $BASEDIR && tar -cf- --exclude='${LOG_DIR}/_build.*.log' build/${TOOLS_DIR} build/bin/sh ${LOG_DIR} | xz ${XZ_OPT} \
+	cd $BASEDIR && tar -cf- --exclude='log/_build.*.log' build/${TOOLS_DIR} build/bin/sh log | xz ${XZ_OPT} \
 		> cache/toolchains/$SNAME-$VERSION-toolchain-$TOOLCHAINVER-${BUILD_ARCH}.tar.xz
 	md5sum cache/toolchains/$SNAME-$VERSION-toolchain-$TOOLCHAINVER-${BUILD_ARCH}.tar.xz \
 		> cache/toolchains/$SNAME-$VERSION-toolchain-$TOOLCHAINVER-${BUILD_ARCH}.md5