X-Git-Url: http://git.ipfire.org/?p=people%2Fpmueller%2Fipfire-2.x.git;a=blobdiff_plain;f=make.sh;h=a7e8068587434ce2fb27022a37c6b35540e22a79;hp=353441b017427107c01592aad29868b654ee8b10;hb=35ce207ff7f94921ddffb23501c28d7095d59a04;hpb=f03e254f39ddb7cacea7a39ec6b98dce77018b0d

diff --git a/make.sh b/make.sh
index 353441b017..a7e8068587 100755
--- a/make.sh
+++ b/make.sh
@@ -24,9 +24,9 @@
 
 NAME="IPFire"							# Software name
 SNAME="ipfire"							# Short name
-VERSION="2.19"							# Version number
-CORE="121"							# Core Level (Filename)
-PAKFIRE_CORE="120"						# Core Level (PAKFIRE)
+VERSION="2.21"							# Version number
+CORE="125"							# Core Level (Filename)
+PAKFIRE_CORE="124"						# Core Level (PAKFIRE)
 GIT_BRANCH=`git rev-parse --abbrev-ref HEAD`			# Git Branch
 SLOGAN="www.ipfire.org"						# Software slogan
 CONFIG_ROOT=/var/ipfire						# Configuration rootdir
@@ -37,7 +37,7 @@ KVER=`grep --max-count=1 VER lfs/linux | awk '{ print $3 }'`
 GIT_TAG=$(git tag | tail -1)					# Git Tag
 GIT_LASTCOMMIT=$(git log | head -n1 | cut -d" " -f2 |head -c8)	# Last commit
 
-TOOLCHAINVER=20180213
+TOOLCHAINVER=20181024
 
 ###############################################################################
 #
@@ -109,10 +109,7 @@ fi
 # This is the directory where make.sh is in
 export BASEDIR=$(echo $FULLPATH | sed "s/\/$BASENAME//g")
 
-LOGFILE=$BASEDIR/log/_build.preparation.log
-export LOGFILE
 DIR_CHK=$BASEDIR/cache/check
-mkdir $BASEDIR/log/ 2>/dev/null
 
 system_processors() {
 	getconf _NPROCESSORS_ONLN 2>/dev/null || echo "1"
@@ -193,6 +190,7 @@ configure_build() {
 
 	BUILD_ARCH="${build_arch}"
 	TOOLS_DIR="/tools_${BUILD_ARCH}"
+	LOG_DIR="log_${BUILD_ARCH}"
 
 	# Enables hardening
 	HARDENING_CFLAGS="-Wp,-D_FORTIFY_SOURCE=2 -Wp,-D_GLIBCXX_ASSERTIONS -fstack-protector-strong"
@@ -225,6 +223,30 @@ configure_build() {
 
 		MAKETUNING="-j${parallelism}"
 	fi
+
+	# Compression parameters
+	# We use mode 8 for reasonable memory usage when decompressing
+	# but with overall good compression
+	XZ_OPT="-8"
+
+	# We try to use as many cores as possible
+	XZ_OPT="${XZ_OPT} -T0"
+
+	# We need to limit memory because XZ uses too much when running
+	# in parallel and it isn't very smart in limiting itself.
+	# We allow XZ to use up to 70% of all system memory.
+	local xz_memory=$(( HOST_MEM * 7 / 10 ))
+
+	# XZ memory cannot be larger than 2GB on 32 bit systems
+	case "${build_arch}" in
+		i*86|armv*)
+			if [ ${xz_memory} -gt 2048 ]; then
+				xz_memory=2048
+			fi
+			;;
+	esac
+
+	XZ_OPT="${XZ_OPT} --memory=${xz_memory}MiB"
 }
 
 configure_build_guess() {
@@ -261,7 +283,7 @@ stdumount() {
 	umount $BASEDIR/build/usr/src/html		2>/dev/null;
 	umount $BASEDIR/build/usr/src/langs	2>/dev/null;
 	umount $BASEDIR/build/usr/src/lfs		2>/dev/null;
-	umount $BASEDIR/build/usr/src/log		2>/dev/null;
+	umount $BASEDIR/build/usr/src/${LOG_DIR}	2>/dev/null;
 	umount $BASEDIR/build/usr/src/src		2>/dev/null;
 }
 
@@ -397,6 +419,9 @@ exiterror() {
 }
 
 prepareenv() {
+	# Create log directory if it doesn't exist, yet
+	mkdir -p "${BASEDIR}/${LOG_DIR}"
+
 	# Are we running the right shell?
 	if [ -z "${BASH}" ]; then
 		exiterror "BASH environment variable is not set.  You're probably running the wrong shell."
@@ -450,7 +475,7 @@ prepareenv() {
 	mkdir -p $BASEDIR/build/{etc,usr/src} 2>/dev/null
 	mkdir -p $BASEDIR/build/{dev/{shm,pts},proc,sys}
 	mkdir -p $BASEDIR/{cache,ccache} 2>/dev/null
-	mkdir -p $BASEDIR/build/usr/src/{cache,config,doc,html,langs,lfs,log,src,ccache}
+	mkdir -p $BASEDIR/build/usr/src/{cache,config,doc,html,langs,lfs,${LOG_DIR},src,ccache}
 
 	mknod -m 600 $BASEDIR/build/dev/console c 5 1 2>/dev/null
 	mknod -m 666 $BASEDIR/build/dev/null c 1 3 2>/dev/null
@@ -468,7 +493,7 @@ prepareenv() {
 	mount --bind $BASEDIR/html   $BASEDIR/build/usr/src/html
 	mount --bind $BASEDIR/langs  $BASEDIR/build/usr/src/langs
 	mount --bind $BASEDIR/lfs    $BASEDIR/build/usr/src/lfs
-	mount --bind $BASEDIR/log    $BASEDIR/build/usr/src/log
+	mount --bind $BASEDIR/${LOG_DIR} $BASEDIR/build/usr/src/${LOG_DIR}
 	mount --bind $BASEDIR/src    $BASEDIR/build/usr/src/src
 
 	# Run LFS static binary creation scripts one by one
@@ -515,6 +540,7 @@ enterchroot() {
 		CORE="${CORE}" \
 		SLOGAN="${SLOGAN}" \
 		TOOLS_DIR="${TOOLS_DIR}" \
+		LOG_DIR="${LOG_DIR}" \
 		CONFIG_ROOT="${CONFIG_ROOT}" \
 		CFLAGS="${CFLAGS} ${HARDENING_CFLAGS}" \
 		CXXFLAGS="${CXXFLAGS} ${HARDENING_CFLAGS}" \
@@ -847,6 +873,28 @@ update_language_list() {
 	done | sort -u > "${path}/LINGUAS"
 }
 
+contributors() {
+	local commits name
+
+	git shortlog --summary --numbered | while read -r commits name; do
+		echo "${name}"
+	done | grep -vE -e "^(alpha197|morlix|root|ummeegge)$" -e "via Development$" -e "@" -e "#$"
+}
+
+update_contributors() {
+	echo -n "Updating list of contributors"
+
+	local contributors="$(contributors | paste -sd , - | sed -e "s/,/&\\\\n/g")"
+
+	# Edit contributors into credits.cgi
+	awk -i inplace \
+		"/<!-- CONTRIBUTORS -->/{ p=1; print; printf \"${contributors}\n\"}/<!-- END -->/{ p=0 } !p" \
+		"${BASEDIR}/html/cgi-bin/credits.cgi"
+
+	print_status DONE
+	return 0
+}
+
 # Load configuration file
 if [ -f .config ]; then
 	. .config
@@ -861,25 +909,6 @@ fi
 # Get the amount of memory in this build system
 HOST_MEM=$(system_memory)
 
-# Checking host memory, tuning XZ_OPT
-if [ ${HOST_MEM} -lt 1024 ]; then
-	exiterror "You will need more than 1GB or host memory to run the build"
-fi
-
-# We compress archives with "xz -8", using all cores and up to 70% of memory
-XZ_MEM=$(( HOST_MEM * 7 / 10 ))
-
-# XZ memory cannot be larger than 2GB on 32 bit systems
-case "${HOST_ARCH}" in
-	i*86|armv*)
-		if [ ${XZ_MEM} -gt 2048 ]; then
-			XZ_MEM=2048
-		fi
-		;;
-esac
-
-XZ_OPT="-T0 -8 --memory=${XZ_MEM}MiB"
-
 if [ -n "${BUILD_ARCH}" ]; then
 	configure_build "${BUILD_ARCH}"
 else
@@ -936,7 +965,7 @@ buildtoolchain() {
 		exiterror "Could not create ${TOOLS_DIR} symbolic link"
 	fi
 
-	LOGFILE="$BASEDIR/log/_build.toolchain.log"
+	LOGFILE="$BASEDIR/${LOG_DIR}/_build.toolchain.log"
 	export LOGFILE
 
 	lfsmake1 stage1
@@ -981,7 +1010,7 @@ buildtoolchain() {
 }
 
 buildbase() {
-	LOGFILE="$BASEDIR/log/_build.base.log"
+	LOGFILE="$BASEDIR/${LOG_DIR}/_build.base.log"
 	export LOGFILE
 	lfsmake2 stage2
 	lfsmake2 linux			KCFG="-headers"
@@ -1012,6 +1041,7 @@ buildbase() {
 	lfsmake2 readline
 	lfsmake2 readline-compat
 	lfsmake2 bzip2
+	lfsmake2 xz
 	lfsmake2 pcre
 	lfsmake2 pcre-compat
 	lfsmake2 bash
@@ -1046,16 +1076,18 @@ buildbase() {
 	lfsmake2 util-linux
 	lfsmake2 udev
 	lfsmake2 vim
-	lfsmake2 xz
-	lfsmake2 paxctl
 }
 
 buildipfire() {
-  LOGFILE="$BASEDIR/log/_build.ipfire.log"
+  LOGFILE="$BASEDIR/${LOG_DIR}/_build.ipfire.log"
   export LOGFILE
   lfsmake2 configroot
   lfsmake2 initscripts
   lfsmake2 backup
+  lfsmake2 openssl
+  [ "${BUILD_ARCH}" = "i586" ] && lfsmake2 openssl KCFG='-sse2'
+  lfsmake2 openssl-compat
+  lfsmake2 popt
   lfsmake2 libusb
   lfsmake2 libusb-compat
   lfsmake2 libpcap
@@ -1073,10 +1105,13 @@ buildipfire() {
   lfsmake2 cpio
   lfsmake2 mdadm
   lfsmake2 dracut
+  lfsmake2 libaio
   lfsmake2 lvm2
   lfsmake2 multipath-tools
   lfsmake2 freetype
   lfsmake2 grub
+  lfsmake2 efivar
+  lfsmake2 efibootmgr
   lfsmake2 libmnl
   lfsmake2 libnfnetlink
   lfsmake2 libnetfilter_queue
@@ -1088,7 +1123,7 @@ buildipfire() {
   lfsmake2 elfutils
 
   case "${BUILD_ARCH}" in
-	x86_64)
+	x86_64|aarch64)
 		lfsmake2 linux			KCFG=""
 #		lfsmake2 backports			KCFG=""
 #		lfsmake2 e1000e			KCFG=""
@@ -1137,10 +1172,8 @@ buildipfire() {
 		lfsmake2 linux-initrd			KCFG="-multi"
 		;;
   esac
+  lfsmake2 intel-microcode
   lfsmake2 xtables-addons			USPACE="1"
-  lfsmake2 openssl
-  [ "${BUILD_ARCH}" = "i586" ] && lfsmake2 openssl KCFG='-sse2'
-  lfsmake2 openssl-compat
   lfsmake2 libgpg-error
   lfsmake2 libgcrypt
   lfsmake2 libassuan
@@ -1179,7 +1212,6 @@ buildipfire() {
   lfsmake2 libtiff
   lfsmake2 libart
   lfsmake2 gd
-  lfsmake2 popt
   lfsmake2 slang
   lfsmake2 newt
   lfsmake2 libsmooth
@@ -1200,7 +1232,9 @@ buildipfire() {
   lfsmake2 bootstrap
   lfsmake2 arping
   lfsmake2 beep
-  lfsmake2 dvdrtools
+  lfsmake2 libarchive
+  lfsmake2 cmake
+  lfsmake2 cdrkit
   lfsmake2 dosfstools
   lfsmake2 reiserfsprogs
   lfsmake2 xfsprogs
@@ -1301,6 +1335,7 @@ buildipfire() {
   lfsmake2 dbus
   lfsmake2 intltool
   lfsmake2 libdaemon
+  lfsmake2 avahi
   lfsmake2 cups
   lfsmake2 lcms2
   lfsmake2 ghostscript
@@ -1317,7 +1352,6 @@ buildipfire() {
   lfsmake2 mc
   lfsmake2 wget
   lfsmake2 bridge-utils
-#  lfsmake2 screen
   lfsmake2 smartmontools
   lfsmake2 htop
   lfsmake2 chkconfig
@@ -1342,12 +1376,11 @@ buildipfire() {
   lfsmake2 libshout
   lfsmake2 xvid
   lfsmake2 libmpeg2
-  lfsmake2 libarchive
-  lfsmake2 cmake
   lfsmake2 gnump3d
   lfsmake2 rsync
   lfsmake2 libtirpc
   lfsmake2 rpcbind
+  lfsmake2 keyutils
   lfsmake2 nfs
   lfsmake2 gnu-netcat
   lfsmake2 ncat
@@ -1461,6 +1494,17 @@ buildipfire() {
   lfsmake2 python-progressbar
   lfsmake2 python-xattr
   lfsmake2 ddns
+  lfsmake2 python3-six
+  lfsmake2 python3-dateutil
+  lfsmake2 python3-jmespath
+  lfsmake2 python3-colorama
+  lfsmake2 python3-docutils
+  lfsmake2 python3-yaml
+  lfsmake2 python3-s3transfer
+  lfsmake2 python3-rsa
+  lfsmake2 python3-pyasn1
+  lfsmake2 python3-botocore
+  lfsmake2 aws-cli
   lfsmake2 transmission
   lfsmake2 dpfhack
   lfsmake2 lcd4linux
@@ -1529,11 +1573,12 @@ buildipfire() {
   lfsmake2 iftop
   lfsmake2 mdns-repeater
   lfsmake2 i2c-tools
+  lfsmake2 nss-myhostname
 }
 
 buildinstaller() {
   # Run installer scripts one by one
-  LOGFILE="$BASEDIR/log/_build.installer.log"
+  LOGFILE="$BASEDIR/${LOG_DIR}/_build.installer.log"
   export LOGFILE
   lfsmake2 memtest
   lfsmake2 installer
@@ -1542,7 +1587,7 @@ buildinstaller() {
 }
 
 buildpackages() {
-  LOGFILE="$BASEDIR/log/_build.packages.log"
+  LOGFILE="$BASEDIR/${LOG_DIR}/_build.packages.log"
   export LOGFILE
   echo "... see detailed log in _build.*.log files" >> $LOGFILE
 
@@ -1550,8 +1595,8 @@ buildpackages() {
   # Generating list of packages used
   print_line "Generating packages list from logs"
   rm -f $BASEDIR/doc/packages-list
-  for i in `ls -1tr $BASEDIR/log/[^_]*`; do
-	if [ "$i" != "$BASEDIR/log/FILES" -a -n $i ]; then
+  for i in `ls -1tr $BASEDIR/${LOG_DIR}/[^_]*`; do
+	if [ "$i" != "$BASEDIR/${LOG_DIR}/FILES" -a -n $i ]; then
 		echo "* `basename $i`" >>$BASEDIR/doc/packages-list
 	fi
   done
@@ -1577,7 +1622,7 @@ buildpackages() {
 	lfsmake2 flash-images
   fi
 
-  mv $LFS/install/images/{*.iso,*.tgz,*.img.xz,*.bz2} $BASEDIR >> $LOGFILE 2>&1
+  mv $LFS/install/images/{*.iso,*.img.xz,*.bz2} $BASEDIR >> $LOGFILE 2>&1
 
   ipfirepackages
 
@@ -1589,7 +1634,7 @@ buildpackages() {
   # remove not useable iso on armv5tel (needed to build flash images)
   [ "${BUILD_ARCH}" = "armv5tel" ] && rm -rf *.iso
 
-  for i in `ls *.bz2 *.img.gz *.iso`; do
+  for i in `ls *.bz2 *.img.xz *.iso`; do
 	md5sum $i > $i.md5
   done
   cd $PWD
@@ -1710,7 +1755,7 @@ clean)
 	rm -rf $BASEDIR/build
 	rm -rf $BASEDIR/cdrom
 	rm -rf $BASEDIR/packages
-	rm -rf $BASEDIR/log
+	rm -rf $BASEDIR/${LOG_DIR}
 	if [ -h "${TOOLS_DIR}" ]; then
 		rm -f "${TOOLS_DIR}"
 	fi
@@ -1721,7 +1766,7 @@ downloadsrc)
 	if [ ! -d $BASEDIR/cache ]; then
 		mkdir $BASEDIR/cache
 	fi
-	mkdir -p $BASEDIR/log
+	mkdir -p $BASEDIR/${LOG_DIR}
 	echo -e "${BOLD}Preload all source files${NORMAL}" | tee -a $LOGFILE
 	FINISHED=0
 	cd $BASEDIR/lfs
@@ -1780,8 +1825,8 @@ toolchain)
 	buildtoolchain
 	echo "`date -u '+%b %e %T'`: Create toolchain image for ${BUILD_ARCH}" | tee -a $LOGFILE
 	test -d $BASEDIR/cache/toolchains || mkdir -p $BASEDIR/cache/toolchains
-	cd $BASEDIR && XZ_OPT="$(XZ_OPT)" tar -Jc --exclude='log/_build.*.log' -f cache/toolchains/$SNAME-$VERSION-toolchain-$TOOLCHAINVER-${BUILD_ARCH}.tar.xz --xz \
-		build/${TOOLS_DIR} build/bin/sh log >> $LOGFILE
+	cd $BASEDIR && tar -cf- --exclude='${LOG_DIR}/_build.*.log' build/${TOOLS_DIR} build/bin/sh ${LOG_DIR} | xz ${XZ_OPT} \
+		> cache/toolchains/$SNAME-$VERSION-toolchain-$TOOLCHAINVER-${BUILD_ARCH}.tar.xz
 	md5sum cache/toolchains/$SNAME-$VERSION-toolchain-$TOOLCHAINVER-${BUILD_ARCH}.tar.xz \
 		> cache/toolchains/$SNAME-$VERSION-toolchain-$TOOLCHAINVER-${BUILD_ARCH}.md5
 	stdumount
@@ -1858,8 +1903,11 @@ lang)
 	update_language_list ${BASEDIR}/src/setup/po
 	print_status DONE
 	;;
+update-contributors)
+	update_contributors
+	;;
 *)
-	echo "Usage: $0 {build|changelog|clean|gettoolchain|downloadsrc|shell|sync|toolchain}"
+	echo "Usage: $0 {build|changelog|clean|gettoolchain|downloadsrc|shell|sync|toolchain|update-contributors}"
 	cat doc/make.sh-usage
 	;;
 esac