X-Git-Url: http://git.ipfire.org/?p=people%2Fpmueller%2Fipfire-2.x.git;a=blobdiff_plain;f=make.sh;h=a7e8068587434ce2fb27022a37c6b35540e22a79;hp=2e7f7c00e682bc628d1b888068c4a8dfc6974c65;hb=35ce207ff7f94921ddffb23501c28d7095d59a04;hpb=e3e17107ba76e7e787bd86bf058c8ac06ea26054 diff --git a/make.sh b/make.sh index 2e7f7c00e6..a7e8068587 100755 --- a/make.sh +++ b/make.sh @@ -24,9 +24,9 @@ NAME="IPFire" # Software name SNAME="ipfire" # Short name -VERSION="2.19" # Version number -CORE="119" # Core Level (Filename) -PAKFIRE_CORE="118" # Core Level (PAKFIRE) +VERSION="2.21" # Version number +CORE="125" # Core Level (Filename) +PAKFIRE_CORE="124" # Core Level (PAKFIRE) GIT_BRANCH=`git rev-parse --abbrev-ref HEAD` # Git Branch SLOGAN="www.ipfire.org" # Software slogan CONFIG_ROOT=/var/ipfire # Configuration rootdir @@ -37,7 +37,7 @@ KVER=`grep --max-count=1 VER lfs/linux | awk '{ print $3 }'` GIT_TAG=$(git tag | tail -1) # Git Tag GIT_LASTCOMMIT=$(git log | head -n1 | cut -d" " -f2 |head -c8) # Last commit -TOOLCHAINVER=20180210 +TOOLCHAINVER=20181024 ############################################################################### # @@ -109,10 +109,7 @@ fi # This is the directory where make.sh is in export BASEDIR=$(echo $FULLPATH | sed "s/\/$BASENAME//g") -LOGFILE=$BASEDIR/log/_build.preparation.log -export LOGFILE DIR_CHK=$BASEDIR/cache/check -mkdir $BASEDIR/log/ 2>/dev/null system_processors() { getconf _NPROCESSORS_ONLN 2>/dev/null || echo "1" @@ -144,14 +141,14 @@ configure_build() { BUILDTARGET="${build_arch}-unknown-linux-gnu" CROSSTARGET="${build_arch}-cross-linux-gnu" BUILD_PLATFORM="x86" - CFLAGS_ARCH="-m64 -mtune=generic" + CFLAGS_ARCH="-m64 -mindirect-branch=thunk -mfunction-return=thunk -mtune=generic" ;; i586) BUILDTARGET="${build_arch}-pc-linux-gnu" CROSSTARGET="${build_arch}-cross-linux-gnu" BUILD_PLATFORM="x86" - CFLAGS_ARCH="-march=i586 -mtune=generic -fomit-frame-pointer" + CFLAGS_ARCH="-march=i586 -mindirect-branch=thunk -mfunction-return=thunk -mtune=generic -fomit-frame-pointer" ;; aarch64) @@ -193,6 +190,7 @@ configure_build() { BUILD_ARCH="${build_arch}" TOOLS_DIR="/tools_${BUILD_ARCH}" + LOG_DIR="log_${BUILD_ARCH}" # Enables hardening HARDENING_CFLAGS="-Wp,-D_FORTIFY_SOURCE=2 -Wp,-D_GLIBCXX_ASSERTIONS -fstack-protector-strong" @@ -225,6 +223,30 @@ configure_build() { MAKETUNING="-j${parallelism}" fi + + # Compression parameters + # We use mode 8 for reasonable memory usage when decompressing + # but with overall good compression + XZ_OPT="-8" + + # We try to use as many cores as possible + XZ_OPT="${XZ_OPT} -T0" + + # We need to limit memory because XZ uses too much when running + # in parallel and it isn't very smart in limiting itself. + # We allow XZ to use up to 70% of all system memory. + local xz_memory=$(( HOST_MEM * 7 / 10 )) + + # XZ memory cannot be larger than 2GB on 32 bit systems + case "${build_arch}" in + i*86|armv*) + if [ ${xz_memory} -gt 2048 ]; then + xz_memory=2048 + fi + ;; + esac + + XZ_OPT="${XZ_OPT} --memory=${xz_memory}MiB" } configure_build_guess() { @@ -261,7 +283,7 @@ stdumount() { umount $BASEDIR/build/usr/src/html 2>/dev/null; umount $BASEDIR/build/usr/src/langs 2>/dev/null; umount $BASEDIR/build/usr/src/lfs 2>/dev/null; - umount $BASEDIR/build/usr/src/log 2>/dev/null; + umount $BASEDIR/build/usr/src/${LOG_DIR} 2>/dev/null; umount $BASEDIR/build/usr/src/src 2>/dev/null; } @@ -397,6 +419,9 @@ exiterror() { } prepareenv() { + # Create log directory if it doesn't exist, yet + mkdir -p "${BASEDIR}/${LOG_DIR}" + # Are we running the right shell? if [ -z "${BASH}" ]; then exiterror "BASH environment variable is not set. You're probably running the wrong shell." @@ -416,7 +441,7 @@ prepareenv() { # Checking if running as root user if [ $(id -u) -ne 0 ]; then - exiterror "Not building as root" + exiterror "root privileges required for building" fi # Checking for necessary temporary space @@ -450,7 +475,7 @@ prepareenv() { mkdir -p $BASEDIR/build/{etc,usr/src} 2>/dev/null mkdir -p $BASEDIR/build/{dev/{shm,pts},proc,sys} mkdir -p $BASEDIR/{cache,ccache} 2>/dev/null - mkdir -p $BASEDIR/build/usr/src/{cache,config,doc,html,langs,lfs,log,src,ccache} + mkdir -p $BASEDIR/build/usr/src/{cache,config,doc,html,langs,lfs,${LOG_DIR},src,ccache} mknod -m 600 $BASEDIR/build/dev/console c 5 1 2>/dev/null mknod -m 666 $BASEDIR/build/dev/null c 1 3 2>/dev/null @@ -468,7 +493,7 @@ prepareenv() { mount --bind $BASEDIR/html $BASEDIR/build/usr/src/html mount --bind $BASEDIR/langs $BASEDIR/build/usr/src/langs mount --bind $BASEDIR/lfs $BASEDIR/build/usr/src/lfs - mount --bind $BASEDIR/log $BASEDIR/build/usr/src/log + mount --bind $BASEDIR/${LOG_DIR} $BASEDIR/build/usr/src/${LOG_DIR} mount --bind $BASEDIR/src $BASEDIR/build/usr/src/src # Run LFS static binary creation scripts one by one @@ -515,6 +540,7 @@ enterchroot() { CORE="${CORE}" \ SLOGAN="${SLOGAN}" \ TOOLS_DIR="${TOOLS_DIR}" \ + LOG_DIR="${LOG_DIR}" \ CONFIG_ROOT="${CONFIG_ROOT}" \ CFLAGS="${CFLAGS} ${HARDENING_CFLAGS}" \ CXXFLAGS="${CXXFLAGS} ${HARDENING_CFLAGS}" \ @@ -526,6 +552,7 @@ enterchroot() { CCACHE_COMPRESS="${CCACHE_COMPRESS}" \ CCACHE_COMPILERCHECK="${CCACHE_COMPILERCHECK}" \ KVER="${KVER}" \ + XZ_OPT="${XZ_OPT}" \ $(fake_environ) \ $(qemu_environ) \ "$@" @@ -846,6 +873,28 @@ update_language_list() { done | sort -u > "${path}/LINGUAS" } +contributors() { + local commits name + + git shortlog --summary --numbered | while read -r commits name; do + echo "${name}" + done | grep -vE -e "^(alpha197|morlix|root|ummeegge)$" -e "via Development$" -e "@" -e "#$" +} + +update_contributors() { + echo -n "Updating list of contributors" + + local contributors="$(contributors | paste -sd , - | sed -e "s/,/&\\\\n/g")" + + # Edit contributors into credits.cgi + awk -i inplace \ + "//{ p=1; print; printf \"${contributors}\n\"}//{ p=0 } !p" \ + "${BASEDIR}/html/cgi-bin/credits.cgi" + + print_status DONE + return 0 +} + # Load configuration file if [ -f .config ]; then . .config @@ -916,7 +965,7 @@ buildtoolchain() { exiterror "Could not create ${TOOLS_DIR} symbolic link" fi - LOGFILE="$BASEDIR/log/_build.toolchain.log" + LOGFILE="$BASEDIR/${LOG_DIR}/_build.toolchain.log" export LOGFILE lfsmake1 stage1 @@ -928,6 +977,7 @@ buildtoolchain() { lfsmake1 gcc PASS=L lfsmake1 binutils PASS=2 lfsmake1 gcc PASS=2 + lfsmake1 zlib lfsmake1 ccache PASS=2 lfsmake1 tcl lfsmake1 expect @@ -960,7 +1010,7 @@ buildtoolchain() { } buildbase() { - LOGFILE="$BASEDIR/log/_build.base.log" + LOGFILE="$BASEDIR/${LOG_DIR}/_build.base.log" export LOGFILE lfsmake2 stage2 lfsmake2 linux KCFG="-headers" @@ -991,6 +1041,7 @@ buildbase() { lfsmake2 readline lfsmake2 readline-compat lfsmake2 bzip2 + lfsmake2 xz lfsmake2 pcre lfsmake2 pcre-compat lfsmake2 bash @@ -1025,16 +1076,18 @@ buildbase() { lfsmake2 util-linux lfsmake2 udev lfsmake2 vim - lfsmake2 xz - lfsmake2 paxctl } buildipfire() { - LOGFILE="$BASEDIR/log/_build.ipfire.log" + LOGFILE="$BASEDIR/${LOG_DIR}/_build.ipfire.log" export LOGFILE lfsmake2 configroot lfsmake2 initscripts lfsmake2 backup + lfsmake2 openssl + [ "${BUILD_ARCH}" = "i586" ] && lfsmake2 openssl KCFG='-sse2' + lfsmake2 openssl-compat + lfsmake2 popt lfsmake2 libusb lfsmake2 libusb-compat lfsmake2 libpcap @@ -1043,20 +1096,22 @@ buildipfire() { lfsmake2 unzip lfsmake2 which lfsmake2 linux-firmware - lfsmake2 ath10k-firmware lfsmake2 dvb-firmwares - lfsmake2 mt7601u-firmware + lfsmake2 xr819-firmware lfsmake2 zd1211-firmware lfsmake2 rpi-firmware lfsmake2 bc - lfsmake2 u-boot + lfsmake2 u-boot MKIMAGE=1 lfsmake2 cpio lfsmake2 mdadm lfsmake2 dracut + lfsmake2 libaio lfsmake2 lvm2 lfsmake2 multipath-tools lfsmake2 freetype lfsmake2 grub + lfsmake2 efivar + lfsmake2 efibootmgr lfsmake2 libmnl lfsmake2 libnfnetlink lfsmake2 libnetfilter_queue @@ -1064,66 +1119,61 @@ buildipfire() { lfsmake2 libnetfilter_cthelper lfsmake2 libnetfilter_cttimeout lfsmake2 iptables + lfsmake2 screen + lfsmake2 elfutils case "${BUILD_ARCH}" in - x86_64) + x86_64|aarch64) lfsmake2 linux KCFG="" - lfsmake2 backports KCFG="" - lfsmake2 e1000e KCFG="" - lfsmake2 igb KCFG="" - lfsmake2 ixgbe KCFG="" +# lfsmake2 backports KCFG="" +# lfsmake2 e1000e KCFG="" +# lfsmake2 igb KCFG="" +# lfsmake2 ixgbe KCFG="" lfsmake2 xtables-addons KCFG="" lfsmake2 linux-initrd KCFG="" ;; i586) # x86-pae (Native and new XEN) kernel build lfsmake2 linux KCFG="-pae" - lfsmake2 backports KCFG="-pae" - lfsmake2 e1000e KCFG="-pae" - lfsmake2 igb KCFG="-pae" - lfsmake2 ixgbe KCFG="-pae" +# lfsmake2 backports KCFG="-pae" +# lfsmake2 e1000e KCFG="-pae" +# lfsmake2 igb KCFG="-pae" +# lfsmake2 ixgbe KCFG="-pae" lfsmake2 xtables-addons KCFG="-pae" lfsmake2 linux-initrd KCFG="-pae" # x86 kernel build lfsmake2 linux KCFG="" - lfsmake2 backports KCFG="" - lfsmake2 e1000e KCFG="" - lfsmake2 igb KCFG="" - lfsmake2 ixgbe KCFG="" +# lfsmake2 backports KCFG="" +# lfsmake2 e1000e KCFG="" +# lfsmake2 igb KCFG="" +# lfsmake2 ixgbe KCFG="" lfsmake2 xtables-addons KCFG="" lfsmake2 linux-initrd KCFG="" ;; armv5tel) - # arm-rpi (Raspberry Pi) kernel build - lfsmake2 linux KCFG="-rpi" - lfsmake2 backports KCFG="-rpi" - lfsmake2 xtables-addons KCFG="-rpi" - lfsmake2 linux-initrd KCFG="-rpi" + # arm-kirkwood (Dreamplug, ICY-Box ...) kernel build + lfsmake2 linux KCFG="-kirkwood" +# lfsmake2 backports KCFG="-kirkwood" +# lfsmake2 e1000e KCFG="-kirkwood" +# lfsmake2 igb KCFG="-kirkwood" +# lfsmake2 ixgbe KCFG="-kirkwood" + lfsmake2 xtables-addons KCFG="-kirkwood" + lfsmake2 linux-initrd KCFG="-kirkwood" # arm multi platform (Panda, Wandboard ...) kernel build lfsmake2 linux KCFG="-multi" - lfsmake2 backports KCFG="-multi" - lfsmake2 e1000e KCFG="-multi" - lfsmake2 igb KCFG="-multi" - lfsmake2 ixgbe KCFG="-multi" +# lfsmake2 backports KCFG="-multi" +# lfsmake2 e1000e KCFG="-multi" +# lfsmake2 igb KCFG="-multi" +# lfsmake2 ixgbe KCFG="-multi" lfsmake2 xtables-addons KCFG="-multi" lfsmake2 linux-initrd KCFG="-multi" - - # arm-kirkwood (Dreamplug, ICY-Box ...) kernel build - lfsmake2 linux KCFG="-kirkwood" - lfsmake2 backports KCFG="-kirkwood" - lfsmake2 e1000e KCFG="-kirkwood" - lfsmake2 igb KCFG="-kirkwood" - lfsmake2 ixgbe KCFG="-kirkwood" - lfsmake2 xtables-addons KCFG="-kirkwood" - lfsmake2 linux-initrd KCFG="-kirkwood" ;; esac + lfsmake2 intel-microcode lfsmake2 xtables-addons USPACE="1" - lfsmake2 openssl - [ "${BUILD_ARCH}" = "i586" ] && lfsmake2 openssl KCFG='-sse2' lfsmake2 libgpg-error lfsmake2 libgcrypt lfsmake2 libassuan @@ -1162,7 +1212,6 @@ buildipfire() { lfsmake2 libtiff lfsmake2 libart lfsmake2 gd - lfsmake2 popt lfsmake2 slang lfsmake2 newt lfsmake2 libsmooth @@ -1174,7 +1223,6 @@ buildipfire() { lfsmake2 libxml2 lfsmake2 libxslt lfsmake2 BerkeleyDB - lfsmake2 mysql lfsmake2 cyrus-sasl lfsmake2 openldap lfsmake2 apache2 @@ -1184,7 +1232,9 @@ buildipfire() { lfsmake2 bootstrap lfsmake2 arping lfsmake2 beep - lfsmake2 dvdrtools + lfsmake2 libarchive + lfsmake2 cmake + lfsmake2 cdrkit lfsmake2 dosfstools lfsmake2 reiserfsprogs lfsmake2 xfsprogs @@ -1253,7 +1303,6 @@ buildipfire() { lfsmake2 python-ipaddress lfsmake2 glib lfsmake2 GeoIP - lfsmake2 noip_updater lfsmake2 ntp lfsmake2 openssh lfsmake2 fontconfig @@ -1282,11 +1331,11 @@ buildipfire() { lfsmake2 lz4 lfsmake2 lzo lfsmake2 openvpn - lfsmake2 pammysql lfsmake2 mpage lfsmake2 dbus lfsmake2 intltool lfsmake2 libdaemon + lfsmake2 avahi lfsmake2 cups lfsmake2 lcms2 lfsmake2 ghostscript @@ -1303,7 +1352,6 @@ buildipfire() { lfsmake2 mc lfsmake2 wget lfsmake2 bridge-utils - lfsmake2 screen lfsmake2 smartmontools lfsmake2 htop lfsmake2 chkconfig @@ -1328,12 +1376,11 @@ buildipfire() { lfsmake2 libshout lfsmake2 xvid lfsmake2 libmpeg2 - lfsmake2 libarchive - lfsmake2 cmake lfsmake2 gnump3d lfsmake2 rsync lfsmake2 libtirpc lfsmake2 rpcbind + lfsmake2 keyutils lfsmake2 nfs lfsmake2 gnu-netcat lfsmake2 ncat @@ -1413,7 +1460,6 @@ buildipfire() { lfsmake2 vdradmin lfsmake2 miau lfsmake2 perl-DBI - lfsmake2 perl-DBD-mysql lfsmake2 perl-DBD-SQLite lfsmake2 perl-File-ReadBackwards lfsmake2 openvmtools @@ -1436,6 +1482,9 @@ buildipfire() { lfsmake2 powertop lfsmake2 parted lfsmake2 swig + lfsmake2 u-boot + lfsmake2 u-boot-kirkwood + lfsmake2 python-typing lfsmake2 python-m2crypto lfsmake2 wireless-regdb lfsmake2 crda @@ -1445,6 +1494,17 @@ buildipfire() { lfsmake2 python-progressbar lfsmake2 python-xattr lfsmake2 ddns + lfsmake2 python3-six + lfsmake2 python3-dateutil + lfsmake2 python3-jmespath + lfsmake2 python3-colorama + lfsmake2 python3-docutils + lfsmake2 python3-yaml + lfsmake2 python3-s3transfer + lfsmake2 python3-rsa + lfsmake2 python3-pyasn1 + lfsmake2 python3-botocore + lfsmake2 aws-cli lfsmake2 transmission lfsmake2 dpfhack lfsmake2 lcd4linux @@ -1465,7 +1525,6 @@ buildipfire() { lfsmake2 sendEmail lfsmake2 sysbench lfsmake2 strace - lfsmake2 elfutils lfsmake2 ltrace lfsmake2 ipfire-netboot lfsmake2 lcdproc @@ -1494,7 +1553,6 @@ buildipfire() { lfsmake2 pigz lfsmake2 tmux lfsmake2 perl-Text-CSV_XS - lfsmake2 swconfig lfsmake2 haproxy lfsmake2 ipset lfsmake2 lua @@ -1514,11 +1572,13 @@ buildipfire() { lfsmake2 wio lfsmake2 iftop lfsmake2 mdns-repeater + lfsmake2 i2c-tools + lfsmake2 nss-myhostname } buildinstaller() { # Run installer scripts one by one - LOGFILE="$BASEDIR/log/_build.installer.log" + LOGFILE="$BASEDIR/${LOG_DIR}/_build.installer.log" export LOGFILE lfsmake2 memtest lfsmake2 installer @@ -1527,7 +1587,7 @@ buildinstaller() { } buildpackages() { - LOGFILE="$BASEDIR/log/_build.packages.log" + LOGFILE="$BASEDIR/${LOG_DIR}/_build.packages.log" export LOGFILE echo "... see detailed log in _build.*.log files" >> $LOGFILE @@ -1535,8 +1595,8 @@ buildpackages() { # Generating list of packages used print_line "Generating packages list from logs" rm -f $BASEDIR/doc/packages-list - for i in `ls -1tr $BASEDIR/log/[^_]*`; do - if [ "$i" != "$BASEDIR/log/FILES" -a -n $i ]; then + for i in `ls -1tr $BASEDIR/${LOG_DIR}/[^_]*`; do + if [ "$i" != "$BASEDIR/${LOG_DIR}/FILES" -a -n $i ]; then echo "* `basename $i`" >>$BASEDIR/doc/packages-list fi done @@ -1560,10 +1620,9 @@ buildpackages() { modprobe loop 2>/dev/null if [ $BUILD_IMAGES == 1 ] && ([ -e /dev/loop/0 ] || [ -e /dev/loop0 ] || [ -e "/dev/loop-control" ]); then lfsmake2 flash-images - lfsmake2 flash-images SCON=1 fi - mv $LFS/install/images/{*.iso,*.tgz,*.img.gz,*.bz2} $BASEDIR >> $LOGFILE 2>&1 + mv $LFS/install/images/{*.iso,*.img.xz,*.bz2} $BASEDIR >> $LOGFILE 2>&1 ipfirepackages @@ -1575,7 +1634,7 @@ buildpackages() { # remove not useable iso on armv5tel (needed to build flash images) [ "${BUILD_ARCH}" = "armv5tel" ] && rm -rf *.iso - for i in `ls *.bz2 *.img.gz *.iso`; do + for i in `ls *.bz2 *.img.xz *.iso`; do md5sum $i > $i.md5 done cd $PWD @@ -1696,7 +1755,7 @@ clean) rm -rf $BASEDIR/build rm -rf $BASEDIR/cdrom rm -rf $BASEDIR/packages - rm -rf $BASEDIR/log + rm -rf $BASEDIR/${LOG_DIR} if [ -h "${TOOLS_DIR}" ]; then rm -f "${TOOLS_DIR}" fi @@ -1707,7 +1766,7 @@ downloadsrc) if [ ! -d $BASEDIR/cache ]; then mkdir $BASEDIR/cache fi - mkdir -p $BASEDIR/log + mkdir -p $BASEDIR/${LOG_DIR} echo -e "${BOLD}Preload all source files${NORMAL}" | tee -a $LOGFILE FINISHED=0 cd $BASEDIR/lfs @@ -1766,8 +1825,8 @@ toolchain) buildtoolchain echo "`date -u '+%b %e %T'`: Create toolchain image for ${BUILD_ARCH}" | tee -a $LOGFILE test -d $BASEDIR/cache/toolchains || mkdir -p $BASEDIR/cache/toolchains - cd $BASEDIR && XZ_OPT="-T0 -8" tar -Jc --exclude='log/_build.*.log' -f cache/toolchains/$SNAME-$VERSION-toolchain-$TOOLCHAINVER-${BUILD_ARCH}.tar.xz \ - build/${TOOLS_DIR} build/bin/sh log >> $LOGFILE + cd $BASEDIR && tar -cf- --exclude='${LOG_DIR}/_build.*.log' build/${TOOLS_DIR} build/bin/sh ${LOG_DIR} | xz ${XZ_OPT} \ + > cache/toolchains/$SNAME-$VERSION-toolchain-$TOOLCHAINVER-${BUILD_ARCH}.tar.xz md5sum cache/toolchains/$SNAME-$VERSION-toolchain-$TOOLCHAINVER-${BUILD_ARCH}.tar.xz \ > cache/toolchains/$SNAME-$VERSION-toolchain-$TOOLCHAINVER-${BUILD_ARCH}.md5 stdumount @@ -1844,8 +1903,11 @@ lang) update_language_list ${BASEDIR}/src/setup/po print_status DONE ;; +update-contributors) + update_contributors + ;; *) - echo "Usage: $0 {build|changelog|clean|gettoolchain|downloadsrc|shell|sync|toolchain}" + echo "Usage: $0 {build|changelog|clean|gettoolchain|downloadsrc|shell|sync|toolchain|update-contributors}" cat doc/make.sh-usage ;; esac