python3-yaml: New paclage as required by aws-cli
[ipfire-2.x.git] / make.sh
diff --git a/make.sh b/make.sh
index 6ac0c8a..e254787 100755 (executable)
--- a/make.sh
+++ b/make.sh
@@ -24,9 +24,9 @@
 
 NAME="IPFire"                                                  # Software name
 SNAME="ipfire"                                                 # Short name
-VERSION="2.19"                                                 # Version number
-CORE="119"                                                     # Core Level (Filename)
-PAKFIRE_CORE="118"                                             # Core Level (PAKFIRE)
+VERSION="2.21"                                                 # Version number
+CORE="123"                                                     # Core Level (Filename)
+PAKFIRE_CORE="122"                                             # Core Level (PAKFIRE)
 GIT_BRANCH=`git rev-parse --abbrev-ref HEAD`                   # Git Branch
 SLOGAN="www.ipfire.org"                                                # Software slogan
 CONFIG_ROOT=/var/ipfire                                                # Configuration rootdir
@@ -37,7 +37,7 @@ KVER=`grep --max-count=1 VER lfs/linux | awk '{ print $3 }'`
 GIT_TAG=$(git tag | tail -1)                                   # Git Tag
 GIT_LASTCOMMIT=$(git log | head -n1 | cut -d" " -f2 |head -c8) # Last commit
 
-TOOLCHAINVER=20180210
+TOOLCHAINVER=20180606
 
 ###############################################################################
 #
@@ -144,14 +144,14 @@ configure_build() {
                        BUILDTARGET="${build_arch}-unknown-linux-gnu"
                        CROSSTARGET="${build_arch}-cross-linux-gnu"
                        BUILD_PLATFORM="x86"
-                       CFLAGS_ARCH="-m64 -mtune=generic"
+                       CFLAGS_ARCH="-m64 -mindirect-branch=thunk -mfunction-return=thunk -mtune=generic"
                        ;;
 
                i586)
                        BUILDTARGET="${build_arch}-pc-linux-gnu"
                        CROSSTARGET="${build_arch}-cross-linux-gnu"
                        BUILD_PLATFORM="x86"
-                       CFLAGS_ARCH="-march=i586 -mtune=generic -fomit-frame-pointer"
+                       CFLAGS_ARCH="-march=i586 -mindirect-branch=thunk -mfunction-return=thunk -mtune=generic -fomit-frame-pointer"
                        ;;
 
                aarch64)
@@ -225,6 +225,30 @@ configure_build() {
 
                MAKETUNING="-j${parallelism}"
        fi
+
+       # Compression parameters
+       # We use mode 8 for reasonable memory usage when decompressing
+       # but with overall good compression
+       XZ_OPT="-8"
+
+       # We try to use as many cores as possible
+       XZ_OPT="${XZ_OPT} -T0"
+
+       # We need to limit memory because XZ uses too much when running
+       # in parallel and it isn't very smart in limiting itself.
+       # We allow XZ to use up to 70% of all system memory.
+       local xz_memory=$(( HOST_MEM * 7 / 10 ))
+
+       # XZ memory cannot be larger than 2GB on 32 bit systems
+       case "${build_arch}" in
+               i*86|armv*)
+                       if [ ${xz_memory} -gt 2048 ]; then
+                               xz_memory=2048
+                       fi
+                       ;;
+       esac
+
+       XZ_OPT="${XZ_OPT} --memory=${xz_memory}MiB"
 }
 
 configure_build_guess() {
@@ -416,7 +440,7 @@ prepareenv() {
 
        # Checking if running as root user
        if [ $(id -u) -ne 0 ]; then
-                       exiterror "Not building as root"
+                       exiterror "root privileges required for building"
        fi
 
        # Checking for necessary temporary space
@@ -526,6 +550,7 @@ enterchroot() {
                CCACHE_COMPRESS="${CCACHE_COMPRESS}" \
                CCACHE_COMPILERCHECK="${CCACHE_COMPILERCHECK}" \
                KVER="${KVER}" \
+               XZ_OPT="${XZ_OPT}" \
                $(fake_environ) \
                $(qemu_environ) \
                "$@"
@@ -846,6 +871,28 @@ update_language_list() {
        done | sort -u > "${path}/LINGUAS"
 }
 
+contributors() {
+       local commits name
+
+       git shortlog --summary --numbered | while read -r commits name; do
+               echo "${name}"
+       done | grep -vE -e "^(alpha197|morlix|root|ummeegge)$" -e "via Development$" -e "@" -e "#$"
+}
+
+update_contributors() {
+       echo -n "Updating list of contributors"
+
+       local contributors="$(contributors | paste -sd , - | sed -e "s/,/&\\\\n/g")"
+
+       # Edit contributors into credits.cgi
+       awk -i inplace \
+               "/<!-- CONTRIBUTORS -->/{ p=1; print; printf \"${contributors}\n\"}/<!-- END -->/{ p=0 } !p" \
+               "${BASEDIR}/html/cgi-bin/credits.cgi"
+
+       print_status DONE
+       return 0
+}
+
 # Load configuration file
 if [ -f .config ]; then
        . .config
@@ -928,6 +975,7 @@ buildtoolchain() {
        lfsmake1 gcc                    PASS=L
        lfsmake1 binutils                       PASS=2
        lfsmake1 gcc                    PASS=2
+       lfsmake1 zlib
        lfsmake1 ccache                 PASS=2
        lfsmake1 tcl
        lfsmake1 expect
@@ -1043,13 +1091,12 @@ buildipfire() {
   lfsmake2 unzip
   lfsmake2 which
   lfsmake2 linux-firmware
-  lfsmake2 ath10k-firmware
   lfsmake2 dvb-firmwares
-  lfsmake2 mt7601u-firmware
+  lfsmake2 xr819-firmware
   lfsmake2 zd1211-firmware
   lfsmake2 rpi-firmware
   lfsmake2 bc
-  lfsmake2 u-boot
+  lfsmake2 u-boot MKIMAGE=1
   lfsmake2 cpio
   lfsmake2 mdadm
   lfsmake2 dracut
@@ -1064,66 +1111,64 @@ buildipfire() {
   lfsmake2 libnetfilter_cthelper
   lfsmake2 libnetfilter_cttimeout
   lfsmake2 iptables
+  lfsmake2 screen
+  lfsmake2 elfutils
 
   case "${BUILD_ARCH}" in
-       x86_64)
+       x86_64|aarch64)
                lfsmake2 linux                  KCFG=""
-               lfsmake2 backports                      KCFG=""
-               lfsmake2 e1000e                 KCFG=""
-               lfsmake2 igb                            KCFG=""
-               lfsmake2 ixgbe                  KCFG=""
+#              lfsmake2 backports                      KCFG=""
+#              lfsmake2 e1000e                 KCFG=""
+#              lfsmake2 igb                            KCFG=""
+#              lfsmake2 ixgbe                  KCFG=""
                lfsmake2 xtables-addons         KCFG=""
                lfsmake2 linux-initrd                   KCFG=""
                ;;
        i586)
                # x86-pae (Native and new XEN) kernel build
                lfsmake2 linux                  KCFG="-pae"
-               lfsmake2 backports                      KCFG="-pae"
-               lfsmake2 e1000e                 KCFG="-pae"
-               lfsmake2 igb                            KCFG="-pae"
-               lfsmake2 ixgbe                  KCFG="-pae"
+#              lfsmake2 backports                      KCFG="-pae"
+#              lfsmake2 e1000e                 KCFG="-pae"
+#              lfsmake2 igb                            KCFG="-pae"
+#              lfsmake2 ixgbe                  KCFG="-pae"
                lfsmake2 xtables-addons         KCFG="-pae"
                lfsmake2 linux-initrd                   KCFG="-pae"
 
                # x86 kernel build
                lfsmake2 linux                  KCFG=""
-               lfsmake2 backports                      KCFG=""
-               lfsmake2 e1000e                 KCFG=""
-               lfsmake2 igb                            KCFG=""
-               lfsmake2 ixgbe                  KCFG=""
+#              lfsmake2 backports                      KCFG=""
+#              lfsmake2 e1000e                 KCFG=""
+#              lfsmake2 igb                            KCFG=""
+#              lfsmake2 ixgbe                  KCFG=""
                lfsmake2 xtables-addons         KCFG=""
                lfsmake2 linux-initrd                   KCFG=""
                ;;
 
        armv5tel)
-               # arm-rpi (Raspberry Pi) kernel build
-               lfsmake2 linux                  KCFG="-rpi"
-               lfsmake2 backports                      KCFG="-rpi"
-               lfsmake2 xtables-addons         KCFG="-rpi"
-               lfsmake2 linux-initrd                   KCFG="-rpi"
+               # arm-kirkwood (Dreamplug, ICY-Box ...) kernel build
+               lfsmake2 linux                  KCFG="-kirkwood"
+#              lfsmake2 backports                      KCFG="-kirkwood"
+#              lfsmake2 e1000e                 KCFG="-kirkwood"
+#              lfsmake2 igb                            KCFG="-kirkwood"
+#              lfsmake2 ixgbe                  KCFG="-kirkwood"
+               lfsmake2 xtables-addons         KCFG="-kirkwood"
+               lfsmake2 linux-initrd                   KCFG="-kirkwood"
 
                # arm multi platform (Panda, Wandboard ...) kernel build
                lfsmake2 linux                  KCFG="-multi"
-               lfsmake2 backports                      KCFG="-multi"
-               lfsmake2 e1000e                 KCFG="-multi"
-               lfsmake2 igb                            KCFG="-multi"
-               lfsmake2 ixgbe                  KCFG="-multi"
+#              lfsmake2 backports                      KCFG="-multi"
+#              lfsmake2 e1000e                 KCFG="-multi"
+#              lfsmake2 igb                            KCFG="-multi"
+#              lfsmake2 ixgbe                  KCFG="-multi"
                lfsmake2 xtables-addons         KCFG="-multi"
                lfsmake2 linux-initrd                   KCFG="-multi"
-
-               # arm-kirkwood (Dreamplug, ICY-Box ...) kernel build
-               lfsmake2 linux                  KCFG="-kirkwood"
-               lfsmake2 backports                      KCFG="-kirkwood"
-               lfsmake2 e1000e                 KCFG="-kirkwood"
-               lfsmake2 igb                            KCFG="-kirkwood"
-               lfsmake2 ixgbe                  KCFG="-kirkwood"
-               lfsmake2 xtables-addons         KCFG="-kirkwood"
-               lfsmake2 linux-initrd                   KCFG="-kirkwood"
                ;;
   esac
+  lfsmake2 intel-microcode
   lfsmake2 xtables-addons                      USPACE="1"
   lfsmake2 openssl
   [ "${BUILD_ARCH}" = "i586" ] && lfsmake2 openssl KCFG='-sse2'
+  lfsmake2 openssl-compat
   lfsmake2 libgpg-error
   lfsmake2 libgcrypt
   lfsmake2 libassuan
@@ -1174,7 +1219,6 @@ buildipfire() {
   lfsmake2 libxml2
   lfsmake2 libxslt
   lfsmake2 BerkeleyDB
-  lfsmake2 mysql
   lfsmake2 cyrus-sasl
   lfsmake2 openldap
   lfsmake2 apache2
@@ -1253,7 +1297,6 @@ buildipfire() {
   lfsmake2 python-ipaddress
   lfsmake2 glib
   lfsmake2 GeoIP
-  lfsmake2 noip_updater
   lfsmake2 ntp
   lfsmake2 openssh
   lfsmake2 fontconfig
@@ -1286,6 +1329,7 @@ buildipfire() {
   lfsmake2 dbus
   lfsmake2 intltool
   lfsmake2 libdaemon
+  lfsmake2 avahi
   lfsmake2 cups
   lfsmake2 lcms2
   lfsmake2 ghostscript
@@ -1302,7 +1346,6 @@ buildipfire() {
   lfsmake2 mc
   lfsmake2 wget
   lfsmake2 bridge-utils
-  lfsmake2 screen
   lfsmake2 smartmontools
   lfsmake2 htop
   lfsmake2 chkconfig
@@ -1412,7 +1455,6 @@ buildipfire() {
   lfsmake2 vdradmin
   lfsmake2 miau
   lfsmake2 perl-DBI
-  lfsmake2 perl-DBD-mysql
   lfsmake2 perl-DBD-SQLite
   lfsmake2 perl-File-ReadBackwards
   lfsmake2 openvmtools
@@ -1435,6 +1477,9 @@ buildipfire() {
   lfsmake2 powertop
   lfsmake2 parted
   lfsmake2 swig
+  lfsmake2 u-boot
+  lfsmake2 u-boot-kirkwood
+  lfsmake2 python-typing
   lfsmake2 python-m2crypto
   lfsmake2 wireless-regdb
   lfsmake2 crda
@@ -1444,6 +1489,14 @@ buildipfire() {
   lfsmake2 python-progressbar
   lfsmake2 python-xattr
   lfsmake2 ddns
+  lfsmake2 python3-six
+  lfsmake2 python3-dateutil
+  lfsmake2 python3-jmespath
+  lfsmake2 python3-colorama
+  lfsmake2 python3-docutils
+  lfsmake2 python3-yaml
+  lfsmake2 python3-botocore
+  lfsmake2 aws-cli
   lfsmake2 transmission
   lfsmake2 dpfhack
   lfsmake2 lcd4linux
@@ -1464,7 +1517,6 @@ buildipfire() {
   lfsmake2 sendEmail
   lfsmake2 sysbench
   lfsmake2 strace
-  lfsmake2 elfutils
   lfsmake2 ltrace
   lfsmake2 ipfire-netboot
   lfsmake2 lcdproc
@@ -1493,7 +1545,6 @@ buildipfire() {
   lfsmake2 pigz
   lfsmake2 tmux
   lfsmake2 perl-Text-CSV_XS
-  lfsmake2 swconfig
   lfsmake2 haproxy
   lfsmake2 ipset
   lfsmake2 lua
@@ -1513,6 +1564,8 @@ buildipfire() {
   lfsmake2 wio
   lfsmake2 iftop
   lfsmake2 mdns-repeater
+  lfsmake2 i2c-tools
+  lfsmake2 nss-myhostname
 }
 
 buildinstaller() {
@@ -1559,10 +1612,9 @@ buildpackages() {
   modprobe loop 2>/dev/null
   if [ $BUILD_IMAGES == 1 ] && ([ -e /dev/loop/0 ] || [ -e /dev/loop0 ] || [ -e "/dev/loop-control" ]); then
        lfsmake2 flash-images
-       lfsmake2 flash-images SCON=1
   fi
 
-  mv $LFS/install/images/{*.iso,*.tgz,*.img.gz,*.bz2} $BASEDIR >> $LOGFILE 2>&1
+  mv $LFS/install/images/{*.iso,*.img.xz,*.bz2} $BASEDIR >> $LOGFILE 2>&1
 
   ipfirepackages
 
@@ -1574,7 +1626,7 @@ buildpackages() {
   # remove not useable iso on armv5tel (needed to build flash images)
   [ "${BUILD_ARCH}" = "armv5tel" ] && rm -rf *.iso
 
-  for i in `ls *.bz2 *.img.gz *.iso`; do
+  for i in `ls *.bz2 *.img.xz *.iso`; do
        md5sum $i > $i.md5
   done
   cd $PWD
@@ -1765,8 +1817,8 @@ toolchain)
        buildtoolchain
        echo "`date -u '+%b %e %T'`: Create toolchain image for ${BUILD_ARCH}" | tee -a $LOGFILE
        test -d $BASEDIR/cache/toolchains || mkdir -p $BASEDIR/cache/toolchains
-       cd $BASEDIR && XZ_OPT="-T0 -8" tar -Jc --exclude='log/_build.*.log' -f cache/toolchains/$SNAME-$VERSION-toolchain-$TOOLCHAINVER-${BUILD_ARCH}.tar.xz \
-               build/${TOOLS_DIR} build/bin/sh log >> $LOGFILE
+       cd $BASEDIR && tar -cf- --exclude='log/_build.*.log' build/${TOOLS_DIR} build/bin/sh log | xz ${XZ_OPT} \
+               > cache/toolchains/$SNAME-$VERSION-toolchain-$TOOLCHAINVER-${BUILD_ARCH}.tar.xz
        md5sum cache/toolchains/$SNAME-$VERSION-toolchain-$TOOLCHAINVER-${BUILD_ARCH}.tar.xz \
                > cache/toolchains/$SNAME-$VERSION-toolchain-$TOOLCHAINVER-${BUILD_ARCH}.md5
        stdumount
@@ -1843,8 +1895,11 @@ lang)
        update_language_list ${BASEDIR}/src/setup/po
        print_status DONE
        ;;
+update-contributors)
+       update_contributors
+       ;;
 *)
-       echo "Usage: $0 {build|changelog|clean|gettoolchain|downloadsrc|shell|sync|toolchain}"
+       echo "Usage: $0 {build|changelog|clean|gettoolchain|downloadsrc|shell|sync|toolchain|update-contributors}"
        cat doc/make.sh-usage
        ;;
 esac