xz: Performance and compression tuning (V1)

Hi,

as mentioned in the "list", we're trying to make the archives as small as possible
using 'xz'-compression.

In order to achieve this, this patch tests the size of the memory available on the
host system.

It sets the xz-memory-limit (--memory=[N]Mib) to 70% of the available working memory,
a maximum of four xz-'worker threads' (-T4) and a compression rate of '-8' (-8).

These options are written to the 'XZ_OPT=' environment.

For details see:
https://linux.die.net/man/1/xz

I have set the number of available xz-'worker threads' (-T) to four (-T4), because during
the final tests the '-T0' parameter led to error messages snd stopped: 'cannot allocate memory'.
It wouldn't even run with 90%.

Furthermore, testing with '-T0' led to countless messages filling  up '_build.packages.log'.

E.g.:
...
xz: Adjusted the number of threads from 8 to 2 to not exceed the memory usage
limit of 1557 MiB
...

Tests took place on a 32bit-Ubuntu 16.04.4-system with 8 GB RAM and an Intel I7-2600.
Build time was about 04:30 hrs. Perhaps a 64bit-system would perform better (higher
values), but my goal was to make this run on as many systems as possible, so I choosed
these averages.

If minimum requirements (1024 MB RAM) are not met, building stops.

Current results:

'next', untuned:
ipfire-2.19.2gb-ext4.i586-full-core121.img.gz	=> 332951687 Bytes
ipfire-2.19.i586-full-core121.iso		=> 228589568 Bytes

'next', XZ_OPT: -T4 -8, 70% RAM:
ipfire-2.19.2gb-ext4.i586-full-core121.img.gz	=> 329725723 Bytes
ipfire-2.19.i586-full-core121.iso		=> 217055232 Bytes

These two resulting archives are 14760300 Bytes smaller than before.

Best,
Matthias

Signed-off-by: Matthias Fischer <matthias.fischer@ipfire.org>
Signed-off-by: Michael Tremer <michael.tremer@ipfire.org>

diff --git a/lfs/Config b/lfs/Config
index d63a1dc9a4a313f8e073201eadce279bdfec8e37..c667a3cc7135814f84da1bf43dda7db3c50e8976 100644 (file)
--- a/lfs/Config
+++ b/lfs/Config
@@ -209,7 +209,7 @@ define PAK
        tar xf /install/packages/package/files.tmp -C /install/packages/package/tmp/ \
                -p --numeric-owner
        rm -f /install/packages/package/files.tmp
        tar xf /install/packages/package/files.tmp -C /install/packages/package/tmp/ \
                -p --numeric-owner
        rm -f /install/packages/package/files.tmp

-       cd /install/packages/package/tmp/ && XZ_OPT=-T0 tar -c -p --numeric-owner -J -f /install/packages/package/files.tar.xz *
+       cd /install/packages/package/tmp/ && XZ_OPT="$(XZ_OPT)" tar -c -p --numeric-owner -J -f /install/packages/package/files.tar.xz *

        rm -r /install/packages/package/tmp
        -cat /install/packages/package/ROOTFILES | grep -v "#" > /install/packages/package/ROOTFILES.tmp
        mv /install/packages/package/ROOTFILES.tmp /install/packages/package/ROOTFILES
        rm -r /install/packages/package/tmp
        -cat /install/packages/package/ROOTFILES | grep -v "#" > /install/packages/package/ROOTFILES.tmp
        mv /install/packages/package/ROOTFILES.tmp /install/packages/package/ROOTFILES

diff --git a/lfs/cdrom b/lfs/cdrom
index 7a7fff166d752c051e76e22477858314cb9fd2fb..820c55b8779e1ec87bf0e1a840b361ce9b4775a5 100644 (file)
--- a/lfs/cdrom
+++ b/lfs/cdrom
@@ -35,9 +35,6 @@ else
        TAR_OPTIONS = --xz
 endif
 
        TAR_OPTIONS = --xz
 endif
 

-# Enable multi-threaded compression for LZMA
-export XZ_OPT = --threads=0
-

 ###############################################################################
 # Top-level Rules
 ###############################################################################
 ###############################################################################
 # Top-level Rules
 ###############################################################################


@@ -74,7 +71,7 @@ $(TARGET) : $(patsubst %,$(DIR_DL)/%,$(objects))
 
        # Compress root filesystem
        # Reason for this tar+untar+tar is removing of entries listed two or more in src/ROOTFILES
 
        # Compress root filesystem
        # Reason for this tar+untar+tar is removing of entries listed two or more in src/ROOTFILES

-       tar -c -C / --files-from=/tmp/ROOTFILES \
+       XZ_OPT="$(XZ_OPT)" tar -c -C / --files-from=/tmp/ROOTFILES \

                -f /$(SNAME).tar --exclude='#*' --exclude='dev/pts/*' \
                --exclude='proc/*' --exclude='tmp/ROOTFILES'
        rm -f /tmp/ROOTFILES
                -f /$(SNAME).tar --exclude='#*' --exclude='dev/pts/*' \
                --exclude='proc/*' --exclude='tmp/ROOTFILES'
        rm -f /tmp/ROOTFILES

diff --git a/make.sh b/make.sh
index 419f9fa95faf2fd17b9b7a73ae1a88a55e778bda..33ebc16bb01bceb0f8fc733d6df10e67f4fb10e0 100755 (executable)
--- a/make.sh
+++ b/make.sh
@@ -526,6 +526,7 @@ enterchroot() {
                CCACHE_COMPRESS="${CCACHE_COMPRESS}" \
                CCACHE_COMPILERCHECK="${CCACHE_COMPILERCHECK}" \
                KVER="${KVER}" \
                CCACHE_COMPRESS="${CCACHE_COMPRESS}" \
                CCACHE_COMPILERCHECK="${CCACHE_COMPILERCHECK}" \
                KVER="${KVER}" \

+               XZ_OPT="${XZ_OPT}" \

                $(fake_environ) \
                $(qemu_environ) \
                "$@"
                $(fake_environ) \
                $(qemu_environ) \
                "$@"


@@ -860,6 +861,21 @@ fi
 # Get the amount of memory in this build system
 HOST_MEM=$(system_memory)
 
 # Get the amount of memory in this build system
 HOST_MEM=$(system_memory)
 

+# Checking host memory, tuning XZ_OPT
+
+if [ $HOST_MEM -lt 1024 ]; then
+       print_build_stage "Host-Memory: $HOST_MEM MiB"
+       print_build_stage "Not enough host memory (less than 1024 MiB, please consider upgrading)"
+
+       exit 1
+
+else
+
+       XZ_MEM="$(( HOST_MEM * 7 / 10 ))MiB"
+       XZ_OPT="-T4 -8 --memory=$XZ_MEM"
+
+fi
+

 if [ -n "${BUILD_ARCH}" ]; then
        configure_build "${BUILD_ARCH}"
 else
 if [ -n "${BUILD_ARCH}" ]; then
        configure_build "${BUILD_ARCH}"
 else


@@ -1760,7 +1776,7 @@ toolchain)
        buildtoolchain
        echo "`date -u '+%b %e %T'`: Create toolchain image for ${BUILD_ARCH}" | tee -a $LOGFILE
        test -d $BASEDIR/cache/toolchains || mkdir -p $BASEDIR/cache/toolchains
        buildtoolchain
        echo "`date -u '+%b %e %T'`: Create toolchain image for ${BUILD_ARCH}" | tee -a $LOGFILE
        test -d $BASEDIR/cache/toolchains || mkdir -p $BASEDIR/cache/toolchains

-       cd $BASEDIR && XZ_OPT="-T0 -8" tar -Jc --exclude='log/_build.*.log' -f cache/toolchains/$SNAME-$VERSION-toolchain-$TOOLCHAINVER-${BUILD_ARCH}.tar.xz \
+       cd $BASEDIR && XZ_OPT="$(XZ_OPT)" tar -Jc --exclude='log/_build.*.log' -f cache/toolchains/$SNAME-$VERSION-toolchain-$TOOLCHAINVER-${BUILD_ARCH}.tar.xz \

                build/${TOOLS_DIR} build/bin/sh log >> $LOGFILE
        md5sum cache/toolchains/$SNAME-$VERSION-toolchain-$TOOLCHAINVER-${BUILD_ARCH}.tar.xz \
                > cache/toolchains/$SNAME-$VERSION-toolchain-$TOOLCHAINVER-${BUILD_ARCH}.md5
                build/${TOOLS_DIR} build/bin/sh log >> $LOGFILE
        md5sum cache/toolchains/$SNAME-$VERSION-toolchain-$TOOLCHAINVER-${BUILD_ARCH}.tar.xz \
                > cache/toolchains/$SNAME-$VERSION-toolchain-$TOOLCHAINVER-${BUILD_ARCH}.md5