xz: Performance and compression tuning (V2) - fine tuning

Changes since V1:

- Tuned some more 'tar'-command lines
- Included 'lfs/core-updates'
- Some fine-tuning

Hi,

Current results with V2 (clean builds):

'next':
'packages' => 255 MB
'ipfire-2.19.2gb-ext4.i586-full-core121.img.gz => 319 MB
'ipfire-2.19.i586-full-core121.iso' => 218 MB
Total => 792 MB

'xz-tuning':
'packages' => 228 MB
'ipfire-2.19.2gb-ext4.i586-full-core121.img.gz' => 318 MB
'ipfire-2.19.i586-full-core121.iso' => 207 MB

Total => 753 MB (-39 MB)

It would be nice if someone could review and test these. ;-)

If necessary, I can upload a complete patch.

Best,
Matthias

Signed-off-by: Matthias Fischer <matthias.fischer@ipfire.org>
Signed-off-by: Michael Tremer <michael.tremer@ipfire.org>

diff --git a/lfs/Config b/lfs/Config
index c667a3cc7135814f84da1bf43dda7db3c50e8976..a726f171d7bdbf5436b2cf3130290638070df922 100644 (file)
--- a/lfs/Config
+++ b/lfs/Config
@@ -47,6 +47,8 @@ else
        PREFIX = /usr
 endif
 
+TAR_OPTIONS = --xz
+
 # URLs that are common sources of downloads.  If you're having trouble with
 # a site you should change its URL to that of a suitable mirror site.
 #
@@ -209,11 +211,11 @@ define PAK
        tar xf /install/packages/package/files.tmp -C /install/packages/package/tmp/ \
                -p --numeric-owner
        rm -f /install/packages/package/files.tmp
-       cd /install/packages/package/tmp/ && XZ_OPT="$(XZ_OPT)" tar -c -p --numeric-owner -J -f /install/packages/package/files.tar.xz *
+       cd /install/packages/package/tmp/ && XZ_OPT="$(XZ_OPT)" tar -c -p --numeric-owner -J -f /install/packages/package/files.tar.xz $(TAR_OPTIONS) *
        rm -r /install/packages/package/tmp
        -cat /install/packages/package/ROOTFILES | grep -v "#" > /install/packages/package/ROOTFILES.tmp
        mv /install/packages/package/ROOTFILES.tmp /install/packages/package/ROOTFILES
-       cd /install/packages/package && tar cf ../$(PROG)-$(VER)-$(PAK_VER).ipfire --files-from=/usr/src/src/paks/files
+       cd /install/packages/package && XZ_OPT="$(XZ_OPT)" tar cf ../$(PROG)-$(VER)-$(PAK_VER).ipfire $(TAR_OPTIONS) --files-from=/usr/src/src/paks/files
        rm -rf /install/packages/package
        sed -e s/NAME/$(PROG)/g \
                        -e s/VER/$(VER)/g \

diff --git a/lfs/cdrom b/lfs/cdrom
index 820c55b8779e1ec87bf0e1a840b361ce9b4775a5..cdb7b07dab15613136d4bf3507de611485f75914 100644 (file)
--- a/lfs/cdrom
+++ b/lfs/cdrom
@@ -71,14 +71,14 @@ $(TARGET) : $(patsubst %,$(DIR_DL)/%,$(objects))
 
        # Compress root filesystem
        # Reason for this tar+untar+tar is removing of entries listed two or more in src/ROOTFILES
-       XZ_OPT="$(XZ_OPT)" tar -c -C / --files-from=/tmp/ROOTFILES \
+       tar -c -C / --files-from=/tmp/ROOTFILES \
                -f /$(SNAME).tar --exclude='#*' --exclude='dev/pts/*' \
                --exclude='proc/*' --exclude='tmp/ROOTFILES'
        rm -f /tmp/ROOTFILES
        tar -x -C /tmp -f /$(SNAME).tar
        rm -f /$(SNAME).tar
        @mkdir /tmp/sys
-       cd /tmp && tar  cf /install/cdrom/distro.img $(TAR_OPTIONS) * && rm -rf *
+       cd /tmp && XZ_OPT="$(XZ_OPT)" tar cf /install/cdrom/distro.img $(TAR_OPTIONS) * && rm -rf *
 
        # Other files
        touch /install/cdrom/$(SNAME)-$(VERSION)-core$(CORE).media

diff --git a/lfs/core-updates b/lfs/core-updates
index 3c847daf2831f55632169eafff3f080a50970a11..d3492696726578c4b015c06e407ea2a88ffeb444 100644 (file)
--- a/lfs/core-updates
+++ b/lfs/core-updates
@@ -26,8 +26,9 @@ include Config
 
 VER = ipfire
 
-THISAPP    = core-updates
-TARGET     = $(DIR_INFO)/$(THISAPP)
+THISAPP     = core-updates
+TARGET      = $(DIR_INFO)/$(THISAPP)
+TAR_OPTIONS = --xz
 
 ###############################################################################
 # Top-level Rules
@@ -78,7 +79,7 @@ core/%:
        mv -f /tmp/ROOTFILES /install/packages/package/ROOTFILES
        tar -p --numeric-owner -x  -C /tmp -f /$(SNAME).tar
        rm -f /$(SNAME).tar
-       cd /tmp && tar cJf /install/packages/package/files.tar.xz * && rm -rf *
+       cd /tmp && XZ_OPT="$(XZ_OPT)" tar cJf /install/packages/package/files.tar.xz $(TAR_OPTIONS) * && rm -rf *
        cat /install/packages/package/ROOTFILES | grep -v "#" > /install/packages/package/ROOTFILES.tmp
        rm /install/packages/package/ROOTFILES
        mv /install/packages/package/ROOTFILES.tmp /install/packages/package/ROOTFILES
@@ -89,7 +90,7 @@ core/%:
        #Change xxxKVERxxx to Kernelversion
        sed -e "s/xxxKVERxxx/$(KVER)/g" -i /install/packages/package/update.sh
 
-       cd /install/packages/package && tar cf ../core-upgrade-$(VERSION)-$$(basename $@).ipfire \
+       cd /install/packages/package && XZ_OPT="$(XZ_OPT)" tar cf ../core-upgrade-$(VERSION)-$$(basename $@).ipfire $(TAR_OPTIONS) \
                update.sh files.tar.xz ROOTFILES
        rm -rf /install/packages/package
        sed -e "s/NAME/core-upgrade/g" \

diff --git a/make.sh b/make.sh
index 33ebc16bb01bceb0f8fc733d6df10e67f4fb10e0..9893fbb20e5a0099510445c8a0094f65e5019e56 100755 (executable)
--- a/make.sh
+++ b/make.sh
@@ -1776,7 +1776,7 @@ toolchain)
        buildtoolchain
        echo "`date -u '+%b %e %T'`: Create toolchain image for ${BUILD_ARCH}" | tee -a $LOGFILE
        test -d $BASEDIR/cache/toolchains || mkdir -p $BASEDIR/cache/toolchains
-       cd $BASEDIR && XZ_OPT="$(XZ_OPT)" tar -Jc --exclude='log/_build.*.log' -f cache/toolchains/$SNAME-$VERSION-toolchain-$TOOLCHAINVER-${BUILD_ARCH}.tar.xz \
+       cd $BASEDIR && XZ_OPT="$(XZ_OPT)" tar -Jc --exclude='log/_build.*.log' -f cache/toolchains/$SNAME-$VERSION-toolchain-$TOOLCHAINVER-${BUILD_ARCH}.tar.xz --xz \
                build/${TOOLS_DIR} build/bin/sh log >> $LOGFILE
        md5sum cache/toolchains/$SNAME-$VERSION-toolchain-$TOOLCHAINVER-${BUILD_ARCH}.tar.xz \
                > cache/toolchains/$SNAME-$VERSION-toolchain-$TOOLCHAINVER-${BUILD_ARCH}.md5