From: Viktor Szakats Date: Fri, 5 Apr 2024 13:27:05 +0000 (+0000) Subject: dist: `set -eu`, fix shellcheck, make reproducible and smaller tarballs X-Git-Tag: curl-8_8_0~285 X-Git-Url: http://git.ipfire.org/?a=commitdiff_plain;h=860cd5fc2dc8e165fadd2c19a9b7c73b3ae5069d;p=thirdparty%2Fcurl.git dist: `set -eu`, fix shellcheck, make reproducible and smaller tarballs - set bash `-eu` and fix fallouts. - fix shellcheck warnings. - set and use `SOURCE_DATE_EPOCH` for reproducibility. Authored-by: Daniel J. H. Ref: #13280 - set `TZ=UTC` and `LC_ALL=C` for reproducibility. - make file timestamps in tarball/zip reproducible. - make directory timestamps in zip reproducible. - make timestamps of tarballs/zip reproducible. - make file order in tarball/zip reproducible. - omit extra file metadata from zip for reproducibility. - use maximum zip compression. - use POSIX `ustar` tarball format to avoid supply chain vulnerability: https://seclists.org/oss-sec/2021/q4/0 - make uid/gid in tarball reproducible. - omit owner user/group names from tarball for reproducibility and privacy. - omit current timestamp from .gz header for reproducibility. - display SHA-256 hashes of produced tarballs/zip. - fix whitespace. `.tar.gz` also became smaller in the process: 4,462,311 -> 4,148,249 bytes (8.7.1) Requires GNU tar, GNU date, `sha256sum`. Reviewed-by: Daniel Stenberg Ref: #13250 Closes #13299 --- diff --git a/maketgz b/maketgz index 602f1071b0..d935d2a242 100755 --- a/maketgz +++ b/maketgz @@ -26,26 +26,33 @@ # ########################################################################### -version=$1 +set -eu + +export LC_ALL=C +export TZ=UTC + +version="${1:-}" if [ -z "$version" ]; then echo "Specify a version number!" exit fi -if [ "xonly" = "x$2" ]; then +if [ "only" = "${2:-}" ]; then echo "Setup version number only!" only=1 +else + only= fi libversion="$version" # we make curl the same version as libcurl -curlversion=$libversion +curlversion="$libversion" -major=`echo $libversion | cut -d. -f1 | sed -e "s/[^0-9]//g"` -minor=`echo $libversion | cut -d. -f2 | sed -e "s/[^0-9]//g"` -patch=`echo $libversion | cut -d. -f3 | cut -d- -f1 | sed -e "s/[^0-9]//g"` +major=$(echo "$libversion" | cut -d. -f1 | sed -e "s/[^0-9]//g") +minor=$(echo "$libversion" | cut -d. -f2 | sed -e "s/[^0-9]//g") +patch=$(echo "$libversion" | cut -d. -f3 | cut -d- -f1 | sed -e "s/[^0-9]//g") if test -z "$patch"; then echo "invalid version number? needs to be z.y.z" @@ -59,7 +66,8 @@ fi echo "removing all old *.dist files" find . -name "*.dist" -a ! -name Makefile.dist -exec rm {} \; -numeric=`perl -e 'printf("%02x%02x%02x\n", '"$major, $minor, $patch);"` +numeric=$(perl -e \ + 'printf("%02x%02x%02x\n", '"$major, $minor, $patch);") HEADER=include/curl/curlver.h CHEADER=src/tool_version.h @@ -70,37 +78,39 @@ if test -z "$only"; then ext=".dist" # when not setting up version numbers locally for a in $HEADER $CHEADER $PLIST; do - cp $a "$a$ext" + cp "$a" "$a$ext" done HEADER="$HEADER$ext" CHEADER="$CHEADER$ext" PLIST="$PLIST$ext" fi -# requires a date command that knows + for format -datestamp=`date +"%F"` +# requires a date command that knows + for format and -d for date input +timestamp=${SOURCE_DATE_EPOCH:-$(date +"%s")} +datestamp=$(date -d "@$timestamp" +"%F") +filestamp=$(date -d "@$timestamp" +"%Y%m%d%H%M.%S") # Replace version number in header file: sed -i.bak \ - -e 's/^#define LIBCURL_VERSION .*/#define LIBCURL_VERSION "'$libversion'"/g' \ - -e 's/^#define LIBCURL_VERSION_NUM .*/#define LIBCURL_VERSION_NUM 0x'$numeric'/g' \ - -e 's/^#define LIBCURL_VERSION_MAJOR .*/#define LIBCURL_VERSION_MAJOR '$major'/g' \ - -e 's/^#define LIBCURL_VERSION_MINOR .*/#define LIBCURL_VERSION_MINOR '$minor'/g' \ - -e 's/^#define LIBCURL_VERSION_PATCH .*/#define LIBCURL_VERSION_PATCH '$patch'/g' \ - -e "s/^#define LIBCURL_TIMESTAMP .*/#define LIBCURL_TIMESTAMP \"$datestamp\"/g" \ - $HEADER + -e "s/^#define LIBCURL_VERSION .*/#define LIBCURL_VERSION \"$libversion\"/g" \ + -e "s/^#define LIBCURL_VERSION_NUM .*/#define LIBCURL_VERSION_NUM 0x$numeric/g" \ + -e "s/^#define LIBCURL_VERSION_MAJOR .*/#define LIBCURL_VERSION_MAJOR $major/g" \ + -e "s/^#define LIBCURL_VERSION_MINOR .*/#define LIBCURL_VERSION_MINOR $minor/g" \ + -e "s/^#define LIBCURL_VERSION_PATCH .*/#define LIBCURL_VERSION_PATCH $patch/g" \ + -e "s/^#define LIBCURL_TIMESTAMP .*/#define LIBCURL_TIMESTAMP \"$datestamp\"/g" \ + "$HEADER" rm -f "$HEADER.bak" # Replace version number in header file: -sed -i.bak 's/#define CURL_VERSION .*/#define CURL_VERSION "'$curlversion'"/g' $CHEADER +sed -i.bak "s/#define CURL_VERSION .*/#define CURL_VERSION \"$curlversion\"/g" "$CHEADER" rm -f "$CHEADER.bak" # Replace version number in plist file: -sed "s/@CURL_PLIST_VERSION@/$curlversion/g" < $PLISTO.in >$PLIST +sed "s/@CURL_PLIST_VERSION@/$curlversion/g" < "$PLISTO.in" > "$PLIST" if test -n "$only"; then # done! - exit; + exit fi echo "curl version $curlversion" @@ -110,7 +120,7 @@ echo "datestamp $datestamp" findprog() { file="$1" - for part in `echo $PATH| tr ':' ' '`; do + for part in $(echo "$PATH" | tr ':' ' '); do path="$part/$file" if [ -x "$path" ]; then # there it is! @@ -137,7 +147,7 @@ echo "Re-running config.status" # if { findprog automake >/dev/null 2>/dev/null; } then - echo "- Could not find or run automake, I hope you know what you're doing!" + echo "- Could not find or run automake, I hope you know what you are doing!" else echo "Runs automake --include-deps" automake --include-deps Makefile >/dev/null @@ -168,14 +178,31 @@ echo "produce RELEASE-TOOLS.md" echo "make dist" targz="curl-$version.tar.gz" -make -sj dist VERSION=$version +make -sj dist "VERSION=$version" res=$? if test "$res" != 0; then - echo "make dist failed" - exit 2 + echo "make dist failed" + exit 2 fi +retar() { + tempdir=$1 + rm -rf "$tempdir" + mkdir "$tempdir" + cd "$tempdir" + gzip -dc "../$targz" | tar -xf - + find curl-* -type f -exec touch -c -t "$filestamp" '{}' + + find curl-* -type f | sort | tar --create --format=ustar --owner=0 --group=0 --numeric-owner --files-from - | gzip --best --no-name > out.tar.gz + mv out.tar.gz ../ + cd .. + rm -rf "$tempdir" +} + +retar ".tarbuild" +echo "replace $targz with out.tar.gz" +mv out.tar.gz "$targz" + ############################################################################ # # Now make a bz2 archive from the tar.gz original @@ -183,7 +210,7 @@ fi bzip2="curl-$version.tar.bz2" echo "Generating $bzip2" -gzip -dc $targz | bzip2 --best > $bzip2 +gzip -dc "$targz" | bzip2 --best > "$bzip2" ############################################################################ # @@ -192,21 +219,22 @@ gzip -dc $targz | bzip2 --best > $bzip2 xz="curl-$version.tar.xz" echo "Generating $xz" -gzip -dc $targz | xz -6e - > $xz +gzip -dc "$targz" | xz -6e - > "$xz" ############################################################################ # # Now make a zip archive from the tar.gz original # makezip() { - rm -rf $tempdir - mkdir $tempdir - cd $tempdir - gzip -dc ../$targz | tar -xf - - find . | zip $zip -@ >/dev/null - mv $zip ../ + rm -rf "$tempdir" + mkdir "$tempdir" + cd "$tempdir" + gzip -dc "../$targz" | tar -xf - + find . -depth -type d -exec touch -c -t "$filestamp" '{}' + + find . | sort | zip -9 -X "$zip" -@ >/dev/null + mv "$zip" ../ cd .. - rm -rf $tempdir + rm -rf "$tempdir" } zip="curl-$version.zip" @@ -214,10 +242,14 @@ echo "Generating $zip" tempdir=".builddir" makezip +# Set deterministic timestamp +touch -c -t "$filestamp" "$targz" "$bzip2" "$xz" "$zip" + echo "------------------" echo "maketgz report:" echo "" -ls -l $targz $bzip2 $zip $xz +ls -l "$targz" "$bzip2" "$xz" "$zip" +sha256sum "$targz" "$bzip2" "$xz" "$zip" echo "Run this:" -echo "gpg -b -a $targz && gpg -b -a $bzip2 && gpg -b -a $zip && gpg -b -a $xz" +echo "gpg -b -a '$targz' && gpg -b -a '$bzip2' && gpg -b -a '$xz' && gpg -b -a '$zip'"