dist: `set -eu`, fix shellcheck, make reproducible and smaller tarballs

- set bash `-eu` and fix fallouts.
- fix shellcheck warnings.
- set and use `SOURCE_DATE_EPOCH` for reproducibility.
  Authored-by: Daniel J. H.
  Ref: #13280
- set `TZ=UTC` and `LC_ALL=C` for reproducibility.
- make file timestamps in tarball/zip reproducible.
- make directory timestamps in zip reproducible.
- make timestamps of tarballs/zip reproducible.
- make file order in tarball/zip reproducible.
- omit extra file metadata from zip for reproducibility.
- use maximum zip compression.
- use POSIX `ustar` tarball format to avoid supply chain vulnerability:
  https://seclists.org/oss-sec/2021/q4/0
- make uid/gid in tarball reproducible.
- omit owner user/group names from tarball for reproducibility and privacy.
- omit current timestamp from .gz header for reproducibility.
- display SHA-256 hashes of produced tarballs/zip.
- fix whitespace.

`.tar.gz` also became smaller in the process: 4,462,311 -> 4,148,249 bytes (8.7.1)

Requires GNU tar, GNU date, `sha256sum`.

Reviewed-by: Daniel Stenberg
Ref: #13250
Closes #13299

diff --git a/maketgz b/maketgz
index 602f1071b0082148035409199de4abcaf9d3ef1b..d935d2a242b538f03f57022092ef7e9a8e5c7728 100755 (executable)
--- a/maketgz
+++ b/maketgz
@@ -26,26 +26,33 @@
 #
 ###########################################################################
 
-version=$1
+set -eu
+
+export LC_ALL=C
+export TZ=UTC
+
+version="${1:-}"
 
 if [ -z "$version" ]; then
   echo "Specify a version number!"
   exit
 fi
 
-if [ "xonly" = "x$2" ]; then
+if [ "only" = "${2:-}" ]; then
   echo "Setup version number only!"
   only=1
+else
+  only=
 fi
 
 libversion="$version"
 
 # we make curl the same version as libcurl
-curlversion=$libversion
+curlversion="$libversion"
 
-major=`echo $libversion | cut -d. -f1 | sed -e "s/[^0-9]//g"`
-minor=`echo $libversion | cut -d. -f2 | sed -e "s/[^0-9]//g"`
-patch=`echo $libversion | cut -d. -f3 | cut -d- -f1 | sed -e "s/[^0-9]//g"`
+major=$(echo "$libversion" | cut -d. -f1 | sed -e "s/[^0-9]//g")
+minor=$(echo "$libversion" | cut -d. -f2 | sed -e "s/[^0-9]//g")
+patch=$(echo "$libversion" | cut -d. -f3 | cut -d- -f1 | sed -e "s/[^0-9]//g")
 
 if test -z "$patch"; then
   echo "invalid version number? needs to be z.y.z"
@@ -59,7 +66,8 @@ fi
 echo "removing all old *.dist files"
 find . -name "*.dist" -a ! -name Makefile.dist -exec rm {} \;
 
-numeric=`perl -e 'printf("%02x%02x%02x\n", '"$major, $minor, $patch);"`
+numeric=$(perl -e \
+  'printf("%02x%02x%02x\n", '"$major, $minor, $patch);")
 
 HEADER=include/curl/curlver.h
 CHEADER=src/tool_version.h
@@ -70,37 +78,39 @@ if test -z "$only"; then
   ext=".dist"
   # when not setting up version numbers locally
   for a in $HEADER $CHEADER $PLIST; do
-    cp $a "$a$ext"
+    cp "$a" "$a$ext"
   done
   HEADER="$HEADER$ext"
   CHEADER="$CHEADER$ext"
   PLIST="$PLIST$ext"
 fi
 
-# requires a date command that knows + for format
-datestamp=`date +"%F"`
+# requires a date command that knows + for format and -d for date input
+timestamp=${SOURCE_DATE_EPOCH:-$(date +"%s")}
+datestamp=$(date -d "@$timestamp" +"%F")
+filestamp=$(date -d "@$timestamp" +"%Y%m%d%H%M.%S")
 
 # Replace version number in header file:
 sed -i.bak \
-    -e 's/^#define LIBCURL_VERSION .*/#define LIBCURL_VERSION "'$libversion'"/g' \
-    -e 's/^#define LIBCURL_VERSION_NUM .*/#define LIBCURL_VERSION_NUM 0x'$numeric'/g' \
-    -e 's/^#define LIBCURL_VERSION_MAJOR .*/#define LIBCURL_VERSION_MAJOR '$major'/g' \
-    -e 's/^#define LIBCURL_VERSION_MINOR .*/#define LIBCURL_VERSION_MINOR '$minor'/g' \
-    -e 's/^#define LIBCURL_VERSION_PATCH .*/#define LIBCURL_VERSION_PATCH '$patch'/g' \
-    -e "s/^#define LIBCURL_TIMESTAMP .*/#define LIBCURL_TIMESTAMP \"$datestamp\"/g" \
-    $HEADER
+  -e "s/^#define LIBCURL_VERSION .*/#define LIBCURL_VERSION \"$libversion\"/g" \
+  -e "s/^#define LIBCURL_VERSION_NUM .*/#define LIBCURL_VERSION_NUM 0x$numeric/g" \
+  -e "s/^#define LIBCURL_VERSION_MAJOR .*/#define LIBCURL_VERSION_MAJOR $major/g" \
+  -e "s/^#define LIBCURL_VERSION_MINOR .*/#define LIBCURL_VERSION_MINOR $minor/g" \
+  -e "s/^#define LIBCURL_VERSION_PATCH .*/#define LIBCURL_VERSION_PATCH $patch/g" \
+  -e "s/^#define LIBCURL_TIMESTAMP .*/#define LIBCURL_TIMESTAMP \"$datestamp\"/g" \
+  "$HEADER"
 rm -f "$HEADER.bak"
 
 # Replace version number in header file:
-sed -i.bak 's/#define CURL_VERSION .*/#define CURL_VERSION "'$curlversion'"/g' $CHEADER
+sed -i.bak "s/#define CURL_VERSION .*/#define CURL_VERSION \"$curlversion\"/g" "$CHEADER"
 rm -f "$CHEADER.bak"
 
 # Replace version number in plist file:
-sed "s/@CURL_PLIST_VERSION@/$curlversion/g" < $PLISTO.in >$PLIST
+sed "s/@CURL_PLIST_VERSION@/$curlversion/g" < "$PLISTO.in" > "$PLIST"
 
 if test -n "$only"; then
   # done!
-  exit;
+  exit
 fi
 
 echo "curl version $curlversion"
@@ -110,7 +120,7 @@ echo "datestamp $datestamp"
 
 findprog() {
   file="$1"
-  for part in `echo $PATH| tr ':' ' '`; do
+  for part in $(echo "$PATH" | tr ':' ' '); do
     path="$part/$file"
     if [ -x "$path" ]; then
       # there it is!
@@ -137,7 +147,7 @@ echo "Re-running config.status"
 #
 
 if { findprog automake >/dev/null 2>/dev/null; } then
-  echo "- Could not find or run automake, I hope you know what you're doing!"
+  echo "- Could not find or run automake, I hope you know what you are doing!"
 else
   echo "Runs automake --include-deps"
   automake --include-deps Makefile >/dev/null
@@ -168,14 +178,31 @@ echo "produce RELEASE-TOOLS.md"
 
 echo "make dist"
 targz="curl-$version.tar.gz"
-make -sj dist VERSION=$version
+make -sj dist "VERSION=$version"
 res=$?
 
 if test "$res" != 0; then
-    echo "make dist failed"
-    exit 2
+  echo "make dist failed"
+  exit 2
 fi
 
+retar() {
+  tempdir=$1
+  rm -rf "$tempdir"
+  mkdir "$tempdir"
+  cd "$tempdir"
+  gzip -dc "../$targz" | tar -xf -
+  find curl-* -type f -exec touch -c -t "$filestamp" '{}' +
+  find curl-* -type f | sort | tar --create --format=ustar --owner=0 --group=0 --numeric-owner --files-from - | gzip --best --no-name > out.tar.gz
+  mv out.tar.gz ../
+  cd ..
+  rm -rf "$tempdir"
+}
+
+retar ".tarbuild"
+echo "replace $targz with out.tar.gz"
+mv out.tar.gz "$targz"
+
 ############################################################################
 #
 # Now make a bz2 archive from the tar.gz original
@@ -183,7 +210,7 @@ fi
 
 bzip2="curl-$version.tar.bz2"
 echo "Generating $bzip2"
-gzip -dc $targz | bzip2 --best > $bzip2
+gzip -dc "$targz" | bzip2 --best > "$bzip2"
 
 ############################################################################
 #
@@ -192,21 +219,22 @@ gzip -dc $targz | bzip2 --best > $bzip2
 
 xz="curl-$version.tar.xz"
 echo "Generating $xz"
-gzip -dc $targz | xz -6e - > $xz
+gzip -dc "$targz" | xz -6e - > "$xz"
 
 ############################################################################
 #
 # Now make a zip archive from the tar.gz original
 #
 makezip() {
-  rm -rf $tempdir
-  mkdir $tempdir
-  cd $tempdir
-  gzip -dc ../$targz | tar -xf -
-  find . | zip $zip -@ >/dev/null
-  mv $zip ../
+  rm -rf "$tempdir"
+  mkdir "$tempdir"
+  cd "$tempdir"
+  gzip -dc "../$targz" | tar -xf -
+  find . -depth -type d -exec touch -c -t "$filestamp" '{}' +
+  find . | sort | zip -9 -X "$zip" -@ >/dev/null
+  mv "$zip" ../
   cd ..
-  rm -rf $tempdir
+  rm -rf "$tempdir"
 }
 
 zip="curl-$version.zip"
@@ -214,10 +242,14 @@ echo "Generating $zip"
 tempdir=".builddir"
 makezip
 
+# Set deterministic timestamp
+touch -c -t "$filestamp" "$targz" "$bzip2" "$xz" "$zip"
+
 echo "------------------"
 echo "maketgz report:"
 echo ""
-ls -l $targz $bzip2 $zip $xz
+ls -l "$targz" "$bzip2" "$xz" "$zip"
+sha256sum "$targz" "$bzip2" "$xz" "$zip"
 
 echo "Run this:"
-echo "gpg -b -a $targz && gpg -b -a $bzip2 && gpg -b -a $zip && gpg -b -a $xz"
+echo "gpg -b -a '$targz' && gpg -b -a '$bzip2' && gpg -b -a '$xz' && gpg -b -a '$zip'"