Further harden tests that might use not-so-compatible tar versions.

Buildfarm testing shows that OpenSUSE (and perhaps related platforms?)
configures GNU tar in such a way that it'll archive sparse WAL files
by default, thus triggering the pax-extension detection code added by
bc30c704a.  Thus, we need something similar to 852de579a but for
GNU tar's option set.  "--format=ustar" seems to do the trick.

Moreover, the buildfarm shows that pg_verifybackup's 003_corruption.pl
test script is also triggering creation of pax-format tar files on
that platform.  We had not noticed because those test cases all fail
(intentionally) before getting to the point of trying to verify WAL
data.

Since that means two TAP scripts need this option-selection logic, and
plausibly more will do so in future, factor it out into a subroutine
in Test::Utils.  We also need to back-patch the 003_corruption.pl fix
into v18, where it's also failing.

While at it, clean up some places where guards for $tar being empty
or undefined were incomplete or even outright backwards.  Presumably,
we missed noticing because the set of machines that run TAP tests
and don't have tar installed is empty.  But if we're going to try
to handle that scenario, we should do it correctly.

Reported-by: Tomas Vondra <tomas@vondra.me>
Author: Tom Lane <tgl@sss.pgh.pa.us>
Discussion: https://postgr.es/m/02770bea-b3f3-4015-8a43-443ae345379c@vondra.me
Backpatch-through: 18

diff --git a/src/bin/pg_verifybackup/t/003_corruption.pl b/src/bin/pg_verifybackup/t/003_corruption.pl
index 882d75d9dc2a204f272527d4c5343aafed2a832d..405ea793b6831c19ace238590b9167e4c026ec4c 100644 (file)
--- a/src/bin/pg_verifybackup/t/003_corruption.pl
+++ b/src/bin/pg_verifybackup/t/003_corruption.pl
@@ -13,6 +13,7 @@ use PostgreSQL::Test::Utils;
 use Test::More;
 
 my $tar = $ENV{TAR};
+my @tar_p_flags = tar_portability_options($tar);
 
 my $primary = PostgreSQL::Test::Cluster->new('primary');
 $primary->init(allows_streaming => 1);
@@ -154,8 +155,8 @@ for my $scenario (@scenario)
                # have a TAR program available. Note that this destructively modifies
                # the backup directory.
                if (   !$scenario->{'needs_unix_permissions'}
-                       || !defined $tar
-                       || $tar eq '')
+                       && defined $tar
+                       && $tar ne '')
                {
                        my $tar_backup_path = $primary->backup_dir . '/tar_' . $name;
                        mkdir($tar_backup_path) || die "mkdir $tar_backup_path: $!";
@@ -171,14 +172,23 @@ for my $scenario (@scenario)
 
                                chdir($tspath) || die "chdir: $!";
                                command_ok(
-                                       [ $tar, '-cf', "$tar_backup_path/$tsoid.tar", '.' ]);
+                                       [
+                                               $tar, @tar_p_flags,
+                                               '-cf' => "$tar_backup_path/$tsoid.tar",
+                                               '.'
+                                       ]);
                                chdir($cwd) || die "chdir: $!";
                                rmtree($tspath);
                        }
 
                        # tar and remove pg_wal
                        chdir($backup_path . '/pg_wal') || die "chdir: $!";
-                       command_ok([ $tar, '-cf', "$tar_backup_path/pg_wal.tar", '.' ]);
+                       command_ok(
+                               [
+                                       $tar, @tar_p_flags,
+                                       '-cf' => "$tar_backup_path/pg_wal.tar",
+                                       '.'
+                               ]);
                        chdir($cwd) || die "chdir: $!";
                        rmtree($backup_path . '/pg_wal');
 
@@ -190,7 +200,12 @@ for my $scenario (@scenario)
 
                        # Construct base.tar with what's left.
                        chdir($backup_path) || die "chdir: $!";
-                       command_ok([ $tar, '-cf' => "$tar_backup_path/base.tar", '.' ]);
+                       command_ok(
+                               [
+                                       $tar, @tar_p_flags,
+                                       '-cf' => "$tar_backup_path/base.tar",
+                                       '.'
+                               ]);
                        chdir($cwd) || die "chdir: $!";
 
                        command_fails_like(

diff --git a/src/bin/pg_waldump/t/001_basic.pl b/src/bin/pg_waldump/t/001_basic.pl
index ce1f6aa30c028615cc6b0082983d0c8cce15595b..a268f0f1dd02ec78f1481b050d1954b88de1e4ab 100644 (file)
--- a/src/bin/pg_waldump/t/001_basic.pl
+++ b/src/bin/pg_waldump/t/001_basic.pl
@@ -11,15 +11,7 @@ use Test::More;
 use List::Util qw(shuffle);
 
 my $tar = $ENV{TAR};
-my @tar_c_flags;
-
-# By default, bsdtar archives sparse files in GNU tar's --format=posix --sparse
-# format, so pg_waldump can't find files that ZFS has decided to store with
-# holes.  Turn that off.
-if (system("$tar --no-read-sparse -c - /dev/null > /dev/null") == 0)
-{
-  push(@tar_c_flags, "--no-read-sparse");
-}
+my @tar_p_flags = tar_portability_options($tar);
 
 program_help_ok('pg_waldump');
 program_version_ok('pg_waldump');
@@ -355,7 +347,7 @@ sub generate_archive
        # move into the WAL directory before archiving files
        my $cwd = getcwd;
        chdir($directory) || die "chdir: $!";
-       command_ok([$tar, @tar_c_flags, $compression_flags, $archive, @files]);
+       command_ok([ $tar, @tar_p_flags, $compression_flags, $archive, @files ]);
        chdir($cwd) || die "chdir: $!";
 }
 
@@ -389,7 +381,7 @@ for my $scenario (@scenarios)
        SKIP:
        {
                skip "tar command is not available", 56
-                 if !defined $tar && $scenario->{'is_archive'};
+                 if (!defined $tar || $tar eq '') && $scenario->{'is_archive'};
                skip "$scenario->{'compression_method'} compression not supported by this build", 56
                  if !$scenario->{'enabled'} && $scenario->{'is_archive'};
 

diff --git a/src/test/perl/PostgreSQL/Test/Utils.pm b/src/test/perl/PostgreSQL/Test/Utils.pm
index 163d9a4beb99fd256ea8aea2eb855e08ddaa8ca3..120999f6ac933fe617722a17f1a2db9cf3455b45 100644 (file)
--- a/src/test/perl/PostgreSQL/Test/Utils.pm
+++ b/src/test/perl/PostgreSQL/Test/Utils.pm
@@ -99,6 +99,8 @@ our @EXPORT = qw(
   command_ok_or_fails_like
   command_checks_all
 
+  tar_portability_options
+
   $windows_os
   $is_msys2
   $use_unix_sockets
@@ -1304,6 +1306,48 @@ sub command_checks_all
 
 =pod
 
+=item tar_portability_options(tar)
+
+Check for non-default options we need to give to tar to create
+a tarfile we can decode (i.e., no "pax" extensions).
+Not needed in tests that only use tar to read tarfiles.
+
+Returns options as an array.
+
+=cut
+
+sub tar_portability_options
+{
+       local $Test::Builder::Level = $Test::Builder::Level + 1;
+
+       my ($tar) = @_;
+
+       my @tar_p_flags = ();
+
+       return @tar_p_flags if (!defined $tar || $tar eq '');
+
+       # GNU tar typically produces gnu-format archives, which we can read fine.
+       # But some platforms configure it to default to posix/pax format, and
+       # apparently they enable --sparse too.  Override that.
+       if (system("$tar --format=ustar -c -O /dev/null >/dev/null 2>/dev/null")
+               == 0)
+       {
+               push(@tar_p_flags, "--format=ustar");
+       }
+
+       # bsdtar also archives sparse files by default, but it spells the switch
+       # to disable that differently.
+       if (system("$tar --no-read-sparse -c - /dev/null >/dev/null 2>/dev/null")
+               == 0)
+       {
+               push(@tar_p_flags, "--no-read-sparse");
+       }
+
+       return @tar_p_flags;
+}
+
+=pod
+
 =back
 
 =cut