Further harden tests that might use not-so-compatible tar versions.

Buildfarm testing shows that OpenSUSE (and perhaps related platforms?)
configures GNU tar in such a way that it'll archive sparse WAL files
by default, thus triggering the pax-extension detection code added by
bc30c704a.  Thus, we need something similar to 852de579a but for
GNU tar's option set.  "--format=ustar" seems to do the trick.

Moreover, the buildfarm shows that pg_verifybackup's 003_corruption.pl
test script is also triggering creation of pax-format tar files on
that platform.  We had not noticed because those test cases all fail
(intentionally) before getting to the point of trying to verify WAL
data.

Since that means two TAP scripts need this option-selection logic, and
plausibly more will do so in future, factor it out into a subroutine
in Test::Utils.  We also need to back-patch the 003_corruption.pl fix
into v18, where it's also failing.

While at it, clean up some places where guards for $tar being empty
or undefined were incomplete or even outright backwards.  Presumably,
we missed noticing because the set of machines that run TAP tests
and don't have tar installed is empty.  But if we're going to try
to handle that scenario, we should do it correctly.

Reported-by: Tomas Vondra <tomas@vondra.me>
Author: Tom Lane <tgl@sss.pgh.pa.us>
Discussion: https://postgr.es/m/02770bea-b3f3-4015-8a43-443ae345379c@vondra.me
Backpatch-through: 18

diff --git a/src/bin/pg_verifybackup/t/003_corruption.pl b/src/bin/pg_verifybackup/t/003_corruption.pl
index 1dd60f709cf66f0c26a464819a083f9af3d029ef..6b3f97471f38132787c64e6165c59fcd5e6fc248 100644 (file)
--- a/src/bin/pg_verifybackup/t/003_corruption.pl
+++ b/src/bin/pg_verifybackup/t/003_corruption.pl
@@ -13,6 +13,7 @@ use PostgreSQL::Test::Utils;
 use Test::More;
 
 my $tar = $ENV{TAR};
+my @tar_p_flags = tar_portability_options($tar);
 
 my $primary = PostgreSQL::Test::Cluster->new('primary');
 $primary->init(allows_streaming => 1);
@@ -154,8 +155,8 @@ for my $scenario (@scenario)
                # have a TAR program available. Note that this destructively modifies
                # the backup directory.
                if (   !$scenario->{'needs_unix_permissions'}
-                       || !defined $tar
-                       || $tar eq '')
+                       && defined $tar
+                       && $tar ne '')
                {
                        my $tar_backup_path = $primary->backup_dir . '/tar_' . $name;
                        mkdir($tar_backup_path) || die "mkdir $tar_backup_path: $!";
@@ -171,14 +172,23 @@ for my $scenario (@scenario)
 
                                chdir($tspath) || die "chdir: $!";
                                command_ok(
-                                       [ $tar, '-cf', "$tar_backup_path/$tsoid.tar", '.' ]);
+                                       [
+                                               $tar, @tar_p_flags,
+                                               '-cf' => "$tar_backup_path/$tsoid.tar",
+                                               '.'
+                                       ]);
                                chdir($cwd) || die "chdir: $!";
                                rmtree($tspath);
                        }
 
                        # tar and remove pg_wal
                        chdir($backup_path . '/pg_wal') || die "chdir: $!";
-                       command_ok([ $tar, '-cf', "$tar_backup_path/pg_wal.tar", '.' ]);
+                       command_ok(
+                               [
+                                       $tar, @tar_p_flags,
+                                       '-cf' => "$tar_backup_path/pg_wal.tar",
+                                       '.'
+                               ]);
                        chdir($cwd) || die "chdir: $!";
                        rmtree($backup_path . '/pg_wal');
 
@@ -190,7 +200,12 @@ for my $scenario (@scenario)
 
                        # Construct base.tar with what's left.
                        chdir($backup_path) || die "chdir: $!";
-                       command_ok([ $tar, '-cf' => "$tar_backup_path/base.tar", '.' ]);
+                       command_ok(
+                               [
+                                       $tar, @tar_p_flags,
+                                       '-cf' => "$tar_backup_path/base.tar",
+                                       '.'
+                               ]);
                        chdir($cwd) || die "chdir: $!";
 
                        # Now check that the backup no longer verifies. We must use -n

diff --git a/src/test/perl/PostgreSQL/Test/Utils.pm b/src/test/perl/PostgreSQL/Test/Utils.pm
index 7d7ca83495f49f3dc58e25baa3d28ea489a60718..e07006d89b51e0d0d9589b835f35ba01f4f91a3e 100644 (file)
--- a/src/test/perl/PostgreSQL/Test/Utils.pm
+++ b/src/test/perl/PostgreSQL/Test/Utils.pm
@@ -92,6 +92,8 @@ our @EXPORT = qw(
   command_ok_or_fails_like
   command_checks_all
 
+  tar_portability_options
+
   $windows_os
   $is_msys2
   $use_unix_sockets
@@ -1151,6 +1153,48 @@ sub command_checks_all
 
 =pod
 
+=item tar_portability_options(tar)
+
+Check for non-default options we need to give to tar to create
+a tarfile we can decode (i.e., no "pax" extensions).
+Not needed in tests that only use tar to read tarfiles.
+
+Returns options as an array.
+
+=cut
+
+sub tar_portability_options
+{
+       local $Test::Builder::Level = $Test::Builder::Level + 1;
+
+       my ($tar) = @_;
+
+       my @tar_p_flags = ();
+
+       return @tar_p_flags if (!defined $tar || $tar eq '');
+
+       # GNU tar typically produces gnu-format archives, which we can read fine.
+       # But some platforms configure it to default to posix/pax format, and
+       # apparently they enable --sparse too.  Override that.
+       if (system("$tar --format=ustar -c -O /dev/null >/dev/null 2>/dev/null")
+               == 0)
+       {
+               push(@tar_p_flags, "--format=ustar");
+       }
+
+       # bsdtar also archives sparse files by default, but it spells the switch
+       # to disable that differently.
+       if (system("$tar --no-read-sparse -c - /dev/null >/dev/null 2>/dev/null")
+               == 0)
+       {
+               push(@tar_p_flags, "--no-read-sparse");
+       }
+
+       return @tar_p_flags;
+}
+
+=pod
+
 =back
 
 =cut