Fix bug in archive streamer with LZ4 decompression

When decompressing some input data, the calculation for the initial
starting point and the initial size were incorrect, potentially leading
to failures when decompressing contents with LZ4.  These initialization
points are fixed in this commit, bringing the logic closer to what
exists for gzip and zstd.

The contents of the compressed data is clear (for example backups taken
with LZ4 can still be decompressed with a "lz4" command), only the
decompression part reading the input data was impacted by this issue.

This code path impacts pg_basebackup and pg_verifybackup, which can use
the LZ4 decompression routines with an archive streamer, or any tools
that try to use the archive streamers in src/fe_utils/.

The issue is easier to reproduce with files that have a low-compression
rate, like ones filled with random data, for a size of at least 512kB,
but this could happen with anything as long as it is stored in a data
folder.  Some tests are added based on this idea, with a file filled
with random bytes grabbed from the backend, written at the root of the
data folder.  This is proving good enough to reproduce the original
problem.

Author: Mikhail Gribkov <youzhick@gmail.com>
Discussion: https://postgr.es/m/CAMEv5_uQS1Hg6KCaEP2JkrTBbZ-nXQhxomWrhYQvbdzR-zy-wA@mail.gmail.com
Backpatch-through: 15

diff --git a/src/bin/pg_basebackup/bbstreamer_lz4.c b/src/bin/pg_basebackup/bbstreamer_lz4.c
index ed2aa01305719e9ad9c28a9df00af9f0916108ef..fb287f5b771148c01ac0a7255eda0eb0c18f1c7b 100644 (file)
--- a/src/bin/pg_basebackup/bbstreamer_lz4.c
+++ b/src/bin/pg_basebackup/bbstreamer_lz4.c
@@ -320,9 +320,9 @@ bbstreamer_lz4_decompressor_content(bbstreamer *streamer,
 
        mystreamer = (bbstreamer_lz4_frame *) streamer;
        next_in = (uint8 *) data;
-       next_out = (uint8 *) mystreamer->base.bbs_buffer.data;
+       next_out = (uint8 *) mystreamer->base.bbs_buffer.data + mystreamer->bytes_written;
        avail_in = len;
-       avail_out = mystreamer->base.bbs_buffer.maxlen;
+       avail_out = mystreamer->base.bbs_buffer.maxlen - mystreamer->bytes_written;
 
        while (avail_in > 0)
        {

diff --git a/src/bin/pg_verifybackup/t/008_untar.pl b/src/bin/pg_verifybackup/t/008_untar.pl
index 4c4959516dd750af1faa574379db7f1ae55be842..0aecb70e53f8fd1337b5e4316a48d30ac09de5d2 100644 (file)
--- a/src/bin/pg_verifybackup/t/008_untar.pl
+++ b/src/bin/pg_verifybackup/t/008_untar.pl
@@ -16,6 +16,22 @@ my $primary = PostgreSQL::Test::Cluster->new('primary');
 $primary->init(allows_streaming => 1);
 $primary->start;
 
+# Create file with some random data and an arbitrary size, useful to check
+# the solidity of the compression and decompression logic.  The size of the
+# file is chosen to be around 640kB.  This has proven to be large enough to
+# detect some issues related to LZ4, and low enough to not impact the runtime
+# of the test significantly.
+my $junk_data = $primary->safe_psql(
+       'postgres', qq(
+               SELECT string_agg(encode(sha256(i::text::bytea), 'hex'), '')
+               FROM generate_series(1, 10240) s(i);));
+my $data_dir = $primary->data_dir;
+my $junk_file = "$data_dir/junk";
+open my $jf, '>', $junk_file
+  or die "Could not create junk file: $!";
+print $jf $junk_data;
+close $jf;
+
 my $backup_path  = $primary->backup_dir . '/server-backup';
 my $extract_path = $primary->backup_dir . '/extracted-backup';
 
@@ -42,6 +58,14 @@ my @test_configuration = (
                'decompress_flags'   => [ '-d', '-m' ],
                'enabled'            => check_pg_config("#define USE_LZ4 1")
        },
+       {
+               'compression_method' => 'lz4',
+               'backup_flags'       => [ '--compress', 'server-lz4:5' ],
+               'backup_archive'     => 'base.tar.lz4',
+               'decompress_program' => $ENV{'LZ4'},
+               'decompress_flags'   => [ '-d', '-m' ],
+               'enabled'            => check_pg_config("#define USE_LZ4 1")
+       },
        {
                'compression_method' => 'zstd',
                'backup_flags'       => [ '--compress', 'server-zstd' ],

diff --git a/src/bin/pg_verifybackup/t/010_client_untar.pl b/src/bin/pg_verifybackup/t/010_client_untar.pl
index 77cb503784c189996fff429d18054e9c36c1d677..e536b83d79cb979b25004a3135e5cceb227c854b 100644 (file)
--- a/src/bin/pg_verifybackup/t/010_client_untar.pl
+++ b/src/bin/pg_verifybackup/t/010_client_untar.pl
@@ -15,6 +15,22 @@ my $primary = PostgreSQL::Test::Cluster->new('primary');
 $primary->init(allows_streaming => 1);
 $primary->start;
 
+# Create file with some random data and an arbitrary size, useful to check
+# the solidity of the compression and decompression logic.  The size of the
+# file is chosen to be around 640kB.  This has proven to be large enough to
+# detect some issues related to LZ4, and low enough to not impact the runtime
+# of the test significantly.
+my $junk_data = $primary->safe_psql(
+       'postgres', qq(
+               SELECT string_agg(encode(sha256(i::text::bytea), 'hex'), '')
+               FROM generate_series(1, 10240) s(i);));
+my $data_dir = $primary->data_dir;
+my $junk_file = "$data_dir/junk";
+open my $jf, '>', $junk_file
+  or die "Could not create junk file: $!";
+print $jf $junk_data;
+close $jf;
+
 my $backup_path  = $primary->backup_dir . '/client-backup';
 my $extract_path = $primary->backup_dir . '/extracted-backup';
 
@@ -42,6 +58,15 @@ my @test_configuration = (
                'output_file'        => 'base.tar',
                'enabled'            => check_pg_config("#define USE_LZ4 1")
        },
+       {
+               'compression_method' => 'lz4',
+               'backup_flags'       => [ '--compress', 'client-lz4:1' ],
+               'backup_archive'     => 'base.tar.lz4',
+               'decompress_program' => $ENV{'LZ4'},
+               'decompress_flags'   => ['-d'],
+               'output_file'        => 'base.tar',
+               'enabled'            => check_pg_config("#define USE_LZ4 1")
+       },
        {
                'compression_method' => 'zstd',
                'backup_flags'       => [ '--compress', 'client-zstd:5' ],