Switch buffile.c/h to use pgoff_t instead of off_t

off_t was previously used for offsets, which is 4 bytes on Windows,
hence limiting the backend code to a hard limit for files longer than
2GB.  This leads to some simplification in these files, removing some
casts based on long, also 4 bytes on Windows.

This commit removes one comment introduced in db3c4c3a2d98, not relevant
anymore as pgoff_t is a safe 8-byte alternative on Windows.

This change is surprisingly not invasive, as the callers of
BufFileTell(), BufFileSeek() and BufFileTruncateFileSet() (worker.c,
tuplestore.c, etc.) track offsets in local structures that just to
switch from off_t to pgoff_t for the most part.

The file is still relying on a maximum file size of
MAX_PHYSICAL_FILESIZE (1GB).  This change allows the code to make this
maximum potentially larger in the future, or larger on a per-demand
basis.

Reviewed-by: Chao Li <li.evan.chao@gmail.com>
Discussion: https://postgr.es/m/aUStrqoOCDRFAq1M@paquier.xyz

diff --git a/src/backend/replication/logical/worker.c b/src/backend/replication/logical/worker.c
index fc64476a9ef13118aecf1369c92466c800e36257..d1ee0261c646fad65c21d8fb733f1d41cab2ee6f 100644 (file)
--- a/src/backend/replication/logical/worker.c
+++ b/src/backend/replication/logical/worker.c
@@ -530,7 +530,7 @@ typedef struct SubXactInfo
 {
        TransactionId xid;                      /* XID of the subxact */
        int                     fileno;                 /* file number in the buffile */
-       off_t           offset;                 /* offset in the file */
+       pgoff_t         offset;                 /* offset in the file */
 } SubXactInfo;
 
 /* Sub-transaction data for the current streaming transaction */
@@ -2226,12 +2226,12 @@ apply_handle_stream_abort(StringInfo s)
  */
 static void
 ensure_last_message(FileSet *stream_fileset, TransactionId xid, int fileno,
-                                       off_t offset)
+                                       pgoff_t offset)
 {
        char            path[MAXPGPATH];
        BufFile    *fd;
        int                     last_fileno;
-       off_t           last_offset;
+       pgoff_t         last_offset;
 
        Assert(!IsTransactionState());
 
@@ -2266,7 +2266,7 @@ apply_spooled_messages(FileSet *stream_fileset, TransactionId xid,
        MemoryContext oldcxt;
        ResourceOwner oldowner;
        int                     fileno;
-       off_t           offset;
+       pgoff_t         offset;
 
        if (!am_parallel_apply_worker())
                maybe_start_skipping_changes(lsn);

diff --git a/src/backend/storage/file/buffile.c b/src/backend/storage/file/buffile.c
index 4e520065ae0001c883f97a7f2921ea628b3847a6..85b316d879dc0a8a6f7e9542ed22d5e8cec2d2d7 100644 (file)
--- a/src/backend/storage/file/buffile.c
+++ b/src/backend/storage/file/buffile.c
@@ -92,7 +92,7 @@ struct BufFile
         * Position as seen by user of BufFile is (curFile, curOffset + pos).
         */
        int                     curFile;                /* file index (0..n) part of current pos */
-       off_t           curOffset;              /* offset part of current pos */
+       pgoff_t         curOffset;              /* offset part of current pos */
        int                     pos;                    /* next read/write position in buffer */
        int                     nbytes;                 /* total # of valid bytes in buffer */
 
@@ -503,7 +503,7 @@ BufFileDumpBuffer(BufFile *file)
         */
        while (wpos < file->nbytes)
        {
-               off_t           availbytes;
+               pgoff_t         availbytes;
                instr_time      io_start;
                instr_time      io_time;
 
@@ -524,7 +524,7 @@ BufFileDumpBuffer(BufFile *file)
                bytestowrite = file->nbytes - wpos;
                availbytes = MAX_PHYSICAL_FILESIZE - file->curOffset;
 
-               if ((off_t) bytestowrite > availbytes)
+               if ((pgoff_t) bytestowrite > availbytes)
                        bytestowrite = (int) availbytes;
 
                thisfile = file->files[file->curFile];
@@ -729,7 +729,7 @@ BufFileFlush(BufFile *file)
  * BufFileSeek
  *
  * Like fseek(), except that target position needs two values in order to
- * work when logical filesize exceeds maximum value representable by off_t.
+ * work when logical filesize exceeds maximum value representable by pgoff_t.
  * We do not support relative seeks across more than that, however.
  * I/O errors are reported by ereport().
  *
@@ -737,10 +737,10 @@ BufFileFlush(BufFile *file)
  * impossible seek is attempted.
  */
 int
-BufFileSeek(BufFile *file, int fileno, off_t offset, int whence)
+BufFileSeek(BufFile *file, int fileno, pgoff_t offset, int whence)
 {
        int                     newFile;
-       off_t           newOffset;
+       pgoff_t         newOffset;
 
        switch (whence)
        {
@@ -754,8 +754,7 @@ BufFileSeek(BufFile *file, int fileno, off_t offset, int whence)
 
                        /*
                         * Relative seek considers only the signed offset, ignoring
-                        * fileno. Note that large offsets (> 1 GB) risk overflow in this
-                        * add, unless we have 64-bit off_t.
+                        * fileno.
                         */
                        newFile = file->curFile;
                        newOffset = (file->curOffset + file->pos) + offset;
@@ -830,7 +829,7 @@ BufFileSeek(BufFile *file, int fileno, off_t offset, int whence)
 }
 
 void
-BufFileTell(BufFile *file, int *fileno, off_t *offset)
+BufFileTell(BufFile *file, int *fileno, pgoff_t *offset)
 {
        *fileno = file->curFile;
        *offset = file->curOffset + file->pos;
@@ -852,7 +851,7 @@ BufFileSeekBlock(BufFile *file, int64 blknum)
 {
        return BufFileSeek(file,
                                           (int) (blknum / BUFFILE_SEG_SIZE),
-                                          (off_t) (blknum % BUFFILE_SEG_SIZE) * BLCKSZ,
+                                          (pgoff_t) (blknum % BUFFILE_SEG_SIZE) * BLCKSZ,
                                           SEEK_SET);
 }
 
@@ -925,11 +924,11 @@ BufFileAppend(BufFile *target, BufFile *source)
  * and the offset.
  */
 void
-BufFileTruncateFileSet(BufFile *file, int fileno, off_t offset)
+BufFileTruncateFileSet(BufFile *file, int fileno, pgoff_t offset)
 {
        int                     numFiles = file->numFiles;
        int                     newFile = fileno;
-       off_t           newOffset = file->curOffset;
+       pgoff_t         newOffset = file->curOffset;
        char            segment_name[MAXPGPATH];
        int                     i;
 

diff --git a/src/backend/utils/sort/tuplestore.c b/src/backend/utils/sort/tuplestore.c
index def945b04541ad6f5f03df23f6ec796f8c927939..9701b8133602bb04b528f173da22b44cd0790015 100644 (file)
--- a/src/backend/utils/sort/tuplestore.c
+++ b/src/backend/utils/sort/tuplestore.c
@@ -94,7 +94,7 @@ typedef struct
        bool            eof_reached;    /* read has reached EOF */
        int                     current;                /* next array index to read */
        int                     file;                   /* temp file# */
-       off_t           offset;                 /* byte offset in file */
+       pgoff_t         offset;                 /* byte offset in file */
 } TSReadPointer;
 
 /*
@@ -179,7 +179,7 @@ struct Tuplestorestate
        int                     readptrsize;    /* allocated length of readptrs array */
 
        int                     writepos_file;  /* file# (valid if READFILE state) */
-       off_t           writepos_offset;        /* offset (valid if READFILE state) */
+       pgoff_t         writepos_offset;        /* offset (valid if READFILE state) */
 };
 
 #define COPYTUP(state,tup)     ((*(state)->copytup) (state, tup))
@@ -1051,7 +1051,7 @@ tuplestore_gettuple(Tuplestorestate *state, bool forward,
                         * Back up to fetch previously-returned tuple's ending length
                         * word. If seek fails, assume we are at start of file.
                         */
-                       if (BufFileSeek(state->myfile, 0, -(long) sizeof(unsigned int),
+                       if (BufFileSeek(state->myfile, 0, -(pgoff_t) sizeof(unsigned int),
                                                        SEEK_CUR) != 0)
                        {
                                /* even a failed backwards fetch gets you out of eof state */
@@ -1072,7 +1072,7 @@ tuplestore_gettuple(Tuplestorestate *state, bool forward,
                                 * Back up to get ending length word of tuple before it.
                                 */
                                if (BufFileSeek(state->myfile, 0,
-                                                               -(long) (tuplen + 2 * sizeof(unsigned int)),
+                                                               -(pgoff_t) (tuplen + 2 * sizeof(unsigned int)),
                                                                SEEK_CUR) != 0)
                                {
                                        /*
@@ -1082,7 +1082,7 @@ tuplestore_gettuple(Tuplestorestate *state, bool forward,
                                         * what in-memory case does).
                                         */
                                        if (BufFileSeek(state->myfile, 0,
-                                                                       -(long) (tuplen + sizeof(unsigned int)),
+                                                                       -(pgoff_t) (tuplen + sizeof(unsigned int)),
                                                                        SEEK_CUR) != 0)
                                                ereport(ERROR,
                                                                (errcode_for_file_access(),
@@ -1099,7 +1099,7 @@ tuplestore_gettuple(Tuplestorestate *state, bool forward,
                         * length word of the tuple, so back up to that point.
                         */
                        if (BufFileSeek(state->myfile, 0,
-                                                       -(long) tuplen,
+                                                       -(pgoff_t) tuplen,
                                                        SEEK_CUR) != 0)
                                ereport(ERROR,
                                                (errcode_for_file_access(),

diff --git a/src/include/storage/buffile.h b/src/include/storage/buffile.h
index a2f4821f240b9cf6d3db3ce50aecf2d36c21a067..4908dc56c7be27ec0621c8538f01df228260be2f 100644 (file)
--- a/src/include/storage/buffile.h
+++ b/src/include/storage/buffile.h
@@ -42,8 +42,8 @@ pg_nodiscard extern size_t BufFileRead(BufFile *file, void *ptr, size_t size);
 extern void BufFileReadExact(BufFile *file, void *ptr, size_t size);
 extern size_t BufFileReadMaybeEOF(BufFile *file, void *ptr, size_t size, bool eofOK);
 extern void BufFileWrite(BufFile *file, const void *ptr, size_t size);
-extern int     BufFileSeek(BufFile *file, int fileno, off_t offset, int whence);
-extern void BufFileTell(BufFile *file, int *fileno, off_t *offset);
+extern int     BufFileSeek(BufFile *file, int fileno, pgoff_t offset, int whence);
+extern void BufFileTell(BufFile *file, int *fileno, pgoff_t *offset);
 extern int     BufFileSeekBlock(BufFile *file, int64 blknum);
 extern int64 BufFileSize(BufFile *file);
 extern int64 BufFileAppend(BufFile *target, BufFile *source);
@@ -54,6 +54,6 @@ extern BufFile *BufFileOpenFileSet(FileSet *fileset, const char *name,
                                                                   int mode, bool missing_ok);
 extern void BufFileDeleteFileSet(FileSet *fileset, const char *name,
                                                                 bool missing_ok);
-extern void BufFileTruncateFileSet(BufFile *file, int fileno, off_t offset);
+extern void BufFileTruncateFileSet(BufFile *file, int fileno, pgoff_t offset);
 
 #endif                                                 /* BUFFILE_H */