From: drh <drh@noemail.net>
Date: Mon, 9 Jun 2014 20:06:01 +0000 (+0000)
Subject: Enhance the unix VFS so that it keeps track of the size of unlinked files
X-Git-Tag: version-3.8.6~124^2
X-Git-Url: http://git.ipfire.org/?a=commitdiff_plain;h=f5154c6a1cb9d91d2cf608dd168bf539f7fa97b9;p=thirdparty%2Fsqlite.git

Enhance the unix VFS so that it keeps track of the size of unlinked files
internally and thus avoids the need to call fstat() on those files, since
fstat() does not work reliably on unlinked files on some implementations of
FuseFS.

FossilOrigin-Name: c41df393c6afbfbfdc4d1b885024e083c6f6de1f
---

diff --git a/manifest b/manifest
index 6555b4ed1a..25fcfdc03d 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Modify\sthe\s%nonassoc\sdirective\sin\slemon\sso\sthat\sit\sgenerates\sa\srun-time\nerror\srather\sthan\sa\sparsing\sconflict.\s\sThis\schanges\sis\sdue\sto\sa\sbug\sreport\non\sthe\smailing\slist.\s\sSQLite\sdoes\snot\suse\sthe\s%nonassoc\sdirective\sin\sits\ngrammar\sso\sthis\schange\sdoes\snot\saffect\sSQLite.
-D 2014-06-09T13:11:40.535
+C Enhance\sthe\sunix\sVFS\sso\sthat\sit\skeeps\strack\sof\sthe\ssize\sof\sunlinked\sfiles\ninternally\sand\sthus\savoids\sthe\sneed\sto\scall\sfstat()\son\sthose\sfiles,\ssince\nfstat()\sdoes\snot\swork\sreliably\son\sunlinked\sfiles\son\ssome\simplementations\sof\nFuseFS.
+D 2014-06-09T20:06:01.535
 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f
 F Makefile.in dd2b1aba364ff9b05de41086f74407f285c57670
 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23
@@ -205,7 +205,7 @@ F src/os.c 1b147e4cf7cc39e618115c14a086aed44bc91ace
 F src/os.h 60d419395e32a8029fa380a80a3da2e9030f635e
 F src/os_common.h 92815ed65f805560b66166e3583470ff94478f04
 F src/os_setup.h c9d4553b5aaa6f73391448b265b89bed0b890faa
-F src/os_unix.c ae4b5240af4619d711301d7992396e182585269f
+F src/os_unix.c e8c5f23bb6e9123b524cd0a87dd1e9263a4dd3be
 F src/os_win.c 8dbf6c11780fe2eb96c1f289e664d0c7b2911d37
 F src/os_win.h 057344a6720b4c8405d9bd98f58cb37a6ee46c25
 F src/pager.c f6bb1fa6cdf2062f2d8aec3e64db302bca519ab8
@@ -1174,7 +1174,10 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1
 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4
 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32
 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f
-P 813469d98519b609669a004b7b604af78ef40c02
-R 17b9b9add2b8e980cb35f959c7029199
+P 1925f3a0a2caa709569df015a8e0d26412f1a9ff
+R ce653ef517f5e803383e83307c89c5db
+T *branch * omit-fstat-after-unlink
+T *sym-omit-fstat-after-unlink *
+T -sym-trunk *
 U drh
-Z c031771471155f639666c4eafbc642f6
+Z c015ab13afaac99134c0355e5c3248ba
diff --git a/manifest.uuid b/manifest.uuid
index 6a1ff5c5e1..918b2aea7b 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-1925f3a0a2caa709569df015a8e0d26412f1a9ff
\ No newline at end of file
+c41df393c6afbfbfdc4d1b885024e083c6f6de1f
\ No newline at end of file
diff --git a/src/os_unix.c b/src/os_unix.c
index fc320a4926..ceb8cdb7f3 100644
--- a/src/os_unix.c
+++ b/src/os_unix.c
@@ -191,6 +191,7 @@ struct unixFile {
   UnixUnusedFd *pUnused;              /* Pre-allocated UnixUnusedFd */
   const char *zPath;                  /* Name of the file */
   unixShm *pShm;                      /* Shared memory segment information */
+  sqlite3_int64 szFile;               /* File size for UNIXFILE_DELETE files */
   int szChunk;                        /* Configured by FCNTL_CHUNK_SIZE */
 #if SQLITE_MAX_MMAP_SIZE>0
   int nFetchOut;                      /* Number of outstanding xFetch refs */
@@ -1321,9 +1322,11 @@ static int fileHasMoved(unixFile *pFile){
 static void verifyDbFile(unixFile *pFile){
   struct stat buf;
   int rc;
-  if( pFile->ctrlFlags & UNIXFILE_WARNED ){
-    /* One or more of the following warnings have already been issued.  Do not
-    ** repeat them so as not to clutter the error log */
+  if( pFile->ctrlFlags & (UNIXFILE_WARNED|UNIXFILE_DELETE) ){
+    /* UNIXFILE_WARNED means that one or more of the following warnings have
+    ** already been issued.  Do not* repeat them so as not to clutter the error
+    ** log.  Do not investigate unlinked files since fstat() does not always
+    ** work following an unlink(). */
     return;
   }
   rc = osFstat(pFile->h, &buf);
@@ -1332,7 +1335,7 @@ static void verifyDbFile(unixFile *pFile){
     pFile->ctrlFlags |= UNIXFILE_WARNED;
     return;
   }
-  if( buf.st_nlink==0 && (pFile->ctrlFlags & UNIXFILE_DELETE)==0 ){
+  if( buf.st_nlink==0 ){
     sqlite3_log(SQLITE_WARNING, "file unlinked while open: %s", pFile->zPath);
     pFile->ctrlFlags |= UNIXFILE_WARNED;
     return;
@@ -3281,6 +3284,10 @@ static int unixWrite(
   assert( id );
   assert( amt>0 );
 
+  /* Update the internally tracked file size.  The internal file size is only
+  ** accurate for unlinked files */
+  if( offset+amt>pFile->szFile ) pFile->szFile = offset+amt;
+
   /* If this is a database file (not a journal, master-journal or temp
   ** file), the bytes in the locking range should never be read or written. */
 #if 0
@@ -3621,6 +3628,7 @@ static int unixTruncate(sqlite3_file *id, i64 nByte){
     }
 #endif
 
+    pFile->szFile = nByte;
     return SQLITE_OK;
   }
 }
@@ -3630,12 +3638,27 @@ static int unixTruncate(sqlite3_file *id, i64 nByte){
 */
 static int unixFileSize(sqlite3_file *id, i64 *pSize){
   int rc;
+  unixFile *pFile = (unixFile*)id;
   struct stat buf;
-  assert( id );
-  rc = osFstat(((unixFile*)id)->h, &buf);
+  assert( pFile );
+
+  /* For files that have been unlinked, simply return the szFile which we keep
+  ** track of internally.  There is no need to fstat() as SQLite has exclusive
+  ** access to the file and no other process can modify the file and thus change
+  ** the file size without our knowing it.  We do this because fstat() will
+  ** fail on unlinked files on some (broken) unix filesystems.
+  */
+  if( pFile->ctrlFlags & UNIXFILE_DELETE ){
+    *pSize = pFile->szFile;
+    /* The following assert() confirms that the internal filesize is correct */
+    assert( osFstat(pFile->h, &buf)!=0 || buf.st_size==pFile->szFile );
+    return SQLITE_OK;
+  }
+
+  rc = osFstat(pFile->h, &buf);
   SimulateIOError( rc=1 );
   if( rc!=0 ){
-    ((unixFile*)id)->lastErrno = errno;
+    pFile->lastErrno = errno;
     return SQLITE_IOERR_FSTAT;
   }
   *pSize = buf.st_size;
@@ -3671,6 +3694,7 @@ static int fcntlSizeHint(unixFile *pFile, i64 nByte){
     i64 nSize;                    /* Required file size */
     struct stat buf;              /* Used to hold return values of fstat() */
    
+    if( pFile->ctrlFlags & UNIXFILE_DELETE ) return SQLITE_OK;
     if( osFstat(pFile->h, &buf) ) return SQLITE_IOERR_FSTAT;
 
     nSize = ((nByte+pFile->szChunk-1) / pFile->szChunk) * pFile->szChunk;
@@ -4219,6 +4243,7 @@ static int unixOpenSharedMemory(unixFile *pDbFd){
   if( p==0 ) return SQLITE_NOMEM;
   memset(p, 0, sizeof(*p));
   assert( pDbFd->pShm==0 );
+  assert( (pDbFd->ctrlFlags & UNIXFILE_DELETE)==0 );
 
   /* Check to see if a unixShmNode object already exists. Reuse an existing
   ** one if present. Create a new one if necessary.
@@ -4765,6 +4790,7 @@ static void unixRemapfile(
   }
   pFd->pMapRegion = (void *)pNew;
   pFd->mmapSize = pFd->mmapSizeActual = nNew;
+  if( nNew>pFd->szFile ) pFd->szFile = nNew;
 }
 
 /*
@@ -4791,12 +4817,10 @@ static int unixMapfile(unixFile *pFd, i64 nByte){
   if( pFd->nFetchOut>0 ) return SQLITE_OK;
 
   if( nMap<0 ){
-    struct stat statbuf;          /* Low-level file information */
-    rc = osFstat(pFd->h, &statbuf);
+    rc = unixFileSize((sqlite3_file*)pFd, &nMap);
     if( rc!=SQLITE_OK ){
       return SQLITE_IOERR_FSTAT;
     }
-    nMap = statbuf.st_size;
   }
   if( nMap>pFd->mmapSizeMax ){
     nMap = pFd->mmapSizeMax;
@@ -5770,6 +5794,7 @@ static int unixOpen(
   }
 
   if( isDelete ){
+    p->szFile = 0;
 #if OS_VXWORKS
     zPath = zName;
 #else