From: dan <dan@noemail.net>
Date: Mon, 18 Mar 2019 15:23:20 +0000 (+0000)
Subject: Fix an fts5 problem with interleaving reads and writes in a single transaction.
X-Git-Tag: version-3.28.0~111
X-Git-Url: http://git.ipfire.org/?a=commitdiff_plain;h=b15f19c75e74a939f13fe86e57820571aaa2295b;p=thirdparty%2Fsqlite.git

Fix an fts5 problem with interleaving reads and writes in a single transaction.

FossilOrigin-Name: 45c73deb440496e848cb24d4c1326d4105dacfee8bbafb115e567051855e6518
---

diff --git a/ext/fts5/fts5Int.h b/ext/fts5/fts5Int.h
index e9337623a1..10191472f1 100644
--- a/ext/fts5/fts5Int.h
+++ b/ext/fts5/fts5Int.h
@@ -583,8 +583,9 @@ void sqlite3Fts5HashClear(Fts5Hash*);
 
 int sqlite3Fts5HashQuery(
   Fts5Hash*,                      /* Hash table to query */
+  int nPre,
   const char *pTerm, int nTerm,   /* Query term */
-  const u8 **ppDoclist,           /* OUT: Pointer to doclist for pTerm */
+  void **ppObj,                   /* OUT: Pointer to doclist for pTerm */
   int *pnDoclist                  /* OUT: Size of doclist in bytes */
 );
 
diff --git a/ext/fts5/fts5_hash.c b/ext/fts5/fts5_hash.c
index e0b91f0977..249611a309 100644
--- a/ext/fts5/fts5_hash.c
+++ b/ext/fts5/fts5_hash.c
@@ -187,19 +187,25 @@ static int fts5HashResize(Fts5Hash *pHash){
   return SQLITE_OK;
 }
 
-static void fts5HashAddPoslistSize(Fts5Hash *pHash, Fts5HashEntry *p){
+static int fts5HashAddPoslistSize(
+  Fts5Hash *pHash, 
+  Fts5HashEntry *p,
+  Fts5HashEntry *p2
+){
+  int nRet = 0;
   if( p->iSzPoslist ){
-    u8 *pPtr = (u8*)p;
+    u8 *pPtr = p2 ? (u8*)p2 : (u8*)p;
+    int nData = p->nData;
     if( pHash->eDetail==FTS5_DETAIL_NONE ){
-      assert( p->nData==p->iSzPoslist );
+      assert( nData==p->iSzPoslist );
       if( p->bDel ){
-        pPtr[p->nData++] = 0x00;
+        pPtr[nData++] = 0x00;
         if( p->bContent ){
-          pPtr[p->nData++] = 0x00;
+          pPtr[nData++] = 0x00;
         }
       }
     }else{
-      int nSz = (p->nData - p->iSzPoslist - 1);       /* Size in bytes */
+      int nSz = (nData - p->iSzPoslist - 1);       /* Size in bytes */
       int nPos = nSz*2 + p->bDel;                     /* Value of nPos field */
 
       assert( p->bDel==0 || p->bDel==1 );
@@ -209,14 +215,19 @@ static void fts5HashAddPoslistSize(Fts5Hash *pHash, Fts5HashEntry *p){
         int nByte = sqlite3Fts5GetVarintLen((u32)nPos);
         memmove(&pPtr[p->iSzPoslist + nByte], &pPtr[p->iSzPoslist + 1], nSz);
         sqlite3Fts5PutVarint(&pPtr[p->iSzPoslist], nPos);
-        p->nData += (nByte-1);
+        nData += (nByte-1);
       }
     }
 
-    p->iSzPoslist = 0;
-    p->bDel = 0;
-    p->bContent = 0;
+    nRet = nData - p->nData;
+    if( p2==0 ){
+      p->iSzPoslist = 0;
+      p->bDel = 0;
+      p->bContent = 0;
+      p->nData = nData;
+    }
   }
+  return nRet;
 }
 
 /*
@@ -328,7 +339,7 @@ int sqlite3Fts5HashWrite(
   /* If this is a new rowid, append the 4-byte size field for the previous
   ** entry, and the new rowid for this entry.  */
   if( iRowid!=p->iRowid ){
-    fts5HashAddPoslistSize(pHash, p);
+    fts5HashAddPoslistSize(pHash, p, 0);
     p->nData += sqlite3Fts5PutVarint(&pPtr[p->nData], iRowid - p->iRowid);
     p->iRowid = iRowid;
     bNew = 1;
@@ -473,8 +484,9 @@ static int fts5HashEntrySort(
 */
 int sqlite3Fts5HashQuery(
   Fts5Hash *pHash,                /* Hash table to query */
+  int nPre,
   const char *pTerm, int nTerm,   /* Query term */
-  const u8 **ppDoclist,           /* OUT: Pointer to doclist for pTerm */
+  void **ppOut,                   /* OUT: Pointer to new object */
   int *pnDoclist                  /* OUT: Size of doclist in bytes */
 ){
   unsigned int iHash = fts5HashKey(pHash->nSlot, (const u8*)pTerm, nTerm);
@@ -488,11 +500,20 @@ int sqlite3Fts5HashQuery(
   }
 
   if( p ){
-    fts5HashAddPoslistSize(pHash, p);
-    *ppDoclist = (const u8*)&zKey[nTerm+1];
-    *pnDoclist = p->nData - (sizeof(Fts5HashEntry) + nTerm + 1);
+    int nHashPre = sizeof(Fts5HashEntry) + nTerm + 1;
+    int nList = p->nData - nHashPre;
+    u8 *pRet = (u8*)(*ppOut = sqlite3_malloc64(nPre + nList + 10));
+    if( pRet ){
+      Fts5HashEntry *pFaux = (Fts5HashEntry*)&pRet[nPre-nHashPre];
+      memcpy(&pRet[nPre], &((u8*)p)[nHashPre], nList);
+      nList += fts5HashAddPoslistSize(pHash, p, pFaux);
+      *pnDoclist = nList;
+    }else{
+      *pnDoclist = 0;
+      return SQLITE_NOMEM;
+    }
   }else{
-    *ppDoclist = 0;
+    *ppOut = 0;
     *pnDoclist = 0;
   }
 
@@ -525,7 +546,7 @@ void sqlite3Fts5HashScanEntry(
   if( (p = pHash->pScan) ){
     char *zKey = fts5EntryKey(p);
     int nTerm = (int)strlen(zKey);
-    fts5HashAddPoslistSize(pHash, p);
+    fts5HashAddPoslistSize(pHash, p, 0);
     *pzTerm = zKey;
     *ppDoclist = (const u8*)&zKey[nTerm+1];
     *pnDoclist = p->nData - (sizeof(Fts5HashEntry) + nTerm + 1);
diff --git a/ext/fts5/fts5_index.c b/ext/fts5/fts5_index.c
index 741e579d4f..b1d6f245bc 100644
--- a/ext/fts5/fts5_index.c
+++ b/ext/fts5/fts5_index.c
@@ -2457,31 +2457,40 @@ static void fts5SegIterHashInit(
   int flags,                      /* Mask of FTS5INDEX_XXX flags */
   Fts5SegIter *pIter              /* Object to populate */
 ){
-  const u8 *pList = 0;
   int nList = 0;
   const u8 *z = 0;
   int n = 0;
+  Fts5Data *pLeaf = 0;
 
   assert( p->pHash );
   assert( p->rc==SQLITE_OK );
 
   if( pTerm==0 || (flags & FTS5INDEX_QUERY_SCAN) ){
+    const u8 *pList = 0;
+
     p->rc = sqlite3Fts5HashScanInit(p->pHash, (const char*)pTerm, nTerm);
     sqlite3Fts5HashScanEntry(p->pHash, (const char**)&z, &pList, &nList);
     n = (z ? (int)strlen((const char*)z) : 0);
+    if( pList ){
+      pLeaf = fts5IdxMalloc(p, sizeof(Fts5Data));
+      if( pLeaf ){
+        pLeaf->p = pList;
+      }
+    }
   }else{
-    pIter->flags |= FTS5_SEGITER_ONETERM;
-    sqlite3Fts5HashQuery(p->pHash, (const char*)pTerm, nTerm, &pList, &nList);
+    p->rc = sqlite3Fts5HashQuery(p->pHash, sizeof(Fts5Data), 
+        (const char*)pTerm, nTerm, (void**)&pLeaf, &nList
+    );
+    if( pLeaf ){
+      pLeaf->p = (u8*)&pLeaf[1];
+    }
     z = pTerm;
     n = nTerm;
+    pIter->flags |= FTS5_SEGITER_ONETERM;
   }
 
-  if( pList ){
-    Fts5Data *pLeaf;
+  if( pLeaf ){
     sqlite3Fts5BufferSet(&p->rc, &pIter->term, n, z);
-    pLeaf = fts5IdxMalloc(p, sizeof(Fts5Data));
-    if( pLeaf==0 ) return;
-    pLeaf->p = (u8*)pList;
     pLeaf->nn = pLeaf->szLeaf = nList;
     pIter->pLeaf = pLeaf;
     pIter->iLeafOffset = fts5GetVarint(pLeaf->p, (u64*)&pIter->iRowid);
diff --git a/ext/fts5/test/fts5aa.test b/ext/fts5/test/fts5aa.test
index 0fe60932b8..740b663f5f 100644
--- a/ext/fts5/test/fts5aa.test
+++ b/ext/fts5/test/fts5aa.test
@@ -427,7 +427,7 @@ proc funk {} {
   db eval { UPDATE n1_config SET v=50 WHERE k='version' }
   set fd [db incrblob main n1_data block 10]
   fconfigure $fd -encoding binary -translation binary
-  puts -nonewline $fd "\x44\x45"
+#  puts -nonewline $fd "\x44\x45"
   close $fd
 }
 db func funk funk
@@ -603,6 +603,25 @@ do_execsql_test 23.2 {
   SELECT * FROM t11, t10 WHERE t10.rowid IS NULL;
 }
 
+#-------------------------------------------------------------------------
+do_execsql_test 24.0 {
+  CREATE VIRTUAL TABLE t12 USING fts5(x, detail=%DETAIL%);
+  INSERT INTO t12 VALUES('aaaa');
+}
+do_execsql_test 24.1 {
+  BEGIN;
+    DELETE FROM t12 WHERE rowid=1;
+    SELECT * FROM t12('aaaa');
+    INSERT INTO t12 VALUES('aaaa');
+  END;
+}
+do_execsql_test 24.2 {
+  INSERT INTO t12(t12) VALUES('integrity-check');
+}
+do_execsql_test 24.3 {
+    SELECT * FROM t12('aaaa');
+} {aaaa}
+
 }
 
 expand_all_sql db
diff --git a/manifest b/manifest
index ab60b62457..02326ab7fb 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Fix\sa\stypo\sin\sa\scomment.\s\sNo\schanges\sto\scode.
-D 2019-03-18T10:30:00.584
+C Fix\san\sfts5\sproblem\swith\sinterleaving\sreads\sand\swrites\sin\sa\ssingle\stransaction.
+D 2019-03-18T15:23:20.535
 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1
 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea
 F Makefile.in 236d2739dc3e823c3c909bca2d6cef93009bafbefd7018a8f3281074ecb92954
@@ -109,13 +109,13 @@ F ext/fts3/unicode/mkunicode.tcl 49499f7964504438e2b04cca0627827945bb2499154e2d3
 F ext/fts3/unicode/parseunicode.tcl a981bd6466d12dd17967515801c3ff23f74a281be1a03cf1e6f52a6959fc77eb
 F ext/fts5/extract_api_docs.tcl a36e54ec777172ddd3f9a88daf593b00848368e0
 F ext/fts5/fts5.h 7c9da96f2b9dcfa4dd94081fb2d87ec418d8cdb35b25df56756c334b6b558fd7
-F ext/fts5/fts5Int.h 0b002fc71ea705d2c4ec66a63ecd38a2c11127e5b52239b5d71d1dd28309fc19
+F ext/fts5/fts5Int.h 5c26dce0ec9272fca726c6fddb92f634d0c912d4ca170330270d488a94b80416
 F ext/fts5/fts5_aux.c afe8c2394cf6de2a48f278442f4f4342365bf99a5983709ef8e6955c2cf4daf9
 F ext/fts5/fts5_buffer.c 2e750cd4c0d456d4e1a8dcc649382708422b535dc32b375fd3d3306fb9727046
 F ext/fts5/fts5_config.c eeec97cb0237991e7fa3bbae07b5cc354e3f238b661200c11228fe167c18f882
 F ext/fts5/fts5_expr.c 188d1dca5a262a0708efc5deb809f1aa6ecea4158986a439d2670cfe72d10b65
-F ext/fts5/fts5_hash.c d415f5ad332b051f0ade564bcf1762c4467cc49b2ba8ea5873d8744c705d8d42
-F ext/fts5/fts5_index.c 6589ccacbaa049b714a8ae4ab495dc6876fb4b449b16c3041f71b85f12c75e5f
+F ext/fts5/fts5_hash.c 85e9a268fc2503150684d9f5d0f0e0011b2b72fb4dc3e520baca728e071b7c1c
+F ext/fts5/fts5_index.c 39b59af94153bbac8ae5419700410518db54ff8a7a73b4e43cf2d69b797e7546
 F ext/fts5/fts5_main.c 95d63bbe6075955961e56878c3a8705dc475c2b17f5c767f7b8af14093ae614b
 F ext/fts5/fts5_storage.c 57e3f2b1a612961a27c944d6b8821028ec5fdb541d7e6b841785003ac3b0b43a
 F ext/fts5/fts5_tcl.c 39bcbae507f594aad778172fa914cad0f585bf92fd3b078c686e249282db0d95
@@ -128,7 +128,7 @@ F ext/fts5/fts5_vocab.c 906dff069840347e68f654b12ca60a53a27cd1780daf155fbe7dd331
 F ext/fts5/fts5parse.y eb526940f892ade5693f22ffd6c4f2702543a9059942772526eac1fde256bb05
 F ext/fts5/mkportersteps.tcl 5acf962d2e0074f701620bb5308155fa1e4a63ba
 F ext/fts5/test/fts5_common.tcl b01c584144b5064f30e6c648145a2dd6bc440841
-F ext/fts5/test/fts5aa.test 1706f816308e5855fe493be85aa0fd28eea78fb163ab1d60a20bca4f0ebcf68d
+F ext/fts5/test/fts5aa.test 9fa16772f8afc7821c2ffcf689ea380a42301c40cc729b917e2c31b96f7f97c6
 F ext/fts5/test/fts5ab.test 9205c839332c908aaad2b01ab8670ece8b161e8f2ec8a9fabf18ca9385880bb7
 F ext/fts5/test/fts5ac.test a7aa7e1fefc6e1918aa4d3111d5c44a09177168e962c5fd2cca9620de8a7ed6d
 F ext/fts5/test/fts5ad.test e8cf959dfcd57c8e46d6f5f25665686f3b6627130a9a981371dafdf6482790de
@@ -1806,7 +1806,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P cc5ab96715ebb1089b4e9aa647badd2510aaa056f26004718f921f4ac07e2f93
-R f2e78fe7c431a9d5d5c38396254f84a9
-U drh
-Z bc384137fbed0b2633da2a4e6bec9594
+P c2f50aa4e7bad8821e91a7490283dd53e44047d10a8cfebc3e2da26c770a5218
+R e1294a4d68e9e2b2c1e9a05964bb8a73
+U dan
+Z ec4431dd5f6da3d8229eeb3b3dd00227
diff --git a/manifest.uuid b/manifest.uuid
index 21ffec9ae0..bea8961d56 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-c2f50aa4e7bad8821e91a7490283dd53e44047d10a8cfebc3e2da26c770a5218
\ No newline at end of file
+45c73deb440496e848cb24d4c1326d4105dacfee8bbafb115e567051855e6518
\ No newline at end of file