Avoid an assert() failure when querying an fts5vocab table that accesses a tokendata...

FossilOrigin-Name: 386ba9e20423fb2f623d6adc9d3c310fb1b135f54a1dad15ef3b593d97886926

diff --git a/ext/fts5/fts5_index.c b/ext/fts5/fts5_index.c
index 735d06c29f67e0fe1815435bca06f57d9da7dfa0..902af552a643e7864a5ce040835a4e2bf71a97df 100644 (file)
--- a/ext/fts5/fts5_index.c
+++ b/ext/fts5/fts5_index.c
@@ -7031,6 +7031,10 @@ int sqlite3Fts5IndexQuery(
     int bTokendata = pConfig->bTokendata;
     if( nToken>0 ) memcpy(&buf.p[1], pToken, nToken);
 
+    if( flags & (FTS5INDEX_QUERY_NOTOKENDATA|FTS5INDEX_QUERY_SCAN) ){
+      bTokendata = 0;
+    }
+
     /* Figure out which index to search and set iIdx accordingly. If this
     ** is a prefix query for which there is no prefix index, set iIdx to
     ** greater than pConfig->nPrefix to indicate that the query will be
@@ -7042,7 +7046,6 @@ int sqlite3Fts5IndexQuery(
     ** for internal sanity checking by the integrity-check in debug 
     ** mode only.  */
 #ifdef SQLITE_DEBUG
-    if( flags & FTS5INDEX_QUERY_NOTOKENDATA ) bTokendata = 0;
     if( pConfig->bPrefixIndex==0 || (flags & FTS5INDEX_QUERY_TEST_NOIDX) ){
       assert( flags & FTS5INDEX_QUERY_PREFIX );
       iIdx = 1+pConfig->nPrefix;

diff --git a/ext/fts5/test/fts5vocab2.test b/ext/fts5/test/fts5vocab2.test
index 6f7aad329c4fd9e569748e2385e1a35c47bae0ec..ecacc50dab8b3169a12ba5c4e0ff98f2e56eed3f 100644 (file)
--- a/ext/fts5/test/fts5vocab2.test
+++ b/ext/fts5/test/fts5vocab2.test
@@ -280,6 +280,30 @@ do_catchsql_test 5.2 {
   INSERT INTO t1 SELECT randomblob(3000) FROM v1
 } {1 {query aborted}}
 
+#-------------------------------------------------------------------------
+reset_db
+sqlite3_fts5_may_be_corrupt 1
+
+do_execsql_test 6.0 {
+  BEGIN TRANSACTION;
+    CREATE VIRTUAL TABLE t1 USING fts5(a,b unindexed,c,tokenize="porter ascii",tokendata=1);
+    REPLACE INTO t1_data VALUES(1,X'03090009');
+    REPLACE INTO t1_data VALUES(10,X'000000000103030003010101020101030101');
+    REPLACE INTO t1_data VALUES(137438953473,X'0000002e023061010202010162010203010163010204010167010601020201016801060102030101690106010204040606060808');
+    REPLACE INTO t1_data VALUES(274877906945,X'0000001f013067020802010202010168020803010203010169020804010204040909');
+    REPLACE INTO t1_data VALUES(412316860417,X'0000002e023061030202010162030203010163030204010167030601020201016803060102030101690306010204040606060808');
+  COMMIT;
+}
+
+do_execsql_test 6.1 {
+  CREATE VIRTUAL TABLE t3 USING fts5vocab('t1', 'row');
+}
+
+do_catchsql_test 6.2 {
+  SELECT * FROM t3;
+} {1 {database disk image is malformed}}
+
+sqlite3_fts5_may_be_corrupt 0
 
 finish_test
 

diff --git a/manifest b/manifest
index f6c5d3e86242f637ad46d19e99e51a17a0074a1b..43fe098e6cc27ca09e75e113756c6a72f43412d0 100644 (file)
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Fix\sa\snull-pointer\sdereference\sin\sfts5\stokendata=1\scode.
-D 2023-12-07T19:08:25.647
+C Avoid\san\sassert()\sfailure\swhen\squerying\san\sfts5vocab\stable\sthat\saccesses\sa\stokendata=1\sfts5\stable\swith\scorrupt\s%_data\srecords.
+D 2023-12-07T20:46:40.220
 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1
 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea
 F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724
@@ -96,7 +96,7 @@ F ext/fts5/fts5_buffer.c 3001fbabb585d6de52947b44b455235072b741038391f830d6b7292
 F ext/fts5/fts5_config.c 8072a207034b51ae9b7694121d1b5715c794e94b275e088f70ae532378ca5cdf
 F ext/fts5/fts5_expr.c b1ec526371b9ffde82341423a5b9753c42cbea629a41b69f26fa377d13b95a8e
 F ext/fts5/fts5_hash.c adda4272be401566a6e0ba1acbe70ee5cb97fce944bc2e04dc707152a0ec91b1
-F ext/fts5/fts5_index.c 1193379f19168e8ccb4831d07355a700459a004bd8ca56c9255c8e2d515b813a
+F ext/fts5/fts5_index.c 3b9f7245d5ff80770ea959030f33c9cb686b6e2c655babee81a849094376742f
 F ext/fts5/fts5_main.c b908696c52410e8383019ac0657c8a5cd0c8f60e78edc169e9b3c4b93f24c933
 F ext/fts5/fts5_storage.c 5d10b9bdcce5b90656cad13c7d12ad4148677d4b9e3fca0481fca56d6601426d
 F ext/fts5/fts5_tcl.c cf0fd0dbe64ec272491b749e0d594f563cda03336aeb60900129e6d18b0aefb8
@@ -238,7 +238,7 @@ F ext/fts5/test/fts5unindexed.test 9021af86a0fb9fc616f7a69a996db0116e7936d0db638
 F ext/fts5/test/fts5update.test b8affd796e45c94a4d19ad5c26606ea06065a0f162a9562d9f005b5a80ccf0bc
 F ext/fts5/test/fts5version.test d6e5a5897550afeccc2f8531d87404dc1c289ee89385dd4318dbdd75e71d7a67
 F ext/fts5/test/fts5vocab.test 7ed80d9af1ddaaa1637da05e406327b5aac250848bc604c1c1cc667908b87760
-F ext/fts5/test/fts5vocab2.test 681980e92e031c9f3fe8d9c149189e876c108da2fb0fb3a25bd8a9b94bff8f68
+F ext/fts5/test/fts5vocab2.test 1b1f0059f762ffb404213d35dac013e010621f08128589b6ec7bec59d9a710f3
 F ext/fts5/tool/fts5speed.tcl b0056f91a55b2d1a3684ec05729de92b042e2f85
 F ext/fts5/tool/fts5txt2db.tcl c0d43c8590656f8240e622b00957b3a0facc49482411a9fdc2870b45c0c82f9f
 F ext/fts5/tool/loadfts5.tcl 95b03429ee6b138645703c6ca192c3ac96eaf093
@@ -2153,8 +2153,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P bc911ab5953532956510c199be72b1d3c556f2d0ddbd7fc0ae6f5f917b337b48
-R 75b0f8b88efb2078e2ea2aad5e87b3ae
+P d69fa8f0504887f968d9a190ecb889ddb40bb1b56d0d4479f9819c106aec719b
+R 296885500e216c301696a13ea32c2874
 U dan
-Z 824fa07c60b09c8dc172e4af722aed18
+Z 332eede4eabfd6d41dcad2a894a2ef97
 # Remove this line to create a well-formed Fossil manifest.

diff --git a/manifest.uuid b/manifest.uuid
index 3806ce1ffe24f14054fe185c53be169b589528b0..fdfae31f28d771aa9266c233325cd6fe120d5f3a 100644 (file)
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-d69fa8f0504887f968d9a190ecb889ddb40bb1b56d0d4479f9819c106aec719b
\ No newline at end of file
+386ba9e20423fb2f623d6adc9d3c310fb1b135f54a1dad15ef3b593d97886926
\ No newline at end of file