From: dan <dan@noemail.net>
Date: Fri, 12 Feb 2016 17:56:27 +0000 (+0000)
Subject: Fix a fairly obscure buffer overread in fts5.
X-Git-Tag: version-3.11.0~4
X-Git-Url: http://git.ipfire.org/?a=commitdiff_plain;h=f2d328fa2542b279aa2642b9c7a19e1b39ff53bf;p=thirdparty%2Fsqlite.git

Fix a fairly obscure buffer overread in fts5.

FossilOrigin-Name: 130580207ab5cee762b2893808acef7c8afad027
---

diff --git a/ext/fts5/fts5Int.h b/ext/fts5/fts5Int.h
index dc1954beba..af40412167 100644
--- a/ext/fts5/fts5Int.h
+++ b/ext/fts5/fts5Int.h
@@ -26,6 +26,7 @@ SQLITE_EXTENSION_INIT1
 typedef unsigned char  u8;
 typedef unsigned int   u32;
 typedef unsigned short u16;
+typedef short i16;
 typedef sqlite3_int64 i64;
 typedef sqlite3_uint64 u64;
 
diff --git a/ext/fts5/fts5_hash.c b/ext/fts5/fts5_hash.c
index 50ca082711..ada8bb16cb 100644
--- a/ext/fts5/fts5_hash.c
+++ b/ext/fts5/fts5_hash.c
@@ -62,10 +62,10 @@ struct Fts5HashEntry {
   int nAlloc;                     /* Total size of allocation */
   int iSzPoslist;                 /* Offset of space for 4-byte poslist size */
   int nData;                      /* Total bytes of data (incl. structure) */
+  int nKey;                       /* Length of zKey[] in bytes */
   u8 bDel;                        /* Set delete-flag @ iSzPoslist */
   u8 bContent;                    /* Set content-flag (detail=none mode) */
-
-  int iCol;                       /* Column of last value written */
+  i16 iCol;                       /* Column of last value written */
   int iPos;                       /* Position of last value written */
   i64 iRowid;                     /* Rowid of last value written */
   char zKey[8];                   /* Nul-terminated entry key */
@@ -245,8 +245,8 @@ int sqlite3Fts5HashWrite(
   iHash = fts5HashKey2(pHash->nSlot, (u8)bByte, (const u8*)pToken, nToken);
   for(p=pHash->aSlot[iHash]; p; p=p->pHashNext){
     if( p->zKey[0]==bByte 
+     && p->nKey==nToken
      && memcmp(&p->zKey[1], pToken, nToken)==0 
-     && p->zKey[nToken+1]==0 
     ){
       break;
     }
@@ -273,6 +273,7 @@ int sqlite3Fts5HashWrite(
     p->zKey[0] = bByte;
     memcpy(&p->zKey[1], pToken, nToken);
     assert( iHash==fts5HashKey(pHash->nSlot, (u8*)p->zKey, nToken+1) );
+    p->nKey = nToken;
     p->zKey[nToken+1] = '\0';
     p->nData = nToken+1 + 1 + FTS5_HASHENTRYSIZE;
     p->pHashNext = pHash->aSlot[iHash];
diff --git a/ext/fts5/test/fts5hash.test b/ext/fts5/test/fts5hash.test
index ac8486f070..f3952d6157 100644
--- a/ext/fts5/test/fts5hash.test
+++ b/ext/fts5/test/fts5hash.test
@@ -66,47 +66,66 @@ proc random_doc {vocab nWord} {
 
 foreach_detail_mode $testprefix {
 
-set vocab [build_vocab1]
-db func r random_doc 
-
-do_execsql_test 1.0 {
-  CREATE VIRTUAL TABLE eee USING fts5(e, ee, detail=%DETAIL%);
-  BEGIN;
-    WITH ii(i) AS (SELECT 1 UNION ALL SELECT i+1 FROM ii WHERE i<100)
-    INSERT INTO eee SELECT r($vocab, 5), r($vocab, 7) FROM ii;
+  set vocab [build_vocab1]
+  db func r random_doc 
+  
+  do_execsql_test 1.0 {
+    CREATE VIRTUAL TABLE eee USING fts5(e, ee, detail=%DETAIL%);
+    BEGIN;
+      WITH ii(i) AS (SELECT 1 UNION ALL SELECT i+1 FROM ii WHERE i<100)
+      INSERT INTO eee SELECT r($vocab, 5), r($vocab, 7) FROM ii;
+      INSERT INTO eee(eee) VALUES('integrity-check');
+    COMMIT;
     INSERT INTO eee(eee) VALUES('integrity-check');
-  COMMIT;
-  INSERT INTO eee(eee) VALUES('integrity-check');
-}
-
-set hash [sqlite3_fts5_token_hash 1024 xyz]
-set vocab [build_vocab1 -prefix xyz -hash $hash]
-lappend vocab xyz
-
-do_execsql_test 1.1 {
-  CREATE VIRTUAL TABLE vocab USING fts5vocab(eee, 'row'); 
-  BEGIN;
-}
-do_test 1.2 {
-  for {set i 1} {$i <= 100} {incr i} {
-    execsql { INSERT INTO eee VALUES( r($vocab, 5), r($vocab, 7) ) }
   }
-} {}
   
-do_test 1.3 {
-  db eval { SELECT term, doc FROM vocab } {
-    set nRow [db one {SELECT count(*) FROM eee WHERE eee MATCH $term}]
-    if {$nRow != $doc} {
-      error "term=$term fts5vocab=$doc cnt=$nRow"
+  set hash [sqlite3_fts5_token_hash 1024 xyz]
+  set vocab [build_vocab1 -prefix xyz -hash $hash]
+  lappend vocab xyz
+  
+  do_execsql_test 1.1 {
+    CREATE VIRTUAL TABLE vocab USING fts5vocab(eee, 'row'); 
+    BEGIN;
+  }
+  do_test 1.2 {
+    for {set i 1} {$i <= 100} {incr i} {
+      execsql { INSERT INTO eee VALUES( r($vocab, 5), r($vocab, 7) ) }
     }
+  } {}
+    
+  do_test 1.3 {
+    db eval { SELECT term, doc FROM vocab } {
+      set nRow [db one {SELECT count(*) FROM eee WHERE eee MATCH $term}]
+      if {$nRow != $doc} {
+        error "term=$term fts5vocab=$doc cnt=$nRow"
+      }
+    }
+    set {} {}
+  } {}
+  
+  do_execsql_test 1.4 {
+    COMMIT;
+    INSERT INTO eee(eee) VALUES('integrity-check');
   }
-  set {} {}
-} {}
 
-do_execsql_test 1.4 {
-  COMMIT;
-  INSERT INTO eee(eee) VALUES('integrity-check');
-}
+  #-----------------------------------------------------------------------
+  # Add a small and very large token with the same hash value to an
+  # empty table. At one point this would provoke an asan error.
+  #
+  do_test 2.0 {
+    set big [string repeat 12345 40]
+    set hash [sqlite3_fts5_token_hash 1024 $big]
+    while {1} {
+      set small [random_token]
+      if {[sqlite3_fts5_token_hash 1024 $small]==$hash} break
+    }
+
+    execsql { CREATE VIRTUAL TABLE t2 USING fts5(x, detail=%DETAIL%) }
+breakpoint
+    execsql {
+      INSERT INTO t2 VALUES($small || ' ' || $big);
+    }
+  } {}
 
 } ;# foreach_detail_mode
 
diff --git a/manifest b/manifest
index ba9bd0d561..0dad1bc0fe 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Fix\sa\sdocumentation\stypo.\s\sNo\schanges\sto\scode.
-D 2016-02-12T17:30:39.594
+C Fix\sa\sfairly\sobscure\sbuffer\soverread\sin\sfts5.
+D 2016-02-12T17:56:27.467
 F Makefile.in 4e90dc1521879022aa9479268a4cd141d1771142
 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434
 F Makefile.msc 30f075dc4f27a07abb76088946b2944178d85347
@@ -98,12 +98,12 @@ F ext/fts3/unicode/mkunicode.tcl 95cf7ec186e48d4985e433ff8a1c89090a774252
 F ext/fts3/unicode/parseunicode.tcl da577d1384810fb4e2b209bf3313074353193e95
 F ext/fts5/extract_api_docs.tcl a36e54ec777172ddd3f9a88daf593b00848368e0
 F ext/fts5/fts5.h ff9c2782e8ed890b0de2f697a8d63971939e70c7
-F ext/fts5/fts5Int.h f9e7772d3ad2b8aac6ad77a5867a254f4422992e
+F ext/fts5/fts5Int.h fa7c17e5c3ec9c8690387ff962f9dc6aee75e114
 F ext/fts5/fts5_aux.c daa57fb45216491814520bbb587e97bf81ced458
 F ext/fts5/fts5_buffer.c 4c1502d4c956cd092c89ce4480867f9d8bf325cd
 F ext/fts5/fts5_config.c 35c5173cae4eb17e82164a7f5aeef56a48903079
 F ext/fts5/fts5_expr.c 8e8e4635f655133eb39018072fc0f0942a2c4337
-F ext/fts5/fts5_hash.c 1b113977296cf4212c6ec667d5e3f2bd18036955
+F ext/fts5/fts5_hash.c f3a7217c86eb8f272871be5f6aa1b6798960a337
 F ext/fts5/fts5_index.c c6cd5190c7e5dd94151cb17acd939c82e5c7be2d
 F ext/fts5/fts5_main.c 0e01ead4e817483e378e7e38e6d902f50b68d29e
 F ext/fts5/fts5_storage.c f8343db90d8c95a4d4b52f6676e354b4649ffd6e
@@ -158,7 +158,7 @@ F ext/fts5/test/fts5fault8.test 430837fe6dd0511fd3aea52bd602ac02441bcb58
 F ext/fts5/test/fts5fault9.test e10e395428a9ea0596ebe752ff7123d16ab78e08
 F ext/fts5/test/fts5faultA.test fa5d59c0ff62b7125cd14eee38ded1c46e15a7ea
 F ext/fts5/test/fts5full.test 6f6143af0c6700501d9fd597189dfab1555bb741
-F ext/fts5/test/fts5hash.test 00668f6fa9b9bffbd7c1be29f408aa2bdade0451
+F ext/fts5/test/fts5hash.test 06f9309ccb4d5050a131594e9e47d0b21456837d
 F ext/fts5/test/fts5integrity.test f5e4f8d284385875068ad0f3e894ce43e9de835d
 F ext/fts5/test/fts5matchinfo.test 86569026d20f1ed748236587ce798de8a96615f1
 F ext/fts5/test/fts5merge.test 8f3cdba2ec9c5e7e568246e81b700ad37f764367
@@ -1427,7 +1427,7 @@ F tool/vdbe_profile.tcl 246d0da094856d72d2c12efec03250d71639d19f
 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4
 F tool/warnings.sh ef6ebc6fd8d2dc35db3b622015c16a023d4fef4f
 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f
-P 6ea84ec1e02f0a48953bc758027ab7148529c36e
-R 32f39dbecc6e1b0fd64cf3f801eef486
-U drh
-Z 253162cb9c4e3721df2895ed3fc41a73
+P d9c985878cf7c74a79ccdb9cf94a2d48fdde3e6f
+R 0a862bd7a75e07028d820b1669759dd1
+U dan
+Z 281d4ece2b41724c1d776859918d3659
diff --git a/manifest.uuid b/manifest.uuid
index ef66c089c1..d57148a112 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-d9c985878cf7c74a79ccdb9cf94a2d48fdde3e6f
\ No newline at end of file
+130580207ab5cee762b2893808acef7c8afad027
\ No newline at end of file