Change the ICU tokenizer so that it does not attempt to call strlen(NULL).

This is a cherry-pick of the fix in [04298f1ac42c40c] on 2012-03-31 first
appearing in release 3.7.12.

FossilOrigin-Name: 49ffbc11c5ea7e3cefd0370f39832dbfc7276ffe

diff --git a/ext/fts3/fts3_icu.c b/ext/fts3/fts3_icu.c
index a10a55d67b9da5799195c4e8863f56a220f1cb6d..5e9c900b09119a21d5916fffe75893c9c356fe94 100644 (file)
--- a/ext/fts3/fts3_icu.c
+++ b/ext/fts3/fts3_icu.c
@@ -110,7 +110,10 @@ static int icuOpen(
 
   *ppCursor = 0;
 
-  if( nInput<0 ){
+  if( zInput==0 ){
+    nInput = 0;
+    zInput = "";
+  }else if( nInput<0 ){
     nInput = strlen(zInput);
   }
   nChar = nInput+1;

diff --git a/manifest b/manifest
index 8746e1fab3d6db5fc951586758a17e33ca5a1d2b..5d39c8732fd55686e0d1b42300a98c7ef344f89c 100644 (file)
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Add\sQNX-specific\sperformance\stweaks\sto\sthe\sunix\sVFS.
-D 2012-10-05T19:10:02.654
+C Change\sthe\sICU\stokenizer\sso\sthat\sit\sdoes\snot\sattempt\sto\scall\sstrlen(NULL).\nThis\sis\sa\scherry-pick\sof\sthe\sfix\sin\s[04298f1ac42c40c]\son\s2012-03-31\sfirst\nappearing\sin\srelease\s3.7.12.
+D 2012-10-15T23:15:45.151
 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f
 F Makefile.in a162fe39e249b8ed4a65ee947c30152786cfe897
 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23
@@ -70,7 +70,7 @@ F ext/fts3/fts3_aux.c 0ebfa7b86cf8ff6a0861605fcc63b83ec1b70691
 F ext/fts3/fts3_expr.c f5df26bddf46a5916b2a5f80c4027996e92b7b15
 F ext/fts3/fts3_hash.c 8dd2d06b66c72c628c2732555a32bc0943114914
 F ext/fts3/fts3_hash.h 8331fb2206c609f9fc4c4735b9ab5ad6137c88ec
-F ext/fts3/fts3_icu.c 6c8f395cdf9e1e3afa7fadb7e523dbbf381c6dfa
+F ext/fts3/fts3_icu.c 62ec177c55f6a5c6e994dd3e5fd3194b4045c347
 F ext/fts3/fts3_porter.c 8d946908f4812c005d3d33fcbe78418b1f4eb70c
 F ext/fts3/fts3_snippet.c 1f9ee6a8e0e242649645968dcec4deb253d86c2a
 F ext/fts3/fts3_term.c a5457992723455a58804cb75c8cbd8978db5c2ef
@@ -472,7 +472,7 @@ F test/fts3corrupt.test 7b0f91780ca36118d73324ec803187208ad33b32
 F test/fts3corrupt2.test 6d96efae2f8a6af3eeaf283aba437e6d0e5447ba
 F test/fts3cov.test e0fb00d8b715ddae4a94c305992dfc3ef70353d7
 F test/fts3d.test bf640d79722b720fa1c81834c48cdaa45d531b1a
-F test/fts3defer.test 2ea3fa028f8d9523f9c33dd8acc4555d567ea4ac
+F test/fts3defer.test 6c2707be1b05b9790ba8ff91d3391d5fb425269e
 F test/fts3defer2.test 35867d33ba6db03f6c73bd6f5fc333ae14f68c81
 F test/fts3drop.test 1b906e293d6773812587b3dc458cb9e8f3f0c297
 F test/fts3e.test 1f6c6ac9cc8b772ca256e6b22aaeed50c9350851
@@ -974,7 +974,7 @@ F tool/tostr.awk e75472c2f98dd76e06b8c9c1367f4ab07e122d06
 F tool/vdbe-compress.tcl d70ea6d8a19e3571d7ab8c9b75cba86d1173ff0f
 F tool/warnings-clang.sh 9f406d66e750e8ac031c63a9ef3248aaa347ef2a
 F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381
-P cd9efeb3eb28efc6a92904bb426c3a50f95fc936
-R 8d36bc7b5efb431bf0071d11802397d7
+P b02849e7bde458dd8a2d91892f18bae7fdc7724d
+R 2a748a26dd1003c041b90c305129751f
 U drh
-Z 13a2888935ea7f47b8176999fa77e5ac
+Z 9c6de044d5854c180b1ac01f5cc72aaa

diff --git a/manifest.uuid b/manifest.uuid
index c2ebebcd1c6341450c337470170b9d119b7f79c4..a67d07b61e5841c3454f4233e3a8a33f6611a2a5 100644 (file)
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-b02849e7bde458dd8a2d91892f18bae7fdc7724d
\ No newline at end of file
+49ffbc11c5ea7e3cefd0370f39832dbfc7276ffe
\ No newline at end of file

diff --git a/test/fts3defer.test b/test/fts3defer.test
index bc50874e4a7b9de07fda627323c31d2d9e4560c7..4c8213d7f777f0e9d2076d8d703e1c08db25cd05 100644 (file)
--- a/test/fts3defer.test
+++ b/test/fts3defer.test
@@ -489,5 +489,39 @@ do_execsql_test 4.2 {
   SELECT * FROM x2 WHERE x2 MATCH 'a b c d e f g h i j k l m n o p q r s';
 } {{a b c d e f g h i j k l m n o p q r s t u v w x y m}}
 
+set tokenizers {1 simple}
+ifcapable icu { lappend tokenizers 2 {icu en_US} }
+foreach {tn tokenizer} $tokenizers {
+  do_execsql_test 5.$tn.1 "
+    CREATE VIRTUAL TABLE x3 USING FTS4(a, b, TOKENIZE $tokenizer)
+  "
+  do_execsql_test 5.$tn.2 {
+    BEGIN;
+    INSERT INTO x3 VALUES('b b b b b b b b b b b', 'b b b b b b b b b b b b b');
+    INSERT INTO x3 SELECT * FROM x3;
+    INSERT INTO x3 SELECT * FROM x3;
+    INSERT INTO x3 SELECT * FROM x3;
+    INSERT INTO x3 SELECT * FROM x3;
+    INSERT INTO x3 SELECT * FROM x3;
+    INSERT INTO x3 SELECT * FROM x3;
+    INSERT INTO x3 SELECT * FROM x3;
+    INSERT INTO x3 SELECT * FROM x3;
+    INSERT INTO x3 SELECT * FROM x3;
+    INSERT INTO x3 SELECT * FROM x3;
+    INSERT INTO x3 SELECT * FROM x3;
+    INSERT INTO x3 SELECT * FROM x3;
+    INSERT INTO x3 SELECT * FROM x3;
+    INSERT INTO x3 SELECT * FROM x3;
+    INSERT INTO x3 SELECT * FROM x3;
+    INSERT INTO x3 SELECT * FROM x3;
+    INSERT INTO x3 VALUES('a b c', NULL);
+    INSERT INTO x3 VALUES('a x c', NULL);
+    COMMIT;
+
+    SELECT * FROM x3 WHERE x3 MATCH 'a b';
+  } {{a b c} {}}
+
+  do_execsql_test 5.$tn.3 { DROP TABLE x3 }
+}
 
 finish_test