Fix problems with combining content= and languageid= in a single fts4 table.

FossilOrigin-Name: 22491e7bc38aee43819b888e04241cb6a6ef73a3

diff --git a/ext/fts3/fts3.c b/ext/fts3/fts3.c
index 6732edaece0187e96fe92bddf2d2fe731ac22d78..dec7f8722c5fe84dd57083dfd897942c025721a4 100644 (file)
--- a/ext/fts3/fts3.c
+++ b/ext/fts3/fts3.c
@@ -800,13 +800,18 @@ static char *fts3ReadExprList(Fts3Table *p, const char *zFunc, int *pRc){
     for(i=0; i<p->nColumn; i++){
       fts3Appendf(pRc, &zRet, ",%s(x.'c%d%q')", zFunction, i, p->azColumn[i]);
     }
-    if( p->zLanguageid ) fts3Appendf(pRc, &zRet, ",langid");
+    if( p->zLanguageid ){
+      fts3Appendf(pRc, &zRet, ", x.%Q", "langid");
+    }
     sqlite3_free(zFree);
   }else{
     fts3Appendf(pRc, &zRet, "rowid");
     for(i=0; i<p->nColumn; i++){
       fts3Appendf(pRc, &zRet, ", x.'%q'", p->azColumn[i]);
     }
+    if( p->zLanguageid ){
+      fts3Appendf(pRc, &zRet, ", x.%Q", p->zLanguageid);
+    }
   }
   fts3Appendf(pRc, &zRet, " FROM '%q'.'%q%s' AS x", 
       p->zDb,
@@ -1215,8 +1220,20 @@ static int fts3InitVtab(
       sqlite3_free((void*)aCol); 
       aCol = 0;
       rc = fts3ContentColumns(db, argv[1], zContent, &aCol, &nCol, &nString);
+
+      /* If a languageid= option was specified, remove the language id
+      ** column from the aCol[] array. */ 
+      if( rc==SQLITE_OK && zLanguageid ){
+        int j;
+        for(j=0; j<nCol; j++){
+          if( sqlite3_stricmp(zLanguageid, aCol[j])==0 ){
+            memmove(&aCol[j], &aCol[j+1], (nCol-j) * sizeof(aCol[0]));
+            nCol--;
+            break;
+          }
+        }
+      }
     }
-    assert( rc!=SQLITE_OK || nCol>0 );
   }
   if( rc!=SQLITE_OK ) goto fts3_init_out;
 
@@ -3045,6 +3062,8 @@ static int fts3ColumnMethod(
     /* The extra column whose name is the same as the table.
     ** Return a blob which is a pointer to the cursor.  */
     sqlite3_result_blob(pCtx, &pCsr, sizeof(pCsr), SQLITE_TRANSIENT);
+  }else if( iCol==p->nColumn+2 && pCsr->pExpr ){
+    sqlite3_result_int64(pCtx, pCsr->iLangid);
   }else{
     /* The requested column is either a user column (one that contains 
     ** indexed data), or the language-id column.  */

diff --git a/manifest b/manifest
index 268d6f64f60ab217d27d2b34cd7d4e64b681d561..15fcd20c9b38b440a060c8c44c9580601bbb3fe8 100644 (file)
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Add\stest\sfor\sFTS\s'rebuild'\scommand.
-D 2012-03-02T16:18:21.845
+C Fix\sproblems\swith\scombining\scontent=\sand\slanguageid=\sin\sa\ssingle\sfts4\stable.
+D 2012-03-02T19:53:02.350
 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f
 F Makefile.in 3f79a373e57c3b92dabf76f40b065e719d31ac34
 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23
@@ -63,7 +63,7 @@ F ext/fts3/README.content fdc666a70d5257a64fee209f97cf89e0e6e32b51
 F ext/fts3/README.syntax a19711dc5458c20734b8e485e75fb1981ec2427a
 F ext/fts3/README.tokenizers 998756696647400de63d5ba60e9655036cb966e9
 F ext/fts3/README.txt 8c18f41574404623b76917b9da66fcb0ab38328d
-F ext/fts3/fts3.c 93a8eb6e6eb4cd0aa4856d841a9d8d0025a2784a
+F ext/fts3/fts3.c fd89caa4169520c32cf46ca5a62df6dd48201422
 F ext/fts3/fts3.h 3a10a0af180d502cecc50df77b1b22df142817fe
 F ext/fts3/fts3Int.h 521d300f2af4e741f53c4e2dd540275fb64533eb
 F ext/fts3/fts3_aux.c 72de4cb43db7bfc2f68fbda04b7d8095ae9a6239
@@ -496,7 +496,7 @@ F test/fts3snippet.test 8e956051221a34c7daeb504f023cb54d5fa5a8b2
 F test/fts3sort.test 95be0b19d7e41c44b29014f13ea8bddd495fd659
 F test/fts4aa.test 6e7f90420b837b2c685f3bcbe84c868492d40a68
 F test/fts4content.test 17b2360f7d1a9a7e5aa8022783f5c5731b6dfd4f
-F test/fts4langid.test 343a65d54419b45a318a1acdb1c37749f26fdc79
+F test/fts4langid.test be989b5cddcd7596b87232af193f6c4560a34272
 F test/func.test 6c5ce11e3a0021ca3c0649234e2d4454c89110ca
 F test/func2.test 772d66227e4e6684b86053302e2d74a2500e1e0f
 F test/func3.test 001021e5b88bd02a3b365a5c5fd8f6f49d39744a
@@ -992,7 +992,7 @@ F tool/tostr.awk e75472c2f98dd76e06b8c9c1367f4ab07e122d06
 F tool/vdbe-compress.tcl d70ea6d8a19e3571d7ab8c9b75cba86d1173ff0f
 F tool/warnings-clang.sh 9f406d66e750e8ac031c63a9ef3248aaa347ef2a
 F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381
-P 65fa693729a336e4d905ce72e6b9ccf4faa772bd
-R 4e2c13b79ff592d73e478288d906b721
+P 181bc35731f19c3e4497ba3338c209918d34ea69
+R acfb20f690a18ac8d67e116ae8c76f7d
 U dan
-Z bcf6ed8432a11d3fdd53825eba08d4d7
+Z b771fbbc3f5dd49e39970a5f917fc8b7

diff --git a/manifest.uuid b/manifest.uuid
index bdb54dd27f232c92f9243c05f88d664da7a9fb98..def5beae2823a3370ed469a193f2826b5de035f0 100644 (file)
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-181bc35731f19c3e4497ba3338c209918d34ea69
\ No newline at end of file
+22491e7bc38aee43819b888e04241cb6a6ef73a3
\ No newline at end of file

diff --git a/test/fts4langid.test b/test/fts4langid.test
index 059a56d08f9d8b15d9c101af626f7716e695a3f6..b9110cb768a26c0f716b6b93ef34e135b78acdd0 100644 (file)
--- a/test/fts4langid.test
+++ b/test/fts4langid.test
@@ -37,17 +37,17 @@ set ::testprefix fts4langid
 #
 #   2.3.* - Same as 2.1.*, after a 'rebuild' command.
 #
-#   3.* - Test that if one is provided, the tokenizer xLanguage method
+#   3.* - Tests with content= tables. Both where there is a real 
+#         underlying content table and where there is not.
+#
+#
+#   4.* - Test that if one is provided, the tokenizer xLanguage method
 #         is called to configure the tokenizer before tokenizing query
 #         or document text.
 #
-#   4.* - Test the fts4aux table when the associated FTS4 table contains
+#   5.* - Test the fts4aux table when the associated FTS4 table contains
 #         multiple languages.
 #
-#   5.* - Tests with content= tables. Both where there is a real 
-#         underlying content table and where there is not.
-#
-
 
 do_execsql_test 1.1 {
   CREATE VIRTUAL TABLE t1 USING fts4(a, b, languageid=lang_id);
@@ -151,6 +151,11 @@ proc build_multilingual_db_1 {db} {
 
     $db eval { INSERT INTO t2(docid, x, y, l) VALUES($i, $x, $y, $iLangid) }
   }
+
+  $db eval {
+    CREATE TABLE data(x, y, l);
+    INSERT INTO data(rowid, x, y, l) SELECT docid, x, y, l FROM t2;
+  }
 }
 
 proc rowid_list_set_langid {langid} {
@@ -159,9 +164,9 @@ proc rowid_list_set_langid {langid} {
 proc rowid_list {pattern} {
   set langid $::rowid_list_langid
   set res [list]
-  db eval {SELECT docid, x, y FROM t2 WHERE l = $langid ORDER BY docid ASC} {
+  db eval {SELECT rowid, x, y FROM data WHERE l = $langid ORDER BY rowid ASC} {
     if {[string match "*$pattern*" $x] || [string match "*$pattern*" $y]} {
-      lappend res $docid
+      lappend res $rowid
     }
   }
   return $res
@@ -235,7 +240,7 @@ do_test 2.0 {
   build_multilingual_db_1 db
 } {}
 
-proc do_test_2 {tn query res_script} {
+proc do_test_query1 {tn query res_script} {
   for {set langid 0} {$langid < 10} {incr langid} {
     rowid_list_set_langid $langid
     set res [eval $res_script]
@@ -243,17 +248,17 @@ proc do_test_2 {tn query res_script} {
     set actual [
       execsql {SELECT docid FROM t2 WHERE t2 MATCH $query AND l = $langid}
     ]
-    do_test 2.$tn.$langid [list set {} $actual] $res
+    do_test $tn.$langid [list set {} $actual] $res
   }
 }
 
 # Run some queries. 
-do_test_2 1.1  {delta}          { rowid_list delta }
-do_test_2 1.2  {"zero one two"} { rowid_list "zero one two" }
-do_test_2 1.3  {zero one two} {
+do_test_query1 2.1.1  {delta}          { rowid_list delta }
+do_test_query1 2.1.2  {"zero one two"} { rowid_list "zero one two" }
+do_test_query1 2.1.3  {zero one two} {
   and_merge_lists [rowid_list zero] [rowid_list one] [rowid_list two]
 }
-do_test_2 1.4  {"zero one" OR "one two"} {
+do_test_query1 2.1.4  {"zero one" OR "one two"} {
   or_merge_lists [rowid_list "zero one"] [rowid_list "one two"]
 }
 
@@ -264,12 +269,12 @@ do_execsql_test 2.2 {
   INSERT INTO t2(t2) VALUES('optimize');
   SELECT count(*) FROM t2_segdir;
 } {9}
-do_test_2 2.1 {delta}          { rowid_list delta }
-do_test_2 2.2 {"zero one two"} { rowid_list "zero one two" }
-do_test_2 2.3 {zero one two} {
+do_test_query1 2.2.1 {delta}          { rowid_list delta }
+do_test_query1 2.2.2 {"zero one two"} { rowid_list "zero one two" }
+do_test_query1 2.2.3 {zero one two} {
   and_merge_lists [rowid_list zero] [rowid_list one] [rowid_list two]
 }
-do_test_2 2.4 {"zero one" OR "one two"} {
+do_test_query1 2.2.4 {"zero one" OR "one two"} {
   or_merge_lists [rowid_list "zero one"] [rowid_list "one two"]
 }
 
@@ -280,14 +285,64 @@ do_test 2.3 {
   build_multilingual_db_1 db
   execsql { INSERT INTO t2(t2) VALUES('rebuild') }
 } {}
-do_test_2 3.1 {delta}          { rowid_list delta }
-do_test_2 3.2 {"zero one two"} { rowid_list "zero one two" }
-do_test_2 3.3 {zero one two} {
+do_test_query1 2.3.1 {delta}          { rowid_list delta }
+do_test_query1 2.3.2 {"zero one two"} { rowid_list "zero one two" }
+do_test_query1 2.3.3 {zero one two} {
+  and_merge_lists [rowid_list zero] [rowid_list one] [rowid_list two]
+}
+do_test_query1 2.3.4 {"zero one" OR "one two"} {
+  or_merge_lists [rowid_list "zero one"] [rowid_list "one two"]
+}
+
+#-------------------------------------------------------------------------
+# Test cases 3.*
+#
+do_test 3.0 {
+  reset_db
+  build_multilingual_db_1 db
+  execsql {
+    CREATE TABLE t3_data(l, x, y);
+    INSERT INTO t3_data(rowid, l, x, y) SELECT docid, l, x, y FROM t2;
+    DROP TABLE t2;
+  }
+} {}
+do_execsql_test 3.1 {
+  CREATE VIRTUAL TABLE t2 USING fts4(content=t3_data, languageid=l);
+  INSERT INTO t2(t2) VALUES('rebuild');
+}
+
+do_test_query1 3.1.1 {delta}          { rowid_list delta }
+do_test_query1 3.1.2 {"zero one two"} { rowid_list "zero one two" }
+do_test_query1 3.1.3 {zero one two} {
   and_merge_lists [rowid_list zero] [rowid_list one] [rowid_list two]
 }
-do_test_2 3.4 {"zero one" OR "one two"} {
+do_test_query1 3.1.4 {"zero one" OR "one two"} {
   or_merge_lists [rowid_list "zero one"] [rowid_list "one two"]
 }
 
+do_execsql_test 3.2.1 {
+  DROP TABLE t2;
+  CREATE VIRTUAL TABLE t2 USING fts4(x, y, languageid=l, content=nosuchtable);
+}
+
+do_execsql_test 3.2.2 {
+  INSERT INTO t2(docid, x, y, l) SELECT rowid, x, y, l FROM t3_data;
+}
+
+do_execsql_test 3.2.3 {
+  DROP TABLE t3_data;
+}
+
+do_test_query1 3.3.1 {delta}          { rowid_list delta }
+do_test_query1 3.3.2 {"zero one two"} { rowid_list "zero one two" }
+do_test_query1 3.3.3 {zero one two} {
+  and_merge_lists [rowid_list zero] [rowid_list one] [rowid_list two]
+}
+do_test_query1 3.3.4 {"zero one" OR "one two"} {
+  or_merge_lists [rowid_list "zero one"] [rowid_list "one two"]
+}
+
+
+
 finish_test