http://www.sqlite.org/cvstrac/tktview?tn=2166,35

Calling UPDATE against an fts table in a UTF-16 database inserts
corrupted data into the database.  The UTF-8 data is being inserted
directly.  This appears to happen because sqlite3_ value_text()
destructively coerces a value to UTF-8, and it's never converted back
when updating the table. This works around the problem by rearranging
things so that the update happens before the coercion. (CVS 3596)

FossilOrigin-Name: 4f2ab4b6320ffc621900049b41f50bc30d76d7f5

diff --git a/ext/fts1/fts1.c b/ext/fts1/fts1.c
index 735b69952ea222c8afe2448da70b321657b01287..ef544a110aa8e346fc7f8577d247605f48f7c1ed 100644 (file)
--- a/ext/fts1/fts1.c
+++ b/ext/fts1/fts1.c
@@ -3089,11 +3089,11 @@ static int index_update(fulltext_vtab *v, sqlite_int64 iRow,
   int rc = deleteTerms(v, pTerms, iRow);
   if( rc!=SQLITE_OK ) return rc;
 
-  /* Now add positions for terms which appear in the updated row. */
-  rc = insertTerms(v, pTerms, iRow, pValues);
+  rc = content_update(v, pValues, iRow);  /* execute an SQL UPDATE */
   if( rc!=SQLITE_OK ) return rc;
 
-  return content_update(v, pValues, iRow);  /* execute an SQL UPDATE */
+  /* Now add positions for terms which appear in the updated row. */
+  return insertTerms(v, pTerms, iRow, pValues);
 }
 
 /* This function implements the xUpdate callback; it's the top-level entry

diff --git a/ext/fts2/fts2.c b/ext/fts2/fts2.c
index b1ca720028544f2cad7ce89fa75a6042c3e95a80..660ba536c1340cc167b7613d0cc1f998ed4b441d 100644 (file)
--- a/ext/fts2/fts2.c
+++ b/ext/fts2/fts2.c
@@ -3611,11 +3611,11 @@ static int index_update(fulltext_vtab *v, sqlite_int64 iRow,
   int rc = deleteTerms(v, pTerms, iRow);
   if( rc!=SQLITE_OK ) return rc;
 
-  /* Now add positions for terms which appear in the updated row. */
-  rc = insertTerms(v, pTerms, iRow, pValues);
+  rc = content_update(v, pValues, iRow);  /* execute an SQL UPDATE */
   if( rc!=SQLITE_OK ) return rc;
 
-  return content_update(v, pValues, iRow);  /* execute an SQL UPDATE */
+  /* Now add positions for terms which appear in the updated row. */
+  return insertTerms(v, pTerms, iRow, pValues);
 }
 
 /*******************************************************************/

diff --git a/manifest b/manifest
index 2dbd83a558059708757c5f19a9a4d6503d0408b4..850210e601ac626119d6cab1252df8761bfa80eb 100644 (file)
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Make\ssure\sthe\sIS\sNULL\soptimization\sintroduced\sby\scheck-in\s(3494)\scorrectly\nhandles\sa\sLEFT\sJOIN\swhere\sthe\sa\sterm\sfrom\sthe\sright\stable\sof\sthe\sjoin\suses\nan\sIS\sNULL\sconstraint.\s\sTicket\s#2177.\s\sThis\scheck-in\salso\sadds\sthe\snew\stest\ncases\sthat\swere\ssuppose\sto\shave\sbeen\sadded\swith\s(3494)\sbut\swhich\swere\nmistakenly\somitted.\s(CVS\s3595)
-D 2007-01-19T01:06:02
+C http://www.sqlite.org/cvstrac/tktview?tn=2166,35\n\nCalling\sUPDATE\sagainst\san\sfts\stable\sin\sa\sUTF-16\sdatabase\sinserts\ncorrupted\sdata\sinto\sthe\sdatabase.\s\sThe\sUTF-8\sdata\sis\sbeing\sinserted\ndirectly.\s\sThis\sappears\sto\shappen\sbecause\ssqlite3_\svalue_text()\ndestructively\scoerces\sa\svalue\sto\sUTF-8,\sand\sit's\snever\sconverted\sback\nwhen\supdating\sthe\stable.\sThis\sworks\saround\sthe\sproblem\sby\srearranging\nthings\sso\sthat\sthe\supdate\shappens\sbefore\sthe\scoercion.\s(CVS\s3596)
+D 2007-01-19T22:59:57
 F Makefile.in 7fa74bf4359aa899da5586e394d17735f221315f
 F Makefile.linux-gcc 2d8574d1ba75f129aba2019f0b959db380a90935
 F README 9c4e2d6706bdcc3efdd773ce752a8cdab4f90028
@@ -21,7 +21,7 @@ F ext/README.txt 913a7bd3f4837ab14d7e063304181787658b14e1
 F ext/fts1/README.txt 20ac73b006a70bcfd80069bdaf59214b6cf1db5e
 F ext/fts1/ft_hash.c 3927bd880e65329bdc6f506555b228b28924921b
 F ext/fts1/ft_hash.h 1a35e654a235c2c662d3ca0dfc3138ad60b8b7d5
-F ext/fts1/fts1.c 18906d1ea54ad1a3cdda186da96aa6e6bdeb04f4
+F ext/fts1/fts1.c 37b91a1a39713220c5411debe46778495fcfc2f2
 F ext/fts1/fts1.h 6060b8f62c1d925ea8356cb1a6598073eb9159a6
 F ext/fts1/fts1_hash.c 3196cee866edbebb1c0521e21672e6d599965114
 F ext/fts1/fts1_hash.h 957d378355ed29f672cd5add012ce8b088a5e089
@@ -33,7 +33,7 @@ F ext/fts1/fulltext.h 08525a47852d1d62a0be81d3fc3fe2d23b094efd
 F ext/fts1/simple_tokenizer.c 1844d72f7194c3fd3d7e4173053911bf0661b70d
 F ext/fts1/tokenizer.h 0c53421b832366d20d720d21ea3e1f6e66a36ef9
 F ext/fts2/README.txt 8c18f41574404623b76917b9da66fcb0ab38328d
-F ext/fts2/fts2.c 5424f41fbc894c14eb87169de003c476827efaf3
+F ext/fts2/fts2.c 5f7247b8eca4c83e95d66a8f6cadb0949bb356dc
 F ext/fts2/fts2.h bbdab26d34f91974d5b9ade8b7836c140a7c4ce1
 F ext/fts2/fts2_hash.c b3f22116d4ef0bc8f2da6e3fdc435c86d0951a9b
 F ext/fts2/fts2_hash.h e283308156018329f042816eb09334df714e105e
@@ -206,6 +206,7 @@ F test/fts1c.test 85a525ce7428907469b4cce13d5563ce542ce64c
 F test/fts1d.test a73deace5c18df4a549b12908bade4f05dcf1a2f
 F test/fts1e.test 77244843e925560b5a0b70069c3e7ab62f181ed2
 F test/fts1f.test cdab90834a7627a26fed2f75d2f451650fb31fad
+F test/fts1i.test 13ddf7481c95d1595f6fbc6d8223de4286151a4c
 F test/fts1porter.test d86e9c3e0c7f8ff95add6582b4b585fb4e02b96d
 F test/fts2a.test 103fc178d134c54c44c1938a4331e9e2030792d9
 F test/fts2b.test 964abc0236c849c07ca1ae496bb25c268ae94816
@@ -215,6 +216,7 @@ F test/fts2e.test 2da13dbc2d009105f42196845c1e1ce136c03d38
 F test/fts2f.test b5f2dde48199d79e859f59d3d857c17dd62a0129
 F test/fts2g.test c69a8ab43ec77d123976ba6cf9422d647ae63032
 F test/fts2h.test 223af921323b409d4b5b18ff4e51619541b174bb
+F test/fts2i.test 7e28587a64056b86941cc6df6dc8a4379111d9bb
 F test/func.test 0ed54b5aeaad319f68016c033acfebef56f5874a
 F test/hook.test 7e7645fd9a033f79cce8fdff151e32715e7ec50a
 F test/in.test 369cb2aa1eab02296b4ec470732fe8c131260b1d
@@ -425,7 +427,7 @@ F www/tclsqlite.tcl bb0d1357328a42b1993d78573e587c6dcbc964b9
 F www/vdbe.tcl 87a31ace769f20d3627a64fa1fade7fed47b90d0
 F www/version3.tcl 890248cf7b70e60c383b0e84d77d5132b3ead42b
 F www/whentouse.tcl 97e2b5cd296f7d8057e11f44427dea8a4c2db513
-P d537aa5edecb5c7b84d8eb876453f385f6f3d91a
-R 99965f5a2fabd1d4a2a67c894b4ff62b
-U drh
-Z e72acdbefc9562f5a7ceb3d6bfdf02c8
+P 335863e4d16113fb9ecebce35d2db043771d98b1
+R 7d82bd5b130abe01b6e3432cba59c03d
+U shess
+Z 5b767fc3587ea545c44144087aefb6be

diff --git a/manifest.uuid b/manifest.uuid
index 227624640fb8a3ae51b222bd8952736351894a1b..5eab5fa1685cfb2537b6b71a9276a04f89255dae 100644 (file)
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-335863e4d16113fb9ecebce35d2db043771d98b1
\ No newline at end of file
+4f2ab4b6320ffc621900049b41f50bc30d76d7f5
\ No newline at end of file

diff --git a/test/fts1i.test b/test/fts1i.test
new file mode 100644 (file)
index 0000000..4a80ed4
--- /dev/null
+++ b/test/fts1i.test
@@ -0,0 +1,81 @@
+# 2007 January 17
+#
+# The author disclaims copyright to this source code.
+#
+#*************************************************************************
+# This file implements regression tests for SQLite fts1 library.  The
+# focus here is testing handling of UPDATE when using UTF-16-encoded
+# databases.
+#
+# $Id: fts1i.test,v 1.1 2007/01/19 22:59:57 shess Exp $
+#
+
+set testdir [file dirname $argv0]
+source $testdir/tester.tcl
+
+# Return the UTF-16 representation of the supplied UTF-8 string $str.
+# If $nt is true, append two 0x00 bytes as a nul terminator.
+# NOTE(shess) Copied from capi3.test.
+proc utf16 {str {nt 1}} {
+  set r [encoding convertto unicode $str]
+  if {$nt} {
+    append r "\x00\x00"
+  }
+  return $r
+}
+
+db eval {
+  PRAGMA encoding = "UTF-16le";
+  CREATE VIRTUAL TABLE t1 USING fts1(content);
+}
+
+do_test fts1i-1.0 {
+  execsql {PRAGMA encoding}
+} {UTF-16le}
+
+do_test fts1i-1.1 {
+  execsql {INSERT INTO t1 (rowid, content) VALUES(1, 'one')}
+  execsql {SELECT content FROM t1 WHERE rowid = 1}
+} {one}
+
+do_test fts1i-1.2 {
+  set sql "INSERT INTO t1 (rowid, content) VALUES(2, 'two')"
+  set STMT [sqlite3_prepare $DB $sql -1 TAIL]
+  sqlite3_step $STMT
+  sqlite3_finalize $STMT
+  execsql {SELECT content FROM t1 WHERE rowid = 2}
+} {two}
+
+do_test fts1i-1.3 {
+  set sql "INSERT INTO t1 (rowid, content) VALUES(3, 'three')"
+  set STMT [sqlite3_prepare $DB $sql -1 TAIL]
+  sqlite3_step $STMT
+  sqlite3_finalize $STMT
+  set sql "UPDATE t1 SET content = 'trois' WHERE rowid = 3"
+  set STMT [sqlite3_prepare $DB $sql -1 TAIL]
+  sqlite3_step $STMT
+  sqlite3_finalize $STMT
+  execsql {SELECT content FROM t1 WHERE rowid = 3}
+} {trois}
+
+do_test fts1i-1.4 {
+  set sql16 [utf16 {INSERT INTO t1 (rowid, content) VALUES(4, 'four')}]
+  set STMT [sqlite3_prepare16 $DB $sql16 -1 TAIL]
+  sqlite3_step $STMT
+  sqlite3_finalize $STMT
+  execsql {SELECT content FROM t1 WHERE rowid = 4}
+} {four}
+
+do_test fts1i-1.5 {
+  set sql16 [utf16 {INSERT INTO t1 (rowid, content) VALUES(5, 'five')}]
+  set STMT [sqlite3_prepare16 $DB $sql16 -1 TAIL]
+  sqlite3_step $STMT
+  sqlite3_finalize $STMT
+  set sql "UPDATE t1 SET content = 'cinq' WHERE rowid = 5"
+  set STMT [sqlite3_prepare $DB $sql -1 TAIL]
+  sqlite3_step $STMT
+  sqlite3_finalize $STMT
+  execsql {SELECT content FROM t1 WHERE rowid = 5}
+} {cinq}
+
+finish_test

diff --git a/test/fts2i.test b/test/fts2i.test
new file mode 100644 (file)
index 0000000..88252fd
--- /dev/null
+++ b/test/fts2i.test
@@ -0,0 +1,81 @@
+# 2007 January 17
+#
+# The author disclaims copyright to this source code.
+#
+#*************************************************************************
+# This file implements regression tests for SQLite fts2 library.  The
+# focus here is testing handling of UPDATE when using UTF-16-encoded
+# databases.
+#
+# $Id: fts2i.test,v 1.1 2007/01/19 22:59:57 shess Exp $
+#
+
+set testdir [file dirname $argv0]
+source $testdir/tester.tcl
+
+# Return the UTF-16 representation of the supplied UTF-8 string $str.
+# If $nt is true, append two 0x00 bytes as a nul terminator.
+# NOTE(shess) Copied from capi3.test.
+proc utf16 {str {nt 1}} {
+  set r [encoding convertto unicode $str]
+  if {$nt} {
+    append r "\x00\x00"
+  }
+  return $r
+}
+
+db eval {
+  PRAGMA encoding = "UTF-16le";
+  CREATE VIRTUAL TABLE t1 USING fts2(content);
+}
+
+do_test fts2i-1.0 {
+  execsql {PRAGMA encoding}
+} {UTF-16le}
+
+do_test fts2i-1.1 {
+  execsql {INSERT INTO t1 (rowid, content) VALUES(1, 'one')}
+  execsql {SELECT content FROM t1 WHERE rowid = 1}
+} {one}
+
+do_test fts2i-1.2 {
+  set sql "INSERT INTO t1 (rowid, content) VALUES(2, 'two')"
+  set STMT [sqlite3_prepare $DB $sql -1 TAIL]
+  sqlite3_step $STMT
+  sqlite3_finalize $STMT
+  execsql {SELECT content FROM t1 WHERE rowid = 2}
+} {two}
+
+do_test fts2i-1.3 {
+  set sql "INSERT INTO t1 (rowid, content) VALUES(3, 'three')"
+  set STMT [sqlite3_prepare $DB $sql -1 TAIL]
+  sqlite3_step $STMT
+  sqlite3_finalize $STMT
+  set sql "UPDATE t1 SET content = 'trois' WHERE rowid = 3"
+  set STMT [sqlite3_prepare $DB $sql -1 TAIL]
+  sqlite3_step $STMT
+  sqlite3_finalize $STMT
+  execsql {SELECT content FROM t1 WHERE rowid = 3}
+} {trois}
+
+do_test fts2i-1.4 {
+  set sql16 [utf16 {INSERT INTO t1 (rowid, content) VALUES(4, 'four')}]
+  set STMT [sqlite3_prepare16 $DB $sql16 -1 TAIL]
+  sqlite3_step $STMT
+  sqlite3_finalize $STMT
+  execsql {SELECT content FROM t1 WHERE rowid = 4}
+} {four}
+
+do_test fts2i-1.5 {
+  set sql16 [utf16 {INSERT INTO t1 (rowid, content) VALUES(5, 'five')}]
+  set STMT [sqlite3_prepare16 $DB $sql16 -1 TAIL]
+  sqlite3_step $STMT
+  sqlite3_finalize $STMT
+  set sql "UPDATE t1 SET content = 'cinq' WHERE rowid = 5"
+  set STMT [sqlite3_prepare $DB $sql -1 TAIL]
+  sqlite3_step $STMT
+  sqlite3_finalize $STMT
+  execsql {SELECT content FROM t1 WHERE rowid = 5}
+} {cinq}
+
+finish_test