From 3ad202dd17200cc6c993ff16569139dd8eb4e07b Mon Sep 17 00:00:00 2001 From: shess Date: Fri, 19 Jan 2007 22:59:56 +0000 Subject: [PATCH] http://www.sqlite.org/cvstrac/tktview?tn=2166,35 Calling UPDATE against an fts table in a UTF-16 database inserts corrupted data into the database. The UTF-8 data is being inserted directly. This appears to happen because sqlite3_ value_text() destructively coerces a value to UTF-8, and it's never converted back when updating the table. This works around the problem by rearranging things so that the update happens before the coercion. (CVS 3596) FossilOrigin-Name: 4f2ab4b6320ffc621900049b41f50bc30d76d7f5 --- ext/fts1/fts1.c | 6 ++-- ext/fts2/fts2.c | 6 ++-- manifest | 18 ++++++----- manifest.uuid | 2 +- test/fts1i.test | 81 +++++++++++++++++++++++++++++++++++++++++++++++++ test/fts2i.test | 81 +++++++++++++++++++++++++++++++++++++++++++++++++ 6 files changed, 179 insertions(+), 15 deletions(-) create mode 100644 test/fts1i.test create mode 100644 test/fts2i.test diff --git a/ext/fts1/fts1.c b/ext/fts1/fts1.c index 735b69952e..ef544a110a 100644 --- a/ext/fts1/fts1.c +++ b/ext/fts1/fts1.c @@ -3089,11 +3089,11 @@ static int index_update(fulltext_vtab *v, sqlite_int64 iRow, int rc = deleteTerms(v, pTerms, iRow); if( rc!=SQLITE_OK ) return rc; - /* Now add positions for terms which appear in the updated row. */ - rc = insertTerms(v, pTerms, iRow, pValues); + rc = content_update(v, pValues, iRow); /* execute an SQL UPDATE */ if( rc!=SQLITE_OK ) return rc; - return content_update(v, pValues, iRow); /* execute an SQL UPDATE */ + /* Now add positions for terms which appear in the updated row. */ + return insertTerms(v, pTerms, iRow, pValues); } /* This function implements the xUpdate callback; it's the top-level entry diff --git a/ext/fts2/fts2.c b/ext/fts2/fts2.c index b1ca720028..660ba536c1 100644 --- a/ext/fts2/fts2.c +++ b/ext/fts2/fts2.c @@ -3611,11 +3611,11 @@ static int index_update(fulltext_vtab *v, sqlite_int64 iRow, int rc = deleteTerms(v, pTerms, iRow); if( rc!=SQLITE_OK ) return rc; - /* Now add positions for terms which appear in the updated row. */ - rc = insertTerms(v, pTerms, iRow, pValues); + rc = content_update(v, pValues, iRow); /* execute an SQL UPDATE */ if( rc!=SQLITE_OK ) return rc; - return content_update(v, pValues, iRow); /* execute an SQL UPDATE */ + /* Now add positions for terms which appear in the updated row. */ + return insertTerms(v, pTerms, iRow, pValues); } /*******************************************************************/ diff --git a/manifest b/manifest index 2dbd83a558..850210e601 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Make\ssure\sthe\sIS\sNULL\soptimization\sintroduced\sby\scheck-in\s(3494)\scorrectly\nhandles\sa\sLEFT\sJOIN\swhere\sthe\sa\sterm\sfrom\sthe\sright\stable\sof\sthe\sjoin\suses\nan\sIS\sNULL\sconstraint.\s\sTicket\s#2177.\s\sThis\scheck-in\salso\sadds\sthe\snew\stest\ncases\sthat\swere\ssuppose\sto\shave\sbeen\sadded\swith\s(3494)\sbut\swhich\swere\nmistakenly\somitted.\s(CVS\s3595) -D 2007-01-19T01:06:02 +C http://www.sqlite.org/cvstrac/tktview?tn=2166,35\n\nCalling\sUPDATE\sagainst\san\sfts\stable\sin\sa\sUTF-16\sdatabase\sinserts\ncorrupted\sdata\sinto\sthe\sdatabase.\s\sThe\sUTF-8\sdata\sis\sbeing\sinserted\ndirectly.\s\sThis\sappears\sto\shappen\sbecause\ssqlite3_\svalue_text()\ndestructively\scoerces\sa\svalue\sto\sUTF-8,\sand\sit's\snever\sconverted\sback\nwhen\supdating\sthe\stable.\sThis\sworks\saround\sthe\sproblem\sby\srearranging\nthings\sso\sthat\sthe\supdate\shappens\sbefore\sthe\scoercion.\s(CVS\s3596) +D 2007-01-19T22:59:57 F Makefile.in 7fa74bf4359aa899da5586e394d17735f221315f F Makefile.linux-gcc 2d8574d1ba75f129aba2019f0b959db380a90935 F README 9c4e2d6706bdcc3efdd773ce752a8cdab4f90028 @@ -21,7 +21,7 @@ F ext/README.txt 913a7bd3f4837ab14d7e063304181787658b14e1 F ext/fts1/README.txt 20ac73b006a70bcfd80069bdaf59214b6cf1db5e F ext/fts1/ft_hash.c 3927bd880e65329bdc6f506555b228b28924921b F ext/fts1/ft_hash.h 1a35e654a235c2c662d3ca0dfc3138ad60b8b7d5 -F ext/fts1/fts1.c 18906d1ea54ad1a3cdda186da96aa6e6bdeb04f4 +F ext/fts1/fts1.c 37b91a1a39713220c5411debe46778495fcfc2f2 F ext/fts1/fts1.h 6060b8f62c1d925ea8356cb1a6598073eb9159a6 F ext/fts1/fts1_hash.c 3196cee866edbebb1c0521e21672e6d599965114 F ext/fts1/fts1_hash.h 957d378355ed29f672cd5add012ce8b088a5e089 @@ -33,7 +33,7 @@ F ext/fts1/fulltext.h 08525a47852d1d62a0be81d3fc3fe2d23b094efd F ext/fts1/simple_tokenizer.c 1844d72f7194c3fd3d7e4173053911bf0661b70d F ext/fts1/tokenizer.h 0c53421b832366d20d720d21ea3e1f6e66a36ef9 F ext/fts2/README.txt 8c18f41574404623b76917b9da66fcb0ab38328d -F ext/fts2/fts2.c 5424f41fbc894c14eb87169de003c476827efaf3 +F ext/fts2/fts2.c 5f7247b8eca4c83e95d66a8f6cadb0949bb356dc F ext/fts2/fts2.h bbdab26d34f91974d5b9ade8b7836c140a7c4ce1 F ext/fts2/fts2_hash.c b3f22116d4ef0bc8f2da6e3fdc435c86d0951a9b F ext/fts2/fts2_hash.h e283308156018329f042816eb09334df714e105e @@ -206,6 +206,7 @@ F test/fts1c.test 85a525ce7428907469b4cce13d5563ce542ce64c F test/fts1d.test a73deace5c18df4a549b12908bade4f05dcf1a2f F test/fts1e.test 77244843e925560b5a0b70069c3e7ab62f181ed2 F test/fts1f.test cdab90834a7627a26fed2f75d2f451650fb31fad +F test/fts1i.test 13ddf7481c95d1595f6fbc6d8223de4286151a4c F test/fts1porter.test d86e9c3e0c7f8ff95add6582b4b585fb4e02b96d F test/fts2a.test 103fc178d134c54c44c1938a4331e9e2030792d9 F test/fts2b.test 964abc0236c849c07ca1ae496bb25c268ae94816 @@ -215,6 +216,7 @@ F test/fts2e.test 2da13dbc2d009105f42196845c1e1ce136c03d38 F test/fts2f.test b5f2dde48199d79e859f59d3d857c17dd62a0129 F test/fts2g.test c69a8ab43ec77d123976ba6cf9422d647ae63032 F test/fts2h.test 223af921323b409d4b5b18ff4e51619541b174bb +F test/fts2i.test 7e28587a64056b86941cc6df6dc8a4379111d9bb F test/func.test 0ed54b5aeaad319f68016c033acfebef56f5874a F test/hook.test 7e7645fd9a033f79cce8fdff151e32715e7ec50a F test/in.test 369cb2aa1eab02296b4ec470732fe8c131260b1d @@ -425,7 +427,7 @@ F www/tclsqlite.tcl bb0d1357328a42b1993d78573e587c6dcbc964b9 F www/vdbe.tcl 87a31ace769f20d3627a64fa1fade7fed47b90d0 F www/version3.tcl 890248cf7b70e60c383b0e84d77d5132b3ead42b F www/whentouse.tcl 97e2b5cd296f7d8057e11f44427dea8a4c2db513 -P d537aa5edecb5c7b84d8eb876453f385f6f3d91a -R 99965f5a2fabd1d4a2a67c894b4ff62b -U drh -Z e72acdbefc9562f5a7ceb3d6bfdf02c8 +P 335863e4d16113fb9ecebce35d2db043771d98b1 +R 7d82bd5b130abe01b6e3432cba59c03d +U shess +Z 5b767fc3587ea545c44144087aefb6be diff --git a/manifest.uuid b/manifest.uuid index 227624640f..5eab5fa168 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -335863e4d16113fb9ecebce35d2db043771d98b1 \ No newline at end of file +4f2ab4b6320ffc621900049b41f50bc30d76d7f5 \ No newline at end of file diff --git a/test/fts1i.test b/test/fts1i.test new file mode 100644 index 0000000000..4a80ed43f6 --- /dev/null +++ b/test/fts1i.test @@ -0,0 +1,81 @@ +# 2007 January 17 +# +# The author disclaims copyright to this source code. +# +#************************************************************************* +# This file implements regression tests for SQLite fts1 library. The +# focus here is testing handling of UPDATE when using UTF-16-encoded +# databases. +# +# $Id: fts1i.test,v 1.1 2007/01/19 22:59:57 shess Exp $ +# + +set testdir [file dirname $argv0] +source $testdir/tester.tcl + +# Return the UTF-16 representation of the supplied UTF-8 string $str. +# If $nt is true, append two 0x00 bytes as a nul terminator. +# NOTE(shess) Copied from capi3.test. +proc utf16 {str {nt 1}} { + set r [encoding convertto unicode $str] + if {$nt} { + append r "\x00\x00" + } + return $r +} + +db eval { + PRAGMA encoding = "UTF-16le"; + CREATE VIRTUAL TABLE t1 USING fts1(content); +} + +do_test fts1i-1.0 { + execsql {PRAGMA encoding} +} {UTF-16le} + +do_test fts1i-1.1 { + execsql {INSERT INTO t1 (rowid, content) VALUES(1, 'one')} + execsql {SELECT content FROM t1 WHERE rowid = 1} +} {one} + +do_test fts1i-1.2 { + set sql "INSERT INTO t1 (rowid, content) VALUES(2, 'two')" + set STMT [sqlite3_prepare $DB $sql -1 TAIL] + sqlite3_step $STMT + sqlite3_finalize $STMT + execsql {SELECT content FROM t1 WHERE rowid = 2} +} {two} + +do_test fts1i-1.3 { + set sql "INSERT INTO t1 (rowid, content) VALUES(3, 'three')" + set STMT [sqlite3_prepare $DB $sql -1 TAIL] + sqlite3_step $STMT + sqlite3_finalize $STMT + set sql "UPDATE t1 SET content = 'trois' WHERE rowid = 3" + set STMT [sqlite3_prepare $DB $sql -1 TAIL] + sqlite3_step $STMT + sqlite3_finalize $STMT + execsql {SELECT content FROM t1 WHERE rowid = 3} +} {trois} + +do_test fts1i-1.4 { + set sql16 [utf16 {INSERT INTO t1 (rowid, content) VALUES(4, 'four')}] + set STMT [sqlite3_prepare16 $DB $sql16 -1 TAIL] + sqlite3_step $STMT + sqlite3_finalize $STMT + execsql {SELECT content FROM t1 WHERE rowid = 4} +} {four} + +do_test fts1i-1.5 { + set sql16 [utf16 {INSERT INTO t1 (rowid, content) VALUES(5, 'five')}] + set STMT [sqlite3_prepare16 $DB $sql16 -1 TAIL] + sqlite3_step $STMT + sqlite3_finalize $STMT + set sql "UPDATE t1 SET content = 'cinq' WHERE rowid = 5" + set STMT [sqlite3_prepare $DB $sql -1 TAIL] + sqlite3_step $STMT + sqlite3_finalize $STMT + execsql {SELECT content FROM t1 WHERE rowid = 5} +} {cinq} + +finish_test diff --git a/test/fts2i.test b/test/fts2i.test new file mode 100644 index 0000000000..88252fdbc4 --- /dev/null +++ b/test/fts2i.test @@ -0,0 +1,81 @@ +# 2007 January 17 +# +# The author disclaims copyright to this source code. +# +#************************************************************************* +# This file implements regression tests for SQLite fts2 library. The +# focus here is testing handling of UPDATE when using UTF-16-encoded +# databases. +# +# $Id: fts2i.test,v 1.1 2007/01/19 22:59:57 shess Exp $ +# + +set testdir [file dirname $argv0] +source $testdir/tester.tcl + +# Return the UTF-16 representation of the supplied UTF-8 string $str. +# If $nt is true, append two 0x00 bytes as a nul terminator. +# NOTE(shess) Copied from capi3.test. +proc utf16 {str {nt 1}} { + set r [encoding convertto unicode $str] + if {$nt} { + append r "\x00\x00" + } + return $r +} + +db eval { + PRAGMA encoding = "UTF-16le"; + CREATE VIRTUAL TABLE t1 USING fts2(content); +} + +do_test fts2i-1.0 { + execsql {PRAGMA encoding} +} {UTF-16le} + +do_test fts2i-1.1 { + execsql {INSERT INTO t1 (rowid, content) VALUES(1, 'one')} + execsql {SELECT content FROM t1 WHERE rowid = 1} +} {one} + +do_test fts2i-1.2 { + set sql "INSERT INTO t1 (rowid, content) VALUES(2, 'two')" + set STMT [sqlite3_prepare $DB $sql -1 TAIL] + sqlite3_step $STMT + sqlite3_finalize $STMT + execsql {SELECT content FROM t1 WHERE rowid = 2} +} {two} + +do_test fts2i-1.3 { + set sql "INSERT INTO t1 (rowid, content) VALUES(3, 'three')" + set STMT [sqlite3_prepare $DB $sql -1 TAIL] + sqlite3_step $STMT + sqlite3_finalize $STMT + set sql "UPDATE t1 SET content = 'trois' WHERE rowid = 3" + set STMT [sqlite3_prepare $DB $sql -1 TAIL] + sqlite3_step $STMT + sqlite3_finalize $STMT + execsql {SELECT content FROM t1 WHERE rowid = 3} +} {trois} + +do_test fts2i-1.4 { + set sql16 [utf16 {INSERT INTO t1 (rowid, content) VALUES(4, 'four')}] + set STMT [sqlite3_prepare16 $DB $sql16 -1 TAIL] + sqlite3_step $STMT + sqlite3_finalize $STMT + execsql {SELECT content FROM t1 WHERE rowid = 4} +} {four} + +do_test fts2i-1.5 { + set sql16 [utf16 {INSERT INTO t1 (rowid, content) VALUES(5, 'five')}] + set STMT [sqlite3_prepare16 $DB $sql16 -1 TAIL] + sqlite3_step $STMT + sqlite3_finalize $STMT + set sql "UPDATE t1 SET content = 'cinq' WHERE rowid = 5" + set STMT [sqlite3_prepare $DB $sql -1 TAIL] + sqlite3_step $STMT + sqlite3_finalize $STMT + execsql {SELECT content FROM t1 WHERE rowid = 5} +} {cinq} + +finish_test -- 2.47.2