Back out the change from [7fab1393c2b22b1f] that tries to convert invalid

surrogate characters in UTF16 into the replacement character 0xfffd, as we
find that this breaks some software.

FossilOrigin-Name: 4218c7b71fb6b227dbe4b852718584c150164af2d84e067cb810aa602554a609

diff --git a/manifest b/manifest
index a5b7b1fcbf0f1b05e026e47329cacb1dfc945e8f..81eb1b4a4acd7bd8c09415b68b41383c8d8b0ac4 100644 (file)
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Add\scompiler\shints\sto\sdisable\sTSAN\sfor\sthe\sroutines\sthat\saccess\sthe\s-shm\nfile\sheader\sin\sWAL\smode\susing\sa\sdouble-read\swith\smemory\sbarrier.
-D 2020-05-19T15:51:10.216
+C Back\sout\sthe\schange\sfrom\s[7fab1393c2b22b1f]\sthat\stries\sto\sconvert\sinvalid\nsurrogate\scharacters\sin\sUTF16\sinto\sthe\sreplacement\scharacter\s0xfffd,\sas\swe\nfind\sthat\sthis\sbreaks\ssome\ssoftware.
+D 2020-05-20T15:02:04.035
 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1
 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea
 F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724
@@ -602,7 +602,7 @@ F src/treeview.c 82c6391a3ba76215d4185fd4719a56ec4caf186a40c8a7b6e6ba4ae4467c274
 F src/trigger.c 4ada1037cc99777f647a882cdacbd1a4deb6567b69daf02946286401b88cdc04
 F src/update.c 3199098455830fc2d8c8fc4ae3ec2ea513eef64339ae9a7048db62b21169bc7a
 F src/upsert.c 2920de71b20f04fe25eb00b655d086f0ba60ea133c59d7fa3325c49838818e78
-F src/utf.c 95fb6e03a5ca679045c5adccd05380f0addccabef5911abddcb06af069500ab7
+F src/utf.c d7a61c1dfdac3eb091d43341a674032dca5a34e122f78ef0b5bd2d5a31967dde
 F src/util.c 3b6cedf7a0c69bd6e1acce832873952d416212d6293b18d03064e07d7a9b5118
 F src/vacuum.c de9780b89fa4ee74c3534f60b94820e3179aca759ffc1338ee53cb4ea7693dd3
 F src/vdbe.c e3dba0dee25bc92e871c13cac655260912b3be4abcc85b439259c9934b208da3
@@ -1446,7 +1446,7 @@ F test/tkt-385a5b56b9.test 5204a7cba0e28c99df0acbf95af5e1af4d32965a7a14de6eccebf
 F test/tkt-38cb5df375.test f3cc8671f1eb604d4ae9cf886ed4366bec656678
 F test/tkt-3998683a16.test 6d1d04d551ed1704eb3396ca87bb9ccc8c5c1eb7
 F test/tkt-3a77c9714e.test 90e3e8455ee945a4076d4c44062b8845708af24a880355328fe7008f2047c9f0
-F test/tkt-3fe897352e.test 6849fde0a87165ff83f54f5047af7c743d72af26908fadb90174f3294450b3f4
+F test/tkt-3fe897352e.test 27e26eb0f1811aeba4d65aba43a4c52e99da5e70
 F test/tkt-4a03edc4c8.test 91c0e135888cdc3d4eea82406a44b05c8c1648d0
 F test/tkt-4c86b126f2.test cbcc611becd0396890169ab23102dd70048bbc9a
 F test/tkt-4dd95f6943.test 3d0ce415d2ee15d3d564121960016b9c7be79407
@@ -1866,7 +1866,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P cb0a18e64f8d81c2ada9f698faaf5ca68a0666687bf6f3abc860324cb1997463
-R 8462379efa542a8195db58fd9ef94a35
+P 3117c1b5a9e348fd8d16ba9d03fdafaad8514567fb3403f72b86d6162ad40bde
+R e941513d71fae9ce3b558d98a564d148
 U drh
-Z 5f8f2ff4f8e169f83e77c91d9875e042
+Z db42a456dee4143dc7c71c6203f4a854

diff --git a/manifest.uuid b/manifest.uuid
index c90a44280462e968c5c6cf1540a5fab1e6e2fa54..c96b83728ef6bda8a77b692dfc238e1cb3f39ff5 100644 (file)
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-3117c1b5a9e348fd8d16ba9d03fdafaad8514567fb3403f72b86d6162ad40bde
\ No newline at end of file
+4218c7b71fb6b227dbe4b852718584c150164af2d84e067cb810aa602554a609
\ No newline at end of file

diff --git a/src/utf.c b/src/utf.c
index f6c8e79e502e4d3d229c44a0231dcf3d8f9103d4..3a2becfbcc10b2a078f57c328a9981b73a1f9572 100644 (file)
--- a/src/utf.c
+++ b/src/utf.c
@@ -284,6 +284,7 @@ SQLITE_NOINLINE int sqlite3VdbeMemTranslate(Mem *pMem, u8 desiredEnc){
         c = *(zIn++);
         c += (*(zIn++))<<8;
         if( c>=0xd800 && c<0xe000 ){
+#ifdef SQLITE_REPLACE_INVALID_UTF
           if( c>=0xdc00 || zIn>=zTerm ){
             c = 0xfffd;
           }else{
@@ -296,6 +297,13 @@ SQLITE_NOINLINE int sqlite3VdbeMemTranslate(Mem *pMem, u8 desiredEnc){
               c = ((c&0x3ff)<<10) + (c2&0x3ff) + 0x10000;
             }
           }
+#else
+          if( zIn<zTerm ){
+            int c2 = (*zIn++);
+            c2 += ((*zIn++)<<8);
+            c = (c2&0x03FF) + ((c&0x003F)<<10) + (((c&0x03C0)+0x0040)<<10);
+          }
+#endif
         }
         WRITE_UTF8(z, c);
       }
@@ -305,6 +313,7 @@ SQLITE_NOINLINE int sqlite3VdbeMemTranslate(Mem *pMem, u8 desiredEnc){
         c = (*(zIn++))<<8;
         c += *(zIn++);
         if( c>=0xd800 && c<0xe000 ){
+#ifdef SQLITE_REPLACE_INVALID_UTF
           if( c>=0xdc00 || zIn>=zTerm ){
             c = 0xfffd;
           }else{
@@ -317,6 +326,13 @@ SQLITE_NOINLINE int sqlite3VdbeMemTranslate(Mem *pMem, u8 desiredEnc){
               c = ((c&0x3ff)<<10) + (c2&0x3ff) + 0x10000;
             }
           }
+#else
+          if( zIn<zTerm ){
+            int c2 = ((*zIn++)<<8);
+            c2 += (*zIn++);
+            c = (c2&0x03FF) + ((c&0x003F)<<10) + (((c&0x03C0)+0x0040)<<10);
+          }
+#endif
         }
         WRITE_UTF8(z, c);
       }

diff --git a/test/tkt-3fe897352e.test b/test/tkt-3fe897352e.test
index fa70ac765831da0a5f6410b6948c65b27cb0d777..bc2b2033e70c7d19d876de04fdac90c8387544b3 100644 (file)
--- a/test/tkt-3fe897352e.test
+++ b/test/tkt-3fe897352e.test
@@ -33,28 +33,28 @@ do_test tkt-3fe89-1.1 {
     INSERT INTO t1 VALUES(hex_to_utf16be('D800'));
     SELECT hex(x) FROM t1;
   }
-} {EFBFBD}
+} {EDA080}
 do_test tkt-3fe89-1.2 {
   db eval {
     DELETE FROM t1;
     INSERT INTO t1 VALUES(hex_to_utf16le('00D8'));
     SELECT hex(x) FROM t1;
   }
-} {EFBFBD}
+} {EDA080}
 do_test tkt-3fe89-1.3 {
   db eval {
     DELETE FROM t1;
     INSERT INTO t1 VALUES(hex_to_utf16be('DFFF'));
     SELECT hex(x) FROM t1;
   }
-} {EFBFBD}
+} {EDBFBF}
 do_test tkt-3fe89-1.4 {
   db eval {
     DELETE FROM t1;
     INSERT INTO t1 VALUES(hex_to_utf16le('FFDF'));
     SELECT hex(x) FROM t1;
   }
-} {EFBFBD}
+} {EDBFBF}
 
 
 finish_test