From: drh <>
Date: Tue, 23 Jan 2024 13:53:45 +0000 (+0000)
Subject: When a JSON input is a blob, but it looks like valid JSON when cast to text,
X-Git-Tag: version-3.45.1~10
X-Git-Url: http://git.ipfire.org/?a=commitdiff_plain;h=5b9e435d7f561a37edbcdad882bd2a79175994ff;p=thirdparty%2Fsqlite.git

When a JSON input is a blob, but it looks like valid JSON when cast to text,
then accept it as valid JSON.  This replicates a long-standing bug in the
behavior of JSON routines, and thus avoids breaking legacy apps.

FossilOrigin-Name: 4c2c1b97dce46a279846380c937ac6de5c367927c6843516641eead7ea6db472
---

diff --git a/manifest b/manifest
index ae0eaa5023..1bf2f7ce7c 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Ensure\sthat\sthe\sxIntegrity\smethods\sof\sfts3\sand\sfts5\swork\son\sread-only\sdatabases.
-D 2024-01-23T10:47:04.969
+C When\sa\sJSON\sinput\sis\sa\sblob,\sbut\sit\slooks\slike\svalid\sJSON\swhen\scast\sto\stext,\nthen\saccept\sit\sas\svalid\sJSON.\s\sThis\sreplicates\sa\slong-standing\sbug\sin\sthe\nbehavior\sof\sJSON\sroutines,\sand\sthus\savoids\sbreaking\slegacy\sapps.
+D 2024-01-23T13:53:45.407
 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1
 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea
 F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724
@@ -697,7 +697,7 @@ F src/hash.h 3340ab6e1d13e725571d7cee6d3e3135f0779a7d8e76a9ce0a85971fa3953c51
 F src/hwtime.h f9c2dfb84dce7acf95ce6d289e46f5f9d3d1afd328e53da8f8e9008e3b3caae6
 F src/in-operator.md 10cd8f4bcd225a32518407c2fb2484089112fd71
 F src/insert.c 3f0a94082d978bbdd33c38fefea15346c6c6bffb70bc645a71dc0f1f87dd3276
-F src/json.c fdb6b417e997d9b45ffd817c8c9d955dba11b99fa1199f8d03cb8fc5a9ee0941
+F src/json.c 19d96d7cae66e9b78b4ef98203e9fd916e35d20f5c8c85f079b66bd883fc9533
 F src/legacy.c d7874bc885906868cd51e6c2156698f2754f02d9eee1bae2d687323c3ca8e5aa
 F src/loadext.c 7432c944ff197046d67a1207790a1b13eec4548c85a9457eb0896bb3641dfb36
 F src/main.c 438b95162acfa17b7d218f586f5bde11d6ae82bcf030c9611fc537556870ad6b
@@ -1341,6 +1341,7 @@ F test/json103.test 53df87f83a4e5fa0c0a56eb29ff6c94055c6eb919f33316d62161a888011
 F test/json104.test 1b844a70cddcfa2e4cd81a5db0657b2e61e7f00868310f24f56a9ba0114348c1
 F test/json105.test 043838b56e68f3252a0dcf5be1689016f6f3f05056f8dcfcdc9d074f4d932988
 F test/json106.test 1d46a9294e2ced35c7f87cebbcb9626d01abab04f1969d7ded7b6f6a1d9be0f2
+F test/json107.test 59054e815c8f6b67d634d44ace421cf975828fb5651c4460aa66015c8e19d562
 F test/json501.test ab168a12eb6eb14d479f8c1cdae3ac062fd5a4679f17f976e96f1af518408330
 F test/json502.test 84634d3dbb521d2814e43624025b760c6198456c8197bbec6c977c0236648f5b
 F test/jsonb01.test cace70765b36a36aec9a85a41ea65667d3bbf647d4400ddc3ac76f8fe7d94f90
@@ -2157,9 +2158,9 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P 950bf9fe7829864e0abe6d71ca0495f346feb5d7943d76c95e55a6b86ea855da
-Q +b855886c4ccce0745af6957943e77be18949722f09821688725d546d3d79b4fb
-R 542e8b3387a0404a78fe04600e4c511f
-U dan
-Z f4b03554379000a3d3fd150f7e515fb8
+P e79b97369fa740f62f695057d4a2cf8dae48a683982ec879f04a19039c9cb418
+Q +e5dc81d5c7ee97866feb688dfa9b6fc225dabff2b020b9b96b49a8fea5640aec
+R b629d4d7fbfecc274ba1dea1a4bad552
+U drh
+Z 2e50b8df024d35696c0f9db8b6090112
 # Remove this line to create a well-formed Fossil manifest.
diff --git a/manifest.uuid b/manifest.uuid
index 9c79168739..4d00054647 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-e79b97369fa740f62f695057d4a2cf8dae48a683982ec879f04a19039c9cb418
\ No newline at end of file
+4c2c1b97dce46a279846380c937ac6de5c367927c6843516641eead7ea6db472
\ No newline at end of file
diff --git a/src/json.c b/src/json.c
index cca2b3d458..34ea058c1d 100644
--- a/src/json.c
+++ b/src/json.c
@@ -3216,6 +3216,38 @@ jsonInsertIntoBlob_patherror:
   return;
 }
 
+/*
+** If pArg is a blob that seems like a JSONB blob, then initialize
+** p to point to that JSONB and return TRUE.  If pArg does not seem like
+** a JSONB blob, then return FALSE;
+**
+** This routine is only called if it is already known that pArg is a
+** blob.  The only open question is whether or not the blob appears
+** to be a JSONB blob.
+*/
+static int jsonArgIsJsonb(sqlite3_value *pArg, JsonParse *p){
+  u32 n, sz = 0;
+  p->aBlob = (u8*)sqlite3_value_blob(pArg);
+  p->nBlob = (u32)sqlite3_value_bytes(pArg);
+  if( p->nBlob==0 ){
+    p->aBlob = 0;
+    return 0;
+  }
+  if( NEVER(p->aBlob==0) ){
+    return 0;
+  }
+  if( (p->aBlob[0] & 0x0f)<=JSONB_OBJECT
+   && (n = jsonbPayloadSize(p, 0, &sz))>0
+   && sz+n==p->nBlob
+   && ((p->aBlob[0] & 0x0f)>JSONB_FALSE || sz==0)
+  ){
+    return 1;
+  }
+  p->aBlob = 0;
+  p->nBlob = 0;
+  return 0;
+}
+
 /*
 ** Generate a JsonParse object, containing valid JSONB in aBlob and nBlob,
 ** from the SQL function argument pArg.  Return a pointer to the new
@@ -3272,29 +3304,24 @@ rebuild_from_cache:
     return p;
   }
   if( eType==SQLITE_BLOB ){
-    u32 n, sz = 0;
-    p->aBlob = (u8*)sqlite3_value_blob(pArg);
-    p->nBlob = (u32)sqlite3_value_bytes(pArg);
-    if( p->nBlob==0 ){
-      goto json_pfa_malformed;
-    }
-    if( NEVER(p->aBlob==0) ){
-      goto json_pfa_oom;
-    }
-    if( (p->aBlob[0] & 0x0f)>JSONB_OBJECT ){
-      goto json_pfa_malformed;
-    }
-    n = jsonbPayloadSize(p, 0, &sz);
-    if( n==0 
-     || sz+n!=p->nBlob
-     || ((p->aBlob[0] & 0x0f)<=JSONB_FALSE && sz>0)
-    ){
-      goto json_pfa_malformed;
-    }
-    if( (flgs & JSON_EDITABLE)!=0 && jsonBlobMakeEditable(p, 0)==0 ){
-      goto json_pfa_oom;
+    if( jsonArgIsJsonb(pArg,p) ){
+      if( (flgs & JSON_EDITABLE)!=0 && jsonBlobMakeEditable(p, 0)==0 ){
+        goto json_pfa_oom;
+      }
+      return p;
     }
-    return p;
+    /* If the blob is not valid JSONB, fall through into trying to cast
+    ** the blob into text which is then interpreted as JSON.  (tag-20240123-a)
+    **
+    ** This goes against all historical documentation about how the SQLite
+    ** JSON functions were suppose to work.  From the beginning, blob was
+    ** reserved for expansion and a blob value should have raised an error.
+    ** But it did not, due to a bug.  And many applications came to depend
+    ** upon this buggy behavior, espeically when using the CLI and reading
+    ** JSON text using readfile(), which returns a blob.  For this reason
+    ** we will continue to support the bug moving forward.
+    ** See for example https://sqlite.org/forum/forumpost/012136abd5292b8d
+    */
   }
   p->zJson = (char*)sqlite3_value_text(pArg);
   p->nJson = sqlite3_value_bytes(pArg);
@@ -4270,12 +4297,12 @@ static void jsonValidFunc(
       return;
     }
     case SQLITE_BLOB: {
-      if( (flags & 0x0c)!=0 && jsonFuncArgMightBeBinary(argv[0]) ){
+      if( jsonFuncArgMightBeBinary(argv[0]) ){
         if( flags & 0x04 ){
           /* Superficial checking only - accomplished by the
           ** jsonFuncArgMightBeBinary() call above. */
           res = 1;
-        }else{
+        }else if( flags & 0x08 ){
           /* Strict checking.  Check by translating BLOB->TEXT->BLOB.  If
           ** no errors occur, call that a "strict check". */
           JsonParse px;
@@ -4286,8 +4313,11 @@ static void jsonValidFunc(
           iErr = jsonbValidityCheck(&px, 0, px.nBlob, 1);
           res = iErr==0;
         }
+        break;
       }
-      break;
+      /* Fall through into interpreting the input as text.  See note
+      ** above at tag-20240123-a. */
+      /* no break */ deliberate_fall_through
     }
     default: {
       JsonParse px;
@@ -5023,13 +5053,9 @@ static int jsonEachFilter(
   memset(&p->sParse, 0, sizeof(p->sParse));
   p->sParse.nJPRef = 1;
   p->sParse.db = p->db;
-  if( sqlite3_value_type(argv[0])==SQLITE_BLOB ){
-    if( jsonFuncArgMightBeBinary(argv[0]) ){
-      p->sParse.nBlob = sqlite3_value_bytes(argv[0]);
-      p->sParse.aBlob = (u8*)sqlite3_value_blob(argv[0]);
-    }else{
-      goto json_each_malformed_input;
-    }
+  if( jsonFuncArgMightBeBinary(argv[0]) ){
+    p->sParse.nBlob = sqlite3_value_bytes(argv[0]);
+    p->sParse.aBlob = (u8*)sqlite3_value_blob(argv[0]);
   }else{
     p->sParse.zJson = (char*)sqlite3_value_text(argv[0]);
     p->sParse.nJson = sqlite3_value_bytes(argv[0]);
diff --git a/test/json107.test b/test/json107.test
new file mode 100644
index 0000000000..779b557fba
--- /dev/null
+++ b/test/json107.test
@@ -0,0 +1,86 @@
+# 2024-01-23
+#
+# The author disclaims copyright to this source code.  In place of
+# a legal notice, here is a blessing:
+#
+#    May you do good and not evil.
+#    May you find forgiveness for yourself and forgive others.
+#    May you share freely, never taking more than you give.
+#
+#***********************************************************************
+# 
+# Legacy JSON bug:  If the input is a BLOB that when cast into TEXT looks
+# like valid JSON, then treat it as valid JSON.
+#
+# The original intent of the JSON functions was to raise an error on any
+# BLOB input.  That intent was clearly documented, but the code failed to
+# to implement it.  Subsequently, many applications began to depend on the
+# incorrect behavior, especially apps that used readfile() to read JSON
+# content, since readfile() returns a BLOB.  So we need to support the
+# bug moving forward.
+#
+# The tests in this fail verify that the original buggy behavior is
+# preserved.
+#
+
+set testdir [file dirname $argv0]
+source $testdir/tester.tcl
+set testprefix json107
+
+if {[db one {PRAGMA encoding}]!="UTF-8"} {
+  # These tests only work for a UTF-8 encoding.
+  finish_test
+  return
+}
+
+do_execsql_test 1.1 {
+  SELECT json_valid( CAST('{"a":1}' AS BLOB) );
+} 1
+do_execsql_test 1.1.1 {
+  SELECT json_valid( CAST('{"a":1}' AS BLOB), 1);
+} 1
+do_execsql_test 1.1.2 {
+  SELECT json_valid( CAST('{"a":1}' AS BLOB), 2);
+} 1
+do_execsql_test 1.1.4 {
+  SELECT json_valid( CAST('{"a":1}' AS BLOB), 4);
+} 0
+do_execsql_test 1.1.8 {
+  SELECT json_valid( CAST('{"a":1}' AS BLOB), 8);
+} 0
+
+do_execsql_test 1.2.1 {
+  SELECT CAST('{"a":123}' AS blob) -> 'a';
+} 123
+do_execsql_test 1.2.2 {
+  SELECT CAST('{"a":123}' AS blob) ->> 'a';
+} 123
+do_execsql_test 1.2.3 {
+  SELECT json_extract(CAST('{"a":123}' AS blob), '$.a');
+} 123
+do_execsql_test 1.3 {
+  SELECT json_insert(CAST('{"a":123}' AS blob),'$.b',456);
+} {{{"a":123,"b":456}}}
+do_execsql_test 1.4 {
+  SELECT json_remove(CAST('{"a":123,"b":456}' AS blob),'$.a');
+} {{{"b":456}}}
+do_execsql_test 1.5 {
+  SELECT json_set(CAST('{"a":123,"b":456}' AS blob),'$.a',789);
+} {{{"a":789,"b":456}}}
+do_execsql_test 1.6 {
+  SELECT json_replace(CAST('{"a":123,"b":456}' AS blob),'$.a',789);
+} {{{"a":789,"b":456}}}
+do_execsql_test 1.7 {
+  SELECT json_type(CAST('{"a":123,"b":456}' AS blob));
+} object
+do_execsql_test 1.8 {
+  SELECT json(CAST('{"a":123,"b":456}' AS blob));
+} {{{"a":123,"b":456}}}
+
+ifcapable vtab {
+  do_execsql_test 2.1 {
+    SELECT key, value FROM json_tree( CAST('{"a":123,"b":456}' AS blob) )
+      WHERE atom;
+  } {a 123 b 456}
+} 
+finish_test