From: drh <> Date: Mon, 25 Sep 2023 13:23:29 +0000 (+0000) Subject: As a temporary measure, try to translate the BLOB JSON format into the X-Git-Tag: version-3.45.0~116^2~145 X-Git-Url: http://git.ipfire.org/?a=commitdiff_plain;h=5933581cf035d1479752ebdd3bce84ae609c215b;p=thirdparty%2Fsqlite.git As a temporary measure, try to translate the BLOB JSON format into the legacy node format for processing. FossilOrigin-Name: 14f2e95a9e531ef0d3fa7f1249f23c073a50c31b2109eefc2f258cada635ac2f --- diff --git a/manifest b/manifest index 1e08af59f7..3fd91c08fe 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\sminor\sparse-to-BLOB\sbugs. -D 2023-09-22T16:20:48.168 +C As\sa\stemporary\smeasure,\stry\sto\stranslate\sthe\sBLOB\sJSON\sformat\sinto\sthe\nlegacy\snode\sformat\sfor\sprocessing. +D 2023-09-25T13:23:29.913 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -670,7 +670,7 @@ F src/hash.h 3340ab6e1d13e725571d7cee6d3e3135f0779a7d8e76a9ce0a85971fa3953c51 F src/hwtime.h f9c2dfb84dce7acf95ce6d289e46f5f9d3d1afd328e53da8f8e9008e3b3caae6 F src/in-operator.md 10cd8f4bcd225a32518407c2fb2484089112fd71 F src/insert.c 3f0a94082d978bbdd33c38fefea15346c6c6bffb70bc645a71dc0f1f87dd3276 -F src/json.c 522ebfebd77224594e2fcd9ea18e201ac8b997d5f1cf2c71d8eba00e874b54b2 +F src/json.c 23efc117c4641118c2154938decc80fb3b29b1a420bcd0eaefe24ef808d1ada4 F src/legacy.c d7874bc885906868cd51e6c2156698f2754f02d9eee1bae2d687323c3ca8e5aa F src/loadext.c 98cfba10989b3da6f1807ad42444017742db7f100a54f1032af7a8b1295912c0 F src/main.c 618aeb399e993cf561864f4b0cf6a331ee4f355cf663635f8d9da3193a46aa40 @@ -2121,8 +2121,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 0b70cb77a4c8e3f17932f1ecca3942e0b0b03de637fb9656a130fe045f7ef826 -R e81e3fd90e402a6e846b93a0d3beab83 +P 8b53b2e6600c324ff7864840d98a3f03896b9792fcb60b70cc1f6227b3bd4ca1 +R 31821ddd534254fa82bb41b5d469060a U drh -Z 285487434c996c07680e15a392e180f6 +Z 25f12fc9cea10e56c4709ee1d21954ff # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 94c65ef3d9..1d3c76b2dd 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -8b53b2e6600c324ff7864840d98a3f03896b9792fcb60b70cc1f6227b3bd4ca1 \ No newline at end of file +14f2e95a9e531ef0d3fa7f1249f23c073a50c31b2109eefc2f258cada635ac2f \ No newline at end of file diff --git a/src/json.c b/src/json.c index ac8976d03c..afa152624e 100644 --- a/src/json.c +++ b/src/json.c @@ -220,6 +220,7 @@ struct JsonParse { u8 hasNonstd; /* True if input uses non-standard features like JSON5 */ u8 useMod; /* Actually use the edits contain inside aNode */ u8 hasMod; /* aNode contains edits from the original zJson */ + u8 isBinary; /* True if zJson is the binary encoding */ u32 nJPRef; /* Number of references to this object */ int nJson; /* Length of the zJson string in bytes */ int nAlt; /* Length of alternative JSON string zAlt, in bytes */ @@ -1809,6 +1810,9 @@ static void jsonParseFillInParentage(JsonParse *pParse, u32 i, u32 iParent){ } } +/* Forward reference */ +static int jsonParseValueFromBinary(JsonParse *pParse, u32 i); + /* ** Parse a complete JSON string. Return 0 on success or non-zero if there ** are any errors. If an error occurs, free all memory held by pParse, @@ -1823,7 +1827,13 @@ static int jsonParse( ){ int i; const char *zJson = pParse->zJson; - i = jsonParseValue(pParse, 0); + if( pParse->isBinary ){ + pParse->aBlob = (u8*)pParse->zJson; + pParse->nBlob = pParse->nJson; + i = jsonParseValueFromBinary(pParse, 0); + }else{ + i = jsonParseValue(pParse, 0); + } if( pParse->oom ) i = -1; if( i>0 ){ assert( pParse->iDepth==0 ); @@ -1872,6 +1882,10 @@ static int jsonParseFindParents(JsonParse *pParse){ #define JSON_CACHE_ID (-429938) /* First cache entry */ #define JSON_CACHE_SZ 4 /* Max number of cache entries */ +/* Forward reference */ +static int jsonFuncArgMightBeBinary(sqlite3_value *pJson); + + /* ** Obtain a complete parse of the JSON found in the pJson argument ** @@ -1906,8 +1920,8 @@ static JsonParse *jsonParseCached( sqlite3_context *pErrCtx, /* Write parse errors here if not NULL */ int bUnedited /* No prior edits allowed */ ){ - char *zJson = (char*)sqlite3_value_text(pJson); - int nJson = sqlite3_value_bytes(pJson); + char *zJson; + int nJson; JsonParse *p; JsonParse *pMatch = 0; int iKey; @@ -1915,6 +1929,16 @@ static JsonParse *jsonParseCached( u32 iMinHold = 0xffffffff; u32 iMaxHold = 0; int bJsonRCStr; + int isBinary; + + if( jsonFuncArgMightBeBinary(pJson) ){ + zJson = (char*)sqlite3_value_blob(pJson); + isBinary = 1; + }else{ + zJson = (char*)sqlite3_value_text(pJson); + isBinary = 0; + } + nJson = sqlite3_value_bytes(pJson); if( zJson==0 ) return 0; for(iKey=0; iKeybJsonIsRCStr = 1; }else{ p->zJson = (char*)&p[1]; - memcpy(p->zJson, zJson, nJson+1); + memcpy(p->zJson, zJson, nJson+(isBinary==0)); } p->nJPRef = 1; + p->isBinary = isBinary; + p->nJson = nJson; if( jsonParse(p, pErrCtx) ){ if( pErrCtx==0 ){ p->nErr = 1; @@ -1985,7 +2011,6 @@ static JsonParse *jsonParseCached( jsonParseFree(p); return 0; } - p->nJson = nJson; p->iHold = iMaxHold+1; /* Transfer ownership of the new JsonParse to the cache */ sqlite3_set_auxdata(pCtx, JSON_CACHE_ID+iMinKey, p, @@ -2951,16 +2976,15 @@ static int jsonParseB( /* The byte at index i is a node type-code. This routine ** determines the payload size for that node and writes that ** payload size in to *pSz. It returns the offset from i to the -** beginning of the payload. +** beginning of the payload. Return 0 on error. */ static u32 jsonbPayloadSize(JsonParse *pParse, u32 i, u32 *pSz){ u8 x; u32 sz; u32 n; if( i>pParse->nBlob ){ - pParse->iErr = 1; *pSz = 0; - return 1; + return 0; } x = pParse->aBlob[i]>>4; if( x<=11 ){ @@ -2968,25 +2992,22 @@ static u32 jsonbPayloadSize(JsonParse *pParse, u32 i, u32 *pSz){ n = 1; }else if( x==12 ){ if( i+1>pParse->nBlob ){ - pParse->iErr = 1; *pSz = 0; - return 1; + return 0; } sz = pParse->aBlob[i+1]; n = 2; }else if( x==13 ){ if( i+2>pParse->nBlob ){ - pParse->iErr = 1; *pSz = 0; - return 1; + return 0; } sz = (pParse->aBlob[i+1]<<8) + pParse->aBlob[i+2]; n = 3; }else{ if( i+4>pParse->nBlob ){ - pParse->iErr = 1; *pSz = 0; - return 1; + return 0; } sz = (pParse->aBlob[i+1]<<24) + (pParse->aBlob[i+2]<<16) + (pParse->aBlob[i+3]<<8) + pParse->aBlob[i+4]; @@ -2994,7 +3015,6 @@ static u32 jsonbPayloadSize(JsonParse *pParse, u32 i, u32 *pSz){ } if( i+sz+n>pParse->nBlob ){ sz = pParse->nBlob - (i+n); - pParse->iErr = 1; } *pSz = sz; return n; @@ -3179,6 +3199,124 @@ static u32 jsonRenderBlob( return i+n+sz; } +/* Return true if the input pJson +** +** For performance reasons, this routine does not do a detailed check of the +** input BLOB to ensure that it is well-formed. Hence, false positives are +** possible. False negatives should never occur, however. +*/ +static int jsonFuncArgMightBeBinary(sqlite3_value *pJson){ + u32 sz, n; + const u8 *aBlob; + int nBlob; + JsonParse s; + if( sqlite3_value_type(pJson)!=SQLITE_BLOB ) return 0; + nBlob = sqlite3_value_bytes(pJson); + if( nBlob<1 ) return 0; + aBlob = sqlite3_value_blob(pJson); + if( (aBlob[0] & 0x0f)>JSONB_OBJECT ) return 0; + memset(&s, 0, sizeof(s)); + s.aBlob = (u8*)aBlob; + s.nBlob = nBlob; + n = jsonbPayloadSize(&s, 0, &sz); + if( n==0 ) return 0; + return sz+n==(u32)nBlob; +} + +/* Parse a single element of binary JSON into the legacy Node structure. +** Return the index of the first byte past the end of the binary JSON element. +** +** This routine is a temporary translator while the legacy Node encoding +** is still in use. It will be removed after all processing translates +** to the new BLOB encoding. +*/ +static int jsonParseValueFromBinary(JsonParse *pParse, u32 i){ + u8 t; /* Node type */ + u32 sz; /* Node size */ + u32 x; /* Index of payload start */ + + const char *zPayload; + x = jsonbPayloadSize(pParse, i, &sz); + if( x==0 ) return -1; + t = pParse->zJson[i] & 0x0f; + zPayload = &pParse->zJson[i+x]; + switch( t ){ + case JSONB_NULL: { + jsonParseAddNode(pParse, JSON_NULL, 0, 0); + break; + } + case JSONB_TRUE: { + jsonParseAddNode(pParse, JSON_TRUE, 0, 0); + break; + } + case JSONB_FALSE: { + jsonParseAddNode(pParse, JSON_FALSE, 0, 0); + break; + } + case JSONB_INT: { + jsonParseAddNode(pParse, JSON_INT, sz, zPayload); + break; + } + case JSONB_INT5: { + pParse->hasNonstd = 1; + jsonParseAddNode(pParse, JSON_INT | (JNODE_JSON5<<8), sz, zPayload); + break; + } + case JSONB_FLOAT: { + jsonParseAddNode(pParse, JSON_REAL, sz, zPayload); + break; + } + case JSONB_FLOAT5: { + pParse->hasNonstd = 1; + jsonParseAddNode(pParse, JSON_REAL | (JNODE_JSON5<<8), sz, zPayload); + break; + } + case JSONB_TEXT: { + jsonParseAddNode(pParse, JSON_STRING | (JNODE_RAW<<8), sz, zPayload); + break; + } + case JSONB_TEXTJ: { + jsonParseAddNode(pParse, JSON_STRING | (JNODE_RAW<<8), sz, zPayload); + break; + } + case JSONB_TEXT5: { + pParse->hasNonstd = 1; + jsonParseAddNode(pParse, JSON_STRING | ((JNODE_ESCAPE|JNODE_JSON5)<<8), + sz, zPayload); + break; + } + case JSONB_TEXTRAW: { + jsonParseAddNode(pParse, JSON_STRING | (JNODE_RAW<<8), sz, zPayload); + break; + } + case JSONB_ARRAY: { + int iThis = jsonParseAddNode(pParse, JSON_ARRAY, 0, 0); + u32 j = i+x; + while( jaNode[iThis].n = pParse->nNode - (u32)iThis - 1; + break; + } + case JSONB_OBJECT: { + int iThis = jsonParseAddNode(pParse, JSON_OBJECT, 0, 0); + u32 j = i+x, k = 0; + while( jaNode[pParse->nNode-1].jnFlags |= JNODE_LABEL; + } + j = (u32)r; + } + pParse->aNode[iThis].n = pParse->nNode - (u32)iThis - 1; + break; + } + } + return i+x+sz; +} /**************************************************************************** ** SQL functions used for testing and debugging @@ -3289,12 +3427,23 @@ static void jsonTest1Func( ** Scalar SQL function implementations ****************************************************************************/ -/* SQL Function: jsonb_test1(TEXT_JSON) +/* SQL Function: jsonb(JSON) ** -** Parse TEXT JSON into the BLOB format and return the resulting BLOB. -** Development testing only. +** Convert the input argument into JSONB (the SQLite binary encoding of +** JSON). +** +** If the input is TEXT, or NUMERIC, try to parse it as JSON. If the fails, +** raise an error. Otherwise, return the resulting BLOB value. +** +** If the input is a BLOB, check to see if the input is a plausible +** JSONB. If it is, return it unchanged. Raise an error if it is not. +** Note that there could be internal inconsistencies in the BLOB - this +** routine does not do a full byte-for-byte validity check of the +** JSON blob. +** +** If the input is NULL, return NULL. */ -static void jsonbTest1( +static void jsonbFunc( sqlite3_context *ctx, int argc, sqlite3_value **argv @@ -3305,20 +3454,28 @@ static void jsonbTest1( JsonParse x; UNUSED_PARAMETER(argc); - zJson = (const char*)sqlite3_value_text(argv[0]); - if( zJson==0 ) return; - nJson = sqlite3_value_bytes(argv[0]); - pParse = &x; - memset(&x, 0, sizeof(x)); - x.zJson = (char*)zJson; - x.nJson = nJson; - if( jsonParseB(pParse, ctx)==0 && pParse->aBlob!=0 ){ - sqlite3_result_blob(ctx, pParse->aBlob, pParse->nBlob, sqlite3_free); - pParse->aBlob = 0; - pParse->nBlob = 0; - pParse->nBlobAlloc = 0; - } - jsonParseReset(pParse); + if( sqlite3_value_type(argv[0])==SQLITE_NULL ){ + /* No-op */ + }else if( jsonFuncArgMightBeBinary(argv[0]) ){ + sqlite3_result_value(ctx, argv[0]); + }else{ + zJson = (const char*)sqlite3_value_text(argv[0]); + if( zJson==0 ) return; + nJson = sqlite3_value_bytes(argv[0]); + pParse = &x; + memset(&x, 0, sizeof(x)); + x.zJson = (char*)zJson; + x.nJson = nJson; + if( jsonParseB(pParse, ctx) ){ + sqlite3_result_error(ctx, "malformed JSON", -1); + }else{ + sqlite3_result_blob(ctx, pParse->aBlob, pParse->nBlob, sqlite3_free); + pParse->aBlob = 0; + pParse->nBlob = 0; + pParse->nBlobAlloc = 0; + } + jsonParseReset(pParse); + } } /* SQL Function: jsonb_test2(BLOB_JSON) @@ -3351,7 +3508,7 @@ static void jsonbTest2( } /* -** Implementation of the json_QUOTE(VALUE) function. Return a JSON value +** Implementation of the json_quote(VALUE) function. Return a JSON value ** corresponding to the SQL value input. Mostly this means putting ** double-quotes around strings and returning the unquoted string "null" ** when given a NULL input. @@ -4757,7 +4914,7 @@ void sqlite3RegisterJsonFunctions(void){ JFUNCTION(json_type, 1, 0, jsonTypeFunc), JFUNCTION(json_type, 2, 0, jsonTypeFunc), JFUNCTION(json_valid, 1, 0, jsonValidFunc), - JFUNCTION(jsonb_test1, 1, 0, jsonbTest1), + JFUNCTION(jsonb, 1, 0, jsonbFunc), JFUNCTION(jsonb_test2, 1, 0, jsonbTest2), #if SQLITE_DEBUG JFUNCTION(json_parse, 1, 0, jsonParseFunc),