From f5ddb9c214b5e6237e9bfb878aa21cfd2ef129fc Mon Sep 17 00:00:00 2001 From: drh Date: Fri, 11 Sep 2015 00:06:41 +0000 Subject: [PATCH] Take out the goofy '$$' path syntax. Instead, use subtypes to communicate when a string is JSON. Add the json() function that validates and minifies the JSON and sets the appropriate subtype. FossilOrigin-Name: 8a80d6459e246ec1b38325e1cbd1e862157138b3 --- ext/misc/json1.c | 89 +++++++++++++++-------------------------------- manifest | 14 ++++---- manifest.uuid | 2 +- test/json101.test | 18 ++++++++-- 4 files changed, 51 insertions(+), 72 deletions(-) diff --git a/ext/misc/json1.c b/ext/misc/json1.c index 755d035c37..a217ce80b8 100644 --- a/ext/misc/json1.c +++ b/ext/misc/json1.c @@ -68,6 +68,9 @@ struct JsonString { #define JSON_ARRAY 6 #define JSON_OBJECT 7 +/* The "subtype" set for JSON values */ +#define JSON_SUBTYPE 74 /* Ascii for "J" */ + /* ** Names of the various JSON types: */ @@ -82,8 +85,7 @@ static const char * const jsonType[] = { #define JNODE_REMOVE 0x04 /* Do not output */ #define JNODE_REPLACE 0x08 /* Replace with JsonNode.iVal */ #define JNODE_APPEND 0x10 /* More ARRAY/OBJECT entries at u.iAppend */ -#define JNODE_JSON 0x20 /* Treat REPLACE as JSON text */ -#define JNODE_LABEL 0x40 /* Is a label of an object */ +#define JNODE_LABEL 0x20 /* Is a label of an object */ /* A single node of parsed JSON @@ -243,8 +245,7 @@ static void jsonAppendString(JsonString *p, const char *zIn, u32 N){ */ static void jsonAppendValue( JsonString *p, /* Append to this JSON string */ - sqlite3_value *pValue, /* Value to append */ - u8 textIsJson /* Try to treat text values as JSON */ + sqlite3_value *pValue /* Value to append */ ){ switch( sqlite3_value_type(pValue) ){ case SQLITE_NULL: { @@ -261,7 +262,7 @@ static void jsonAppendValue( case SQLITE_TEXT: { const char *z = (const char*)sqlite3_value_text(pValue); u32 n = (u32)sqlite3_value_bytes(pValue); - if( textIsJson ){ + if( sqlite3_value_subtype(pValue)==JSON_SUBTYPE ){ jsonAppendRaw(p, z, n); }else{ jsonAppendString(p, z, n); @@ -365,8 +366,7 @@ static void jsonRenderNode( if( pNode[j].jnFlags & (JNODE_REMOVE|JNODE_REPLACE) ){ if( pNode[j].jnFlags & JNODE_REPLACE ){ jsonAppendSeparator(pOut); - jsonAppendValue(pOut, aReplace[pNode[j].iVal], - (pNode[j].jnFlags & JNODE_JSON)!=0); + jsonAppendValue(pOut, aReplace[pNode[j].iVal]); } }else{ jsonAppendSeparator(pOut); @@ -391,8 +391,7 @@ static void jsonRenderNode( jsonRenderNode(&pNode[j], pOut, aReplace); jsonAppendChar(pOut, ':'); if( pNode[j+1].jnFlags & JNODE_REPLACE ){ - jsonAppendValue(pOut, aReplace[pNode[j+1].iVal], - (pNode[j+1].jnFlags & JNODE_JSON)!=0); + jsonAppendValue(pOut, aReplace[pNode[j+1].iVal]); }else{ jsonRenderNode(&pNode[j+1], pOut, aReplace); } @@ -421,6 +420,7 @@ static void jsonReturnJson( jsonInit(&s, pCtx); jsonRenderNode(pNode, &s, aReplace); jsonResult(&s); + sqlite3_result_subtype(pCtx, JSON_SUBTYPE); } /* @@ -951,21 +951,15 @@ static char *jsonPathSyntaxError(const char *zErr){ ** ** If pApnd!=NULL then try to append missing nodes and set *pApnd = 1 if ** nodes are appended. -** -** If the path starts with $$ then set *pFlags to JNODE_REPLACE|JNODE_JSON -** as a single to the caller that the input text to be inserted should be -** interpreted as JSON rather than as ordinary text. */ static JsonNode *jsonLookup( JsonParse *pParse, /* The JSON to search */ const char *zPath, /* The path to search */ int *pApnd, /* Append nodes to complete path if not NULL */ - sqlite3_context *pCtx, /* Report errors here, if not NULL */ - u8 *pFlags /* Write JNODE_REPLACE or _REPLACE|_JSON here */ + sqlite3_context *pCtx /* Report errors here, if not NULL */ ){ const char *zErr = 0; JsonNode *pNode = 0; - u8 fg = JNODE_REPLACE; if( zPath==0 ) return 0; if( zPath[0]!='$' ){ @@ -973,15 +967,6 @@ static JsonNode *jsonLookup( goto lookup_err; } zPath++; - if( zPath[0]=='$' ){ - if( pFlags==0 ){ - zErr = zPath; - goto lookup_err; - } - zPath++; - fg = JNODE_REPLACE|JNODE_JSON; - } - if( pFlags ) *pFlags = fg; pNode = jsonLookupStep(pParse, 0, zPath, pApnd, &zErr); return pNode; @@ -996,7 +981,6 @@ lookup_err: sqlite3_result_error_nomem(pCtx); } } - if( pFlags ) *pFlags = fg; return 0; } @@ -1060,32 +1044,16 @@ static void jsonParseFunc( } /* -** The json_test1(JSON) function parses and rebuilds the JSON string. +** The json_test1(JSON) function return true (1) if the input is JSON +** text generated by another json function. It returns (0) if the input +** is not known to be JSON. */ static void jsonTest1Func( sqlite3_context *ctx, int argc, sqlite3_value **argv ){ - JsonParse x; /* The parse */ - if( jsonParse(&x, ctx, (const char*)sqlite3_value_text(argv[0])) ) return; - jsonReturnJson(x.aNode, ctx, 0); - jsonParseReset(&x); -} - -/* -** The json_nodecount(JSON) function returns the number of nodes in the -** input JSON string. -*/ -static void jsonNodeCountFunc( - sqlite3_context *ctx, - int argc, - sqlite3_value **argv -){ - JsonParse x; /* The parse */ - if( jsonParse(&x, ctx, (const char*)sqlite3_value_text(argv[0])) ) return; - sqlite3_result_int64(ctx, (sqlite3_int64)x.nNode); - jsonParseReset(&x); + sqlite3_result_int(ctx, sqlite3_value_subtype(argv[0])==JSON_SUBTYPE); } #endif /* SQLITE_DEBUG */ @@ -1110,10 +1078,11 @@ static void jsonArrayFunc( jsonAppendChar(&jx, '['); for(i=0; i2 ){ jsonAppendSeparator(&jx); @@ -1196,6 +1165,7 @@ static void jsonExtractFunc( if( argc>2 && i==argc ){ jsonAppendChar(&jx, ']'); jsonResult(&jx); + sqlite3_result_subtype(ctx, JSON_SUBTYPE); } jsonReset(&jx); jsonParseReset(&x); @@ -1234,10 +1204,11 @@ static void jsonObjectFunc( n = (u32)sqlite3_value_bytes(argv[i]); jsonAppendString(&jx, z, n); jsonAppendChar(&jx, ':'); - jsonAppendValue(&jx, argv[i+1], 0); + jsonAppendValue(&jx, argv[i+1]); } jsonAppendChar(&jx, '}'); jsonResult(&jx); + sqlite3_result_subtype(ctx, JSON_SUBTYPE); } @@ -1263,7 +1234,7 @@ static void jsonRemoveFunc( for(i=1; i<(u32)argc; i++){ zPath = (const char*)sqlite3_value_text(argv[i]); if( zPath==0 ) goto remove_done; - pNode = jsonLookup(&x, zPath, 0, ctx, 0); + pNode = jsonLookup(&x, zPath, 0, ctx); if( x.nErr ) goto remove_done; if( pNode ) pNode->jnFlags |= JNODE_REMOVE; } @@ -1299,13 +1270,11 @@ static void jsonReplaceFunc( if( jsonParse(&x, ctx, (const char*)sqlite3_value_text(argv[0])) ) return; if( x.nNode ){ for(i=1; i<(u32)argc; i+=2){ - u8 jnFlags = JNODE_REPLACE; zPath = (const char*)sqlite3_value_text(argv[i]); - pNode = jsonLookup(&x, zPath, 0, ctx, &jnFlags); + pNode = jsonLookup(&x, zPath, 0, ctx); if( x.nErr ) goto replace_err; if( pNode ){ - pNode->jnFlags &= ~JNODE_JSON; - pNode->jnFlags |= jnFlags; + pNode->jnFlags |= JNODE_REPLACE; pNode->iVal = i+1; } } @@ -1351,18 +1320,16 @@ static void jsonSetFunc( if( jsonParse(&x, ctx, (const char*)sqlite3_value_text(argv[0])) ) return; if( x.nNode ){ for(i=1; i<(u32)argc; i+=2){ - u8 jnFlags = JNODE_REPLACE; zPath = (const char*)sqlite3_value_text(argv[i]); bApnd = 0; - pNode = jsonLookup(&x, zPath, &bApnd, ctx, &jnFlags); + pNode = jsonLookup(&x, zPath, &bApnd, ctx); if( x.oom ){ sqlite3_result_error_nomem(ctx); goto jsonSetDone; }else if( x.nErr ){ goto jsonSetDone; }else if( pNode && (bApnd || bIsSet) ){ - pNode->jnFlags &= ~JNODE_JSON; - pNode->jnFlags |= jnFlags; + pNode->jnFlags |= JNODE_REPLACE; pNode->iVal = i+1; } } @@ -1396,7 +1363,7 @@ static void jsonTypeFunc( JsonNode *pNode; if( argc==2 ){ zPath = (const char*)sqlite3_value_text(argv[1]); - pNode = jsonLookup(&x, zPath, 0, ctx, 0); + pNode = jsonLookup(&x, zPath, 0, ctx); }else{ pNode = x.aNode; } @@ -1917,6 +1884,7 @@ int sqlite3_json_init( int flag; void (*xFunc)(sqlite3_context*,int,sqlite3_value**); } aFunc[] = { + { "json", 1, 0, jsonRemoveFunc }, { "json_array", -1, 0, jsonArrayFunc }, { "json_array_length", 1, 0, jsonArrayLengthFunc }, { "json_array_length", 2, 0, jsonArrayLengthFunc }, @@ -1934,7 +1902,6 @@ int sqlite3_json_init( /* DEBUG and TESTING functions */ { "json_parse", 1, 0, jsonParseFunc }, { "json_test1", 1, 0, jsonTest1Func }, - { "json_nodecount", 1, 0, jsonNodeCountFunc }, #endif }; #ifndef SQLITE_OMIT_VIRTUALTABLE diff --git a/manifest b/manifest index 285e65a2cd..525605fc0d 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Make\sthe\ssqlite3_value_subtype()\sand\ssqlite3_result_subtype()\sinterfaces\navailable\sto\sloadable\sextensions. -D 2015-09-10T20:40:21.038 +C Take\sout\sthe\sgoofy\s'$$'\spath\ssyntax.\s\sInstead,\suse\ssubtypes\sto\scommunicate\swhen\na\sstring\sis\sJSON.\s\sAdd\sthe\sjson()\sfunction\sthat\svalidates\sand\sminifies\sthe\nJSON\sand\ssets\sthe\sappropriate\ssubtype. +D 2015-09-11T00:06:41.899 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in f85066ce844a28b671aaeeff320921cd0ce36239 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -195,7 +195,7 @@ F ext/misc/eval.c f971962e92ebb8b0a4e6b62949463ee454d88fa2 F ext/misc/fileio.c d4171c815d6543a9edef8308aab2951413cd8d0f F ext/misc/fuzzer.c 4c84635c71c26cfa7c2e5848cf49fe2d2cfcd767 F ext/misc/ieee754.c b0362167289170627659e84173f5d2e8fee8566e -F ext/misc/json1.c 4387d091c0ec0f4d9ed05560960f03d366db4fe0 +F ext/misc/json1.c 96490b8e34299a416ab221f827e0369344d95c53 F ext/misc/nextchar.c 35c8b8baacb96d92abbb34a83a997b797075b342 F ext/misc/percentile.c bcbee3c061b884eccb80e21651daaae8e1e43c63 F ext/misc/regexp.c af92cdaa5058fcec1451e49becc7ba44dba023dc @@ -814,7 +814,7 @@ F test/journal3.test ff8af941f9e06161d3db1b46bb9f965ff0e7f307 F test/jrnlmode.test 7864d59cf7f6e552b9b99ba0f38acd167edc10fa F test/jrnlmode2.test 81610545a4e6ed239ea8fa661891893385e23a1d F test/jrnlmode3.test 556b447a05be0e0963f4311e95ab1632b11c9eaa -F test/json101.test 11535d8986184500f4c30cc2f0b154b4ab05cc4e +F test/json101.test e20d2421c531db32fad59c5e06e80af0b1b002c8 F test/json102.test 12ef6d7d7c02c526fa3c2be1e933e7eb2a497cea F test/keyword1.test 37ef6bba5d2ed5b07ecdd6810571de2956599dff F test/lastinsert.test 42e948fd6442f07d60acbd15d33fb86473e0ef63 @@ -1386,7 +1386,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 48bd54594752d5be3337f12c72f28d2080cb630b F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 7b5be299c617a3d3ed327ed30ef0a66a62c85b1f -R 3b51d146c21b59a26b4bd6730fa2e17a +P c6fca0be11f7414292279e2ea1b004260e6f4bb6 +R 6298ef57aadddbd507dac61cb4572ea0 U drh -Z 3c2729e75edd6da9e7ec99089fe2f222 +Z f00bae494bb0cd397ae4843df54988dc diff --git a/manifest.uuid b/manifest.uuid index d8c4b3eba0..0c45a9ed67 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -c6fca0be11f7414292279e2ea1b004260e6f4bb6 \ No newline at end of file +8a80d6459e246ec1b38325e1cbd1e862157138b3 \ No newline at end of file diff --git a/test/json101.test b/test/json101.test index 1a84a5fc5e..9543ccfd24 100644 --- a/test/json101.test +++ b/test/json101.test @@ -16,9 +16,21 @@ set testdir [file dirname $argv0] source $testdir/tester.tcl load_static_extension db json -do_execsql_test json1-1.1 { +do_execsql_test json1-1.1.00 { SELECT json_array(1,2.5,null,'hello'); } {[1,2.5,null,"hello"]} +do_execsql_test json1-1.1.01 { + SELECT json_array(1,'{"abc":2.5,"def":null,"ghi":hello}',99); + -- the second term goes in as a string: +} {[1,"{\\"abc\\":2.5,\\"def\\":null,\\"ghi\\":hello}",99]} +do_execsql_test json1-1.1.02 { + SELECT json_array(1,json('{"abc":2.5,"def":null,"ghi":"hello"}'),99); + -- the second term goes in as JSON +} {[1,{"abc":2.5,"def":null,"ghi":"hello"},99]} +do_execsql_test json1-1.1.03 { + SELECT json_array(1,json_object('abc',2.5,'def',null,'ghi','hello'),99); + -- the second term goes in as JSON +} {[1,{"abc":2.5,"def":null,"ghi":"hello"},99]} do_execsql_test json1-1.2 { SELECT hex(json_array('String "\ Test')); } {5B22537472696E67205C225C5C2054657374225D} @@ -54,13 +66,13 @@ do_execsql_test json1-3.1 { SELECT json_replace('{"a":1,"b":2}','$.a','[3,4,5]'); } {{{"a":"[3,4,5]","b":2}}} do_execsql_test json1-3.2 { - SELECT json_replace('{"a":1,"b":2}','$$.a','[3,4,5]'); + SELECT json_replace('{"a":1,"b":2}','$.a',json('[3,4,5]')); } {{{"a":[3,4,5],"b":2}}} do_execsql_test json1-3.3 { SELECT json_type(json_set('{"a":1,"b":2}','$.b','{"x":3,"y":4}'),'$.b'); } {text} do_execsql_test json1-3.4 { - SELECT json_type(json_set('{"a":1,"b":2}','$$.b','{"x":3,"y":4}'),'$.b'); + SELECT json_type(json_set('{"a":1,"b":2}','$.b',json('{"x":3,"y":4}')),'$.b'); } {object} # Per rfc7159, any JSON value is allowed at the top level, and whitespace -- 2.39.5