From: drh <> Date: Fri, 18 Nov 2022 17:50:52 +0000 (+0000) Subject: Add the SQLITE_DBCONFIG_LENIENT_JSON configuration option. Modify the X-Git-Url: http://git.ipfire.org/?a=commitdiff_plain;h=refs%2Fheads%2Flenient-json;p=thirdparty%2Fsqlite.git Add the SQLITE_DBCONFIG_LENIENT_JSON configuration option. Modify the built-in JSON routines such that when this setting is active, arguments that that ought to be JSON but still give a reasonable result (ex: NULL) rather than raising an error. FossilOrigin-Name: 186db57d263be09624cac84c97723d05813d70fe2e88d45e15b9f3b6d8c63935 --- diff --git a/manifest b/manifest index 00bc7b6ef8..fddbab7df3 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\scorner\scases\sin\sUTF8\shandling\sin\sthe\sREGEXP\sextension.\n[forum:/forumpost/3ffe058b04|Forum\spost\s3ffe058b04]. -D 2022-11-17T19:24:39.375 +C Add\sthe\sSQLITE_DBCONFIG_LENIENT_JSON\sconfiguration\soption.\s\sModify\sthe\nbuilt-in\sJSON\sroutines\ssuch\sthat\swhen\sthis\ssetting\sis\sactive,\sarguments\sthat\nthat\sought\sto\sbe\sJSON\sbut\sstill\sgive\sa\sreasonable\sresult\s(ex:\sNULL)\srather\nthan\sraising\san\serror. +D 2022-11-18T17:50:52.177 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -597,10 +597,10 @@ F src/hash.h 3340ab6e1d13e725571d7cee6d3e3135f0779a7d8e76a9ce0a85971fa3953c51 F src/hwtime.h cb1d7e3e1ed94b7aa6fde95ae2c2daccc3df826be26fc9ed7fd90d1750ae6144 F src/in-operator.md 10cd8f4bcd225a32518407c2fb2484089112fd71 F src/insert.c 90a32bc7faa755cd5292ade21d2b3c6edba8fd1d70754a364caccabfde2c3bb2 -F src/json.c 7749b98c62f691697c7ee536b570c744c0583cab4a89200fdd0fc2aa8cc8cbd6 +F src/json.c 7634894944baeab438665d792401a159b5e4ae45735337ccfbfa2224f00faf70 F src/legacy.c d7874bc885906868cd51e6c2156698f2754f02d9eee1bae2d687323c3ca8e5aa F src/loadext.c 8086232d10e51e183a7f64199815bad1c579896354db69435347665f62f481e9 -F src/main.c fa53bb2ae09549dab5629271c3cfd681f89059f5192afaaaf5c0d396bb3957fe +F src/main.c 88e65046323fd5ac7e11278f6a45043e519c4b7c5a1ed561c89fb1f13ec2b013 F src/malloc.c dfddca1e163496c0a10250cedeafaf56dff47673e0f15888fb0925340a8e3f90 F src/mem0.c 6a55ebe57c46ca1a7d98da93aaa07f99f1059645 F src/mem1.c c12a42539b1ba105e3707d0e628ad70e611040d8f5e38cf942cee30c867083de @@ -638,16 +638,16 @@ F src/random.c 606b00941a1d7dd09c381d3279a058d771f406c5213c9932bbd93d5587be4b9c F src/resolve.c efea4e5fbecfd6d0a9071b0be0d952620991673391b6ffaaf4c277b0bb674633 F src/rowset.c ba9515a922af32abe1f7d39406b9d35730ed65efab9443dc5702693b60854c92 F src/select.c 9886d6669f5787471aab6ae52af76fad90b53edb1c218fc9ed9d953363bc5184 -F src/shell.c.in a0ba4a297f8134fef1a6b618ac57604a6c4f1eadeab4f6950b99a8bc151f3c1f -F src/sqlite.h.in 100fc660c2f19961b8ed8437b9d53d687de2f8eb2b96437ec6da216adcb643ca +F src/shell.c.in d912025ccd71060b7344c876f4d5b193407047e3b172f098cd69611b66dd0d55 +F src/sqlite.h.in 3455a8d57cb50353a66f4f8d3566e04ee3959337ce2ab6d1d5ab7362576ee532 F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8 F src/sqlite3ext.h c4b9fa7a7e2bcdf850cfeb4b8a91d5ec47b7a00033bc996fd2ee96cbf2741f5f -F src/sqliteInt.h 2c24ba38f78e32fe5d7ec136321a6ad827698b33ca98664970a8b7274d69ef7c +F src/sqliteInt.h d924a3cbfcc146dcc5cde2d2a83eb88201fd4565149502f9c6718ba1d775cdd0 F src/sqliteLimit.h d7323ffea5208c6af2734574bae933ca8ed2ab728083caa117c9738581a31657 F src/status.c 160c445d7d28c984a0eae38c144f6419311ed3eace59b44ac6dafc20db4af749 F src/table.c 0f141b58a16de7e2fbe81c308379e7279f4c6b50eb08efeec5892794a0ba30d1 F src/tclsqlite.c 4e64ba300a5a26e0f1170e09032429faeb65e45e8f3d1a7833e8edb69fc2979e -F src/test1.c 40c9a40975512985762f87b83d0c63e4904833a9fe78cbcca664a37095301b1d +F src/test1.c e2953a23ad637abe90f60679bb07a261934315451583d079d961ba686cc2c5c8 F src/test2.c 827446e259a3b7ab949da1542953edda7b5117982576d3e6f1c24a0dd20a5cef F src/test3.c 61798bb0d38b915067a8c8e03f5a534b431181f802659a6616f9b4ff7d872644 F src/test4.c 4533b76419e7feb41b40582554663ed3cd77aaa54e135cf76b3205098cd6e664 @@ -1257,6 +1257,7 @@ F test/json102.test 327e77275f338c028faefa2da5164daf6b142a165e3015ff2a6e4251ddc6 F test/json103.test 53df87f83a4e5fa0c0a56eb29ff6c94055c6eb919f33316d62161a8880112dbe F test/json104.test a502dc01853aada95d721b3b275afbe2dc18fffdac1fea6e96fb20c13586bbb5 F test/json105.test 11670a4387f4308ae0318cadcbd6a918ea7edcd19fbafde020720a073952675d +F test/json106.test fc906df241664e0b27b68c714a04239116c9e281f9ffaabaef0da23d05c7e45a F test/keyword1.test 37ef6bba5d2ed5b07ecdd6810571de2956599dff F test/kvtest.c feb4358fb022da8ebd098c45811f2f6507688bb6c43aa72b3e840df19026317b F test/lastinsert.test 42e948fd6442f07d60acbd15d33fb86473e0ef63 @@ -2055,8 +2056,11 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 7c572d02e60a83b36543ba4d9d45f61e9fc111b61fee085410c2d87558c732d6 -R f93830c4b318c6359968570a2d6be5a6 +P abb18f61c5cec0f524acc41453b4c06b61c5af51ff46417588837fc0c3967288 +R e4559648d28c24dd2cc0a0eb6174583b +T *branch * lenient-json +T *sym-lenient-json * +T -sym-trunk * U drh -Z 502cd80eb5e32a4170e530e0ee906c79 +Z daa8f079a41c86c9792c1e9898a8a53b # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 13c0efb61f..193465cd9a 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -abb18f61c5cec0f524acc41453b4c06b61c5af51ff46417588837fc0c3967288 \ No newline at end of file +186db57d263be09624cac84c97723d05813d70fe2e88d45e15b9f3b6d8c63935 \ No newline at end of file diff --git a/src/json.c b/src/json.c index 3636d36655..f4dc469eca 100644 --- a/src/json.c +++ b/src/json.c @@ -916,6 +916,14 @@ static int jsonParseValue(JsonParse *pParse, u32 i){ } } +/* +** Return true if we are in lenient-JSON mode +*/ +static int jsonLenientMode(sqlite3_context *pCtx){ + sqlite3 *db = sqlite3_context_db_handle(pCtx); + return (db->flags & SQLITE_LenientJSON)!=0; +} + /* ** Parse a complete JSON string. Return 0 on success or non-zero if there ** are any errors. If an error occurs, free all memory associated with @@ -926,7 +934,8 @@ static int jsonParseValue(JsonParse *pParse, u32 i){ static int jsonParse( JsonParse *pParse, /* Initialize and fill this JsonParse object */ sqlite3_context *pCtx, /* Report errors here */ - const char *zJson /* Input JSON text to be parsed */ + const char *zJson, /* Input JSON text to be parsed */ + int noErr /* If bad JSON, act as if input was "null" */ ){ int i; memset(pParse, 0, sizeof(*pParse)); @@ -943,11 +952,15 @@ static int jsonParse( if( pCtx!=0 ){ if( pParse->oom ){ sqlite3_result_error_nomem(pCtx); - }else{ + }else if( !jsonLenientMode(pCtx) ){ sqlite3_result_error(pCtx, "malformed JSON", -1); + noErr = 0; } } jsonParseReset(pParse); + if( noErr ){ + return jsonParse(pParse, pCtx, "null", 0); + } return 1; } return 0; @@ -1056,7 +1069,7 @@ static JsonParse *jsonParseCached( memset(p, 0, sizeof(*p)); p->zJson = (char*)&p[1]; memcpy((char*)p->zJson, zJson, nJson+1); - if( jsonParse(p, pErrCtx, p->zJson) ){ + if( jsonParse(p, pErrCtx, p->zJson, 0) ){ sqlite3_free(p); return 0; } @@ -1374,7 +1387,7 @@ static void jsonParseFunc( u32 i; assert( argc==1 ); - if( jsonParse(&x, ctx, (const char*)sqlite3_value_text(argv[0])) ) return; + if( jsonParse(&x, ctx, (const char*)sqlite3_value_text(argv[0]), 0) ) return; jsonParseFindParents(&x); jsonInit(&s, ctx); for(i=0; inNode ); if( argc==2 ){ const char *zPath = (const char*)sqlite3_value_text(argv[1]); @@ -1492,6 +1508,7 @@ static void jsonArrayLengthFunc( pNode = p->aNode; } if( pNode==0 ){ + if( jsonLenientMode(ctx) ) goto end_json_array_length; return; } if( pNode->eType==JSON_ARRAY ){ @@ -1500,6 +1517,7 @@ static void jsonArrayLengthFunc( i += jsonNodeSize(&pNode[i]); } } +end_json_array_length: sqlite3_result_int64(ctx, n); } @@ -1707,8 +1725,8 @@ static void jsonPatchFunc( JsonNode *pResult; /* The result of the merge */ UNUSED_PARAMETER(argc); - if( jsonParse(&x, ctx, (const char*)sqlite3_value_text(argv[0])) ) return; - if( jsonParse(&y, ctx, (const char*)sqlite3_value_text(argv[1])) ){ + if( jsonParse(&x, ctx, (const char*)sqlite3_value_text(argv[0]), 1) ) return; + if( jsonParse(&y, ctx, (const char*)sqlite3_value_text(argv[1]), 0) ){ jsonParseReset(&x); return; } @@ -1782,7 +1800,7 @@ static void jsonRemoveFunc( u32 i; if( argc<1 ) return; - if( jsonParse(&x, ctx, (const char*)sqlite3_value_text(argv[0])) ) return; + if( jsonParse(&x, ctx, (const char*)sqlite3_value_text(argv[0]), 0) ) return; assert( x.nNode ); for(i=1; i<(u32)argc; i++){ zPath = (const char*)sqlite3_value_text(argv[i]); @@ -1819,7 +1837,7 @@ static void jsonReplaceFunc( jsonWrongNumArgs(ctx, "replace"); return; } - if( jsonParse(&x, ctx, (const char*)sqlite3_value_text(argv[0])) ) return; + if( jsonParse(&x, ctx, (const char*)sqlite3_value_text(argv[0]), 1) ) return; assert( x.nNode ); for(i=1; i<(u32)argc; i+=2){ zPath = (const char*)sqlite3_value_text(argv[i]); @@ -1873,7 +1891,7 @@ static void jsonSetFunc( jsonWrongNumArgs(ctx, bIsSet ? "set" : "insert"); return; } - if( jsonParse(&x, ctx, (const char*)sqlite3_value_text(argv[0])) ) return; + if( jsonParse(&x, ctx, (const char*)sqlite3_value_text(argv[0]), 1) ) return; assert( x.nNode ); for(i=1; i<(u32)argc; i+=2){ zPath = (const char*)sqlite3_value_text(argv[i]); @@ -2521,7 +2539,7 @@ static int jsonEachFilter( p->zJson = sqlite3_malloc64( n+1 ); if( p->zJson==0 ) return SQLITE_NOMEM; memcpy(p->zJson, z, (size_t)n+1); - if( jsonParse(&p->sParse, 0, p->zJson) ){ + if( jsonParse(&p->sParse, 0, p->zJson, 0) ){ int rc = SQLITE_NOMEM; if( p->sParse.oom==0 ){ sqlite3_free(cur->pVtab->zErrMsg); diff --git a/src/main.c b/src/main.c index b0645efe61..9d6d6af131 100644 --- a/src/main.c +++ b/src/main.c @@ -954,6 +954,7 @@ int sqlite3_db_config(sqlite3 *db, int op, ...){ { SQLITE_DBCONFIG_DQS_DML, SQLITE_DqsDML }, { SQLITE_DBCONFIG_LEGACY_FILE_FORMAT, SQLITE_LegacyFileFmt }, { SQLITE_DBCONFIG_TRUSTED_SCHEMA, SQLITE_TrustedSchema }, + { SQLITE_DBCONFIG_LENIENT_JSON, SQLITE_LenientJSON }, }; unsigned int i; rc = SQLITE_ERROR; /* IMP: R-42790-23372 */ diff --git a/src/shell.c.in b/src/shell.c.in index 6b038495f5..6cddd9cf80 100644 --- a/src/shell.c.in +++ b/src/shell.c.in @@ -7929,6 +7929,7 @@ static int do_meta_command(char *zLine, ShellState *p){ { "fts3_tokenizer", SQLITE_DBCONFIG_ENABLE_FTS3_TOKENIZER }, { "legacy_alter_table", SQLITE_DBCONFIG_LEGACY_ALTER_TABLE }, { "legacy_file_format", SQLITE_DBCONFIG_LEGACY_FILE_FORMAT }, + { "lenient_json", SQLITE_DBCONFIG_LENIENT_JSON }, { "load_extension", SQLITE_DBCONFIG_ENABLE_LOAD_EXTENSION }, { "no_ckpt_on_close", SQLITE_DBCONFIG_NO_CKPT_ON_CLOSE }, { "reset_database", SQLITE_DBCONFIG_RESET_DATABASE }, diff --git a/src/sqlite.h.in b/src/sqlite.h.in index c2fc4e5a6a..f1cfa7b1fb 100644 --- a/src/sqlite.h.in +++ b/src/sqlite.h.in @@ -2424,6 +2424,19 @@ struct sqlite3_mem_methods { ** process a table with generated columns and a descending index. This is ** not considered a bug since SQLite versions 3.3.0 and earlier do not support ** either generated columns or decending indexes. +** +** [[SQLITE_DBCONFIG_LENIENT_JSON]] +**
SQLITE_DBCONFIG_LENIENT_JSON +**
The SQLITE_DBCONFIG_LENIENT_JSON option affects the operation of +** the [JSON SQL functions]. When SQLITE_DBCONFIG_LENIENT_JSON is disabled +** (the default) then the JSON functions will typically (depending on the +** specific function) raise an error when given malformed JSON inputs. +** When SQLITE_DBCONFIG_LENIENT_JSON is enabled, the JSON functions are more +** forgiving of malformed inputs and will press on and try to return some kind +** of sensible answer in spite of the malformed input. The lenient JSON mode +** is useful when dealing with unvalidated inputs as it avoids having to +** test every input using [json_valid()] prior to using it, thus greatly +** simplifying complex queries. **
** */ @@ -2445,7 +2458,8 @@ struct sqlite3_mem_methods { #define SQLITE_DBCONFIG_ENABLE_VIEW 1015 /* int int* */ #define SQLITE_DBCONFIG_LEGACY_FILE_FORMAT 1016 /* int int* */ #define SQLITE_DBCONFIG_TRUSTED_SCHEMA 1017 /* int int* */ -#define SQLITE_DBCONFIG_MAX 1017 /* Largest DBCONFIG */ +#define SQLITE_DBCONFIG_LENIENT_JSON 1018 /* int int* */ +#define SQLITE_DBCONFIG_MAX 1018 /* Largest DBCONFIG */ /* ** CAPI3REF: Enable Or Disable Extended Result Codes diff --git a/src/sqliteInt.h b/src/sqliteInt.h index e4b74f6d0b..fe1b616139 100644 --- a/src/sqliteInt.h +++ b/src/sqliteInt.h @@ -1707,8 +1707,7 @@ struct sqlite3 { #define SQLITE_ShortColNames 0x00000040 /* Show short columns names */ #define SQLITE_TrustedSchema 0x00000080 /* Allow unsafe functions and ** vtabs in the schema definition */ -#define SQLITE_NullCallback 0x00000100 /* Invoke the callback once if the */ - /* result set is empty */ +#define SQLITE_LenientJSON 0x00000100 /* Be less strict about JSON inputs */ #define SQLITE_IgnoreChecks 0x00000200 /* Do not enforce check constraints */ #define SQLITE_ReadUncommit 0x00000400 /* READ UNCOMMITTED in shared-cache */ #define SQLITE_NoCkptOnClose 0x00000800 /* No checkpoint on close()/DETACH */ @@ -1736,6 +1735,8 @@ struct sqlite3 { /* DELETE, or UPDATE and return */ /* the count using a callback. */ #define SQLITE_CorruptRdOnly HI(0x00002) /* Prohibit writes due to error */ +#define SQLITE_NullCallback HI(0x00004) /* Invoke the callback once if the */ + /* result set is empty */ /* Flags used only if debugging */ #ifdef SQLITE_DEBUG diff --git a/src/test1.c b/src/test1.c index ba1a2252fd..4a63fb87cb 100644 --- a/src/test1.c +++ b/src/test1.c @@ -8154,6 +8154,7 @@ static int SQLITE_TCLAPI test_sqlite3_db_config( { "DQS_DML", SQLITE_DBCONFIG_DQS_DML }, { "DQS_DDL", SQLITE_DBCONFIG_DQS_DDL }, { "LEGACY_FILE_FORMAT", SQLITE_DBCONFIG_LEGACY_FILE_FORMAT }, + { "LENIENT_JSON", SQLITE_DBCONFIG_LENIENT_JSON }, }; int i; int v = 0; diff --git a/test/json106.test b/test/json106.test new file mode 100644 index 0000000000..7427928a88 --- /dev/null +++ b/test/json106.test @@ -0,0 +1,90 @@ +# 2022-11-18 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# This file implements tests for JSON SQL functions extension to the +# SQLite library. This file focuses on SQLITE_DBCONFIG_LENIENT_JSON. +# + +set testdir [file dirname $argv0] +source $testdir/tester.tcl + +sqlite3_db_config db LENIENT_JSON 0 +do_catchsql_test json108-100 { + SELECT json_array_length(NULL); +} {0 {{}}} +do_catchsql_test json108-110 { + SELECT json_array_length('[1,2,3]'); +} {0 3} +do_catchsql_test json108-120 { + SELECT json_array_length('not-json'); +} {1 {malformed JSON}} +do_catchsql_test json108-130 { + SELECT json_array_length('null'); +} {0 0} + +sqlite3_db_config db LENIENT_JSON 1 +do_catchsql_test json108-150 { + SELECT json_array_length(NULL); +} {0 0} +do_catchsql_test json108-160 { + SELECT json_array_length('[1,2,3]'); +} {0 3} +do_catchsql_test json108-170 { + SELECT json_array_length('not-json'); +} {0 0} +do_catchsql_test json108-180 { + SELECT json_array_length('null'); +} {0 0} + +sqlite3_db_config db LENIENT_JSON 0 +do_catchsql_test json108-200 { + SELECT null ->> 'x'; +} {0 {{}}} +do_catchsql_test json108-210 { + SELECT 'null' ->> 'x'; +} {0 {{}}} +do_catchsql_test json108-220 { + SELECT 'nullx' ->> 'x'; +} {1 {malformed JSON}} +do_catchsql_test json108-230 { + SELECT '{"x":5}' ->> 'x'; +} {0 5} + +sqlite3_db_config db LENIENT_JSON 1 +do_catchsql_test json108-250 { + SELECT null ->> 'x'; +} {0 {{}}} +do_catchsql_test json108-260 { + SELECT 'null' ->> 'x'; +} {0 {{}}} +do_catchsql_test json108-270 { + SELECT 'nullx' ->> 'x'; +} {0 {{}}} +do_catchsql_test json108-280 { + SELECT '{"x":5}' ->> 'x'; +} {0 5} + +sqlite3_db_config db LENIENT_JSON 0 +do_catchsql_test json108-300 { + SELECT json_patch('nullx','{"x":5,"y":10}'); +} {1 {malformed JSON}} +do_catchsql_test json108-310 { + SELECT json_patch('null','{"x":5,"y":10}'); +} {0 {{{"x":5,"y":10}}}} + +sqlite3_db_config db LENIENT_JSON 1 +do_catchsql_test json108-350 { + SELECT json_patch('nullx','{"x":5,"y":10}'); +} {0 {{{"x":5,"y":10}}}} +do_catchsql_test json108-360 { + SELECT json_patch('null','{"x":5,"y":10}'); +} {0 {{{"x":5,"y":10}}}} + +finish_test