From 063d0d4c3a453939237e64c8f9b3b6d96cad0a65 Mon Sep 17 00:00:00 2001 From: drh <> Date: Fri, 1 Dec 2023 18:46:14 +0000 Subject: [PATCH] Fix up the JSON cache to work better. FossilOrigin-Name: 1fdbc39521f63aedc6f08ecaafa54ea467b8c6316a692a18ad01eecbf22a0977 --- manifest | 15 ++-- manifest.uuid | 2 +- src/json.c | 201 +++++++++++++++++++++++++------------------------- 3 files changed, 109 insertions(+), 109 deletions(-) diff --git a/manifest b/manifest index 8aa6ceaf3b..9b30ebcd15 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Cache\sis\sworking\sbetter,\sbut\sdoes\snot\spreserve\sthe\shasJson5\sflag. -D 2023-12-01T13:28:13.897 +C Fix\sup\sthe\sJSON\scache\sto\swork\sbetter. +D 2023-12-01T18:46:14.485 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -688,7 +688,7 @@ F src/hash.h 3340ab6e1d13e725571d7cee6d3e3135f0779a7d8e76a9ce0a85971fa3953c51 F src/hwtime.h f9c2dfb84dce7acf95ce6d289e46f5f9d3d1afd328e53da8f8e9008e3b3caae6 F src/in-operator.md 10cd8f4bcd225a32518407c2fb2484089112fd71 F src/insert.c 3f0a94082d978bbdd33c38fefea15346c6c6bffb70bc645a71dc0f1f87dd3276 -F src/json.c 32e3741e5ff9f8380ebf84dcd6d3c28ddcd7f6a2923b72a05165ad108ca0c278 +F src/json.c 3556e879386b0af1c542ed13ebb2f58c48d3b8b3ab13a96ab77ad2a9493cc715 F src/legacy.c d7874bc885906868cd51e6c2156698f2754f02d9eee1bae2d687323c3ca8e5aa F src/loadext.c 7432c944ff197046d67a1207790a1b13eec4548c85a9457eb0896bb3641dfb36 F src/main.c 1b89f3de98d1b59fec5bac1d66d6ece21f703821b8eaa0d53d9604c35309f6f9 @@ -2145,11 +2145,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 25ed295f300fea6185104a73721076bccd2b2a6e411c78564266fa6dca4ff70c -R b145e990ca1c690b2dc7d50ed4bd4abc -T *branch * jsonb-cache -T *sym-jsonb-cache * -T -sym-jsonb * +P a12add7ab9f5aee5bb2ede0c4d22e599dd28f7a107dce72b2ea48ef92d233e8a +R bed96169a434a08ba8168cff3e09a739 U drh -Z 663fd17309f10179421e079add21165c +Z 599839251e392336311bcdbf8af02666 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 697f12705c..c67da5411d 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -a12add7ab9f5aee5bb2ede0c4d22e599dd28f7a107dce72b2ea48ef92d233e8a \ No newline at end of file +1fdbc39521f63aedc6f08ecaafa54ea467b8c6316a692a18ad01eecbf22a0977 \ No newline at end of file diff --git a/src/json.c b/src/json.c index 5f02a1c8ed..08e0f4fb63 100644 --- a/src/json.c +++ b/src/json.c @@ -201,7 +201,6 @@ static const char jsonIsOk[256] = { /* Objects */ typedef struct JsonCache JsonCache; -typedef struct JsonCacheLine JsonCacheLine; typedef struct JsonString JsonString; typedef struct JsonParse JsonParse; @@ -217,16 +216,10 @@ typedef struct JsonParse JsonParse; ** All content, both JSON text and the JSONB blobs, is stored as RCStr ** objects. */ -struct JsonCacheLine { - u32 nJson; /* Size of the JSON text, in bytes */ - u32 nBlob; /* Size of the corresponding JSONB, in bytes */ - char *zJson; /* RCStr holding the JSON text */ - char *aBlob; /* RCStr holding the corresponding JSONB */ -}; struct JsonCache { - sqlite3 *db; /* Database connection */ - int nUsed; /* Number of active entries in the cache */ - JsonCacheLine a[JSON_CACHE_SIZE]; /* One line for each cache entry */ + sqlite3 *db; /* Database connection */ + int nUsed; /* Number of active entries in the cache */ + JsonParse *a[JSON_CACHE_SIZE]; /* One line for each cache entry */ }; /* An instance of this object represents a JSON string @@ -285,13 +278,14 @@ struct JsonParse { u32 nBlob; /* Bytes of aBlob[] actually used */ u32 nBlobAlloc; /* Bytes allocated to aBlob[]. 0 if aBlob is external */ char *zJson; /* Json text used for parsing */ + int nJson; /* Length of the zJson string in bytes */ u16 iDepth; /* Nesting depth */ u8 nErr; /* Number of errors seen */ u8 oom; /* Set to true if out of memory */ - u8 bBlobIsRCStr; /* True if aBlob is an RCStr */ + u8 bJsonIsRCStr; /* True if zJson is an RCStr */ u8 hasNonstd; /* True if input uses non-standard features like JSON5 */ + u8 bReadOnly; /* Do not modify. */ u32 nJPRef; /* Number of references to this object */ - int nJson; /* Length of the zJson string in bytes */ u32 iErr; /* Error location in zJson[] */ /* Search and edit information. See jsonLookupBlobStep() */ u8 eEdit; /* Edit operation to apply */ @@ -330,6 +324,9 @@ static int jsonFuncArgMightBeBinary(sqlite3_value *pJson); static u32 jsonXlateBlobToText(const JsonParse*,u32,JsonString*); static void jsonReturnParse(sqlite3_context*,JsonParse*); static JsonParse *jsonParseFuncArg(sqlite3_context*,sqlite3_value*,u32); +static void jsonParseFree(JsonParse*); + + /************************************************************************** ** Utility routines for dealing with JsonCache objects **************************************************************************/ @@ -340,8 +337,7 @@ static JsonParse *jsonParseFuncArg(sqlite3_context*,sqlite3_value*,u32); static void jsonCacheDelete(JsonCache *p){ int i; for(i=0; inUsed; i++){ - sqlite3RCStrUnref(p->a[i].zJson); - sqlite3RCStrUnref(p->a[i].aBlob); + jsonParseFree(p->a[i]); } sqlite3DbFree(p->db, p); } @@ -358,16 +354,12 @@ static void jsonCacheDeleteGeneric(void *p){ */ static int jsonCacheInsert( sqlite3_context *ctx, /* The SQL statement context holding the cache */ - char *zJson, /* The key. */ - u32 nJson, /* Number of bytes in zJson */ - int bJsonIsRCStr, /* True if zJson is already an RCStr */ - u8 *aBlob, /* The value. */ - u32 nBlob /* Number of bytes in aBlob */ + JsonParse *pParse /* The parse object to be added to the cache */ ){ JsonCache *p; - char *aRCBlob; - char *zRCJson; + assert( pParse->zJson!=0 ); + assert( pParse->bJsonIsRCStr ); p = sqlite3_get_auxdata(ctx, JSON_CACHE_ID); if( p==0 ){ sqlite3 *db = sqlite3_context_db_handle(ctx); @@ -378,76 +370,61 @@ static int jsonCacheInsert( p = sqlite3_get_auxdata(ctx, JSON_CACHE_ID); if( p==0 ) return SQLITE_NOMEM; } - aRCBlob = sqlite3RCStrNew( nBlob ); - if( aRCBlob==0 ) return SQLITE_NOMEM; - memcpy(aRCBlob, aBlob, nBlob); - if( bJsonIsRCStr ){ - zRCJson = sqlite3RCStrRef(zJson); - }else{ - zRCJson = sqlite3RCStrNew( nJson ); - if( zRCJson==0 ){ - sqlite3RCStrUnref(aRCBlob); - return SQLITE_NOMEM; - } - memcpy(zRCJson, zJson, nJson); - zRCJson[nJson] = 0; - } if( p->nUsed >= JSON_CACHE_SIZE ){ - sqlite3RCStrUnref(p->a[0].zJson); - sqlite3RCStrUnref(p->a[0].aBlob); + jsonParseFree(p->a[0]); memmove(p->a, &p->a[1], (JSON_CACHE_SIZE-1)*sizeof(p->a[0])); p->nUsed = JSON_CACHE_SIZE-1; } - p->a[p->nUsed].nJson = nJson; - p->a[p->nUsed].nBlob = nBlob; - p->a[p->nUsed].zJson = zRCJson; - p->a[p->nUsed].aBlob = aRCBlob; + pParse->eEdit = 0; + pParse->nJPRef++; + pParse->bReadOnly = 1; + p->a[p->nUsed] = pParse; p->nUsed++; return SQLITE_OK; } /* -** Search for a cached translation of zJson (size: nJson bytes) into -** JSONB. Return it if found. -** -** The returned value is an RCStr object if it is not NULL. -** The caller is responsible for incrementing the reference count. +** Search for a cached translation the json text supplied by pArg. Return +** the JsonParse object if found. */ -static u8 *jsonCacheSearch( +static JsonParse *jsonCacheSearch( sqlite3_context *ctx, /* The SQL statement context holding the cache */ - char *zJson, /* The key. Might or might not be an RCStr */ - u32 nJson, /* Size of the key in bytes */ - u32 *pnBlob /* OUT: Size of the result in bytes */ + sqlite3_value *pArg /* Function argument containing SQL text */ ){ JsonCache *p; int i; + const char *zJson; + int nJson; + + if( sqlite3_value_type(pArg)!=SQLITE_TEXT ){ + return 0; + } + zJson = (const char*)sqlite3_value_text(pArg); + if( zJson==0 ) return 0; + nJson = sqlite3_value_bytes(pArg); - assert( pnBlob!=0 ); p = sqlite3_get_auxdata(ctx, JSON_CACHE_ID); if( p==0 ){ - *pnBlob = 0; return 0; } for(i=0; inUsed; i++){ - if( p->a[i].zJson==zJson ) break; + if( p->a[i]->zJson==zJson ) break; } if( i>=p->nUsed ){ for(i=0; inUsed; i++){ - if( p->a[i].nJson!=nJson ) continue; - if( memcmp(p->a[i].zJson, zJson, nJson)==0 ) break; + if( p->a[i]->nJson!=nJson ) continue; + if( memcmp(p->a[i]->zJson, zJson, nJson)==0 ) break; } } if( inUsed ){ if( inUsed-1 ){ - JsonCacheLine tmp = p->a[i]; + JsonParse *tmp = p->a[i]; memmove(&p->a[i], &p->a[i+1], (p->nUsed-i-1)*sizeof(tmp)); p->a[p->nUsed-1] = tmp; i = p->nUsed - 1; } - *pnBlob = p->a[i].nBlob; - return (u8*)p->a[i].aBlob; + return p->a[i]; }else{ - *pnBlob = 0; return 0; } } @@ -717,9 +694,9 @@ static void jsonAppendSqlValue( ** ** The JsonString is reset. ** -** If pParse and ctx are both non-NULL and if pParse->aBlob is valid -** then an attempt is made to cache the translation from JSON text into -** the blob. +** If pParse and ctx are both non-NULL, then the SQL string in p is +** loaded into the zJson field of the pParse object as a RCStr and the +** pParse is added to the cache. */ static void jsonReturnString( JsonString *p, /* String to return */ @@ -736,18 +713,19 @@ static void jsonReturnString( sqlite3_result_text64(p->pCtx, p->zBuf, p->nUsed, SQLITE_TRANSIENT, SQLITE_UTF8); }else if( jsonForceRCStr(p) ){ - sqlite3RCStrRef(p->zBuf); - if( pParse ){ - int rc = jsonCacheInsert(ctx, p->zBuf, p->nUsed, 1, - pParse->aBlob, pParse->nBlob); + if( pParse && pParse->bJsonIsRCStr==0 ){ + int rc; + pParse->zJson = sqlite3RCStrRef(p->zBuf); + pParse->nJson = p->nUsed; + pParse->bJsonIsRCStr = 1; + rc = jsonCacheInsert(ctx, pParse); if( rc==SQLITE_NOMEM ){ - sqlite3RCStrUnref(p->zBuf); sqlite3_result_error_nomem(ctx); jsonStringReset(p); return; } } - sqlite3_result_text64(p->pCtx, p->zBuf, p->nUsed, + sqlite3_result_text64(p->pCtx, sqlite3RCStrRef(p->zBuf), p->nUsed, sqlite3RCStrUnref, SQLITE_UTF8); }else{ @@ -771,12 +749,11 @@ static void jsonReturnString( */ static void jsonParseReset(JsonParse *pParse){ assert( pParse->nJPRef<=1 ); - if( pParse->bBlobIsRCStr ){ - assert( pParse->nBlobAlloc==0 ); - sqlite3RCStrUnref((char*)pParse->aBlob); - pParse->aBlob = 0; - pParse->nBlob = 0; - pParse->bBlobIsRCStr = 0; + if( pParse->bJsonIsRCStr ){ + sqlite3RCStrUnref(pParse->zJson); + pParse->zJson = 0; + pParse->nJson = 0; + pParse->bJsonIsRCStr = 0; } if( pParse->nBlobAlloc ){ sqlite3_free(pParse->aBlob); @@ -1048,6 +1025,7 @@ static int jsonBlobExpand(JsonParse *pParse, u32 N){ static int jsonBlobMakeEditable(JsonParse *pParse, u32 nExtra){ u8 *aOld; u32 nSize; + assert( !pParse->bReadOnly ); if( pParse->nBlobAlloc>0 ) return 1; aOld = pParse->aBlob; nSize = pParse->nBlob + nExtra; @@ -2671,6 +2649,11 @@ jsonInsertIntoBlob_patherror: return; } +/* +** Make a copy of a JsonParse object. The copy will be editable. +*/ + + /* ** Generate a JsonParse object, containing valid JSONB in aBlob and nBlob, ** from the SQL function argument pArg. Return a pointer to the new @@ -2692,17 +2675,40 @@ static JsonParse *jsonParseFuncArg( sqlite3_value *pArg, u32 flgs ){ - int eType; /* Datatype of pArg */ - JsonParse *p = 0; /* Value to be returned */ + int eType; /* Datatype of pArg */ + JsonParse *p = 0; /* Value to be returned */ + JsonParse *pFromCache = 0; /* Value taken from cache */ assert( ctx!=0 ); eType = sqlite3_value_type(pArg); if( eType==SQLITE_NULL ){ return 0; } + pFromCache = jsonCacheSearch(ctx, pArg); + if( pFromCache ){ + pFromCache->nJPRef++; + if( (flgs & JSON_EDITABLE)==0 ){ + return pFromCache; + } + } +rebuild_from_cache: p = sqlite3_malloc64( sizeof(*p) ); if( p==0 ) goto json_pfa_oom; memset(p, 0, sizeof(*p)); + p->nJPRef = 1; + if( pFromCache!=0 ){ + u32 nBlob = pFromCache->nBlob; + p->aBlob = sqlite3_malloc64( nBlob ); + if( p->aBlob==0 ) goto json_pfa_oom; + memcpy(p->aBlob, pFromCache->aBlob, nBlob); + p->nBlobAlloc = p->nBlob = nBlob; + p->hasNonstd = pFromCache->hasNonstd; + jsonParseFree(pFromCache); + return p; + }else{ + jsonParseFree(pFromCache); + pFromCache = 0; + } if( eType==SQLITE_BLOB ){ u32 n, sz = 0; p->aBlob = (u8*)sqlite3_value_blob(pArg); @@ -2729,29 +2735,11 @@ static JsonParse *jsonParseFuncArg( } return p; } - /* TODO: Check in the cache */ p->zJson = (char*)sqlite3_value_text(pArg); p->nJson = sqlite3_value_bytes(pArg); if( p->nJson==0 ) goto json_pfa_malformed; if( p->zJson==0 ) goto json_pfa_oom; - - p->aBlob = jsonCacheSearch(ctx, p->zJson, p->nJson, &p->nBlob); - if( p->aBlob ){ - if( flgs & JSON_EDITABLE ){ - u8 *pNew = sqlite3_malloc64( p->nBlob ); - if( pNew==0 ) goto json_pfa_oom; - memcpy(pNew, p->aBlob, p->nBlob); - p->aBlob = pNew; - p->nBlobAlloc = p->nBlob; - }else{ - sqlite3RCStrRef((char*)p->aBlob); - p->bBlobIsRCStr = 1; - } - return p; - } - - if( flgs & JSON_KEEPERROR ) ctx = 0; - if( jsonConvertTextToBlob(p, ctx) ){ + if( jsonConvertTextToBlob(p, (flgs & JSON_KEEPERROR) ? 0 : ctx) ){ if( flgs & JSON_KEEPERROR ){ p->nErr = 1; return p; @@ -2759,12 +2747,26 @@ static JsonParse *jsonParseFuncArg( jsonParseFree(p); return 0; } - } - if( ctx ){ + }else{ int isRCStr = sqlite3ValueIsOfClass(pArg, sqlite3RCStrUnref); - int rc = jsonCacheInsert(ctx, p->zJson, p->nJson, isRCStr, - p->aBlob, p->nBlob); + int rc; + if( !isRCStr ){ + char *zNew = sqlite3RCStrNew( p->nJson ); + if( zNew==0 ) goto json_pfa_oom; + memcpy(zNew, p->zJson, p->nJson); + p->zJson = zNew; + p->zJson[p->nJson] = 0; + }else{ + sqlite3RCStrRef(p->zJson); + } + p->bJsonIsRCStr = 1; + rc = jsonCacheInsert(ctx, p); if( rc==SQLITE_NOMEM ) goto json_pfa_oom; + if( flgs & JSON_EDITABLE ){ + pFromCache = p; + p = 0; + goto rebuild_from_cache; + } } return p; @@ -2779,6 +2781,7 @@ json_pfa_malformed: } json_pfa_oom: + jsonParseFree(pFromCache); jsonParseFree(p); sqlite3_result_error_nomem(ctx); return 0; -- 2.39.5