From: dan Date: Thu, 18 Apr 2019 21:14:11 +0000 (+0000) Subject: Add the sqlite_dbptr virtual table to the dbdata extension. For querying the links... X-Git-Tag: version-3.29.0~176^2~15 X-Git-Url: http://git.ipfire.org/?a=commitdiff_plain;h=3b412ac2473707fc7bc4ce29466d7b7dc4988905;p=thirdparty%2Fsqlite.git Add the sqlite_dbptr virtual table to the dbdata extension. For querying the links between b-tree pages. FossilOrigin-Name: 3213a15f2133afbb0a4fec3b8f6e0eeca8c0befafd6658c41074e84f589d5d32 --- diff --git a/ext/misc/dbdata.c b/ext/misc/dbdata.c index c0ad8c1f9e..eb2bc01600 100644 --- a/ext/misc/dbdata.c +++ b/ext/misc/dbdata.c @@ -23,6 +23,9 @@ ** schema TEXT HIDDEN ** ); ** +** IMPORTANT: THE VIRTUAL TABLE SCHEMA ABOVE IS SUBJECT TO CHANGE. IN THE +** FUTURE NEW NON-HIDDEN COLUMNS MAY BE ADDED BETWEEN "value" AND "schema". +** ** Each page of the database is inspected. If it cannot be interpreted as a ** b-tree page, or if it is a b-tree page containing 0 entries, the ** sqlite_dbdata table contains no rows for that page. Otherwise, the table @@ -52,6 +55,13 @@ ** ** This module requires that the "sqlite_dbpage" eponymous virtual table be ** available. +** +** +** CREATE TABLE sqlite_dbptr( +** pgno INTEGER, +** child INTEGER, +** schema TEXT HIDDEN +** ); */ #if !defined(SQLITEINT_H) #include "sqlite3ext.h" @@ -67,6 +77,7 @@ SQLITE_EXTENSION_INIT1 typedef struct DbdataTable DbdataTable; typedef struct DbdataCursor DbdataCursor; + /* A cursor for the sqlite_dbdata table */ struct DbdataCursor { sqlite3_vtab_cursor base; /* Base class. Must be first */ @@ -77,19 +88,22 @@ struct DbdataCursor { int nPage; /* Size of aPage[] in bytes */ int nCell; /* Number of cells on aPage[] */ int iCell; /* Current cell number */ + int bOnePage; /* True to stop after one page */ + sqlite3_int64 iRowid; + + /* Only for the sqlite_dbdata table */ u8 *pRec; /* Buffer containing current record */ int nRec; /* Size of pRec[] in bytes */ int nField; /* Number of fields in pRec */ int iField; /* Current field number */ sqlite3_int64 iIntkey; /* Integer key value */ - - sqlite3_int64 iRowid; }; /* The sqlite_dbdata table */ struct DbdataTable { sqlite3_vtab base; /* Base class. Must be first */ sqlite3 *db; /* The database connection */ + int bPtr; /* True for sqlite3_dbptr table */ }; #define DBDATA_COLUMN_PGNO 0 @@ -98,6 +112,10 @@ struct DbdataTable { #define DBDATA_COLUMN_VALUE 3 #define DBDATA_COLUMN_SCHEMA 4 +#define DBPTR_COLUMN_PGNO 0 +#define DBPTR_COLUMN_CHILD 1 +#define DBPTR_COLUMN_SCHEMA 2 + #define DBDATA_SCHEMA \ "CREATE TABLE x(" \ " pgno INTEGER," \ @@ -107,6 +125,13 @@ struct DbdataTable { " schema TEXT HIDDEN" \ ")" +#define DBPTR_SCHEMA \ + "CREATE TABLE x(" \ + " pgno INTEGER," \ + " child INTEGER," \ + " schema TEXT HIDDEN" \ + ")" + /* ** Connect to the sqlite_dbdata virtual table. */ @@ -118,7 +143,7 @@ static int dbdataConnect( char **pzErr ){ DbdataTable *pTab = 0; - int rc = sqlite3_declare_vtab(db, DBDATA_SCHEMA); + int rc = sqlite3_declare_vtab(db, pAux ? DBPTR_SCHEMA : DBDATA_SCHEMA); if( rc==SQLITE_OK ){ pTab = (DbdataTable*)sqlite3_malloc64(sizeof(DbdataTable)); @@ -127,6 +152,7 @@ static int dbdataConnect( }else{ memset(pTab, 0, sizeof(DbdataTable)); pTab->db = db; + pTab->bPtr = (pAux!=0); } } @@ -157,14 +183,16 @@ static int dbdataDisconnect(sqlite3_vtab *pVtab){ ** position 1. */ static int dbdataBestIndex(sqlite3_vtab *tab, sqlite3_index_info *pIdxInfo){ + DbdataTable *pTab = (DbdataTable*)tab; int i; int iSchema = -1; int iPgno = -1; + int colSchema = (pTab->bPtr ? DBPTR_COLUMN_SCHEMA : DBDATA_COLUMN_SCHEMA); for(i=0; inConstraint; i++){ struct sqlite3_index_constraint *p = &pIdxInfo->aConstraint[i]; if( p->op==SQLITE_INDEX_CONSTRAINT_EQ ){ - if( p->iColumn==DBDATA_COLUMN_SCHEMA ){ + if( p->iColumn==colSchema ){ if( p->usable==0 ) return SQLITE_CONSTRAINT; iSchema = i; } @@ -210,6 +238,7 @@ static void dbdataResetCursor(DbdataCursor *pCsr){ pCsr->iPgno = 1; pCsr->iCell = 0; pCsr->iField = 0; + pCsr->bOnePage = 0; } /* @@ -281,132 +310,152 @@ static int dbdataGetVarint(const u8 *z, sqlite3_int64 *pVal){ ** Move a dbdata cursor to the next entry in the file. */ static int dbdataNext(sqlite3_vtab_cursor *pCursor){ - DbdataCursor *pCsr = (DbdataCursor *)pCursor; + DbdataCursor *pCsr = (DbdataCursor*)pCursor; + DbdataTable *pTab = (DbdataTable*)pCursor->pVtab; pCsr->iRowid++; while( 1 ){ int rc; + int iOff = (pCsr->iPgno==1 ? 100 : 0); if( pCsr->aPage==0 ){ rc = dbdataLoadPage(pCsr, pCsr->iPgno, &pCsr->aPage, &pCsr->nPage); - if( rc!=SQLITE_OK ) return rc; - pCsr->iCell = 0; - pCsr->nCell = get_uint16(&pCsr->aPage[pCsr->iPgno==1 ? 103 : 3]); + if( rc!=SQLITE_OK || pCsr->aPage==0 ) return rc; + pCsr->iCell = pTab->bPtr ? -2 : 0; + pCsr->nCell = get_uint16(&pCsr->aPage[iOff+3]); } - /* If there is no record loaded, load it now. */ - if( pCsr->pRec==0 ){ - int iOff = (pCsr->iPgno==1 ? 100 : 0); - int bHasRowid = 0; - int nPointer = 0; - sqlite3_int64 nPayload = 0; - sqlite3_int64 nHdr = 0; - int iHdr; - int U, X; - int nLocal; - - switch( pCsr->aPage[iOff] ){ - case 0x02: - nPointer = 4; - break; - case 0x0a: - break; - case 0x0d: - bHasRowid = 1; - break; - default: - pCsr->iCell = pCsr->nCell; - break; + if( pTab->bPtr ){ + if( pCsr->aPage[iOff]!=0x02 && pCsr->aPage[iOff]!=0x05 ){ + pCsr->iCell = pCsr->nCell; } + pCsr->iCell++; if( pCsr->iCell>=pCsr->nCell ){ sqlite3_free(pCsr->aPage); pCsr->aPage = 0; + if( pCsr->bOnePage ) return SQLITE_OK; + pCsr->iPgno++; + }else{ return SQLITE_OK; } + }else{ + /* If there is no record loaded, load it now. */ + if( pCsr->pRec==0 ){ + int bHasRowid = 0; + int nPointer = 0; + sqlite3_int64 nPayload = 0; + sqlite3_int64 nHdr = 0; + int iHdr; + int U, X; + int nLocal; + + switch( pCsr->aPage[iOff] ){ + case 0x02: + nPointer = 4; + break; + case 0x0a: + break; + case 0x0d: + bHasRowid = 1; + break; + default: + /* This is not a b-tree page with records on it. Continue. */ + pCsr->iCell = pCsr->nCell; + break; + } - iOff += 8 + nPointer + pCsr->iCell*2; - iOff = get_uint16(&pCsr->aPage[iOff]); - - /* For an interior node cell, skip past the child-page number */ - iOff += nPointer; - - /* Load the "byte of payload including overflow" field */ - iOff += dbdataGetVarint(&pCsr->aPage[iOff], &nPayload); - - /* If this is a leaf intkey cell, load the rowid */ - if( bHasRowid ){ - iOff += dbdataGetVarint(&pCsr->aPage[iOff], &pCsr->iIntkey); - } - - /* Allocate space for payload */ - pCsr->pRec = (u8*)sqlite3_malloc64(nPayload); - if( pCsr->pRec==0 ) return SQLITE_NOMEM; - pCsr->nRec = nPayload; - - U = pCsr->nPage; - if( bHasRowid ){ - X = U-35; - }else{ - X = ((U-12)*64/255)-23; - } - if( nPayload<=X ){ - nLocal = nPayload; - }else{ - int M, K; - M = ((U-12)*32/255)-23; - K = M+((nPayload-M)%(U-4)); - if( K<=X ){ - nLocal = K; + if( pCsr->iCell>=pCsr->nCell ){ + sqlite3_free(pCsr->aPage); + pCsr->aPage = 0; + if( pCsr->bOnePage ) return SQLITE_OK; + pCsr->iPgno++; + continue; + } + + iOff += 8 + nPointer + pCsr->iCell*2; + iOff = get_uint16(&pCsr->aPage[iOff]); + + /* For an interior node cell, skip past the child-page number */ + iOff += nPointer; + + /* Load the "byte of payload including overflow" field */ + iOff += dbdataGetVarint(&pCsr->aPage[iOff], &nPayload); + + /* If this is a leaf intkey cell, load the rowid */ + if( bHasRowid ){ + iOff += dbdataGetVarint(&pCsr->aPage[iOff], &pCsr->iIntkey); + } + + /* Allocate space for payload */ + pCsr->pRec = (u8*)sqlite3_malloc64(nPayload); + if( pCsr->pRec==0 ) return SQLITE_NOMEM; + pCsr->nRec = nPayload; + + U = pCsr->nPage; + if( bHasRowid ){ + X = U-35; }else{ - nLocal = M; + X = ((U-12)*64/255)-23; } - } - - /* Load the nLocal bytes of payload */ - memcpy(pCsr->pRec, &pCsr->aPage[iOff], nLocal); - iOff += nLocal; - - /* Load content from overflow pages */ - if( nPayload>nLocal ){ - sqlite3_int64 nRem = nPayload - nLocal; - u32 pgnoOvfl = get_uint32(&pCsr->aPage[iOff]); - while( nRem>0 ){ - u8 *aOvfl = 0; - int nOvfl = 0; - int nCopy; - rc = dbdataLoadPage(pCsr, pgnoOvfl, &aOvfl, &nOvfl); - assert( rc!=SQLITE_OK || nOvfl==pCsr->nPage ); - if( rc!=SQLITE_OK ) return rc; - - nCopy = U-4; - if( nCopy>nRem ) nCopy = nRem; - memcpy(&pCsr->pRec[nPayload-nRem], &aOvfl[4], nCopy); - nRem -= nCopy; - - sqlite3_free(aOvfl); + if( nPayload<=X ){ + nLocal = nPayload; + }else{ + int M, K; + M = ((U-12)*32/255)-23; + K = M+((nPayload-M)%(U-4)); + if( K<=X ){ + nLocal = K; + }else{ + nLocal = M; + } } + + /* Load the nLocal bytes of payload */ + memcpy(pCsr->pRec, &pCsr->aPage[iOff], nLocal); + iOff += nLocal; + + /* Load content from overflow pages */ + if( nPayload>nLocal ){ + sqlite3_int64 nRem = nPayload - nLocal; + u32 pgnoOvfl = get_uint32(&pCsr->aPage[iOff]); + while( nRem>0 ){ + u8 *aOvfl = 0; + int nOvfl = 0; + int nCopy; + rc = dbdataLoadPage(pCsr, pgnoOvfl, &aOvfl, &nOvfl); + assert( rc!=SQLITE_OK || nOvfl==pCsr->nPage ); + if( rc!=SQLITE_OK ) return rc; + + nCopy = U-4; + if( nCopy>nRem ) nCopy = nRem; + memcpy(&pCsr->pRec[nPayload-nRem], &aOvfl[4], nCopy); + nRem -= nCopy; + + sqlite3_free(aOvfl); + } + } + + /* Figure out how many fields in the record */ + pCsr->nField = 0; + iHdr = dbdataGetVarint(pCsr->pRec, &nHdr); + while( iHdrpRec[iHdr], &iDummy); + pCsr->nField++; + } + + pCsr->iField = (bHasRowid ? -2 : -1); } - - /* Figure out how many fields in the record */ - pCsr->nField = 0; - iHdr = dbdataGetVarint(pCsr->pRec, &nHdr); - while( iHdrpRec[iHdr], &iDummy); - pCsr->nField++; - } - - pCsr->iField = (bHasRowid ? -2 : -1); + + pCsr->iField++; + if( pCsr->iFieldnField ) return SQLITE_OK; + + /* Advance to the next cell. The next iteration of the loop will load + ** the record and so on. */ + sqlite3_free(pCsr->pRec); + pCsr->pRec = 0; + pCsr->iCell++; } - - pCsr->iField++; - if( pCsr->iFieldnField ) return SQLITE_OK; - - /* Advance to the next cell. The next iteration of the loop will load - ** the record and so on. */ - sqlite3_free(pCsr->pRec); - pCsr->pRec = 0; - pCsr->iCell++; } assert( !"can't get here" ); @@ -440,6 +489,7 @@ static int dbdataFilter( } if( idxNum & 0x02 ){ pCsr->iPgno = sqlite3_value_int(argv[(idxNum & 0x01)]); + pCsr->bOnePage = 1; } rc = sqlite3_prepare_v2(pTab->db, @@ -533,34 +583,54 @@ static int dbdataColumn( int i ){ DbdataCursor *pCsr = (DbdataCursor*)pCursor; - switch( i ){ - case DBDATA_COLUMN_PGNO: - sqlite3_result_int64(ctx, pCsr->iPgno); - break; - case DBDATA_COLUMN_CELL: - sqlite3_result_int(ctx, pCsr->iCell); - break; - case DBDATA_COLUMN_FIELD: - sqlite3_result_int(ctx, pCsr->iField); - break; - case DBDATA_COLUMN_VALUE: { - if( pCsr->iField<0 ){ - sqlite3_result_int64(ctx, pCsr->iIntkey); - }else{ - int iHdr; - sqlite3_int64 iType; - sqlite3_int64 iOff; - int i; - iHdr = dbdataGetVarint(pCsr->pRec, &iOff); - for(i=0; iiField; i++){ - iHdr += dbdataGetVarint(&pCsr->pRec[iHdr], &iType); - iOff += dbdataValueBytes(iType); + DbdataTable *pTab = (DbdataTable*)pCursor->pVtab; + if( pTab->bPtr ){ + switch( i ){ + case DBPTR_COLUMN_PGNO: + sqlite3_result_int64(ctx, pCsr->iPgno); + break; + case DBPTR_COLUMN_CHILD: { + int iOff = pCsr->iPgno==1 ? 100 : 0; + if( pCsr->iCell<0 ){ + iOff += 8; + }else{ + iOff += 12 + pCsr->iCell*2; + iOff = get_uint16(&pCsr->aPage[iOff]); } - dbdataGetVarint(&pCsr->pRec[iHdr], &iType); - - dbdataValue(ctx, iType, &pCsr->pRec[iOff]); + sqlite3_result_int64(ctx, get_uint32(&pCsr->aPage[iOff])); + break; + } + } + }else{ + switch( i ){ + case DBDATA_COLUMN_PGNO: + sqlite3_result_int64(ctx, pCsr->iPgno); + break; + case DBDATA_COLUMN_CELL: + sqlite3_result_int(ctx, pCsr->iCell); + break; + case DBDATA_COLUMN_FIELD: + sqlite3_result_int(ctx, pCsr->iField); + break; + case DBDATA_COLUMN_VALUE: { + if( pCsr->iField<0 ){ + sqlite3_result_int64(ctx, pCsr->iIntkey); + }else{ + int iHdr; + sqlite3_int64 iType; + sqlite3_int64 iOff; + int i; + iHdr = dbdataGetVarint(pCsr->pRec, &iOff); + for(i=0; iiField; i++){ + iHdr += dbdataGetVarint(&pCsr->pRec[iHdr], &iType); + iOff += dbdataValueBytes(iType); + } + dbdataGetVarint(&pCsr->pRec[iHdr], &iType); + + dbdataValue(ctx, iType, &pCsr->pRec[iOff]); + } + break; } - break; } } return SQLITE_OK; @@ -604,7 +674,12 @@ static int sqlite3DbdataRegister(sqlite3 *db){ 0, /* xRollbackTo */ 0 /* xShadowName */ }; - return sqlite3_create_module(db, "sqlite_dbdata", &dbdata_module, 0); + + int rc = sqlite3_create_module(db, "sqlite_dbdata", &dbdata_module, 0); + if( rc==SQLITE_OK ){ + rc = sqlite3_create_module(db, "sqlite_dbptr", &dbdata_module, (void*)1); + } + return rc; } #ifdef _WIN32 diff --git a/manifest b/manifest index 59bdc64972..d6a2b045e8 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\sthe\sexperimental\sdbdata\sextension. -D 2019-04-17T21:17:22.795 +C Add\sthe\ssqlite_dbptr\svirtual\stable\sto\sthe\sdbdata\sextension.\sFor\squerying\sthe\slinks\sbetween\sb-tree\spages. +D 2019-04-18T21:14:11.260 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -284,7 +284,7 @@ F ext/misc/closure.c dbfd8543b2a017ae6b1a5843986b22ddf99ff126ec9634a2f4047cd14c8 F ext/misc/completion.c cec672d40604075bb341a7f11ac48393efdcd90a979269b8fe7977ea62d0547f F ext/misc/compress.c dd4f8a6d0baccff3c694757db5b430f3bbd821d8686d1fc24df55cf9f035b189 F ext/misc/csv.c 7f047aeb68f5802e7ce6639292095d622a488bb43526ed04810e0649faa71ceb -F ext/misc/dbdata.c 436a7883a7f1455c5d2853bd927c5ac31826b21c6e16ca11a21d7262d25ff838 +F ext/misc/dbdata.c 20d85d7d7503817a5f9ac7d93c4ae7bf5fcc40570a77adc417acab0500058c99 F ext/misc/dbdump.c baf6e37447c9d6968417b1cd34cbedb0b0ab3f91b5329501d8a8d5be3287c336 F ext/misc/eval.c 4b4757592d00fd32e44c7a067e6a0e4839c81a4d57abc4131ee7806d1be3104e F ext/misc/explain.c d5c12962d79913ef774b297006872af1fccda388f61a11d37758f9179a09551f @@ -787,7 +787,7 @@ F test/cursorhint2.test 6f3aa9cb19e7418967a10ec6905209bcbb5968054da855fc36c8beee F test/dataversion1.test 6e5e86ac681f0782e766ebcb56c019ae001522d114e0e111e5ebf68ccf2a7bb8 F test/date.test 9b73bbeb1b82d9c1f44dec5cf563bf7da58d2373 F test/date2.test 74c234bece1b016e94dd4ef9c8cc7a199a8806c0e2291cab7ba64bace6350b10 -F test/dbdata.test 6e791619d18e0cff2c79392de980ca0594368cdaa05326f043f866beb0c82614 +F test/dbdata.test 573fa3347744863e47d011e4e8e9b87c6795ba92a759bf5d5c68da975900ddd4 F test/dbfuzz.c 73047c920d6210e5912c87cdffd9a1c281d4252e F test/dbfuzz001.test e32d14465f1c77712896fda6a1ccc0f037b481c191c1696a9c44f6c9e4964faf F test/dbfuzz2-seed1.db e6225c6f3d7b63f9c5b6867146a5f329d997ab105bee64644dc2b3a2f2aebaee @@ -1820,10 +1820,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 7ac500fb5abfe1ad60f2ffdcc8fbe5ccc1c641bbeed53f00940e9ff78788e53d -R 9b65ccecf461800383c04a268416aaa8 -T *branch * dbdata -T *sym-dbdata * -T -sym-trunk * +P a3ab58832935e1399ecc7e4d8daefa3a6afa6b301792ce7176bc5d7c173510fb +R 901e9e476021499846a29c847f49c578 U dan -Z f99e8e6e33bab5d12ccf1d6f1a7cae6f +Z e0c2ac18f5bd53328c7a0485e8c7ef97 diff --git a/manifest.uuid b/manifest.uuid index dc49ef98d8..6ee3f97f4e 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -a3ab58832935e1399ecc7e4d8daefa3a6afa6b301792ce7176bc5d7c173510fb \ No newline at end of file +3213a15f2133afbb0a4fec3b8f6e0eeca8c0befafd6658c41074e84f589d5d32 \ No newline at end of file diff --git a/test/dbdata.test b/test/dbdata.test index c4412a5de9..5a0df5dc86 100644 --- a/test/dbdata.test +++ b/test/dbdata.test @@ -43,12 +43,62 @@ do_execsql_test 1.1 { 2 1 1 'ten' } -set big [string repeat big 2000] +breakpoint do_execsql_test 1.2 { + SELECT pgno, cell, field, quote(value) FROM sqlite_dbdata; +} { + 1 0 -1 1 + 1 0 0 'table' + 1 0 1 'T1' + 1 0 2 'T1' + 1 0 3 2 + 1 0 4 {'CREATE TABLE T1(a, b)'} + 2 0 -1 5 + 2 0 0 'v' + 2 0 1 'five' + 2 1 -1 10 + 2 1 0 'x' + 2 1 1 'ten' +} + +set big [string repeat big 2000] +do_execsql_test 1.3 { INSERT INTO t1 VALUES(NULL, $big); SELECT value FROM sqlite_dbdata WHERE pgno=2 AND cell=2 AND field=1; } $big +#------------------------------------------------------------------------- +reset_db +db enable_load_extension 1 +db eval { SELECT load_extension('../dbdata') } + +do_execsql_test 2.0 { + CREATE TABLE t1(a); + CREATE INDEX i1 ON t1(a); + WITH s(i) AS ( + SELECT 1 UNION ALL SELECT i+1 FROM s WHERE i<10 + ) + INSERT INTO t1 SELECT randomblob(900) FROM s; +} + +do_execsql_test 2.1 { + SELECT * FROM sqlite_dbptr WHERE pgno=2; +} { + 2 25 2 6 2 7 2 9 2 11 2 13 2 15 2 17 2 19 2 21 +} + +do_execsql_test 2.2 { + SELECT * FROM sqlite_dbptr WHERE pgno=3; +} { + 3 24 3 23 +} + +do_execsql_test 2.3 { + SELECT * FROM sqlite_dbptr +} { + 2 25 2 6 2 7 2 9 2 11 2 13 2 15 2 17 2 19 2 21 + 3 24 3 23 +} finish_test