From: drh Date: Mon, 22 Jan 2001 00:31:53 +0000 (+0000) Subject: :-) (CVS 180) X-Git-Tag: version-3.6.10~5891 X-Git-Url: http://git.ipfire.org/?a=commitdiff_plain;h=acbcdc49c64677d1a77e548d5f1b7a6be6c2b524;p=thirdparty%2Fsqlite.git :-) (CVS 180) FossilOrigin-Name: 98da825312fd4bb8a20ff33293131c02beb3ae63 --- diff --git a/doc/report1.txt b/doc/report1.txt new file mode 100644 index 0000000000..cf5668d1b8 --- /dev/null +++ b/doc/report1.txt @@ -0,0 +1,48 @@ +The SQL database used for ACD contains 113 tables and indices implemented +in GDBM. The following are statistics on the sizes of keys and data +within these tables and indices. + +Entries: 962080 +Size: 45573853 +Avg Size: 48 +Key Size: 11045299 +Avg Key Size: 12 +Max Key Size: 99 + + 0..8 266 0% + 9..12 5485 0% + 13..16 73633 8% + 17..24 180918 27% + 25..32 209823 48% + 33..40 148995 64% + 41..48 76304 72% + 49..56 14346 73% + 57..64 15725 75% + 65..80 44916 80% + 81..96 127815 93% + 97..112 34769 96% + 113..128 13314 98% + 129..144 8098 99% + 145..160 3355 99% + 161..176 1159 99% + 177..192 629 99% + 193..208 221 99% + 209..224 210 99% + 225..240 129 99% + 241..256 57 99% + 257..288 496 99% + 289..320 60 99% + 321..352 37 99% + 353..384 46 99% + 385..416 22 99% + 417..448 24 99% + 449..480 26 99% + 481..512 27 99% + 513..1024 471 99% + 1025..2048 389 99% + 2049..4096 182 99% + 4097..8192 74 99% + 8193..16384 34 99% +16385..32768 17 99% +32769..65536 5 99% +65537..131073 3 100% diff --git a/manifest b/manifest index d9433317c4..045776eb8d 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C :-)\s(CVS\s1711) -D 2001-01-21T22:03:30 +C :-)\s(CVS\s180) +D 2001-01-22T00:31:53 F COPYRIGHT 74a8a6531a42e124df07ab5599aad63870fa0bd4 F Makefile.in 7efa81e2985b45ba73db27d55b70cc927f5abfd7 F README 51f6a4e7408b34afa5bc1c0485f61b6a4efb6958 @@ -7,9 +7,10 @@ F VERSION 05e17b646a817240c206186f94f8f4c70974d5dc F configure 3dc1edb9dcf60215e31ff72b447935ab62211442 x F configure.in d892ca33db7e88a055519ce2f36dcb11020e8fff F doc/lemon.html e233a3e97a779c7a87e1bc4528c664a58e49dd47 -F src/TODO 435a4ea4f760375f6821f390ea4ee0767f455b40 +F doc/report1.txt ad0a41513479f1be0355d1f3f074e66779ff2282 +F src/TODO 38a68a489e56e9fd4a96263e0ff9404a47368ad4 F src/build.c 7aa5879bf58ea6bbff22c26c59d1130021fa6ca4 -F src/db.c af04a75b521be61f5d7f3fab12ba6d477937854e +F src/db.c 8e841538cc1079c99b050ba8694c7bd544b0a355 F src/db.h 488f01d3c0182568b0ec1186149603e271e79c43 F src/dbbe.c 162d29b09ac379f160892c5795efc14099dcc8eb F src/dbbe.h 0435a36906a839cce062608f51bd9d3e79878fec @@ -57,7 +58,7 @@ F test/select5.test e2b9d51d88cbd6c307c2c05b0ef55fe7ba811ac2 F test/sort.test d582086c4bb7df3fbf50aa72e69d7e235e9f8e31 F test/subselect.test bf8b251a92fb091973c1c469ce499dc9648a41d5 F test/table.test eaa25951c0f18615763cd3dc248ea4bc38739c05 -F test/tester.tcl 446b88283b12efb12691479a403cde15d64fbb82 +F test/tester.tcl e053e14aa986c05a87de0b5635e76566f1e022ae F test/update.test 62f6ce99ff31756aab0ca832ff6d34c5a87b6250 F test/vacuum.test 2127748ff4ddb409212efbb6d9fb9c469ea1b49c F test/where.test bbab5a308055fb6087dc23d600b4ad2b72797397 @@ -83,7 +84,7 @@ F www/opcode.tcl cb3a1abf8b7b9be9f3a228d097d6bf8b742c2b6f F www/sqlite.tcl cb0d23d8f061a80543928755ec7775da6e4f362f F www/tclsqlite.tcl 06f81c401f79a04f2c5ebfb97e7c176225c0aef2 F www/vdbe.tcl 0c8aaa529dd216ccbf7daaabd80985e413d5f9ad -P d5f2a668978c0d108045237f19b0a7efa07678f2 -R 1dea96afaac1107dc23974be6b858cb9 +P 0529c979fd17995aff82e21b91b5cc833f23d8ef +R 190eb04a63af8235aad9df6267107e4c U drh -Z 7168eb4919f6604238c3251129477d38 +Z da6be622c2a2caa5949500b0c0127edd diff --git a/manifest.uuid b/manifest.uuid index 77de891b64..fd1845f763 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -0529c979fd17995aff82e21b91b5cc833f23d8ef \ No newline at end of file +98da825312fd4bb8a20ff33293131c02beb3ae63 \ No newline at end of file diff --git a/src/TODO b/src/TODO index 50e1b9fbad..31610a2678 100644 --- a/src/TODO +++ b/src/TODO @@ -22,6 +22,7 @@ - Transactions * Modify sqlite_master to store the table number. * Add a cache in DbCursor to speed up the sqliteDbReadOvfl() routine. + * Add cache information to speed up sqliteDbCursorMoveTo(). Longer term: * Document all the changes and release Sqlite 2.0. @@ -29,4 +30,6 @@ Longer term: indices. * "OPTIMIZE select" statement to automatically create and/or tune indices. + * "CREATE INDEX FOR select" to automatically generate needed indices. + * "VACUUM table USING index". * Parse and use constraints. diff --git a/src/db.c b/src/db.c index c5fd8711bb..74c3dfdf7b 100644 --- a/src/db.c +++ b/src/db.c @@ -21,7 +21,7 @@ ** http://www.hwaci.com/drh/ ** ************************************************************************* -** $Id: db.c,v 1.2 2001/01/21 00:58:08 drh Exp $ +** $Id: db.c,v 1.3 2001/01/22 00:31:53 drh Exp $ */ #include "sqliteInt.h" #include "pg.h" @@ -92,16 +92,16 @@ struct DbCursor { ** entries.... ** 0. size of this entry (measured in u32's) ** 1. hash -** 2. keysize (in bytes. bit 31 set if uses overflow) -** 3. datasize (in bytes. bit 31 set if uses overflow pages) -** 4. key -** 5+. data +** 2. keysize (in bytes) +** 3. datasize (in bytes) +** 4. payload ** -** Overflow block: +** Payload area: ** -** 0. BLOCK_MAGIC | BLOCK_OVERFLOW -** 1. address of next block in overflow buffer -** data... +** * up to LOCAL_PAYLOAD bytes of data +** * 10 page number of direct blocks +** * 1 indirect block +** * 1 double-indirect block ** ** Index block: ** @@ -114,12 +114,15 @@ struct DbCursor { ** ** Contents block: (The first page in the file) ** 0. BLOCK_MAGIC | BLOCK_CONTENTS -** 1. overflow page list -** 2. freelist -** 3. number of tables -** table root page numbers... +** 1. zero +** 2. number of bytes of payload +** 3. freelist +** 4... root pages numbers of tables */ +#define U32_PER_PAGE (SQLITE_PAGE_SIZE/sizeof(u32)) +#deifne LOCAL_PAYLOAD (SQLITE_PAGE_SIZE - 18*sizeof(u32)) + /* ** Byte swapping code. */ @@ -142,6 +145,262 @@ static u32 sqliteDbSwapBytes(u32 x){ #endif +/* +** Return the number of bytes of payload storage required on the leaf +** node to hold the key and data given. Overflow pages do not count. +** The argument is the total size of the payload. +*/ +static int payloadLocalSize(int nTotal){ + int nLocal, i; + if( nTotal<0 ) nTotal = 0; + if( nTotal <= LOCAL_PAYLOAD ){ + /* All the data fits on the leaf page */ + return (nTotal + 3)/4; + } + nLocal = LOCAL_PAYLOAD; + nTotal -= LOCAL_PAYLOAD; + if( nTotal < 10*SQLITE_PAGE_SIZE ){ + return nLocal + ((nTotal+SQLITE_PAGE_SIZE-1)/SQLITE_PAGE_SIZE)*sizeof(u32); + } + nLocal += 10*sizeof(u32); + nTotal -= 10*SQLITE_PAGE_SIZE; + if( nTotal < U32_PER_PAGE*SQLITE_PAGE_SIZE ){ + return nLocal + sizeof(u32); + } + nLocal += sizeof(u32); + nTotal -= U32_PER_PAGE*SQLITE_PAGE_SIZE; + if( nTotal < U32_PER_PAGE*U32_PER_PAGE*SQLITE_PAGE_SIZE ){ + return nLocal + sizeof(u32); + } + return -1; /* This payload will not fit. */ +} + +/* +** Read data from the payload area. +** +** aPage points directly at the beginning of the payload. No bounds +** checking is done on offset or amt -- it is assumed that the payload +** area is big enough to accomodate. +*/ +static int payloadRead(Db *pDb, u32 *aPage, int offset, int amt, void *pBuf){ + int rc; + int toread, more; + + assert( offset>=0 && amt>=0 ); + if( offset < LOCAL_PAYLOAD ){ + /* Data stored directly in the leaf block of the BTree */ + if( amt+offset>LOCAL_PAYLOAD ){ + toread = LOCAL_PAYLOAD - offset; + more = 1; + }else{ + toread = amt; + more = 0; + } + memcpy(pBuf, &((char*)aPage)[offset], toread); + if( !more ) return SQLITE_OK; + pBuf = &((char*)pBuf)[toread]; + offset += toread; + amt -= toread; + } + offset -= LOCAL_PAYLOAD; + aPage += LOCAL_PAYLOAD/sizeof(aPage[0]); + while( offset < 10*SQLITE_PAGE_SIZE ){ + /* Data stored in one of 10 direct pages */ + int iDir; + char *aData; + iDir = offset/SQLITE_PAGE_SIZE; + base = offset - iDir*SQLITE_PAGE_SIZE; + rc = sqlitePgGet(pDb->pPgr, aPage[iDir], &aData); + if( rc!=SQLITE_OK ) return rc; + if( amt+base > SQLITE_PAGE_SIZE ){ + toread = SQLITE_PAGE_SIZE - base; + more = 1; + }else{ + toread = amt; + more = 0; + } + memcpy(pBuf, &aData[base], toread); + sqlitePgUnref(aData); + if( !more ) return SQLITE_OK; + pBuf = &((char*)pBuf)[toread]; + amt -= toread; + offset += toread; + } + offset -= 10*SQLITE_PAGE_SIZE; + aPage += 10; + if( offset < U32_PER_PAGE*SQLITE_PAGE_SIZE ){ + /* Data stored in an indirect page */ + u32 *indirPage; + rc = sqlitePgGet(pDb->pPgr, aPage[0], &indirPage); + if( rc!=SQLITE_OK ) return rc; + while( amt>0 && offset < U32_PER_PAGE*SQLITE_PAGE_SIZE ){ + int idx, base; + char *aData; + idx = offset/SQLITE_PAGE_SIZE; + base = offset - idx*SQLITE_PAGE_SIZE; + rc = sqlitePgGet(pDb->pPgr, indirPage[idx], &aData); + if( rc!=SQLITE_OK ) break; + if( amt+base > SQLITE_PAGE_SIZE ){ + toread = SQLITE_PAGE_SIZE - base; + }else{ + toread = amt; + } + memcpy(pBuf, &aData[base], toread); + sqlitePgUnref(aData); + pBuf = &((char*)pBuf)[toread]; + amt -= toread; + offset += toread; + } + sqlitePgUnref(indirPage); + if( rc!=SQLITE_OK ) return rc; + } + offset -= U32_PER_PAGE*SQLITE_PAGE_SIZE; + aPage++; + if( offset < U32_PER_PAGE*U32_PER_PAGE*SQLITE_PAGE_SIZE ){ + /* Data stored in a double-indirect page */ + u32 *dblIndirPage; + rc = sqlitePgGet(pDb->pPgr, aPage[0], &dblIndirPage); + if( rc!=SQLITE_OK ) return rc; + while( amt>0 && offset < U32_PER_PAGE*U32_PER_PAGE*SQLITE_PAGE_SIZE ){ + int dblidx; + u32 *indirPage; + int basis; + dblidx = offset/(U32_PER_PAGE*SQLITE_PAGE_SIZE); + rc = sqlitePgGet(pDb->pPgr, dblIndirPage[dblidx], &indirPage); + if( rc!=SQLITE_OK ) break; + basis = dblidx*U32_PER_PAGE*SQLITE_PAGE_SIZE; + while( amt>0 && offset < basis + U32_PER_PAGE*SQLITE_PAGE_SIZE ){ + int idx, base; + char *aData; + idx = (offset - basis)/SQLITE_PAGE_SIZE; + base = (offset - basis) - idx*SQLITE_PAGE_SIZE; + rc = sqlitePgGet(pDb->pPgr, indirPage[idx], &aData); + if( rc!=SQLITE_OK ) break; + if( amt+base > SQLITE_PAGE_SIZE ){ + toread = SQLITE_PAGE_SIZE - base; + }else{ + toread = amt; + } + memcpy(pBuf, &aData[base], toread); + sqlitePgUnref(aData); + pBuf = &((char*)pBuf)[toread]; + amt -= toread; + offset += toread; + } + sqlitePgUnref(indirPage); + if( rc!=SQLITE_OK ) break; + } + sqlitePgUnref(dblIndirPage); + return rc; + } + memset(pBuf, 0, amt); + return SQLITE_OK; +} + +/* +** Write data into the payload area. +** +** If pages have already been allocated for the payload, they are +** simply overwritten. New pages are allocated as necessary to +** fill in gaps. sqlitePgTouch() is called on all overflow pages, +** but the calling function must invoke sqlitePgTouch() for aPage +** itself. +*/ +static int payloadWrite(Db *pDb, u32 *aPage, int offset, int amt, void *pBuf){ + assert( offset>=0 && amt>=0 ); + if( offset < LOCAL_PAYLOAD ){ + if( amt+offset>LOCAL_PAYLOAD ){ + towrite = LOCAL_PAYLOAD - offset; + more = 1; + }else{ + towrite = amt; + more = 0; + } + memcpy(&((char*)aPage)[offset], pBuf, towrite); + if( !more ) return SQLITE_OK; + pBuf = &((char*)pBuf)[towrite]; + offset += toread; + amt -= toread; + } + offset -= LOCAL_PAYLOAD; + aPage += LOCAL_PAYLOAD/sizeof(aPage[0]); + while( offset < 10*SQLITE_PAGE_SIZE ){ + int iDir; + char *aData; + iDir = offset/SQLITE_PAGE_SIZE; + base = offset - iDir*SQLITE_PAGE_SIZE; + if( aPage[iDir] ){ + rc = sqliteGet(pDb->pPgr, aPage[iDir], &aData); + }else{ + rc = sqliteDbAllocPage(pDb, &aPage[iDir], &aData); + } + if( rc!=SQLITE_OK ) return rc; + if( amt+base > SQLITE_PAGE_SIZE ){ + towrite = SQLITE_PAGE_SIZE - base; + more = 1; + }else{ + towrite = amt; + more = 0; + } + memcpy(&aData[base], pBuf, towrite); + sqlitePgUnref(aData); + if( !more ) return SQLITE_OK; + pBuf = &((char*)pBuf)[towrite]; + amt -= towrite; + offset += towrite; + } + /* TBD.... */ +} + +/* +** Release any and all overflow pages associated with data starting +** with byte "newSize" up to but not including "oldSize". +*/ +static int payloadFree(Db *pDb, u32 *aPage, int newSize, int oldSize){ + int i; + + if( newSize>=oldSize ) return; + oldSize -= LOCAL_PAYLOAD; + if( oldSize<=0 ) return SQLITE_OK; + newSize -= LOCAL_PAYLOAD; + if( newSize<0 ) newSize = 0; + aPage += LOCAL_PAYLOAD/sizeof(u32); +************* + for(i=0; i<10; i++){ + sqliteDbFreePage(pDb, aPage[0], 0); + amt -= SQLITE_PAGE_SIZE; + if( amt<=0 ) return SQLITE_OK; + aPage++; + } + rc = sqlitePgGet(pDb->pPgr, aPage[0], &indirPage); + if( rc!=SQLITE_OK ) return rc; + for(i=0; ipPgr, aPage[0], &dblIndirPage); + if( rc!=SQLITE_OK ) return rc; + for(i=0; ipPgr, dblIndirPage[i], &indirPage); + if( rc!=SQLITE_OK ) break; + for(j=0; j0 ){ - int toWrite, rc; - u32 *nxPage, nxPgno; - if( nData > SQLITE_PAGE_SIZE - 2*sizeof(u32) ){ - toWrite = SQLITE_PAGE_SIZE - 2*sizeof(u32); - }else{ - toWrite = nData; - } - memcpy(&aPage[2], pData, toWrite); - nData -= toWrite; - pData = &((char*)pData)[toWrite]; - if( nData<=0 ) break; - rc = sqliteDbAllocPage(pDb, &nxPgno, &nxPage); - if( rc!=SQLITE_OK ) return rc; /* Be smarter here */ - aPage[1] = SWB(nxPgno); - nxPage[0] = SWB(BLOCK_MAGIC|BLOCK_OVERFLOW); - nxPage[1] = 0; - sqlitePgTouch(aPage); - sqlitePgUnref(aPage); - aPage = nxPage; - } - return SQLITE_OK; -} - /* ** Open a database. */ @@ -254,18 +483,18 @@ int sqliteDbOpen(const char *filename, Db **ppDb){ if( rc!=0 ) goto open_err; if( nPage==0 ){ sqlitePgBeginTransaction(pDb->pPgr); - aPage1[0] = SWB(BLOCK_MAGIC|BLOCK_CONTENT); + aPage1[0] = BLOCK_MAGIC|BLOCK_CONTENT; + aPage1[2] = sizeof(u32)*10; sqlitePgTouch(aPage1); sqlitePgCommit(pDb->pPgr); } - pDb->nContent = SWB(aPage1[3]) + 2; + pDb->nContent = aPage1[2]/sizeof(u32); pDb->nAlloc = 0; rc = sqliteDbExpandContent(pDb, pDb->nContent); if( rc!=SQLITE_OK ) goto open_err; - rc = sqliteDbReadOvfl(pDb, 1, aPage1, 0, pDb->nContent*sizeof(u32), - pDb->aContent); - if( rc!=SQLITE_OK ) goto open_err; + rc = payloadRead(pDb, &aPage1[3], 0, aPage[2], pDb->aContent); sqlitePgUnref(aPage1); + if( rc!=SQLITE_OK ) goto open_err; *ppDb = pDb; return SQLITE_OK; @@ -313,14 +542,16 @@ int sqliteDbBeginTransaction(Db *pDb){ ** Commit changes to the database */ int sqliteDbCommit(Db *pDb){ - u32 *aPage; + u32 *aPage1; int rc; if( !pDb->inTransaction ){ return SQLITE_OK; } - rc = sqlitePgGet(pDb->pPgr, 1, &aPage); + rc = sqlitePgGet(pDb->pPgr, 1, &aPage1); if( rc!=SQLITE_OK ) return rc; - sqliteDbWriteOvfl(pDb, aPage, pDb->nContent*sizeof(u32), pDb->aContent); + aPage1[2] = pDb->nContent*sizeof(u32); + payloadWrite(pDb, 0, aPage1[2], pDb->aContent); + sqlitePgUnref(aPage1); rc = sqlitePgCommit(pDb->pPgr); if( rc!=SQLITE_OK ) return rc; pDb->inTransaction = 0; diff --git a/test/tester.tcl b/test/tester.tcl index 3690c1636d..c235a8c181 100644 --- a/test/tester.tcl +++ b/test/tester.tcl @@ -23,7 +23,7 @@ # This file implements some common TCL routines used for regression # testing the SQLite library # -# $Id: tester.tcl,v 1.8 2000/12/10 18:23:52 drh Exp $ +# $Id: tester.tcl,v 1.9 2001/01/22 00:31:53 drh Exp $ # Create a test database # @@ -139,6 +139,7 @@ proc finish_test {} { # A procedure to execute SQL # proc execsql {sql} { + # puts "SQL = $sql" return [db eval $sql] }