From: dan Date: Thu, 5 Oct 2023 19:09:23 +0000 (+0000) Subject: Add the sqlite3changegroup_schema() API. To allow changegroups to handle differences... X-Git-Tag: version-3.44.0~147^2~4 X-Git-Url: http://git.ipfire.org/?a=commitdiff_plain;h=16381d062aa6b769a68359b1eec20ba881c40888;p=thirdparty%2Fsqlite.git Add the sqlite3changegroup_schema() API. To allow changegroups to handle differences in schema created by ALTER TABLE ADD COLUMN. FossilOrigin-Name: 309deee2dd8dd07623fce79f6bb62d5279d140dd0be3b34bc42af20b0507726b --- diff --git a/ext/session/sqlite3session.c b/ext/session/sqlite3session.c index cb40f25fd3..1c291f51c0 100644 --- a/ext/session/sqlite3session.c +++ b/ext/session/sqlite3session.c @@ -119,6 +119,10 @@ struct sqlite3_changeset_iter { ** The data associated with each hash-table entry is a structure containing ** a subset of the initial values that the modified row contained at the ** start of the session. Or no initial values if the row was inserted. +** +** pDfltStmt: +** This is only used by the sqlite3changegroup_xxx() APIs, not by +** regular sqlite3_session objects. */ struct SessionTable { SessionTable *pNext; @@ -132,6 +136,8 @@ struct SessionTable { int nEntry; /* Total number of entries in hash table */ int nChange; /* Size of apChange[] array */ SessionChange **apChange; /* Hash table buckets */ + + sqlite3_stmt *pDfltStmt; }; /* @@ -326,7 +332,7 @@ static int sessionVarintLen(int iVal){ ** Read a varint value from aBuf[] into *piVal. Return the number of ** bytes read. */ -static int sessionVarintGet(u8 *aBuf, int *piVal){ +static int sessionVarintGet(const u8 *aBuf, int *piVal){ return getVarint32(aBuf, *piVal); } @@ -589,7 +595,7 @@ static int sessionPreupdateHash( ** Return the number of bytes of space occupied by the value (including ** the type byte). */ -static int sessionSerialLen(u8 *a){ +static int sessionSerialLen(const u8 *a){ int e = *a; int n; if( e==0 || e==0xFF ) return 1; @@ -1161,15 +1167,22 @@ static int sessionTableInfo( ** indicate that updates on this table should be ignored. SessionTable.abPK ** is set to NULL in this case. */ -static int sessionInitTable(sqlite3_session *pSession, SessionTable *pTab){ +static int sessionInitTable( + sqlite3_session *pSession, + SessionTable *pTab, + sqlite3 *db, + const char *zDb +){ + int rc = SQLITE_OK; + if( pTab->nCol==0 ){ u8 *abPK; assert( pTab->azCol==0 || pTab->abPK==0 ); - pSession->rc = sessionTableInfo(pSession, pSession->db, pSession->zDb, + rc = sessionTableInfo(pSession, db, zDb, pTab->zName, &pTab->nCol, 0, &pTab->azCol, &pTab->azDflt, &abPK, - (pSession->bImplicitPK ? &pTab->bRowid : 0) + ((pSession==0 || pSession->bImplicitPK) ? &pTab->bRowid : 0) ); - if( pSession->rc==SQLITE_OK ){ + if( rc==SQLITE_OK ){ int i; for(i=0; inCol; i++){ if( abPK[i] ){ @@ -1181,14 +1194,19 @@ static int sessionInitTable(sqlite3_session *pSession, SessionTable *pTab){ pTab->bStat1 = 1; } - if( pSession->bEnableSize ){ + if( pSession && pSession->bEnableSize ){ pSession->nMaxChangesetSize += ( 1 + sessionVarintLen(pTab->nCol) + pTab->nCol + strlen(pTab->zName)+1 ); } } } - return (pSession->rc || pTab->abPK==0); + + if( pSession ){ + pSession->rc = rc; + return (rc || pTab->abPK==0); + } + return rc; } static int sessionReinitTable(sqlite3_session *pSession, SessionTable *pTab){ @@ -1417,14 +1435,28 @@ static void sessionAppendPrintf( } } -static int sessionUpdateChanges(sqlite3_session *pSession, SessionTable *pTab){ - sqlite3 *db = pSession->db; +/* +** Prepare a statement against database handle db that SELECTs a single +** row containing the default values for each column in table pTab. For +** example, if pTab is declared as: +** +** CREATE TABLE pTab(a PRIMARY KEY, b DEFAULT 123, c DEFAULT 'abcd'); +** +** Then this function prepares and returns the SQL statement: +** +** SELECT NULL, 123, 'abcd'; +*/ +static int sessionPrepareDfltStmt( + sqlite3 *db, /* Database handle */ + SessionTable *pTab, /* Table to prepare statement for */ + sqlite3_stmt **ppStmt /* OUT: Statement handle */ +){ SessionBuffer sql = {0,0,0}; - sqlite3_stmt *pStmt = 0; + int rc = SQLITE_OK; const char *zSep = " "; int ii = 0; - int rc = pSession->rc; + *ppStmt = 0; sessionAppendPrintf(&sql, &rc, "SELECT"); for(ii=0; iinCol; ii++){ const char *zDflt = pTab->azDflt[ii] ? pTab->azDflt[ii] : "NULL"; @@ -1432,9 +1464,22 @@ static int sessionUpdateChanges(sqlite3_session *pSession, SessionTable *pTab){ zSep = ", "; } if( rc==SQLITE_OK ){ - rc = sqlite3_prepare_v2(db, (const char*)sql.aBuf, -1, &pStmt, 0); + rc = sqlite3_prepare_v2(db, (const char*)sql.aBuf, -1, ppStmt, 0); } + sqlite3_free(sql.aBuf); + + return rc; +} + +static int sessionUpdateChanges(sqlite3_session *pSession, SessionTable *pTab){ + sqlite3 *db = pSession->db; + sqlite3_stmt *pStmt = 0; + int ii = 0; + int rc = pSession->rc; + + rc = sessionPrepareDfltStmt(pSession->db, pTab, &pStmt); if( rc==SQLITE_OK && SQLITE_ROW==sqlite3_step(pStmt) ){ + int ii = 0; SessionChange **pp = 0; for(ii=0; iinChange; ii++){ for(pp=&pTab->apChange[ii]; *pp; pp=&((*pp)->pNext)){ @@ -1445,7 +1490,6 @@ static int sessionUpdateChanges(sqlite3_session *pSession, SessionTable *pTab){ } } - sqlite3_free(sql.aBuf); pSession->rc = rc; rc = sqlite3_finalize(pStmt); if( pSession->rc==SQLITE_OK ) pSession->rc = rc; @@ -1618,7 +1662,7 @@ static void sessionPreupdateOneChange( if( pSession->rc ) return; /* Load table details if required */ - if( sessionInitTable(pSession, pTab) ) return; + if( sessionInitTable(pSession, pTab, pSession->db, pSession->zDb) ) return; /* Check the number of columns in this xPreUpdate call matches the ** number of columns in the table. */ @@ -2117,7 +2161,7 @@ int sqlite3session_diff( /* Locate and if necessary initialize the target table object */ rc = sessionFindTable(pSession, zTbl, &pTo); if( pTo==0 ) goto diff_out; - if( sessionInitTable(pSession, pTo) ){ + if( sessionInitTable(pSession, pTo, pSession->db, pSession->zDb) ){ rc = pSession->rc; goto diff_out; } @@ -5352,6 +5396,9 @@ struct sqlite3_changegroup { int rc; /* Error code */ int bPatch; /* True to accumulate patchsets */ SessionTable *pList; /* List of tables in current patch */ + + sqlite3 *db; /* Configured by changegroup_schema() */ + const char *zDb; /* Configured by changegroup_schema() */ }; /* @@ -5537,6 +5584,111 @@ static int sessionChangeMerge( return rc; } +/* +** Check if a changeset entry with nCol columns and the PK array passed +** as the final argument to this function is compatible with SessionTable +** pTab. If so, return 1. Otherwise, if they are incompatible in some way, +** return 0. +*/ +static int sessionChangesetCheckCompat( + SessionTable *pTab, + int nCol, + u8 *abPK +){ + if( pTab->azCol && nColnCol ){ + int ii; + for(ii=0; iinCol; ii++){ + u8 bPK = (ii < nCol) ? abPK[ii] : 0; + if( pTab->abPK[ii]!=bPK ) return 0; + } + return 1; + } + return (pTab->nCol==nCol && 0==memcmp(abPK, pTab->abPK, nCol)); +} + +static int sessionChangesetExtendRecord( + sqlite3_changegroup *pGrp, + SessionTable *pTab, + int nCol, + int op, + const u8 *aRec, + int nRec, + SessionBuffer *pOut +){ + int rc = SQLITE_OK; + int ii = 0; + + assert( pTab->azCol ); + assert( nColnCol ); + + pOut->nBuf = 0; + if( op==SQLITE_INSERT || (op==SQLITE_DELETE && pGrp->bPatch==0) ){ + /* Append the missing default column values to the record. */ + sessionAppendBlob(pOut, aRec, nRec, &rc); + if( pTab->pDfltStmt==0 ){ + rc = sessionPrepareDfltStmt(pGrp->db, pTab, &pTab->pDfltStmt); + } + for(ii=nCol; rc==SQLITE_OK && iinCol; ii++){ + int eType = sqlite3_column_type(pTab->pDfltStmt, ii); + sessionAppendByte(pOut, eType, &rc); + switch( eType ){ + case SQLITE_FLOAT: + case SQLITE_INTEGER: { + i64 iVal; + if( eType==SQLITE_INTEGER ){ + iVal = sqlite3_column_int64(pTab->pDfltStmt, ii); + }else{ + double rVal = sqlite3_column_int64(pTab->pDfltStmt, ii); + memcpy(&iVal, &rVal, sizeof(i64)); + } + if( SQLITE_OK==sessionBufferGrow(pOut, 8, &rc) ){ + sessionPutI64(&pOut->aBuf[pOut->nBuf], iVal); + } + break; + } + + case SQLITE_BLOB: + case SQLITE_TEXT: { + int n = sqlite3_column_bytes(pTab->pDfltStmt, ii); + sessionAppendVarint(pOut, n, &rc); + if( eType==SQLITE_TEXT ){ + const u8 *z = (const u8*)sqlite3_column_text(pTab->pDfltStmt, ii); + sessionAppendBlob(pOut, z, n, &rc); + }else{ + const u8 *z = (const u8*)sqlite3_column_blob(pTab->pDfltStmt, ii); + sessionAppendBlob(pOut, z, n, &rc); + } + break; + } + + default: + assert( eType==SQLITE_NULL ); + break; + } + } + }else{ + /* Append missing "undefined" entries to the old.* record. And, if this + ** is an UPDATE, to the new.* record as well. */ + int iOff = 0; + if( op==SQLITE_UPDATE ){ + for(ii=0; iinCol-nCol); ii++){ + sessionAppendByte(pOut, 0x00, &rc); + } + } + + sessionAppendBlob(pOut, &aRec[iOff], nRec-iOff, &rc); + for(ii=0; ii<(pTab->nCol-nCol); ii++){ + sessionAppendByte(pOut, 0x00, &rc); + } + } + + return rc; +} + /* ** Add all changes in the changeset traversed by the iterator passed as ** the first argument to the changegroup hash tables. @@ -5550,6 +5702,7 @@ static int sessionChangesetToHash( int nRec; int rc = SQLITE_OK; SessionTable *pTab = 0; + SessionBuffer rec = {0, 0, 0}; while( SQLITE_ROW==sessionChangesetNext(pIter, &aRec, &nRec, 0) ){ const char *zNew; @@ -5561,6 +5714,9 @@ static int sessionChangesetToHash( SessionChange *pExist = 0; SessionChange **pp; + /* Ensure that only changesets, or only patchsets, but not a mixture + ** of both, are being combined. It is an error to try to combine a + ** changeset and a patchset. */ if( pGrp->pList==0 ){ pGrp->bPatch = pIter->bPatchset; }else if( pIter->bPatchset!=pGrp->bPatch ){ @@ -5593,18 +5749,33 @@ static int sessionChangesetToHash( pTab->zName = (char*)&pTab->abPK[nCol]; memcpy(pTab->zName, zNew, nNew+1); + if( pGrp->db ){ + pTab->nCol = 0; + rc = sessionInitTable(0, pTab, pGrp->db, pGrp->zDb); + } + /* The new object must be linked on to the end of the list, not ** simply added to the start of it. This is to ensure that the ** tables within the output of sqlite3changegroup_output() are in ** the right order. */ for(ppTab=&pGrp->pList; *ppTab; ppTab=&(*ppTab)->pNext); *ppTab = pTab; - }else if( pTab->nCol!=nCol || memcmp(pTab->abPK, abPK, nCol) ){ + } + + if( !sessionChangesetCheckCompat(pTab, nCol, abPK) ){ rc = SQLITE_SCHEMA; break; } } + if( nColnCol ){ + assert( pGrp->db ); + rc = sessionChangesetExtendRecord(pGrp, pTab, nCol, op, aRec, nRec, &rec); + if( rc ) break; + aRec = rec.aBuf; + nRec = rec.nBuf; + } + if( sessionGrowHash(0, pIter->bPatchset, pTab) ){ rc = SQLITE_NOMEM; break; @@ -5642,6 +5813,7 @@ static int sessionChangesetToHash( } } + sqlite3_free(rec.aBuf); if( rc==SQLITE_OK ) rc = pIter->rc; return rc; } @@ -5728,6 +5900,28 @@ int sqlite3changegroup_new(sqlite3_changegroup **pp){ return rc; } +int sqlite3changegroup_schema( + sqlite3_changegroup *pGrp, + sqlite3 *db, + const char *zDb +){ + int rc = SQLITE_OK; + + if( pGrp->pList || pGrp->db ){ + /* Cannot add a schema after one or more calls to sqlite3changegroup_add(), + ** or after sqlite3changegroup_schema() has already been called. */ + rc = SQLITE_MISUSE; + }else{ + pGrp->zDb = sqlite3_mprintf("%s", zDb); + if( pGrp->zDb==0 ){ + rc = SQLITE_NOMEM; + }else{ + pGrp->db = db; + } + } + return rc; +} + /* ** Add the changeset currently stored in buffer pData, size nData bytes, ** to changeset-group p. diff --git a/manifest b/manifest index 7f7d8d946b..cf27c7fb81 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Allow\sa\ssession\sobject\sto\sgenerate\sa\schangeset,\seven\sif\scolumns\swere\sadded\sto\sone\sof\sthe\stables\susing\sALTER\sTABLE\sADD\sCOLUMN\swhile\sthe\schangeset\swas\sbeing\scollected. -D 2023-10-04T21:15:24.286 +C Add\sthe\ssqlite3changegroup_schema()\sAPI.\sTo\sallow\schangegroups\sto\shandle\sdifferences\sin\sschema\screated\sby\sALTER\sTABLE\sADD\sCOLUMN. +D 2023-10-05T19:09:23.510 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -544,7 +544,7 @@ F ext/session/sessionrowid.test 85187c2f1b38861a5844868126f69f9ec62223a03449a98a F ext/session/sessionsize.test 8fcf4685993c3dbaa46a24183940ab9f5aa9ed0d23e5fb63bfffbdb56134b795 F ext/session/sessionstat1.test b039e38e2ba83767b464baf39b297cc0b1cc6f3292255cb467ea7e12d0d0280c F ext/session/sessionwor.test 6fd9a2256442cebde5b2284936ae9e0d54bde692d0f5fd009ecef8511f4cf3fc -F ext/session/sqlite3session.c a6ede47e1dbb2ff61180858559502020738e2f978a9648b07fa74a8ce5a776b4 +F ext/session/sqlite3session.c 8a0886dc8772c00ccbfe10012e8d21175a96ee18428a58fae9be81d924b89d03 F ext/session/sqlite3session.h 4d1f69f1d8bfd4798e8f6431de301d17bb2e4097de2f77ca4dad494bb6c60dc0 F ext/session/test_session.c 5285482f83cd92b4c1fe12fcf88210566a18312f4f2aa110f6399dae46aeccbb F ext/userauth/sqlite3userauth.h 7f3ea8c4686db8e40b0a0e7a8e0b00fac13aa7a3 @@ -2124,11 +2124,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 770308db9776b8c0a70b8807463e89a9eddfe5552e25e67324cd303dc974f50e -R 649352ef20ed525adadedc4edb926bc7 -T *branch * session-alter -T *sym-session-alter * -T -sym-trunk * +P a3f435eccf3a2aa11cb7420e94af5efcdfa04e9c169c5aaf61fa5cdcb165ceef +R a796b1e127975d7f78d17f701882d2f3 U dan -Z 220e071d4024660dea84b49e6cbcfb2f +Z afe7ef1533b48179b0f6fa3866dc9cb3 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index d83a120cf7..1d23b0122d 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -a3f435eccf3a2aa11cb7420e94af5efcdfa04e9c169c5aaf61fa5cdcb165ceef \ No newline at end of file +309deee2dd8dd07623fce79f6bb62d5279d140dd0be3b34bc42af20b0507726b \ No newline at end of file