do_test $tn3.1.$tn2.$tn.1 {
create_ota1 ota.db
- breakpoint
$cmd test.db ota.db
} {SQLITE_DONE}
}
}
+ catch { db close }
eval $destroy_vfs
}
} {1 {SQLITE_BUSY - database modified during ota update}}
}
-for {set nStep 7} {$nStep < 8} {incr nStep} {
- do_test 1.$nStep.1 {
- setup_test
- sqlite3ota ota test.db ota.db
- for {set i 0} {$i<$nStep} {incr i} {ota step}
- ota close
- sqlite3 db test.db
- execsql { INSERT INTO t1 VALUES(5, 'hello') }
- sqlite3ota ota test.db ota.db
- ota step
- } {SQLITE_OK}
- do_test 1.$nStep.2 {
- ota step
- } {SQLITE_OK}
- do_test 1.$nStep.3 {
- list [file exists test.db-oal] [file exists test.db-wal]
- } {0 1}
- do_test 1.$nStep.4 {
- list [catch { ota close } msg] $msg
- } {0 SQLITE_OK}
-}
-
# Test the outcome of some other client writing the db after the *-oal
# file has been copied to the *-wal path. Once this has happened, any
# other client writing to the db causes OTA to consider its job finished.
#
-for {set nStep 8} {$nStep < 20} {incr nStep} {
+for {set nStep 7} {$nStep < 20} {incr nStep} {
do_test 1.$nStep.1 {
setup_test
sqlite3ota ota test.db ota.db
#define OTA_STATE_COOKIE 7
#define OTA_STAGE_OAL 1
+#define OTA_STAGE_CAPTURE 2
#define OTA_STAGE_CKPT 3
#define OTA_STAGE_DONE 4
typedef struct ota_vfs ota_vfs;
typedef struct ota_file ota_file;
+#if !defined(SQLITE_AMALGAMATION)
+typedef unsigned int u32;
+typedef unsigned char u8;
+typedef sqlite3_int64 i64;
+#endif
+
+/*
+** These values must match the values defined in wal.c for the equivalent
+** locks. These are not magic numbers as they are part of the SQLite file
+** format.
+*/
+#define WAL_LOCK_WRITE 0
+#define WAL_LOCK_CKPT 1
+#define WAL_LOCK_READ0 3
+
/*
** A structure to store values read from the ota_state table in memory.
*/
int eStage;
char *zTbl;
char *zIdx;
- unsigned char *pCkptState;
- int nCkptState;
+ i64 iWalCksum;
int nRow;
- sqlite3_int64 nProgress;
+ i64 nProgress;
};
/*
char **azTblCol; /* Array of unquoted target column names */
char **azTblType; /* Array of target column types */
int *aiSrcOrder; /* src table col -> target table col */
- unsigned char *abTblPk; /* Array of flags, set on target PK columns */
- unsigned char *abNotNull; /* Array of flags, set on NOT NULL columns */
+ u8 *abTblPk; /* Array of flags, set on target PK columns */
+ u8 *abNotNull; /* Array of flags, set on NOT NULL columns */
int eType; /* Table type - an OTA_PK_XXX value */
/* Output variables. zTbl==0 implies EOF. */
#define OTA_PK_VTAB 5
+typedef struct OtaFrame OtaFrame;
+struct OtaFrame {
+ u32 iDbPage;
+ u32 iWalFrame;
+};
+
/*
** OTA handle.
*/
int nStep; /* Rows processed for current object */
int nProgress; /* Rows processed for all objects */
OtaObjIter objiter; /* Iterator for skipping through tbl/idx */
- sqlite3_ckpt *pCkpt; /* Incr-checkpoint handle */
- ota_file *pTargetFd; /* File handle open on target db */
const char *zVfsName; /* Name of automatically created ota vfs */
+ ota_file *pTargetFd; /* File handle open on target db */
+
+ /* The following state variables are used as part of the incremental
+ ** checkpoint stage (eStage==OTA_STAGE_CKPT). See function otaSetupCkpt()
+ ** for details. */
+ u32 iMaxFrame; /* Largest iWalFrame value in aFrame[] */
+ u32 mLock;
+ int nFrame; /* Entries in aFrame[] array */
+ int nFrameAlloc; /* Allocated size of aFrame[] array */
+ OtaFrame *aFrame;
+ int pgsz;
+ u8 *aBuf;
+ i64 iWalCksum;
};
struct ota_vfs {
- sqlite3_vfs base; /* ota VFS shim methods */
- sqlite3_vfs *pRealVfs; /* Underlying VFS */
- sqlite3_mutex *mutex;
- const char *zOtaWal;
+ sqlite3_vfs base; /* ota VFS shim methods */
+ sqlite3_vfs *pRealVfs; /* Underlying VFS */
+ sqlite3_mutex *mutex; /* Mutex to protect pMain */
+ ota_file *pMain; /* Linked list of main db files */
};
struct ota_file {
sqlite3ota *pOta; /* Pointer to ota object (ota target only) */
int openFlags; /* Flags this file was opened with */
- unsigned int iCookie; /* Cookie value for main db files */
- unsigned char iWriteVer; /* "write-version" value for main db files */
+ u32 iCookie; /* Cookie value for main db files */
+ u8 iWriteVer; /* "write-version" value for main db files */
int nShm; /* Number of entries in apShm[] array */
char **apShm; /* Array of mmap'd *-shm regions */
- const char *zWal; /* Wal filename for this db file */
char *zDel; /* Delete this when closing file */
+
+ const char *zWal; /* Wal filename for this main db file */
+ ota_file *pWalFd; /* Wal file descriptor for this main db */
+ ota_file *pMainNext; /* Next MAIN_DB file */
};
** error code in the OTA handle passed as the first argument.
*/
static void otaAllocateIterArrays(sqlite3ota *p, OtaObjIter *pIter, int nCol){
- int nByte = (2*sizeof(char*) + sizeof(int) + 2*sizeof(unsigned char)) * nCol;
+ int nByte = (2*sizeof(char*) + sizeof(int) + 2*sizeof(u8)) * nCol;
char **azNew;
azNew = (char**)otaMalloc(p, nByte);
pIter->azTblCol = azNew;
pIter->azTblType = &azNew[nCol];
pIter->aiSrcOrder = (int*)&pIter->azTblType[nCol];
- pIter->abTblPk = (unsigned char*)&pIter->aiSrcOrder[nCol];
- pIter->abNotNull = (unsigned char*)&pIter->abTblPk[nCol];
+ pIter->abTblPk = (u8*)&pIter->aiSrcOrder[nCol];
+ pIter->abNotNull = (u8*)&pIter->abTblPk[nCol];
}
}
zSql = 0;
if( pStmt==0 ) goto otaTableType_end;
while( sqlite3_step(pStmt)==SQLITE_ROW ){
- const unsigned char *zOrig = sqlite3_column_text(pStmt,3);
+ const u8 *zOrig = sqlite3_column_text(pStmt,3);
if( zOrig && zOrig[0]=='p' ){
zSql = sqlite3_mprintf("SELECT rootpage FROM main.sqlite_master"
" WHERE name=%Q", sqlite3_column_text(pStmt,1));
pIter->azTblType[iOrder] = otaStrndup(zType, -1, &p->rc);
pIter->abTblPk[iOrder] = (iPk!=0);
- pIter->abNotNull[iOrder] = (unsigned char)bNotNull || (iPk!=0);
+ pIter->abNotNull[iOrder] = (u8)bNotNull || (iPk!=0);
iOrder++;
}
}
assert( p->rc==SQLITE_OK );
assert( p->db==0 );
+ p->eStage = 0;
p->rc = sqlite3_open_v2(p->zTarget, &p->db, flags, p->zVfsName);
if( p->rc ){
p->zErrmsg = sqlite3_mprintf("%s", sqlite3_errmsg(p->db));
}else{
- otaMPrintfExec(p, "ATTACH %Q AS ota", p->zOta);
+ p->rc = sqlite3_file_control(p->db, "main", SQLITE_FCNTL_OTA, (void*)p);
+ if( p->rc==SQLITE_OK ){
+ otaMPrintfExec(p, "ATTACH %Q AS ota", p->zOta);
+ }
/* Mark the database file just opened as an OTA target database. If
** this call returns SQLITE_NOTFOUND, then the OTA vfs is not in use.
** This is an error. */
if( p->rc==SQLITE_OK ){
p->rc = sqlite3_file_control(p->db, "main", SQLITE_FCNTL_OTA, (void*)p);
- if( p->rc==SQLITE_NOTFOUND ){
- p->rc = SQLITE_ERROR;
- p->zErrmsg = sqlite3_mprintf("ota vfs not found");
- }
+ }
+
+ if( p->rc==SQLITE_NOTFOUND ){
+ p->rc = SQLITE_ERROR;
+ p->zErrmsg = sqlite3_mprintf("ota vfs not found");
}
}
}
#endif
}
+/*
+** Return the current wal-index header checksum for the target database
+** as a 64-bit integer.
+**
+** The checksum is store in the first page of xShmMap memory as an 8-byte
+** blob starting at byte offset 40.
+*/
+static i64 otaShmChecksum(sqlite3ota *p){
+ i64 iRet;
+ if( p->rc==SQLITE_OK ){
+ sqlite3_file *pDb = p->pTargetFd->pReal;
+ u32 volatile *ptr;
+ p->rc = pDb->pMethods->xShmMap(pDb, 0, 32*1024, 0, (void volatile**)&ptr);
+ if( p->rc==SQLITE_OK ){
+ iRet = ((i64)ptr[10] << 32) + ptr[11];
+ }
+ }
+ return iRet;
+}
+
+static void otaSetupCheckpoint(sqlite3ota *p, OtaState *pState){
+
+ if( pState==0 ){
+ p->eStage = 0;
+ if( p->rc==SQLITE_OK ){
+ p->rc = sqlite3_exec(p->db, "SELECT * FROM sqlite_master", 0, 0, 0);
+ }
+ }
+
+ if( p->rc==SQLITE_OK ){
+ int rc2;
+ p->eStage = OTA_STAGE_CAPTURE;
+ rc2 = sqlite3_exec(p->db, "PRAGMA main.wal_checkpoint=restart", 0, 0, 0);
+ if( rc2!=SQLITE_INTERNAL ) p->rc = rc2;
+ }
+
+ if( p->rc==SQLITE_OK ){
+ p->eStage = OTA_STAGE_CKPT;
+ p->nStep = 0;
+ p->aBuf = otaMalloc(p, p->pgsz);
+ p->iWalCksum = otaShmChecksum(p);
+ }
+
+ if( p->rc==SQLITE_OK && pState && pState->iWalCksum!=p->iWalCksum ){
+ p->rc = SQLITE_DONE;
+ p->eStage = OTA_STAGE_DONE;
+ }
+}
+
+static int otaCaptureWalRead(sqlite3ota *pOta, i64 iOff, int iAmt){
+ const u32 mReq = (1<<WAL_LOCK_WRITE)|(1<<WAL_LOCK_CKPT)|(1<<WAL_LOCK_READ0);
+ u32 iFrame;
+
+ if( pOta->mLock!=mReq ){
+ return SQLITE_BUSY;
+ }
+
+ pOta->pgsz = iAmt;
+ if( pOta->nFrame==pOta->nFrameAlloc ){
+ int nNew = (pOta->nFrameAlloc ? pOta->nFrameAlloc : 64) * 2;
+ OtaFrame *aNew;
+ aNew = (OtaFrame*)sqlite3_realloc(pOta->aFrame, nNew * sizeof(OtaFrame));
+ if( aNew==0 ) return SQLITE_NOMEM;
+ pOta->aFrame = aNew;
+ pOta->nFrameAlloc = nNew;
+ }
+
+ iFrame = (u32)((iOff-32) / (i64)(iAmt+24)) + 1;
+ if( pOta->iMaxFrame<iFrame ) pOta->iMaxFrame = iFrame;
+ pOta->aFrame[pOta->nFrame].iWalFrame = iFrame;
+ pOta->aFrame[pOta->nFrame].iDbPage = 0;
+ pOta->nFrame++;
+ return SQLITE_OK;
+}
+
+static int otaCaptureDbWrite(sqlite3ota *pOta, i64 iOff){
+ pOta->aFrame[pOta->nFrame-1].iDbPage = (u32)(iOff / pOta->pgsz) + 1;
+ return SQLITE_OK;
+}
+
+static void otaCheckpointFrame(sqlite3ota *p, OtaFrame *pFrame){
+ if( p->rc==SQLITE_OK ){
+ sqlite3_file *pWal = p->pTargetFd->pWalFd->pReal;
+ sqlite3_file *pDb = p->pTargetFd->pReal;
+ i64 iOff;
+
+ iOff = (i64)(pFrame->iWalFrame-1) * (p->pgsz + 24) + 32 + 24;
+ p->rc = pWal->pMethods->xRead(pWal, p->aBuf, p->pgsz, iOff);
+ if( p->rc ) return;
+
+ iOff = (i64)(pFrame->iDbPage-1) * p->pgsz;
+ p->rc = pDb->pMethods->xWrite(pDb, p->aBuf, p->pgsz, iOff);
+ }
+}
+
/*
** The OTA handle is currently in OTA_STAGE_OAL state, with a SHARED lock
** on the database file. This proc moves the *-oal file to the *-wal path,
otaObjIterFinalize(&p->objiter);
sqlite3_close(p->db);
p->db = 0;
- p->eStage = OTA_STAGE_CKPT;
otaOpenDatabase(p);
+ otaSetupCheckpoint(p, 0);
}
sqlite3_free(zWal);
}
case OTA_STAGE_CKPT: {
+ if( p->nStep>=p->nFrame ){
+ sqlite3_file *pDb = p->pTargetFd->pReal;
- if( p->rc==SQLITE_OK && p->pCkpt==0 ){
- p->rc = sqlite3_ckpt_open(p->db, 0, 0, &p->pCkpt);
- }
- if( p->rc==SQLITE_OK ){
- if( p->pCkpt==0 ){
- p->eStage = OTA_STAGE_DONE;
- p->rc = SQLITE_DONE;
- }else if( SQLITE_OK!=sqlite3_ckpt_step(p->pCkpt) ){
- p->rc = sqlite3_ckpt_close(p->pCkpt, 0, 0);
- p->pCkpt = 0;
+ /* Sync the db file */
+ p->rc = pDb->pMethods->xSync(pDb, SQLITE_SYNC_NORMAL);
+
+ /* Update nBackfill */
+ if( p->rc==SQLITE_OK ){
+ void volatile *ptr;
+ p->rc = pDb->pMethods->xShmMap(pDb, 0, 32*1024, 0, &ptr);
if( p->rc==SQLITE_OK ){
- p->eStage = OTA_STAGE_DONE;
- p->rc = SQLITE_DONE;
+ ((u32*)ptr)[12] = p->iMaxFrame;
}
}
- p->nProgress++;
+
+ if( p->rc==SQLITE_OK ){
+ p->eStage = OTA_STAGE_DONE;
+ p->rc = SQLITE_DONE;
+ }
+ }else{
+ OtaFrame *pFrame = &p->aFrame[p->nStep];
+ otaCheckpointFrame(p, pFrame);
+ p->nStep++;
}
+ p->nProgress++;
break;
}
}
}
-static void otaSaveTransactionState(sqlite3ota *p){
+static void otaSaveState(sqlite3ota *p){
sqlite3_stmt *pInsert;
int rc;
"(%d, %Q), "
"(%d, %d), "
"(%d, %lld), "
- "(%d, ?), "
+ "(%d, %lld), "
"(%d, %lld) ",
OTA_STATE_STAGE, p->eStage,
OTA_STATE_TBL, p->objiter.zTbl,
OTA_STATE_IDX, p->objiter.zIdx,
OTA_STATE_ROW, p->nStep,
OTA_STATE_PROGRESS, p->nProgress,
- OTA_STATE_CKPT,
- OTA_STATE_COOKIE, (sqlite3_int64)p->pTargetFd->iCookie
+ OTA_STATE_CKPT, p->iWalCksum,
+ OTA_STATE_COOKIE, (i64)p->pTargetFd->iCookie
)
);
assert( pInsert==0 || rc==SQLITE_OK );
- if( rc==SQLITE_OK ){
- if( p->pCkpt ){
- unsigned char *pCkptState = 0;
- int nCkptState = 0;
- rc = sqlite3_ckpt_close(p->pCkpt, &pCkptState, &nCkptState);
- p->pCkpt = 0;
- sqlite3_bind_blob(pInsert, 1, pCkptState, nCkptState, SQLITE_TRANSIENT);
- sqlite3_free(pCkptState);
- }
- }
+
if( rc==SQLITE_OK ){
sqlite3_step(pInsert);
rc = sqlite3_finalize(pInsert);
if( p ){
sqlite3_free(p->zTbl);
sqlite3_free(p->zIdx);
- sqlite3_free(p->pCkptState);
sqlite3_free(p);
}
}
break;
case OTA_STATE_CKPT:
- pRet->nCkptState = sqlite3_column_bytes(pStmt, 1);
- pRet->pCkptState = (unsigned char*)otaStrndup(
- (char*)sqlite3_column_blob(pStmt, 1), pRet->nCkptState, &rc
- );
+ pRet->iWalCksum = sqlite3_column_int64(pStmt, 1);
break;
case OTA_STATE_COOKIE:
** committed in rollback mode) currently stored on page 1 of the
** database file. */
if( pRet->eStage==OTA_STAGE_OAL
- && p->pTargetFd->iCookie!=(unsigned int)sqlite3_column_int64(pStmt, 1)
+ && p->pTargetFd->iCookie!=(u32)sqlite3_column_int64(pStmt, 1)
){
rc = SQLITE_BUSY;
p->zErrmsg = sqlite3_mprintf("database modified during ota update");
if( p ){
OtaState *pState = 0;
- /* Create the custom VFS */
+ /* Create the custom VFS. */
memset(p, 0, sizeof(sqlite3ota));
otaCreateVfs(p, 0);
p->rc = sqlite3_exec(p->db, OTA_CREATE_STATE, 0, 0, &p->zErrmsg);
}
+ /* Check that this is not a wal mode database. If it is, it cannot be
+ ** updated. There is also a check for a live *-wal file in otaVfsAccess()
+ ** function, on the off chance that the target is a wal database for
+ ** which the first page of the db file has been overwritten by garbage
+ ** during an earlier failed checkpoint. */
if( p->rc==SQLITE_OK && p->pTargetFd->iWriteVer>1 ){
p->rc = SQLITE_ERROR;
p->zErrmsg = sqlite3_mprintf("cannot update wal mode database");
if( p->rc==SQLITE_OK ){
if( pState->eStage==0 ){
otaDeleteOalFile(p);
- p->eStage = 1;
+ p->eStage = OTA_STAGE_OAL;
}else{
p->eStage = pState->eStage;
}
if( p->rc==SQLITE_OK ){
if( p->eStage==OTA_STAGE_OAL ){
- ota_vfs *pOtaVfs = p->pTargetFd->pOtaVfs;
-
- sqlite3_mutex_enter(pOtaVfs->mutex);
- assert( pOtaVfs->zOtaWal==0 );
- pOtaVfs->zOtaWal = p->pTargetFd->zWal;
- p->rc = sqlite3_exec(p->db, "BEGIN IMMEDIATE", 0, 0, &p->zErrmsg);
- pOtaVfs->zOtaWal = 0;
- sqlite3_mutex_leave(pOtaVfs->mutex);
+
+ /* Open the transaction */
+ if( p->rc==SQLITE_OK ){
+ p->rc = sqlite3_exec(p->db, "BEGIN IMMEDIATE", 0, 0, &p->zErrmsg);
+ }
/* Point the object iterator at the first object */
if( p->rc==SQLITE_OK ){
otaLoadTransactionState(p, pState);
}
}else if( p->eStage==OTA_STAGE_CKPT ){
- p->rc = sqlite3_ckpt_open(
- p->db, pState->pCkptState, pState->nCkptState, &p->pCkpt
- );
- if( p->rc==SQLITE_MISMATCH || (p->rc==SQLITE_OK && p->pCkpt==0) ){
- p->eStage = OTA_STAGE_DONE;
- p->rc = SQLITE_DONE;
- }
+ otaSetupCheckpoint(p, pState);
+ p->nStep = pState->nRow;
}else if( p->eStage==OTA_STAGE_DONE ){
p->rc = SQLITE_DONE;
}
assert( p->rc!=SQLITE_ROW );
if( p->rc==SQLITE_OK || p->rc==SQLITE_DONE ){
assert( p->zErrmsg==0 );
- otaSaveTransactionState(p);
+ otaSaveState(p);
}
/* Close any open statement handles. */
}
/* Close the open database handle and VFS object. */
- if( p->pCkpt ) sqlite3_ckpt_close(p->pCkpt, 0, 0);
sqlite3_close(p->db);
otaDeleteVfs(p);
+ sqlite3_free(p->aBuf);
+ sqlite3_free(p->aFrame);
otaEditErrmsg(p);
rc = p->rc;
p->apShm = 0;
sqlite3_free(p->zDel);
+ if( p->openFlags & SQLITE_OPEN_MAIN_DB ){
+ ota_file **pp;
+ sqlite3_mutex_enter(p->pOtaVfs->mutex);
+ for(pp=&p->pOtaVfs->pMain; *pp!=p; pp=&((*pp)->pMainNext));
+ *pp = p->pMainNext;
+ sqlite3_mutex_leave(p->pOtaVfs->mutex);
+ }
+
/* Close the underlying file handle */
rc = p->pReal->pMethods->xClose(p->pReal);
return rc;
** Read and return an unsigned 32-bit big-endian integer from the buffer
** passed as the only argument.
*/
-static unsigned int otaGetU32(unsigned char *aBuf){
- return ((unsigned int)aBuf[0] << 24)
- + ((unsigned int)aBuf[1] << 16)
- + ((unsigned int)aBuf[2] << 8)
- + ((unsigned int)aBuf[3]);
+static u32 otaGetU32(u8 *aBuf){
+ return ((u32)aBuf[0] << 24)
+ + ((u32)aBuf[1] << 16)
+ + ((u32)aBuf[2] << 8)
+ + ((u32)aBuf[3]);
}
/*
sqlite_int64 iOfst
){
ota_file *p = (ota_file*)pFile;
- int rc = p->pReal->pMethods->xRead(p->pReal, zBuf, iAmt, iOfst);
- if( rc==SQLITE_OK && iOfst==0 && (p->openFlags & SQLITE_OPEN_MAIN_DB) ){
- /* These look like magic numbers. But they are stable, as they are part
- ** of the definition of the SQLite file format, which may not change. */
- unsigned char *pBuf = (unsigned char*)zBuf;
- p->iCookie = otaGetU32(&pBuf[24]);
- p->iWriteVer = pBuf[19];
+ int rc;
+
+ if( p->pOta
+ && p->pOta->eStage==OTA_STAGE_CAPTURE
+ && (p->openFlags & SQLITE_OPEN_WAL)
+ ){
+ rc = otaCaptureWalRead(p->pOta, iOfst, iAmt);
+ }else{
+ rc = p->pReal->pMethods->xRead(p->pReal, zBuf, iAmt, iOfst);
+ if( rc==SQLITE_OK && iOfst==0 && (p->openFlags & SQLITE_OPEN_MAIN_DB) ){
+ /* These look like magic numbers. But they are stable, as they are part
+ ** of the definition of the SQLite file format, which may not change. */
+ u8 *pBuf = (u8*)zBuf;
+ p->iCookie = otaGetU32(&pBuf[24]);
+ p->iWriteVer = pBuf[19];
+ }
}
return rc;
}
sqlite_int64 iOfst
){
ota_file *p = (ota_file*)pFile;
- int rc = p->pReal->pMethods->xWrite(p->pReal, zBuf, iAmt, iOfst);
- if( rc==SQLITE_OK && iOfst==0 && (p->openFlags & SQLITE_OPEN_MAIN_DB) ){
- /* These look like magic numbers. But they are stable, as they are part
- ** of the definition of the SQLite file format, which may not change. */
- unsigned char *pBuf = (unsigned char*)zBuf;
- p->iCookie = otaGetU32(&pBuf[24]);
- p->iWriteVer = pBuf[19];
+ int rc;
+ if( p->pOta
+ && p->pOta->eStage==OTA_STAGE_CAPTURE
+ && (p->openFlags & SQLITE_OPEN_MAIN_DB)
+ ){
+ rc = otaCaptureDbWrite(p->pOta, iOfst);
+ }else{
+ rc = p->pReal->pMethods->xWrite(p->pReal, zBuf, iAmt, iOfst);
+ if( rc==SQLITE_OK && iOfst==0 && (p->openFlags & SQLITE_OPEN_MAIN_DB) ){
+ /* These look like magic numbers. But they are stable, as they are part
+ ** of the definition of the SQLite file format, which may not change. */
+ u8 *pBuf = (u8*)zBuf;
+ p->iCookie = otaGetU32(&pBuf[24]);
+ p->iWriteVer = pBuf[19];
+ }
}
return rc;
}
*/
static int otaVfsSync(sqlite3_file *pFile, int flags){
ota_file *p = (ota_file *)pFile;
+ if( p->pOta && p->pOta->eStage==OTA_STAGE_CAPTURE ){
+ if( p->openFlags & SQLITE_OPEN_MAIN_DB ){
+ return SQLITE_INTERNAL;
+ }
+ return SQLITE_OK;
+ }
return p->pReal->pMethods->xSync(p->pReal, flags);
}
sqlite3ota *pOta = p->pOta;
int rc = SQLITE_OK;
- if( pOta && eLock==SQLITE_LOCK_EXCLUSIVE
- && (pOta->eStage==OTA_STAGE_OAL || pOta->eStage==OTA_STAGE_CKPT)
- ){
+ assert( p->openFlags & (SQLITE_OPEN_MAIN_DB|SQLITE_OPEN_TEMP_DB) );
+ if( pOta && eLock==SQLITE_LOCK_EXCLUSIVE && pOta->eStage!=OTA_STAGE_DONE ){
/* Do not allow EXCLUSIVE locks. Preventing SQLite from taking this
** prevents it from checkpointing the database from sqlite3_close(). */
rc = SQLITE_BUSY;
ota_file *p = (ota_file *)pFile;
int (*xControl)(sqlite3_file*,int,void*) = p->pReal->pMethods->xFileControl;
+ assert( p->openFlags & (SQLITE_OPEN_MAIN_DB|SQLITE_OPEN_TEMP_DB) );
if( op==SQLITE_FCNTL_OTA ){
int rc;
sqlite3ota *pOta = (sqlite3ota*)pArg;
}else if( rc==SQLITE_NOTFOUND ){
pOta->pTargetFd = p;
p->pOta = pOta;
+ if( p->pWalFd ) p->pWalFd->pOta = pOta;
rc = SQLITE_OK;
}
}
*/
static int otaVfsShmLock(sqlite3_file *pFile, int ofst, int n, int flags){
ota_file *p = (ota_file*)pFile;
+ sqlite3ota *pOta = p->pOta;
int rc = SQLITE_OK;
#ifdef SQLITE_AMALGAMATION
assert( WAL_CKPT_LOCK==1 );
#endif
- if( p->pOta && p->pOta->eStage==OTA_STAGE_OAL ){
+ assert( p->openFlags & (SQLITE_OPEN_MAIN_DB|SQLITE_OPEN_TEMP_DB) );
+ if( pOta && pOta->eStage==OTA_STAGE_OAL ){
/* Magic number 1 is the WAL_CKPT_LOCK lock. Preventing SQLite from
** taking this lock also prevents any checkpoints from occurring.
** todo: really, it's not clear why this might occur, as
** wal_autocheckpoint ought to be turned off. */
- if( ofst==1 && n==1 ) rc = SQLITE_BUSY;
+ if( ofst==WAL_LOCK_CKPT && n==1 ) rc = SQLITE_BUSY;
}else{
- assert( p->nShm==0 );
- rc = p->pReal->pMethods->xShmLock(p->pReal, ofst, n, flags);
+ int bCapture = 0;
+ if( n==1 && (flags & SQLITE_SHM_EXCLUSIVE)
+ && p->pOta && p->pOta->eStage==OTA_STAGE_CAPTURE
+ && (ofst==WAL_LOCK_WRITE || ofst==WAL_LOCK_CKPT || ofst==WAL_LOCK_READ0)
+ ){
+ bCapture = 1;
+ }
+
+ if( bCapture==0 || 0==(flags & SQLITE_SHM_UNLOCK) ){
+ rc = p->pReal->pMethods->xShmLock(p->pReal, ofst, n, flags);
+ if( bCapture && rc==SQLITE_OK ){
+ p->pOta->mLock |= (1 << ofst);
+ }
+ }
}
return rc;
/* If not in OTA_STAGE_OAL, allow this call to pass through. Or, if this
** ota is in the OTA_STAGE_OAL state, use heap memory for *-shm space
** instead of a file on disk. */
+ assert( p->openFlags & (SQLITE_OPEN_MAIN_DB|SQLITE_OPEN_TEMP_DB) );
if( p->pOta && p->pOta->eStage==OTA_STAGE_OAL ){
if( iRegion<=p->nShm ){
int nByte = (iRegion+1) * sizeof(char*);
ota_file *p = (ota_file*)pFile;
int rc = SQLITE_OK;
+ assert( p->openFlags & (SQLITE_OPEN_MAIN_DB|SQLITE_OPEN_TEMP_DB) );
if( p->pOta && p->pOta->eStage==OTA_STAGE_OAL ){
/* no-op */
}else{
return rc;
}
+static ota_file *otaFindMaindb(ota_vfs *pOtaVfs, const char *zWal){
+ ota_file *pDb;
+ sqlite3_mutex_enter(pOtaVfs->mutex);
+ for(pDb=pOtaVfs->pMain; pDb && pDb->zWal!=zWal; pDb=pDb->pMainNext);
+ sqlite3_mutex_leave(pOtaVfs->mutex);
+ return pDb;
+}
+
/*
** Open an ota file handle.
*/
z += (n + 8 + 1);
pFd->zWal = z;
}
- else if( (flags & SQLITE_OPEN_WAL) && zName==pOtaVfs->zOtaWal ){
- char *zCopy = otaStrndup(zName, -1, &rc);
- if( zCopy ){
- int nCopy = strlen(zCopy);
- zCopy[nCopy-3] = 'o';
- zOpen = (const char*)(pFd->zDel = zCopy);
+ else if( flags & SQLITE_OPEN_WAL ){
+ ota_file *pDb = otaFindMaindb(pOtaVfs, zName);
+ if( pDb ){
+ if( pDb->pOta && pDb->pOta->eStage==OTA_STAGE_OAL ){
+ char *zCopy = otaStrndup(zName, -1, &rc);
+ if( zCopy ){
+ int nCopy = strlen(zCopy);
+ zCopy[nCopy-3] = 'o';
+ zOpen = (const char*)(pFd->zDel = zCopy);
+ }
+ pFd->pOta = pDb->pOta;
+ }
+ pDb->pWalFd = pFd;
}
}
}
rc = pRealVfs->xOpen(pRealVfs, zOpen, pFd->pReal, flags, pOutFlags);
}
if( pFd->pReal->pMethods ){
+ /* The xOpen() operation has succeeded. Set the sqlite3_file.pMethods
+ ** pointer and, if the file is a main database file, link it into the
+ ** mutex protected linked list of all such files. */
pFile->pMethods = &otavfs_io_methods;
+ if( flags & SQLITE_OPEN_MAIN_DB ){
+ sqlite3_mutex_enter(pOtaVfs->mutex);
+ pFd->pMainNext = pOtaVfs->pMain;
+ pOtaVfs->pMain = pFd;
+ sqlite3_mutex_leave(pOtaVfs->mutex);
+ }
}
return rc;
rc = pRealVfs->xAccess(pRealVfs, zPath, flags, pResOut);
- if( rc==SQLITE_OK && flags==SQLITE_ACCESS_EXISTS && pOtaVfs->zOtaWal==zPath ){
- if( *pResOut ){
- rc = SQLITE_CANTOPEN;
- }else{
- *pResOut = 1;
+ /* If this call is to check if a *-wal file associated with an OTA target
+ ** database connection exists, and the OTA update is in OTA_STAGE_OAL,
+ ** the following special handling is activated:
+ **
+ ** a) if the *-wal file does exist, return SQLITE_CANTOPEN. This
+ ** ensures that the OTA extension never tries to update a database
+ ** in wal mode, even if the first page of the database file has
+ ** been damaged.
+ **
+ ** b) if the *-wal file does not exist, claim that it does anyway,
+ ** causing SQLite to call xOpen() to open it. This call will also
+ ** be intercepted (see the otaVfsOpen() function) and the *-oal
+ ** file opened instead.
+ */
+ if( rc==SQLITE_OK && flags==SQLITE_ACCESS_EXISTS ){
+ ota_file *pDb = otaFindMaindb(pOtaVfs, zPath);
+ if( pDb && pDb->pOta && pDb->pOta->eStage==OTA_STAGE_OAL ){
+ if( *pResOut ){
+ rc = SQLITE_CANTOPEN;
+ }else{
+ *pResOut = 1;
+ }
}
}
-C Merge\sthe\sota-update-no-pager_ota_mode\sbranch\sinto\sthis\sone.
-D 2015-02-11T17:05:17.871
+C Change\sthe\sway\sthe\s"incremental\scheckpoint"\sfunction\sof\sOTA\sworks\sin\sorder\sto\sreduce\sthe\seffect\son\sthe\sSQLite\score\scode.
+D 2015-02-14T18:58:22.415
F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f
F Makefile.in 6b9e7677829aa94b9f30949656e27312aefb9a46
F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23
F ext/misc/wholenumber.c 784b12543d60702ebdd47da936e278aa03076212
F ext/ota/README.txt 2ce4ffbb0aaa6731b041c27a7359f9a5f1c69152
F ext/ota/ota.c c11a85af71dccc45976622fe7a51169a481caa91
-F ext/ota/ota1.test d50ba4ded2edeba99740bc7dd0b7284c1894127c
+F ext/ota/ota1.test dee5b852353642a243e0bf414d332b1bccd5324f
F ext/ota/ota10.test 85e0f6e7964db5007590c1b299e75211ed4240d4
F ext/ota/ota3.test a77efbce7723332eb688d2b28bf18204fc9614d7
F ext/ota/ota5.test ad0799daf8923ddebffe75ae8c5504ca90b7fadb
-F ext/ota/ota6.test 1fbba5fd46e3e0bfa5ae1d0caf9da27d15cb7cdf
+F ext/ota/ota6.test 40996b7716dee72a6c5d28c3bee436717a438d3d
F ext/ota/ota7.test 1fe2c5761705374530e29f70c39693076028221a
F ext/ota/ota8.test cd70e63a0c29c45c0906692827deafa34638feda
F ext/ota/ota9.test d3eee95dd836824d07a22e5efcdb7bf6e869358b
F ext/ota/otaA.test 95566a8d193113867b960eadf85b310937f2fe03
F ext/ota/otafault.test 508ba87c83d632670ac0f94371a465d4bb4d49dd
-F ext/ota/sqlite3ota.c 466546d41d9b09136216349db98647d0afd77816
+F ext/ota/sqlite3ota.c 0cf2a1b5ac7009050159a39d938f1334ce7072b8
F ext/ota/sqlite3ota.h 1cc7201086fe65a36957740381485a24738c4077
F ext/rtree/README 6315c0d73ebf0ec40dedb5aa0e942bc8b54e3761
F ext/rtree/rtree.c 14e6239434d4e3f65d3e90320713f26aa24e167f
F src/legacy.c ba1863ea58c4c840335a84ec276fc2b25e22bc4e
F src/lempar.c 7274c97d24bb46631e504332ccd3bd1b37841770
F src/loadext.c 86bd4e2fccd520b748cba52492ab60c4a770f660
-F src/main.c c4cb192ebf0bcc975648ae05ac40bc1f40018c52
+F src/main.c 17e3a37374f3c13e27311773c30720b61584f5b9
F src/malloc.c 740db54387204c9a2eb67c6d98e68b08e9ef4eab
F src/mem0.c 6a55ebe57c46ca1a7d98da93aaa07f99f1059645
F src/mem1.c abe6ee469b6c5a35c7f22bfeb9c9bac664a1c987
F src/os_unix.c aefeaf915aaef9f81aa2645e0d5d06fa1bd83beb
F src/os_win.c 8223e7db5b7c4a81d8b161098ac3959400434cdb
F src/os_win.h eb7a47aa17b26b77eb97e4823f20a00b8bda12ca
-F src/pager.c 9d29fb3dfd99d16896d839a511b467784d72f4da
-F src/pager.h 20954a3fa1bbf05d39063d94e789ad9efd15e5d1
+F src/pager.c 4120a49ecd37697e28f5ed807f470b9c0b88410c
+F src/pager.h c3476e7c89cdf1c6914e50a11f3714e30b4e0a77
F src/parse.y 0f8e7d60f0ab3cb53d270adef69259ac307d83a8
F src/pcache.c d210cf90d04365a74f85d21374dded65af67b0cb
F src/pcache.h b44658c9c932d203510279439d891a2a83e12ba8
F src/rowset.c eccf6af6d620aaa4579bd3b72c1b6395d9e9fa1e
F src/select.c e46cef4c224549b439384c88fc7f57ba064dad54
F src/shell.c 82c25508dac802b32198af6f5256ca1597c6a1af
-F src/sqlite.h.in c49acd2daa6e54110ab0cc607eb73ff32720a269
+F src/sqlite.h.in ad5cdebfdd8bcef2534f93de6b8466b4c455a032
F src/sqlite3.rc 992c9f5fb8285ae285d6be28240a7e8d3a7f2bad
F src/sqlite3ext.h 17d487c3c91b0b8c584a32fbeb393f6f795eea7d
F src/sqliteInt.h 57a405ae6d2ed10fff52de376d18f21e04d96609
F src/vdbesort.c 6d64c5448b64851b99931ede980addc3af70d5e2
F src/vdbetrace.c 7e4222955e07dd707a2f360c0eb73452be1cb010
F src/vtab.c c08ec66f45919eaa726bf88aa53eb08379d607f9
-F src/wal.c 7a8a4e7a40d693d44dbfc4d1f2bcb7e2b620f530
-F src/wal.h 0d3ba0c3f1b4c25796cb213568a84b9f9063f465
+F src/wal.c 39303f2c9db02a4e422cd8eb2c8760420c6a51fe
+F src/wal.h df01efe09c5cb8c8e391ff1715cca294f89668a4
F src/walker.c c253b95b4ee44b21c406e2a1052636c31ea27804
F src/where.c d46de821bc604a4fd36fa3928c086950e91aafb1
F src/whereInt.h d3633e9b592103241b74b0ec76185f3e5b8b62e0
F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4
F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32
F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f
-P c3931db560ab4a2601c7f7318fb02c8d5e6862b1 0b63e8dcbaec5043e353734e684c2a46552a3409
-R f20bdc7a7c21af7b4cb60f763acf418a
+P 71887cd9b38def398d48eaf0ec34eeac3c7c5177
+R 9f9fcb7ec94597bb519ead7cc6d77d1b
U dan
-Z 0a34d9bc8960589f0668f7402b31b0ce
+Z b1cfeba62f30a42f6b3ffd27f7078fc0
-71887cd9b38def398d48eaf0ec34eeac3c7c5177
\ No newline at end of file
+b64a11a754dc56f3406d3b703531ebe9e4af4908
\ No newline at end of file
#endif
}
-#ifdef SQLITE_ENABLE_OTA
-/*
-** Open an incremental checkpoint handle.
-*/
-int sqlite3_ckpt_open(
- sqlite3 *db,
- unsigned char *a, int n,
- sqlite3_ckpt **ppCkpt
-){
- Pager *pPager = 0;
- int rc;
-
- *ppCkpt = 0;
- sqlite3_mutex_enter(db->mutex);
-
- /* Find the Pager object. */
- rc = sqlite3_file_control(db,"main",SQLITE_FCNTL_ZIPVFS_PAGER,(void*)&pPager);
- if( rc!=SQLITE_OK ){
- pPager = sqlite3BtreePager(db->aDb[0].pBt);
- }
-
- rc = sqlite3PagerWalCheckpointStart(db, pPager, a, n, ppCkpt);
- sqlite3_mutex_leave(db->mutex);
- return rc;
-}
-#endif /* SQLITE_ENABLE_OTA */
-
/*
** Checkpoint database zDb. If zDb is NULL, or if the buffer zDb points
** mode. Otherwise, the following function call is a no-op.
*/
rc = pagerOpenWalIfPresent(pPager);
-
#ifndef SQLITE_OMIT_WAL
assert( pPager->pWal==0 || rc==SQLITE_OK );
#endif
}
#endif
-#ifdef SQLITE_ENABLE_OTA
-
-/*
-** Open an incremental checkpoint handle.
-*/
-int sqlite3PagerWalCheckpointStart(
- sqlite3 *db,
- Pager *pPager,
- u8 *a, int n,
- sqlite3_ckpt **ppCkpt
-){
- if( pPager->pWal==0 ){
- *ppCkpt = 0;
- return SQLITE_OK;
- }else{
- return sqlite3WalCheckpointStart(db, pPager->pWal, a, n,
- pPager->xBusyHandler, pPager->pBusyHandlerArg,
- pPager->ckptSyncFlags, ppCkpt
- );
- }
-}
-#endif /* !SQLITE_ENABLE_OTA */
#endif /* SQLITE_OMIT_DISKIO */
# define enable_simulated_io_errors()
#endif
-int sqlite3PagerWalCheckpointStart(sqlite3*, Pager*, u8*, int, sqlite3_ckpt**);
-
#endif /* _PAGER_H_ */
*/
SQLITE_EXPERIMENTAL void sqlite3_stmt_scanstatus_reset(sqlite3_stmt*);
-/*
-** Incremental checkpoint API.
-**
-** An incremental checkpoint handle is opened using the sqlite3_ckpt_open()
-** API. To begin a new checkpoint, the second and third arguments should both
-** be passed zero. To resume an earlier checkpoint, the second and third
-** arguments should specify a buffer returned by an earlier call to
-** sqlite3_ckpt_close(). When resuming a checkpoint, if the database or WAL
-** file has been modified since the checkpoint was suspended, the
-** sqlite3_ckpt_open() call fails with SQLITE_MISMATCH.
-**
-** Each time sqlite3_ckpt_step() is called on an open checkpoint handle, a
-** single page is copied from the WAL file to the database. If no error
-** occurs, but the checkpoint is not finished, SQLITE_OK is returned. If the
-** checkpoint has been finished (and so sqlite3_ckpt_step() should not be
-** called again), SQLITE_DONE is returned. Otherwise, if an error occurs,
-** some other SQLite error code is returned.
-**
-** Calling sqlite3_ckpt_close() closes an open checkpoint handle. If the
-** checkpoint has finished and no error has occurred, SQLITE_OK is returned
-** and the two output parameters zeroed. Or, if an error has occurred, an
-** error code is returned and the two output parameters are zeroed. Finally,
-** if the checkpoint is not finished but no error has occurred, SQLITE_OK is
-** returned and the first output variable set to point to a buffer allocated
-** using sqlite3_malloc() containing the serialized state of the checkpoint.
-** The contents of this buffer may be passed to a later call to
-** sqlite3_ckpt_open() to restart the checkpoint. The second output variable
-** is set to the size of the buffer in bytes.
-**
-** These APIs are only available if SQLITE_ENABLE_OTA is defined at compile
-** time. They are intended for use by the OTA extension only. As such, they
-** are subject to change or removal at any point.
-*/
-typedef struct sqlite3_ckpt sqlite3_ckpt;
-int sqlite3_ckpt_open(sqlite3*, unsigned char*, int n, sqlite3_ckpt **ppCkpt);
-int sqlite3_ckpt_step(sqlite3_ckpt*);
-int sqlite3_ckpt_close(sqlite3_ckpt*, unsigned char **pa, int *pn);
/*
** Undo the hack that converts floating point types to integer for
} aSegment[1]; /* One for every 32KB page in the wal-index */
};
-/*
-** An object of the following type is used to store state information for
-** an ongoing checkpoint operation. For normal checkpoints, the instance
-** is allocated on the stack by the walCheckpoint() function. For the special
-** incremental checkpoints performed by OTA clients, it is allocated in
-** heap memory by sqlite3WalCheckpointStart().
-**
-** See the implementations of walCheckpointStart(), walCheckpointStep() and
-** walCheckpointFinalize() for further details.
-*/
-typedef struct WalCkpt WalCkpt;
-struct WalCkpt {
- sqlite3 *db; /* Database pointer (incremental only) */
- int szPage; /* Database page-size */
- int sync_flags; /* Flags for OsSync() (or 0) */
- u32 mxSafeFrame; /* Max frame that can be backfilled */
- u32 mxPage; /* Max database page to write */
- volatile WalCkptInfo *pInfo; /* The checkpoint status information */
- WalIterator *pIter; /* Wal iterator context */
- Wal *pWal; /* Pointer to owner object */
- u8 *aBuf; /* Temporary page-sized buffer to use */
- int rc; /* Error code. SQLITE_DONE -> finished */
- int nStep; /* Number of times pIter has been stepped */
-};
-
/*
** Define the parameters of the hash tables in the wal-index file. There
** is a hash-table following every HASHTABLE_NPAGE page numbers in the
if( rc!=SQLITE_OK ){
walIteratorFree(p);
- p = 0;
}
*pp = p;
return rc;
return (pWal->hdr.szPage&0xfe00) + ((pWal->hdr.szPage&0x0001)<<16);
}
-/*
-** Initialize the contents of the WalCkpt object indicated by the final
-** argument and begin a checkpoint operation. The CKPT lock must already
-** be held when this function is called.
-**
-** Return SQLITE_OK if successful or an error code otherwise.
-*/
-static int walCheckpointStart(
- Wal *pWal, /* Wal connection */
- u8 *aBuf, /* Page-sized temporary buffer */
- int nBuf, /* Size of aBuf[] in bytes */
- int (*xBusy)(void*), /* Function to call when busy (or NULL) */
- void *pBusyArg, /* Context argument for xBusyHandler */
- int sync_flags, /* Flags for OsSync() (or 0) */
- WalCkpt *p /* Allocated object to populate */
-){
- int rc; /* Return code */
- int i; /* Iterator variable */
-
- memset(p, 0, sizeof(WalCkpt));
- if( pWal->hdr.mxFrame && walPagesize(pWal)!=nBuf ){
- return SQLITE_CORRUPT_BKPT;
- }
-
- p->szPage = walPagesize(pWal);
- p->pWal = pWal;
- p->aBuf = aBuf;
- p->sync_flags = sync_flags;
- testcase( p->szPage<=32768 );
- testcase( p->szPage>=65536 );
- p->pInfo = walCkptInfo(pWal);
- if( p->pInfo->nBackfill>=pWal->hdr.mxFrame ) return SQLITE_OK;
-
- /* Allocate the iterator */
- rc = walIteratorInit(pWal, &p->pIter);
- if( rc!=SQLITE_OK ) return rc;
- assert( p->pIter );
-
- /* Compute in mxSafeFrame the index of the last frame of the WAL that is
- ** safe to write into the database. Frames beyond mxSafeFrame might
- ** overwrite database pages that are in use by active readers and thus
- ** cannot be backfilled from the WAL.
- */
- p->mxSafeFrame = pWal->hdr.mxFrame;
- p->mxPage = pWal->hdr.nPage;
- for(i=1; i<WAL_NREADER; i++){
- u32 y = p->pInfo->aReadMark[i];
- if( p->mxSafeFrame>y ){
- assert( y<=pWal->hdr.mxFrame );
- rc = walBusyLock(pWal, xBusy, pBusyArg, WAL_READ_LOCK(i), 1);
- if( rc==SQLITE_OK ){
- p->pInfo->aReadMark[i] = (i==1 ? p->mxSafeFrame : READMARK_NOT_USED);
- walUnlockExclusive(pWal, WAL_READ_LOCK(i), 1);
- }else if( rc==SQLITE_BUSY ){
- p->mxSafeFrame = y;
- xBusy = 0;
- }else{
- walIteratorFree(p->pIter);
- p->pIter = 0;
- return rc;
- }
- }
- }
-
- if( p->pInfo->nBackfill>=p->mxSafeFrame
- || (rc = walBusyLock(pWal, xBusy, pBusyArg, WAL_READ_LOCK(0), 1))!=SQLITE_OK
- ){
- walIteratorFree(p->pIter);
- p->pIter = 0;
- }
- if( rc==SQLITE_BUSY ) rc = SQLITE_OK;
-
- if( rc==SQLITE_OK && p->pIter ){
- /* Sync the WAL to disk */
- if( sync_flags ){
- rc = sqlite3OsSync(pWal->pWalFd, sync_flags);
- }
-
- /* If the database may grow as a result of this checkpoint, hint
- ** about the eventual size of the db file to the VFS layer. */
- if( rc==SQLITE_OK ){
- i64 nSize; /* Current size of database file */
- i64 nReq = ((i64)p->mxPage * p->szPage);
- rc = sqlite3OsFileSize(pWal->pDbFd, &nSize);
- if( rc==SQLITE_OK && nSize<nReq ){
- sqlite3OsFileControlHint(pWal->pDbFd, SQLITE_FCNTL_SIZE_HINT, &nReq);
- }
- }
- }
-
- return rc;
-}
-
-/*
-** Attempt to copy the next frame from the wal file to the database file. If
-** there are no more frames to copy to the database file return SQLITE_DONE.
-** If the frame is successfully copied, return SQLITE_OK. Or, if an error
-** occurs, return an SQLite error code.
-*/
-static int walCheckpointStep(WalCkpt *p){
- u32 iDbpage = 0; /* Next database page to write */
- u32 iFrame = 0; /* Wal frame containing data for iDbpage */
- int rc = SQLITE_DONE;
-
- assert( p->rc==SQLITE_OK );
- while( p->pIter && 0==walIteratorNext(p->pIter, &iDbpage, &iFrame) ){
- i64 iOffset;
- assert( walFramePgno(p->pWal, iFrame)==iDbpage );
- p->nStep++;
- if( iFrame<=p->pInfo->nBackfill
- || iFrame>p->mxSafeFrame
- || iDbpage>p->mxPage
- ){
- continue;
- }
-
- iOffset = walFrameOffset(iFrame, p->szPage) + WAL_FRAME_HDRSIZE;
- /* testcase( IS_BIG_INT(iOffset) ); // requires a 4GiB WAL file */
- rc = sqlite3OsRead(p->pWal->pWalFd, p->aBuf, p->szPage, iOffset);
- if( rc!=SQLITE_OK ) break;
- iOffset = (iDbpage-1)*(i64)p->szPage;
- testcase( IS_BIG_INT(iOffset) );
- rc = sqlite3OsWrite(p->pWal->pDbFd, p->aBuf, p->szPage, iOffset);
- break;
- }
-
- p->rc = rc;
- return rc;
-}
-
-/*
-** The current round of checkpointing work using the object indicated by
-** the only argument is now finished. If no error occcurred, this function
-** saves the results to shared memory (i.e. updates the WalCkptInfo.nBackfill
-** variable), and truncates and syncs the database file as required.
-**
-** All dynamic resources currently held by the WalCkpt object are released.
-** It is the responsibility of the caller to delete the WalCkpt itself if
-** required.
-*/
-static int walCheckpointFinalize(WalCkpt *p){
- if( p->pIter ){
- int rc = p->rc;
- Wal *pWal = p->pWal;
-
- if( rc==SQLITE_DONE ){
- /* If work was completed */
- rc = SQLITE_OK;
- if( p->mxSafeFrame==walIndexHdr(pWal)->mxFrame ){
- i64 szDb = pWal->hdr.nPage*(i64)p->szPage;
- testcase( IS_BIG_INT(szDb) );
- rc = sqlite3OsTruncate(pWal->pDbFd, szDb);
- if( rc==SQLITE_OK && p->sync_flags ){
- rc = sqlite3OsSync(pWal->pDbFd, p->sync_flags);
- }
- }
- if( rc==SQLITE_OK ){
- p->pInfo->nBackfill = p->mxSafeFrame;
- }
- p->rc = rc;
- }else{
-#ifdef SQLITE_ENABLE_OTA
- if( rc==SQLITE_OK && p->sync_flags ){
- /* If work was not completed, but no error has occured. */
- p->rc = sqlite3OsSync(pWal->pDbFd, p->sync_flags);
- }
-#else
- assert( rc!=SQLITE_OK );
-#endif
- }
-
- /* Release the reader lock held while backfilling */
- walUnlockExclusive(pWal, WAL_READ_LOCK(0), 1);
- walIteratorFree(p->pIter);
- p->pIter = 0;
- }else if( p->rc==SQLITE_DONE ){
- p->rc = SQLITE_OK;
- }
-
- return p->rc;
-}
-
/*
** The following is guaranteed when this function is called:
**
int (*xBusy)(void*), /* Function to call when busy */
void *pBusyArg, /* Context argument for xBusyHandler */
int sync_flags, /* Flags for OsSync() (or 0) */
- u8 *zBuf, /* Temporary buffer to use */
- int nBuf /* Size of zBuf in bytes */
+ u8 *zBuf /* Temporary buffer to use */
){
- int rc; /* Return code */
- WalCkpt sC;
+ int rc = SQLITE_OK; /* Return code */
+ int szPage; /* Database page-size */
+ WalIterator *pIter = 0; /* Wal iterator context */
+ u32 iDbpage = 0; /* Next database page to write */
+ u32 iFrame = 0; /* Wal frame containing data for iDbpage */
+ u32 mxSafeFrame; /* Max frame that can be backfilled */
+ u32 mxPage; /* Max database page to write */
+ int i; /* Loop counter */
+ volatile WalCkptInfo *pInfo; /* The checkpoint status information */
- /* EVIDENCE-OF: R-62920-47450 The busy-handler callback is never invoked
- ** in the SQLITE_CHECKPOINT_PASSIVE mode. */
- assert( eMode!=SQLITE_CHECKPOINT_PASSIVE || xBusy==0 );
+ szPage = walPagesize(pWal);
+ testcase( szPage<=32768 );
+ testcase( szPage>=65536 );
+ pInfo = walCkptInfo(pWal);
+ if( pInfo->nBackfill<pWal->hdr.mxFrame ){
+
+ /* Allocate the iterator */
+ rc = walIteratorInit(pWal, &pIter);
+ if( rc!=SQLITE_OK ){
+ return rc;
+ }
+ assert( pIter );
+
+ /* EVIDENCE-OF: R-62920-47450 The busy-handler callback is never invoked
+ ** in the SQLITE_CHECKPOINT_PASSIVE mode. */
+ assert( eMode!=SQLITE_CHECKPOINT_PASSIVE || xBusy==0 );
+
+ /* Compute in mxSafeFrame the index of the last frame of the WAL that is
+ ** safe to write into the database. Frames beyond mxSafeFrame might
+ ** overwrite database pages that are in use by active readers and thus
+ ** cannot be backfilled from the WAL.
+ */
+ mxSafeFrame = pWal->hdr.mxFrame;
+ mxPage = pWal->hdr.nPage;
+ for(i=1; i<WAL_NREADER; i++){
+ u32 y = pInfo->aReadMark[i];
+ if( mxSafeFrame>y ){
+ assert( y<=pWal->hdr.mxFrame );
+ rc = walBusyLock(pWal, xBusy, pBusyArg, WAL_READ_LOCK(i), 1);
+ if( rc==SQLITE_OK ){
+ pInfo->aReadMark[i] = (i==1 ? mxSafeFrame : READMARK_NOT_USED);
+ walUnlockExclusive(pWal, WAL_READ_LOCK(i), 1);
+ }else if( rc==SQLITE_BUSY ){
+ mxSafeFrame = y;
+ xBusy = 0;
+ }else{
+ goto walcheckpoint_out;
+ }
+ }
+ }
+
+ if( pInfo->nBackfill<mxSafeFrame
+ && (rc = walBusyLock(pWal, xBusy, pBusyArg, WAL_READ_LOCK(0),1))==SQLITE_OK
+ ){
+ i64 nSize; /* Current size of database file */
+ u32 nBackfill = pInfo->nBackfill;
+
+ /* Sync the WAL to disk */
+ if( sync_flags ){
+ rc = sqlite3OsSync(pWal->pWalFd, sync_flags);
+ }
+
+ /* If the database may grow as a result of this checkpoint, hint
+ ** about the eventual size of the db file to the VFS layer.
+ */
+ if( rc==SQLITE_OK ){
+ i64 nReq = ((i64)mxPage * szPage);
+ rc = sqlite3OsFileSize(pWal->pDbFd, &nSize);
+ if( rc==SQLITE_OK && nSize<nReq ){
+ sqlite3OsFileControlHint(pWal->pDbFd, SQLITE_FCNTL_SIZE_HINT, &nReq);
+ }
+ }
- rc = walCheckpointStart(pWal, zBuf, nBuf, xBusy, pBusyArg, sync_flags, &sC);
- if( rc!=SQLITE_OK ) goto walcheckpoint_out;
- /* Step the checkpoint object until it reports something other than
- ** SQLITE_OK. */
- while( SQLITE_OK==(rc = walCheckpointStep(&sC)) );
- rc = walCheckpointFinalize(&sC);
+ /* Iterate through the contents of the WAL, copying data to the db file */
+ while( rc==SQLITE_OK && 0==walIteratorNext(pIter, &iDbpage, &iFrame) ){
+ i64 iOffset;
+ assert( walFramePgno(pWal, iFrame)==iDbpage );
+ if( iFrame<=nBackfill || iFrame>mxSafeFrame || iDbpage>mxPage ){
+ continue;
+ }
+ iOffset = walFrameOffset(iFrame, szPage) + WAL_FRAME_HDRSIZE;
+ /* testcase( IS_BIG_INT(iOffset) ); // requires a 4GiB WAL file */
+ rc = sqlite3OsRead(pWal->pWalFd, zBuf, szPage, iOffset);
+ if( rc!=SQLITE_OK ) break;
+ iOffset = (iDbpage-1)*(i64)szPage;
+ testcase( IS_BIG_INT(iOffset) );
+ rc = sqlite3OsWrite(pWal->pDbFd, zBuf, szPage, iOffset);
+ if( rc!=SQLITE_OK ) break;
+ }
+
+ /* If work was actually accomplished... */
+ if( rc==SQLITE_OK ){
+ if( mxSafeFrame==walIndexHdr(pWal)->mxFrame ){
+ i64 szDb = pWal->hdr.nPage*(i64)szPage;
+ testcase( IS_BIG_INT(szDb) );
+ rc = sqlite3OsTruncate(pWal->pDbFd, szDb);
+ if( rc==SQLITE_OK && sync_flags ){
+ rc = sqlite3OsSync(pWal->pDbFd, sync_flags);
+ }
+ }
+ if( rc==SQLITE_OK ){
+ pInfo->nBackfill = mxSafeFrame;
+ }
+ }
+
+ /* Release the reader lock held while backfilling */
+ walUnlockExclusive(pWal, WAL_READ_LOCK(0), 1);
+ }
+
+ if( rc==SQLITE_BUSY ){
+ /* Reset the return code so as not to report a checkpoint failure
+ ** just because there are active readers. */
+ rc = SQLITE_OK;
+ }
+ }
/* If this is an SQLITE_CHECKPOINT_RESTART or TRUNCATE operation, and the
** entire wal file has been copied into the database file, then block
*/
if( rc==SQLITE_OK && eMode!=SQLITE_CHECKPOINT_PASSIVE ){
assert( pWal->writeLock );
- if(
sC.pInfo->nBackfill<pWal->hdr.mxFrame ){
+ if( pInfo->nBackfill<pWal->hdr.mxFrame ){
rc = SQLITE_BUSY;
}else if( eMode>=SQLITE_CHECKPOINT_RESTART ){
u32 salt1;
sqlite3_randomness(4, &salt1);
- assert( sC.pInfo->nBackfill==pWal->hdr.mxFrame );
+ assert( pInfo->nBackfill==pWal->hdr.mxFrame );
rc = walBusyLock(pWal, xBusy, pBusyArg, WAL_READ_LOCK(1), WAL_NREADER-1);
if( rc==SQLITE_OK ){
if( eMode==SQLITE_CHECKPOINT_TRUNCATE ){
}
walcheckpoint_out:
- walIteratorFree(sC.pIter);
+ walIteratorFree(pIter);
return rc;
}
/*
** Close a connection to a log file.
-**
-** If parameter zBuf is not NULL, also attempt to obtain an exclusive
-** lock and run a checkpoint.
*/
int sqlite3WalClose(
Wal *pWal, /* Wal to close */
**
** The EXCLUSIVE lock is not released before returning.
*/
-#ifdef SQLITE_ENABLE_OTA
- if( zBuf ) /* In non-OTA builds, zBuf is always non-NULL */
-#endif
- {
- rc = sqlite3OsLock(pWal->pDbFd, SQLITE_LOCK_EXCLUSIVE);
+ rc = sqlite3OsLock(pWal->pDbFd, SQLITE_LOCK_EXCLUSIVE);
+ if( rc==SQLITE_OK ){
+ if( pWal->exclusiveMode==WAL_NORMAL_MODE ){
+ pWal->exclusiveMode = WAL_EXCLUSIVE_MODE;
+ }
+ rc = sqlite3WalCheckpoint(
+ pWal, SQLITE_CHECKPOINT_PASSIVE, 0, 0, sync_flags, nBuf, zBuf, 0, 0
+ );
if( rc==SQLITE_OK ){
- if( pWal->exclusiveMode==WAL_NORMAL_MODE ){
- pWal->exclusiveMode = WAL_EXCLUSIVE_MODE;
- }
- rc = sqlite3WalCheckpoint(
- pWal, SQLITE_CHECKPOINT_PASSIVE, 0, 0, sync_flags, nBuf, zBuf, 0, 0
+ int bPersist = -1;
+ sqlite3OsFileControlHint(
+ pWal->pDbFd, SQLITE_FCNTL_PERSIST_WAL, &bPersist
);
- if( rc==SQLITE_OK ){
- int bPersist = -1;
- sqlite3OsFileControlHint(
- pWal->pDbFd, SQLITE_FCNTL_PERSIST_WAL, &bPersist
- );
- if( bPersist!=1 ){
- /* Try to delete the WAL file if the checkpoint completed and
- ** fsyned (rc==SQLITE_OK) and if we are not in persistent-wal
- ** mode (!bPersist) */
- isDelete = 1;
- }else if( pWal->mxWalSize>=0 ){
- /* Try to truncate the WAL file to zero bytes if the checkpoint
- ** completed and fsynced (rc==SQLITE_OK) and we are in persistent
- ** WAL mode (bPersist) and if the PRAGMA journal_size_limit is a
- ** non-negative value (pWal->mxWalSize>=0). Note that we truncate
- ** to zero bytes as truncating to the journal_size_limit might
- ** leave a corrupt WAL file on disk. */
- walLimitSize(pWal, 0);
- }
+ if( bPersist!=1 ){
+ /* Try to delete the WAL file if the checkpoint completed and
+ ** fsyned (rc==SQLITE_OK) and if we are not in persistent-wal
+ ** mode (!bPersist) */
+ isDelete = 1;
+ }else if( pWal->mxWalSize>=0 ){
+ /* Try to truncate the WAL file to zero bytes if the checkpoint
+ ** completed and fsynced (rc==SQLITE_OK) and we are in persistent
+ ** WAL mode (bPersist) and if the PRAGMA journal_size_limit is a
+ ** non-negative value (pWal->mxWalSize>=0). Note that we truncate
+ ** to zero bytes as truncating to the journal_size_limit might
+ ** leave a corrupt WAL file on disk. */
+ walLimitSize(pWal, 0);
}
}
}
/* Copy data from the log to the database file. */
if( rc==SQLITE_OK ){
- rc = walCheckpoint(pWal, eMode2, xBusy2, pBusyArg, sync_flags, zBuf, nBuf);
+ if( pWal->hdr.mxFrame && walPagesize(pWal)!=nBuf ){
+ rc = SQLITE_CORRUPT_BKPT;
+ }else{
+ rc = walCheckpoint(pWal, eMode2, xBusy2, pBusyArg, sync_flags, zBuf);
+ }
/* If no error occurred, set the output variables. */
if( rc==SQLITE_OK || rc==SQLITE_BUSY ){
return (rc==SQLITE_OK && eMode!=eMode2 ? SQLITE_BUSY : rc);
}
-#ifdef SQLITE_ENABLE_OTA
-
-/*
-** Step the checkpoint object passed as the first argument.
-*/
-int sqlite3_ckpt_step(sqlite3_ckpt *pCkpt){
- int rc;
- WalCkpt *p = (WalCkpt*)pCkpt;
- sqlite3_mutex_enter(p->db->mutex);
- rc = walCheckpointStep(p);
- sqlite3_mutex_leave(p->db->mutex);
- return rc;
-}
-
-/*
-** Close the checkpoint object passed as the first argument. If the checkpoint
-** was completed, zero the two output variables. Otherwise, set *paState to
-** point to a buffer containing data that may be passed to a subsequent
-** call to ckpt_open() to resume the checkpoint. In this case *pnState is
-** set to the size of the buffer in bytes. The buffer should be eventually
-** freed by the caller using sqlite3_free().
-*/
-int sqlite3_ckpt_close(sqlite3_ckpt *pCkpt, u8 **paState, int *pnState){
- int rc;
- WalCkpt *p = (WalCkpt*)pCkpt;
- sqlite3 *db = p->db;
- Wal *pWal = p->pWal;
- sqlite3_mutex_enter(db->mutex);
- if( paState ){
- *paState = 0;
- *pnState = 0;
- if( p->rc==SQLITE_OK ){
- u8 *aState = sqlite3_malloc(sizeof(u32) * 3);
- if( aState==0 ){
- p->rc = SQLITE_NOMEM;
- }else{
- *pnState = sizeof(u32)*3;
- sqlite3Put4byte(&aState[0], p->nStep);
- sqlite3Put4byte(&aState[4], p->pWal->hdr.aCksum[0]);
- sqlite3Put4byte(&aState[8], p->pWal->hdr.aCksum[1]);
- *paState = aState;
- }
- }
- }
- rc = walCheckpointFinalize(p);
- walUnlockExclusive(pWal, WAL_CKPT_LOCK, 1);
- pWal->ckptLock = 0;
- sqlite3_free(p);
- memset(&pWal->hdr, 0, sizeof(WalIndexHdr));
- sqlite3_mutex_leave(db->mutex);
- return rc;
-}
-
-/*
-** Open an incremental checkpoint handle.
-*/
-int sqlite3WalCheckpointStart(
- sqlite3 *db, /* Database connection */
- Wal *pWal, /* Wal connection */
- u8 *aState, int nState, /* Checkpoint state to restore */
- int (*xBusy)(void*), /* Function to call when busy */
- void *pBusyArg, /* Context argument for xBusyHandler */
- int sync_flags, /* Flags to sync db file with (or 0) */
- sqlite3_ckpt **ppCkpt /* OUT: Incremental checkpoint object */
-){
- WalCkpt *p = 0;
- int isChanged = 0;
- int rc;
- int pgsz;
-
- *ppCkpt = 0;
- if( pWal->readOnly ) return SQLITE_READONLY;
- WALTRACE(("WAL%p: checkpoint begins\n", pWal));
- rc = walLockExclusive(pWal, WAL_CKPT_LOCK, 1);
- if( rc ){
- /* Usually this is SQLITE_BUSY meaning that another thread or process
- ** is already running a checkpoint, or maybe a recovery. But it might
- ** also be SQLITE_IOERR. */
- return rc;
- }
- pWal->ckptLock = 1;
-
- /* Read the wal-index header. */
- rc = walIndexReadHdr(pWal, &isChanged);
- if( rc!=SQLITE_OK ) goto ckptstart_out;
- if( isChanged && pWal->pDbFd->pMethods->iVersion>=3 ){
- sqlite3OsUnfetch(pWal->pDbFd, 0, 0);
- }
-
- pgsz = walPagesize(pWal);
- p = sqlite3_malloc(sizeof(WalCkpt) + pgsz);
- if( p==0 ){
- rc = SQLITE_NOMEM;
- goto ckptstart_out;
- }
-
- rc = walCheckpointStart(
- pWal, (u8*)&p[1], pgsz, xBusy, pBusyArg, sync_flags, p
- );
- p->db = db;
-
- if( rc==SQLITE_OK && aState ){
- if( nState!=sizeof(u32)*3 ){
- rc = SQLITE_CORRUPT_BKPT;
- }else{
- int i;
- if( pWal->hdr.aCksum[0]!=sqlite3Get4byte(&aState[4])
- || pWal->hdr.aCksum[1]!=sqlite3Get4byte(&aState[8])
- ){
- rc = SQLITE_MISMATCH;
- }else{
- p->nStep = (int)sqlite3Get4byte(aState);
- sqlite3Put4byte(&aState[4], pWal->hdr.aCksum[0]);
- sqlite3Put4byte(&aState[8], pWal->hdr.aCksum[1]);
- for(i=0; rc==SQLITE_OK && i<p->nStep; i++){
- u32 dummy1, dummy2;
- rc = walIteratorNext(p->pIter, &dummy1, &dummy2);
- }
- }
- }
- }
-
- ckptstart_out:
- if( rc!=SQLITE_OK ){
- if( p ) walIteratorFree(p->pIter);
- walUnlockExclusive(pWal, WAL_CKPT_LOCK, 1);
- pWal->ckptLock = 0;
- sqlite3_free(p);
- p = 0;
- }
- *ppCkpt = (sqlite3_ckpt*)p;
- return rc;
-}
-#endif /* SQLITE_ENABLE_OTA */
-
-/*
-** Unless the wal file is empty, check that the 8 bytes of salt stored in
-** the wal header are identical to those in the buffer indicated by the
-** second argument. If they are not, return SQLITE_BUSY_SNAPSHOT. Otherwise,
-** if the buffers match or the WAL file is empty, return SQLITE_OK.
-*/
-int sqlite3WalCheckSalt(Wal *pWal, sqlite3_file *pFd){
- int rc = SQLITE_OK;
- if( pWal->hdr.mxFrame>0 ){
- u8 aData[16];
- rc = sqlite3OsRead(pFd, aData, sizeof(aData), 24);
- if( rc==SQLITE_OK && memcmp(pWal->hdr.aSalt, aData, 8) ){
- rc = SQLITE_BUSY_SNAPSHOT;
- }
- }
- return rc;
-}
-
/* Return the value to pass to a sqlite3_wal_hook callback, the
** number of frames in the WAL at the point of the last commit since
** sqlite3WalCallback() was called. If no commits have occurred since
*/
int sqlite3WalHeapMemory(Wal *pWal);
-int sqlite3WalCheckSalt(Wal *pWal, sqlite3_file*);
-
-int sqlite3WalCheckpointStart(sqlite3 *,
- Wal *pWal, /* Wal connection */
- u8 *aState, int nState, /* Checkpoint state to restore */
- int (*xBusy)(void*), /* Function to call when busy */
- void *pBusyArg, /* Context argument for xBusyHandler */
- int sync_flags, /* Flags to sync db file with (or 0) */
- sqlite3_ckpt **ppCkpt /* OUT: Incremental checkpoint object */
-);
-
#ifdef SQLITE_ENABLE_ZIPVFS
/* If the WAL file is not empty, return the number of bytes of content
** stored in each frame (i.e. the db page-size when the WAL was created).
projects / thirdparty / sqlite.git / commitdiff
