From: danielk1977 Date: Tue, 3 Jan 2006 13:39:25 +0000 (+0000) Subject: Add test files for the asynchronous IO concept. (CVS 2853) X-Git-Tag: version-3.6.10~3314 X-Git-Url: http://git.ipfire.org/?a=commitdiff_plain;h=34e84942027b937d3e9513b97ed5ba97ddee0e40;p=thirdparty%2Fsqlite.git Add test files for the asynchronous IO concept. (CVS 2853) FossilOrigin-Name: 7c3492c8404c39c808af4429b4fcdb7413539ab3 --- diff --git a/manifest b/manifest index 4665d73e30..32cfa8a706 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C The\ssqlite\sTCL\scommand\sno\slonger\sreturns\sthe\shex\saddress\sof\sthe\ssqlite3*\nstructure.\s\sInstead\sthere\sis\sa\snew\scommand\sin\stestfixture\sto\sfind\sthat\ninformation.\s(CVS\s2852) -D 2006-01-03T00:33:50 +C Add\stest\sfiles\sfor\sthe\sasynchronous\sIO\sconcept.\s(CVS\s2853) +D 2006-01-03T13:39:26 F Makefile.in e3c6b3a38d734d41574c04f2fc90d18de2b87102 F Makefile.linux-gcc aee18d8a05546dcf1888bd4547e442008a49a092 F README 9c4e2d6706bdcc3efdd773ce752a8cdab4f90028 @@ -78,6 +78,7 @@ F src/test3.c 9742aa146eb750cab81c1d5605286c3a0eb88054 F src/test4.c a8fd681e139e1c61f22a77d07fc3a99cb28fff3f F src/test5.c 7162f8526affb771c4ed256826eee7bb9eca265f F src/test6.c cb811391ec0b7c75f29e545d4820a9cf19f3637e +F src/test_async.c a383aed2753e47d2ac4b5397c43f6ac216a237ae F src/tokenize.c 7a3a3d3cc734f684a77c4dfd09eb46fcee25394c F src/trigger.c 2925ba96d964d9b717e74006bf7e64b8a6b70d97 F src/update.c ec8e540617b116725b5a55c8d6b4db8bc67fdd7d @@ -99,6 +100,7 @@ F test/alter2.test cc0b8832e4e98605dbc26910efd4bb89abe59cb2 F test/alter3.test 6e144ea3dcc395afcc28e794bb532be83dc8fdcb F test/altermalloc.test 6e1f404ec021eb2ba6582e3c77b0a35cf206b7af F test/analyze.test 2f55535aa335785db1a2f97d3f3831c16c09f8b0 +F test/async.test 522826f998b24328ce221971dc30ebbc7c0c4adc F test/attach.test dae07fa1554b618b9cc4c7bc349b3bc1a532180e F test/attach2.test 0e6a7c54343c85dd877a1e86073a05176043ed40 F test/attach3.test 63013383adc4380af69779f34f4af19bd49f7cbe @@ -333,7 +335,7 @@ F www/tclsqlite.tcl ddcf912ea48695603c8ed7efb29f0812ef8d1b49 F www/vdbe.tcl 87a31ace769f20d3627a64fa1fade7fed47b90d0 F www/version3.tcl a99cf5f6d8bd4d5537584a2b342f0fb9fa601d8b F www/whentouse.tcl 97e2b5cd296f7d8057e11f44427dea8a4c2db513 -P a582b159595ff8d31c81e9b3044711d6590d3f0e -R 3443cda8c76c7eb6b12365da80f51486 -U drh -Z e8e69e33240623256267b54701f3d2e2 +P 70b228575e045bc56013aab945334203ceb31d8b +R 3ad9f3b93cc87edd3692f844738f28c0 +U danielk1977 +Z d1aa958d08ac6132fcf74135e20aafaf diff --git a/manifest.uuid b/manifest.uuid index 1af600fd5c..635a4f29a6 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -70b228575e045bc56013aab945334203ceb31d8b \ No newline at end of file +7c3492c8404c39c808af4429b4fcdb7413539ab3 \ No newline at end of file diff --git a/src/test_async.c b/src/test_async.c new file mode 100644 index 0000000000..8ba10ee28b --- /dev/null +++ b/src/test_async.c @@ -0,0 +1,929 @@ +/* +** 2005 December 14 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +************************************************************************* +** +** This file contains an example implementation of an asynchronous IO +** backend for SQLite. It is used to test that the concept of asynchronous +** IO in SQLite is valid. +*/ + +#include "sqliteInt.h" +#include "os.h" + +#define MIN(x,y) ((x)<(y)?(x):(y)) +#define MAX(x,y) ((x)>(y)?(x):(y)) + +typedef struct AsyncWrite AsyncWrite; +typedef struct AsyncFile AsyncFile; + +/* +** TODO: +** * File locks... +*/ + +/* +** THREAD SAFETY NOTES +** +** Basic rules: +** +** * Both read and write access to the global write-op queue must be +** protected by the sqlite3Os mutex functions. +** * The file handles from the underlying system are assumed not to +** be thread safe. +** * See the last paragraph under "sqlite3_async_flush() Threads" for +** an assumption to do with file-handle synchronization by the Os. +** +** File system operations (invoked by SQLite thread): +** +** xOpenXXX (three versions) +** xDelete +** xFileExists +** +** Todo: +** xSyncDirectory +** +** File handle operations (invoked by SQLite thread): +** +** The following operations add an entry to the global write-op list. They +** prepare the entry, aquire the mutex momentarily while list pointers are +** manipulated to insert the new entry, and release the mutex. +** +** asyncWrite, asyncClose, asyncTruncate, asyncSync, +** asyncSetFullSync, asyncOpenDirectory. +** +** Read operations. Both of these read from both the underlying file and +** the write-op list. So we grab the mutex for the whole call (even +** while performing a blocking read on the file). +** +** asyncRead, asyncFileSize. +** +** These locking primitives become no-ops. Files are always opened for +** exclusive access when using this IO backend: +** +** asyncLock, asyncUnlock, asyncLockState, asyncCheckReservedLock +** +** The sqlite3OsFileHandle() function is currently only used when +** debugging the pager module. Unless sqlite3OsClose() is called on the +** file (shouldn't be possible for other reasons), the underlying +** implementations are safe to call without grabbing any mutex. So we just +** go ahead and call it no matter what any other thread is doing. +** +** asyncFileHandle. +** +** Calling this method just manipulates the AsyncFile.iOffset variable. +** Since this variable is never accessed by an sqlite3_async_flush() thread, +** this function does not require the mutex. Actual calls to OsSeek() take +** place just before OsWrite() or OsRead(), which are always protected by +** the mutex. +** +** asyncSeek. +** +** sqlite3_async_flush() (any thread): +** +** A pseudo-mutex (a global boolean variable) is used to make sure only +** one thread is inside the sqlite3_async_flush() thread at any one time. +** If the variable is set when a thread enters _flush(), then it +** immediately returns SQLITE_BUSY. Otherwise, it sets the variable, +** executes the body of the function, and clears the variable just before +** returning. Both read and write access to said global variable +** (sqlite3_asyncIoBusy) is protected by sqlite3Os mutex, of course. +** +** Inside sqlite3_async_flush() is a loop that works like this: +** +** WHILE (write-op list is not empty) +** Do IO operation at head of write-op list +** Remove entry from head of write-op list +** END WHILE +** +** The mutex is always obtained during the +** test, and when the entry is removed from the head of the write-op +** list. Sometimes it is held for the interim period (while the IO is +** performed), and sometimes it is relinquished. It is relinquished if +** (a) the IO op is an ASYNC_CLOSE or (b) when the file handle was +** opened, two of the underlying systems handles were opened on the +** same file-system entry. +** +** If condition (b) above is true, then one file-handle +** (AsyncFile.pBaseRead) is used exclusively by sqlite threads to read the +** file, the other (AsyncFile.pBaseWrite) by sqlite3_async_flush() +** threads to perform write() operations. This means that read +** operations are not blocked by asynchronous writes (although +** asynchronous writes may still be blocked by reads). +** +** This assumes that the OS keeps two handles open on the same file +** properly in sync. That is, any read operation that starts after a +** write operation on the same file system entry has completed returns +** data consistent with the write. We also assume that if one thread +** reads a file while another is writing it all bytes other than the +** ones actually being written contain valid data. +** +** If the above assumptions are not true, set the preprocessor symbol +** SQLITE_ASYNC_TWO_FILEHANDLES to 0. +*/ + +#ifndef SQLITE_ASYNC_TWO_FILEHANDLES +/* #define SQLITE_ASYNC_TWO_FILEHANDLES 0 */ +#define SQLITE_ASYNC_TWO_FILEHANDLES 1 +#endif + +/* +** First and last elements of the global write-op list. +** +** Whenever an OsWrite(), OsSync(), OsTrunc() or OsClose() operation is +** requested, instead of performing the file IO immediately, a new AsyncWrite +** structure is allocated and added to the global linked list that starts at +** sqlite3_asyncListFirst. The next time to sqlite3_async_flush() is called, +** all operations are realised and the list elements deleted. +*/ +static AsyncWrite *sqlite3_asyncListFirst = 0; +static AsyncWrite *sqlite3_asyncListLast = 0; + +/* True after an IO error has occured */ +/* static int *sqlite3_asyncIoError = 0; */ + +/* True if some thread is currently inside sqlite3_async_flush() */ +static int sqlite3_asyncIoBusy = 0; + +/* Possible values of AsyncWrite.op */ +#define ASYNC_WRITE 1 +#define ASYNC_SYNC 2 +#define ASYNC_TRUNCATE 3 +#define ASYNC_CLOSE 4 +#define ASYNC_OPENDIRECTORY 5 +#define ASYNC_SETFULLSYNC 6 + +#define ASYNC_DELETE 7 +#define ASYNC_OPENEXCLUSIVE 8 +#define ASYNC_SYNCDIRECTORY 9 + +/* +** The interpretation of the iOffset and nByte variables varies depending +** on the value of AsyncWrite.op: +** +** ASYNC_WRITE: +** iOffset -> Offset in file to write to. +** nByte -> Number of bytes of data to write (pointed to by zBuf). +** +** ASYNC_SYNC: +** iOffset -> Unused. +** nByte -> Value of "fullsync" flag to pass to sqlite3OsSync(). +** +** ASYNC_TRUNCATE: +** iOffset -> Size to truncate file to. +** nByte -> Unused. +** +** ASYNC_CLOSE: +** iOffset -> Unused. +** nByte -> Unused. +** +** ASYNC_OPENDIRECTORY: +** iOffset -> Unused. +** nByte -> Number of bytes of zBuf points to (directory name). +** +** ASYNC_SETFULLSYNC: +** iOffset -> Unused. +** nByte -> New value for the full-sync flag. +** +** +** ASYNC_DELETE: +** iOffset -> Unused. +** nByte -> Number of bytes of zBuf points to (file name). +** +** ASYNC_OPENEXCLUSIVE: +** iOffset -> Value of "delflag". +** nByte -> Number of bytes of zBuf points to (file name). +** +** For an ASYNC_WRITE operation, zBuf points to the data to write to the file. +** This space is sqliteMalloc()d along with the AsyncWrite structure in a +** single blob, so is deleted when sqliteFree() is called on the parent +** structure. +*/ +struct AsyncWrite { + AsyncFile *pFile; /* File to write data to or sync */ + int op; /* One of ASYNC_xxx etc. */ + i64 iOffset; /* See above */ + int nByte; /* See above */ + char *zBuf; /* Data to write to file (or NULL if op!=ASYNC_WRITE) */ + AsyncWrite *pNext; /* Next write operation (to any file) */ +}; + +/* +** The AsyncFile structure is a subclass of OsFile used for asynchronous IO. +*/ +struct AsyncFile { + IoMethod *pMethod; /* Must be first */ + i64 iOffset; /* Current seek() offset in file */ + OsFile *pBaseRead; /* Read handle to the underlying Os file */ + OsFile *pBaseWrite; /* Write handle to the underlying Os file */ +}; + +/* +** Add an entry to the end of the global write-op list. pWrite should point +** to an AsyncWrite structure allocated using sqliteMalloc(). A future call +** to sqlite3_async_flush() is responsible for calling sqliteFree(). +** +** Once an AsyncWrite structure has been added to the list, it must not be +** read or modified by the caller (in case another thread calls +** sqlite3_async_flush() ). +*/ +static void addAsyncWrite(AsyncWrite *pWrite){ + sqlite3_os_enter_mutex(); + assert( !pWrite->pNext ); + if( sqlite3_asyncListLast ){ + assert( sqlite3_asyncListFirst ); + sqlite3_asyncListLast->pNext = pWrite; + }else{ + sqlite3_asyncListFirst = pWrite; + } + sqlite3_asyncListLast = pWrite; + sqlite3_os_leave_mutex(); +} + +/* +** The caller should already hold the mutex when this is called. +*/ +static void removeAsyncWrite(AsyncWrite *p){ + assert( p==sqlite3_asyncListFirst ); + assert( sqlite3_asyncListLast ); + if( sqlite3_asyncListFirst==sqlite3_asyncListLast ){ + assert( !sqlite3_asyncListFirst->pNext ); + sqlite3_asyncListLast = 0; + } + sqlite3_asyncListFirst = sqlite3_asyncListFirst->pNext; +} + +/* +** This is a utility function to allocate and populate a new AsyncWrite +** structure and insert it (via addAsyncWrite() ) into the global list. +*/ +static int addNewAsyncWrite( + AsyncFile *pFile, + int op, + i64 iOffset, + int nByte, + const char *zByte +){ + AsyncWrite *p = sqlite3Os.xMalloc(sizeof(AsyncWrite) + (zByte?nByte:0)); + if( !p ){ + return SQLITE_NOMEM; + } + p->op = op; + p->iOffset = iOffset; + p->nByte = nByte; + p->pFile = pFile; + p->pNext = 0; + if( zByte ){ + p->zBuf = (char *)&p[1]; + memcpy(p->zBuf, zByte, nByte); + }else{ + p->zBuf = 0; + } + addAsyncWrite(p); + return SQLITE_OK; +} + +/* +** Close the file. This just adds an entry to the write-op list, the file is +** not actually closed. +*/ +static int asyncClose(OsFile **pId){ + return addNewAsyncWrite((AsyncFile *)*pId, ASYNC_CLOSE, 0, 0, 0); +} + +/* +** Implementation of sqlite3OsWrite() for asynchronous files. Instead of +** writing to the underlying file, this function adds an entry to the end of +** the global AsyncWrite list. Either SQLITE_OK or SQLITE_NOMEM may be +** returned. +*/ +static int asyncWrite(OsFile *id, const void *pBuf, int amt){ + AsyncFile *pFile = (AsyncFile *)id; + int rc = addNewAsyncWrite(pFile, ASYNC_WRITE, pFile->iOffset, amt, pBuf); + pFile->iOffset += (i64)amt; + return rc; +} + +/* +** Truncate the file to nByte bytes in length. This just adds an entry to +** the write-op list, no IO actually takes place. +*/ +static int asyncTruncate(OsFile *id, i64 nByte){ + return addNewAsyncWrite((AsyncFile *)id, ASYNC_TRUNCATE, nByte, 0, 0); +} + +/* +** Open the directory identified by zName and associate it with the +** specified file. This just adds an entry to the write-op list, the +** directory is opened later by sqlite3_async_flush(). +*/ +static int asyncOpenDirectory(OsFile *id, const char *zName){ + AsyncFile *pFile = (AsyncFile *)id; + return addNewAsyncWrite(pFile, ASYNC_OPENDIRECTORY, 0, strlen(zName)+1,zName); +} + +/* +** Sync the file. This just adds an entry to the write-op list, the +** sync() is done later by sqlite3_async_flush(). +*/ +static int asyncSync(OsFile *id, int fullsync){ + return addNewAsyncWrite((AsyncFile *)id, ASYNC_SYNC, 0, fullsync, 0); +} + +/* +** Set (or clear) the full-sync flag on the underlying file. This operation +** is queued and performed later by sqlite3_async_flush(). +*/ +static void asyncSetFullSync(OsFile *id, int value){ + addNewAsyncWrite((AsyncFile *)id, ASYNC_SETFULLSYNC, 0, value, 0); +} + +/* +** Read data from the file. First we read from the filesystem, then adjust +** the contents of the buffer based on ASYNC_WRITE operations in the +** write-op queue. Todo: Do we need to think about ASYNC_TRUNCATE in +** this method as well? +** +** This method holds the mutex from start to finish. +*/ +static int asyncRead(OsFile *id, void *obuf, int amt){ + int rc = SQLITE_OK; + i64 filesize; + int nRead; + AsyncFile *pFile = (AsyncFile *)id; + + /* Grab the mutex for the duration of the call */ + sqlite3_os_enter_mutex(); + + if( pFile->pBaseRead ){ + rc = sqlite3OsFileSize(pFile->pBaseRead, &filesize); + if( rc!=SQLITE_OK ){ + goto asyncread_out; + } + rc = sqlite3OsSeek(pFile->pBaseRead, pFile->iOffset); + if( rc!=SQLITE_OK ){ + goto asyncread_out; + } + nRead = MIN(filesize - pFile->iOffset, amt); + if( nRead>0 ){ + rc = sqlite3OsRead(((AsyncFile *)id)->pBaseRead, obuf, nRead); + } + } + + if( rc==SQLITE_OK ){ + AsyncWrite *p; + i64 iOffset = pFile->iOffset; /* Current seek offset */ + + for(p=sqlite3_asyncListFirst; p; p = p->pNext){ + if( p->pFile==pFile && p->op==ASYNC_WRITE ){ + int iBeginIn = (p->iOffset - iOffset); + int iBeginOut = (iOffset - p->iOffset); + int nCopy; + + if( iBeginIn<0 ) iBeginIn = 0; + if( iBeginOut<0 ) iBeginOut = 0; + nCopy = MIN(p->nByte-iBeginIn, amt-iBeginOut); + + if( nCopy>0 ){ + memcpy(&((char *)obuf)[iBeginOut], &p->zBuf[iBeginIn], nCopy); + } + } + } + + pFile->iOffset += (i64)amt; + } + +asyncread_out: + sqlite3_os_leave_mutex(); + return rc; +} + +/* +** Seek to the specified offset. This just adjusts the AsyncFile.iOffset +** variable - calling seek() on the underlying file is defered until the +** next read() or write() operation. +*/ +static int asyncSeek(OsFile *id, i64 offset){ + AsyncFile *pFile = (AsyncFile *)id; + pFile->iOffset = offset; + return SQLITE_OK; +} + +/* +** Read the size of the file. First we read the size of the file system +** entry, then adjust for any ASYNC_WRITE or ASYNC_TRUNCATE operations +** currently in the write-op list. +** +** This method holds the mutex from start to finish. +*/ +int asyncFileSize(OsFile *id, i64 *pSize){ + int rc = SQLITE_OK; + i64 s = 0; + OsFile *pBase; + sqlite3_os_enter_mutex(); + + /* Read the filesystem size from the base file. If pBaseRead is NULL, this + ** means the file hasn't been opened yet. In this case all relevant data + ** must be in the write-op queue anyway, so we can omit reading from the + ** file-system. + */ + pBase = ((AsyncFile *)id)->pBaseRead; + if( pBase ){ + rc = sqlite3OsFileSize(pBase, &s); + } + + if( rc==SQLITE_OK ){ + AsyncWrite *p; + for(p=sqlite3_asyncListFirst; p; p = p->pNext){ + if( p->pFile==(AsyncFile *)id ){ + switch( p->op ){ + case ASYNC_WRITE: + s = MAX(p->iOffset + (i64)(p->nByte), s); + break; + case ASYNC_TRUNCATE: + s = MIN(s, p->nByte); + break; + } + } + } + *pSize = s; + } + sqlite3_os_leave_mutex(); + return rc; +} + +/* +** Return the operating system file handle. This is only used for debugging +** at the moment anyway. +*/ +static int asyncFileHandle(OsFile *id){ + return sqlite3OsFileHandle(((AsyncFile *)id)->pBaseRead); +} + +static int asyncLock(OsFile *id, int lockType){ + return SQLITE_OK; +} +static int asyncUnlock(OsFile *id, int lockType){ + return SQLITE_OK; +} + +/* +** This function is called when the pager layer first opens a database file +** and is checking for a hot-journal. +*/ +static int asyncCheckReservedLock(OsFile *id){ + return SQLITE_OK; +} + +/* +** This is broken. But sqlite3OsLockState() is only used for testing anyway. +*/ +static int asyncLockState(OsFile *id){ + return SQLITE_OK; +} + +/* +** The three file-open functions for the underlying file system layer. +*/ +static int (*xOrigOpenReadWrite)(const char*, OsFile**, int*) = 0; +static int (*xOrigOpenExclusive)(const char*, OsFile**, int) = 0; +static int (*xOrigOpenReadOnly)(const char*, OsFile**) = 0; + +/* +** Pointers to the original versions of other overridden file-system +** operations. +*/ +static int (*xOrigDelete)(const char*) = 0; +static int (*xOrigFileExists)(const char*) = 0; +static int (*xOrigSyncDirectory)(const char*) = 0; + +static int asyncOpenFile( + const char *zName, + OsFile **pFile, + OsFile *pBaseRead, + int openSecondFile +){ + int rc; + AsyncFile *p; + OsFile *pBaseWrite = 0; + + static IoMethod iomethod = { + asyncClose, + asyncOpenDirectory, + asyncRead, + asyncWrite, + asyncSeek, + asyncTruncate, + asyncSync, + asyncSetFullSync, + asyncFileHandle, + asyncFileSize, + asyncLock, + asyncUnlock, + asyncLockState, + asyncCheckReservedLock + }; + + if( openSecondFile && SQLITE_ASYNC_TWO_FILEHANDLES ){ + int dummy; + rc = xOrigOpenReadWrite(zName, &pBaseWrite, &dummy); + if( rc!=SQLITE_OK ){ + goto error_out; + } + } + + p = (AsyncFile *)sqlite3Os.xMalloc(sizeof(AsyncFile)); + if( !p ){ + rc = SQLITE_NOMEM; + goto error_out; + } + memset(p, 0, sizeof(AsyncFile)); + + p->pMethod = &iomethod; + p->pBaseRead = pBaseRead; + p->pBaseWrite = pBaseWrite; + + *pFile = (OsFile *)p; + return SQLITE_OK; + +error_out: + assert(!p); + sqlite3OsClose(&pBaseRead); + sqlite3OsClose(&pBaseWrite); + *pFile = 0; + return rc; +} + +/* +** The async-IO backends implementation of the three functions used to open +** a file (xOpenExclusive, xOpenReadWrite and xOpenReadOnly). Most of the +** work is done in function asyncOpenFile() - see above. +*/ +static int asyncOpenExclusive(const char *z, OsFile **ppFile, int delFlag){ + int rc = asyncOpenFile(z, ppFile, 0, 0); + if( rc==SQLITE_OK ){ + AsyncFile *pFile = (AsyncFile *)(*ppFile); + int nByte = strlen(z)+1; + i64 i = (i64)(delFlag); + rc = addNewAsyncWrite(pFile, ASYNC_OPENEXCLUSIVE, i, nByte, z); + if( rc!=SQLITE_OK ){ + sqlite3Os.xFree(pFile); + *ppFile = 0; + } + } + return rc; +} +static int asyncOpenReadOnly(const char *z, OsFile **ppFile){ + OsFile *pBase = 0; + int rc = xOrigOpenReadOnly(z, &pBase); + if( rc==SQLITE_OK ){ + rc = asyncOpenFile(z, ppFile, pBase, 0); + } + return rc; +} +static int asyncOpenReadWrite(const char *z, OsFile **ppFile, int *pReadOnly){ + OsFile *pBase = 0; + int rc = xOrigOpenReadWrite(z, &pBase, pReadOnly); + if( rc==SQLITE_OK ){ + rc = asyncOpenFile(z, ppFile, pBase, (*pReadOnly ? 0 : 1)); + } + return rc; +} + +/* +** Implementation of sqlite3Os.xDelete. Add an entry to the end of the +** write-op queue to perform the delete. +*/ +static int asyncDelete(const char *z){ + return addNewAsyncWrite(0, ASYNC_DELETE, 0, strlen(z)+1, z); +} + +/* +** Implementation of sqlite3Os.xDelete. Add an entry to the end of the +** write-op queue to perform the delete. +*/ +static int asyncSyncDirectory(const char *z){ + return addNewAsyncWrite(0, ASYNC_SYNCDIRECTORY, 0, strlen(z)+1, z); +} + +/* +** Implementation of sqlite3Os.xFileExists. Return true if file 'z' exists +** in the file system. +** +** This method holds the mutex from start to finish. +*/ +static int asyncFileExists(const char *z){ + int ret; + AsyncWrite *p; + sqlite3_os_enter_mutex(); + + /* See if the real file system contains the specified file. */ + ret = xOrigFileExists(z); + + for(p=sqlite3_asyncListFirst; p; p = p->pNext){ + if( p->op==ASYNC_DELETE && 0==strcmp(p->zBuf, z) ){ + ret = 0; + }else if( p->op==ASYNC_OPENEXCLUSIVE && 0==strcmp(p->zBuf, z) ){ + ret = 1; + } + } + + sqlite3_os_leave_mutex(); + return ret; +} + +/* +** The following routine is one of two exported symbols in this module (along +** with sqlite3_async_flush(), see below). This routine should be called +** once to enable the asynchronous IO features implemented in this file. If +** the features are successfully enabled (or if they have already been +** enabled) then SQLITE_OK is returned. Otherwise, SQLITE_MISUSE. +*/ +int sqlite3_async_enable(void){ + if( xOrigOpenReadWrite==0 ){ +#define ROUTINE(a,b,c) {(void**)&a,SQLITE_OS_ROUTINE_ ## b,(void *)c} + struct ReplacementOp { + void ** pOldRoutine; + int eRoutine; + void * pNewRoutine; + } aRoutines[] = { + ROUTINE(xOrigOpenReadWrite, OPENREADWRITE, asyncOpenReadWrite), + ROUTINE(xOrigOpenReadOnly, OPENREADONLY, asyncOpenReadOnly), + ROUTINE(xOrigOpenExclusive, OPENEXCLUSIVE, asyncOpenExclusive), + ROUTINE(xOrigDelete, DELETE, asyncDelete), + ROUTINE(xOrigFileExists, FILEEXISTS, asyncFileExists), + ROUTINE(xOrigSyncDirectory, SYNCDIRECTORY, asyncSyncDirectory) + }; +#undef ROUTINE + int i; + + sqlite3_os_enter_mutex(); + for(i=0; ipOldRoutine) = sqlite3_os_routine_set(p->eRoutine, p->pNewRoutine); + } + sqlite3_os_leave_mutex(); + } + return SQLITE_OK; +} + +/* +** This function is called externally to perform queued write and sync +** operations. It returns when an IO error occurs or there are no more queued +** operations to perform. +*/ +int sqlite3_async_flush(void){ + AsyncWrite *p = 0; + int rc = SQLITE_OK; + + /* Grab the mutex and set the sqlite3_asyncIoBusy flag to make sure this + ** is the only thread performing an sqlite3_async_flush() at this time. + ** Or, if some other thread is already inside this function, return + ** SQLITE_BUSY to the caller. + */ + sqlite3_os_enter_mutex(); + if( sqlite3_asyncIoBusy ){ + sqlite3_os_leave_mutex(); + return SQLITE_BUSY; + } + sqlite3_asyncIoBusy = 1; + + while( (p = sqlite3_asyncListFirst) && rc==SQLITE_OK ){ + int isInsideMutex = 1; + + /* Right now this thread is holding the global mutex. Variable 'p' points + ** to the first entry in the write-op queue. In the general case, we + ** hold on to the mutex for the entire body of the loop. + ** + ** However in the cases enumerated below, we relinquish the mutex, + ** perform the IO, and then re-request the mutex before removing 'p' from + ** the head of the write-op queue. The idea is to increase concurrency with + ** sqlite threads. + ** + ** * An ASYNC_CLOSE operation. + ** * An ASYNC_OPENEXCLUSIVE operation. For this one, we relinquish + ** the mutex, call the underlying xOpenExclusive() function, then + ** re-aquire the mutex before seting the AsyncFile.pBaseRead + ** variable. + ** * ASYNC_SYNC and ASYNC_WRITE operations, if + ** SQLITE_ASYNC_TWO_FILEHANDLES was set at compile time and two + ** file-handles are open for the particular file being "synced". + */ + OsFile *pBase = 0; + if( p->pFile ){ + pBase = p->pFile->pBaseWrite; + if( + p->op==ASYNC_CLOSE || + p->op==ASYNC_OPENEXCLUSIVE || + (pBase && (p->op==ASYNC_SYNC || p->op==ASYNC_WRITE) ) + ){ + sqlite3_os_leave_mutex(); + isInsideMutex = 0; + } + if( !pBase ){ + pBase = p->pFile->pBaseRead; + } + } + + switch( p->op ){ + case ASYNC_WRITE: + assert( pBase ); + rc = sqlite3OsSeek(pBase, p->iOffset); + if( rc==SQLITE_OK ){ + rc = sqlite3OsWrite(pBase, (const void *)(p->zBuf), p->nByte); + } + break; + + case ASYNC_SYNC: + assert( pBase ); + rc = sqlite3OsSync(pBase, p->nByte); + break; + + case ASYNC_TRUNCATE: + assert( pBase ); + rc = sqlite3OsTruncate(pBase, p->nByte); + break; + + case ASYNC_CLOSE: + sqlite3OsClose(&p->pFile->pBaseRead); + sqlite3OsClose(&p->pFile->pBaseWrite); + sqlite3Os.xFree(p->pFile); + break; + + case ASYNC_OPENDIRECTORY: + assert( pBase ); + sqlite3OsOpenDirectory(pBase, p->zBuf); + break; + + case ASYNC_SETFULLSYNC: + assert( pBase ); + sqlite3OsSetFullSync(pBase, p->nByte); + break; + + case ASYNC_DELETE: + rc = xOrigDelete(p->zBuf); + break; + + case ASYNC_SYNCDIRECTORY: + rc = xOrigSyncDirectory(p->zBuf); + break; + + case ASYNC_OPENEXCLUSIVE: { + AsyncFile *pFile = p->pFile; + int delFlag = ((p->iOffset)?1:0); + OsFile *pBase = 0; + rc = xOrigOpenExclusive(p->zBuf, &pBase, delFlag); + + sqlite3_os_enter_mutex(); + isInsideMutex = 1; + if( rc==SQLITE_OK ){ + pFile->pBaseRead = pBase; + } + break; + } + + default: assert(!"Illegal value for AsyncWrite.op"); + } + + /* If we didn't hang on to the mutex during the IO op, obtain it now + ** so that the AsyncWrite structure can be safely removed from the + ** global write-op queue. + */ + if( !isInsideMutex ){ + sqlite3_os_enter_mutex(); + } + if( rc==SQLITE_OK ){ + removeAsyncWrite(p); + sqlite3Os.xFree(p); + } + } + + /* Clear the io-busy flag and exit the mutex */ + assert( sqlite3_asyncIoBusy ); + sqlite3_asyncIoBusy = 0; + sqlite3_os_leave_mutex(); + + return rc; +} + +/* +** The following code defines a Tcl interface for testing the asynchronous +** IO implementation in this file. +*/ +#if defined(SQLITE_TEST) && defined(TCLSH) + +#include + +/* +** sqlite3_async_enable +*/ +static int testAsyncEnable( + void * clientData, + Tcl_Interp *interp, + int objc, + Tcl_Obj *CONST objv[] +){ + if( sqlite3_async_enable() ){ + Tcl_SetResult(interp, "sqlite3_async_enable() failed", TCL_STATIC); + return TCL_ERROR; + } + return TCL_OK; +} + +/* +** This is the main proc for a thread spawned by the Tcl command +** [sqlite3_async_flush -start]. The client data is a pointer to an integer +** variable that will be set to non-zero when this thread should exit. +*/ +static void testAsyncFlushThread(ClientData clientData){ + int *pStop = (int *)clientData; + int rc = 0; + + /* Run in a loop until an IO error occurs or we are told to stop via + ** the *pStop variable. Each iteration of the loop, call + ** sqlite3_async_flush() and then sleep for a tenth of a second. + */ + while( !(*pStop) && !rc ){ + rc = sqlite3_async_flush(); + assert( rc==SQLITE_OK ); + Tcl_Sleep(100); + } + if( rc==0 ){ + rc = sqlite3_async_flush(); + } + Tcl_ExitThread(rc); +} + +/* +** sqlite3_async_flush +** sqlite3_async_flush -start +** sqlite3_async_flush -stop +*/ +static int testAsyncFlush( + void * clientData, + Tcl_Interp *interp, + int objc, + Tcl_Obj *CONST objv[] +){ + static Tcl_ThreadId thread_id = 0; + static int stop = 0; + + assert(stop==0); + + if( objc!=1 && objc!=2 ){ + Tcl_WrongNumArgs(interp, 1, objv, "?-start | -stop?"); + return TCL_ERROR; + } + + if( objc==2 ){ + char *zOpt = Tcl_GetString(objv[1]); + if( 0==strcmp(zOpt, "-start") ){ + /* Unless it is already running, kick off the _flush() thread */ + if( thread_id ){ + Tcl_AppendResult(interp, "Thread has already started", 0); + return TCL_ERROR; + }else{ + int rc = Tcl_CreateThread( + &thread_id, + testAsyncFlushThread, + &stop, + TCL_THREAD_STACK_DEFAULT, + TCL_THREAD_JOINABLE + ); + if( rc!=TCL_OK ){ + Tcl_AppendResult(interp, "Tcl_CreateThread() failed", 0); + return TCL_ERROR; + } + } + }else if( 0==strcmp(zOpt, "-stop") ){ + int dummy; + stop = 1; + Tcl_JoinThread(thread_id, &dummy); + stop = 0; + thread_id = 0; + }else{ + Tcl_AppendResult(interp, "Invalid option: \"", zOpt, "\"", 0); + return TCL_ERROR; + } + }else if( sqlite3_async_flush() ){ + Tcl_SetResult(interp, "sqlite3_async_flush() failed", TCL_STATIC); + return TCL_ERROR; + } + return TCL_OK; +} + +int Sqlitetestasync_Init(Tcl_Interp *interp){ + Tcl_CreateObjCommand(interp,"sqlite3_async_enable",testAsyncEnable,0,0); + Tcl_CreateObjCommand(interp,"sqlite3_async_flush",testAsyncFlush,0,0); + return TCL_OK; +} + +#endif + diff --git a/test/async.test b/test/async.test new file mode 100644 index 0000000000..43835bc097 --- /dev/null +++ b/test/async.test @@ -0,0 +1,74 @@ +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# This file runs all tests. +# +# $Id: async.test,v 1.1 2006/01/03 13:39:26 danielk1977 Exp $ + +set testdir [file dirname $argv0] +source $testdir/tester.tcl +rename finish_test really_finish_test +proc finish_test {} {} +set ISQUICK 1 + +if {[info exists ::tcl_platform(threaded)]} { + set USE_THREAD 1 +} else { + set USE_THREAD 0 +} + +set INCLUDE { + select1.test + select2.test + select3.test + select4.test + insert.test + insert2.test + insert3.test +} +#set INCLUDE [lrange $INCLUDE 0 0] + +# Enable asynchronous IO. +sqlite3_async_enable + +# set USE_THREAD 0 +if {$USE_THREAD} { + sqlite3_async_flush -start +} else { + rename do_test really_do_test + proc do_test {name args} { + uplevel really_do_test async_io-$name $args + sqlite3_async_flush + } +} + +foreach testfile [lsort -dictionary [glob $testdir/*.test]] { + set tail [file tail $testfile] + if {[lsearch -exact $INCLUDE $tail]<0} continue + source $testfile + catch {db close} +# if {$sqlite_open_file_count>0} { +# puts "$tail did not close all files: $sqlite_open_file_count" +# incr nErr +# lappend ::failList $tail +# } +} +#source $testdir/misuse.test + + +if {$USE_THREAD} { + sqlite3_async_flush -stop +} else { + rename do_test {} + rename really_do_test do_test + puts "Warning: Tcl library was without --enable-threads." + puts " Cannot test async IO properly." + sqlite3_async_flush +} + +set sqlite_open_file_count 0 +really_finish_test +