From: dan Date: Fri, 4 Aug 2017 16:16:32 +0000 (+0000) Subject: Add test cases and associated fixes for swarmvtab. X-Git-Tag: version-3.21.0~188^2~2 X-Git-Url: http://git.ipfire.org/?a=commitdiff_plain;h=0ff2217035501f00d1c3137016721f0c5a5da106;p=thirdparty%2Fsqlite.git Add test cases and associated fixes for swarmvtab. FossilOrigin-Name: 7ae20eac83fc053dc1bbc42501dd41f77445a6b9a33cfa42b899fc7a18c637ab --- diff --git a/ext/misc/unionvtab.c b/ext/misc/unionvtab.c index 243152b261..2df3c30e06 100644 --- a/ext/misc/unionvtab.c +++ b/ext/misc/unionvtab.c @@ -10,8 +10,8 @@ ** ************************************************************************* ** -** This file contains the implementation of the "unionvtab" virtual -** table. This module provides read-only access to multiple tables, +** This file contains the implementation of the "unionvtab" and "swarmvtab" +** virtual tables. These modules provide read-only access to multiple tables, ** possibly in multiple database files, via a single database object. ** The source tables must have the following characteristics: ** @@ -25,25 +25,44 @@ ** ** * Each table must contain a distinct range of rowid values. ** -** A "unionvtab" virtual table is created as follows: +** The difference between the two virtual table modules is that for +** "unionvtab", all source tables must be located in the main database or +** in databases ATTACHed to the main database by the user. For "swarmvtab", +** the tables may be located in any database file on disk. The "swarmvtab" +** implementation takes care of opening and closing database files +** automatically. ** -** CREATE VIRTUAL TABLE USING unionvtab(); +** UNIONVTAB ** -** The implementation evalutes whenever a unionvtab virtual -** table is created or opened. It should return one row for each source -** database table. The four columns required of each row are: +** A "unionvtab" virtual table is created as follows: ** -** 1. The name of the database containing the table ("main" or "temp" or -** the name of an attached database). Or NULL to indicate that all -** databases should be searched for the table in the usual fashion. +** CREATE VIRTUAL TABLE USING unionvtab(); ** -** 2. The name of the database table. +** The implementation evalutes whenever a unionvtab virtual +** table is created or opened. It should return one row for each source +** database table. The four columns required of each row are: ** -** 3. The smallest rowid in the range of rowids that may be stored in the -** database table (an integer). +** 1. The name of the database containing the table ("main" or "temp" or +** the name of an attached database). Or NULL to indicate that all +** databases should be searched for the table in the usual fashion. ** -** 4. The largest rowid in the range of rowids that may be stored in the -** database table (an integer). +** 2. The name of the database table. +** +** 3. The smallest rowid in the range of rowids that may be stored in the +** database table (an integer). +** +** 4. The largest rowid in the range of rowids that may be stored in the +** database table (an integer). +** +** SWARMVTAB +** +** A "swarmvtab" virtual table is created similarly to a unionvtab table: +** +** CREATE VIRTUAL TABLE USING swarmvtab(); +** +** The difference is that for a swarmvtab table, the first column returned +** by the must return a path or URI that can be used to open +** the database file containing the source table. ** */ @@ -65,7 +84,12 @@ SQLITE_EXTENSION_INIT1 # define SMALLEST_INT64 (((sqlite3_int64)-1) - LARGEST_INT64) #endif -#define SWARMVTAB_MAX_ATTACHED 9 +/* +** The swarmvtab module attempts to keep the number of open database files +** at or below this limit. This may not be possible if there are too many +** simultaneous queries. +*/ +#define SWARMVTAB_MAX_OPEN 9 typedef struct UnionCsr UnionCsr; typedef struct UnionTab UnionTab; @@ -83,7 +107,7 @@ struct UnionSrc { sqlite3_int64 iMax; /* Maximum rowid */ /* Fields used by swarmvtab only */ - char *zFile; /* File to ATTACH */ + char *zFile; /* Database file containing table zTab */ int nUser; /* Current number of users */ sqlite3 *db; /* Database handle */ UnionSrc *pNextClosable; /* Next in list of closable sources */ @@ -119,6 +143,12 @@ struct UnionCsr { int iTab; /* Index of table read by pStmt */ }; +/* +** Given UnionTab table pTab and UnionSrc object pSrc, return the database +** handle that should be used to access the table identified by pSrc. This +** is the main db handle for "unionvtab" tables, or the source-specific +** handle for "swarmvtab". +*/ #define unionGetDb(pTab, pSrc) ((pTab)->bSwarm ? (pSrc)->db : (pTab)->db) /* @@ -298,25 +328,23 @@ static void unionFinalize(int *pRc, sqlite3_stmt *pStmt, char **pzErr){ } /* -** Close one database from the closable list. +** This function is a no-op for unionvtab. For swarmvtab, it attempts to +** close open database files until at most nMax are open. An SQLite error +** code is returned if an error occurs, or SQLITE_OK otherwise. */ -static int unionCloseDatabase(UnionTab *pTab, char **pzErr){ +static int unionCloseSources(UnionTab *pTab, int nMax){ int rc = SQLITE_OK; - UnionSrc **pp; - - assert( pTab->pClosable && pTab->bSwarm ); - - pp = &pTab->pClosable; - while( (*pp)->pNextClosable ){ - pp = &(*pp)->pNextClosable; + if( pTab->bSwarm ){ + while( rc==SQLITE_OK && pTab->pClosable && pTab->nOpen>nMax ){ + UnionSrc **pp; + for(pp=&pTab->pClosable; (*pp)->pNextClosable; pp=&(*pp)->pNextClosable); + assert( (*pp)->db ); + rc = sqlite3_close((*pp)->db); + (*pp)->db = 0; + *pp = 0; + pTab->nOpen--; + } } - - assert( (*pp)->db ); - rc = sqlite3_close((*pp)->db); - (*pp)->db = 0; - *pp = 0; - pTab->nOpen--; - return rc; } @@ -447,18 +475,31 @@ static int unionSourceCheck(UnionTab *pTab, char **pzErr){ return rc; } +/* +** This function may only be called for swarmvtab tables. The results of +** calling it on a unionvtab table are undefined. +** +** For a swarmvtab table, this function ensures that source database iSrc +** is open. If the database is opened successfully and the schema is as +** expected, or if it is already open when this function is called, SQLITE_OK +** is returned. +** +** Alternatively If an error occurs while opening the databases, or if the +** database schema is unsuitable, an SQLite error code is returned and (*pzErr) +** may be set to point to an English language error message. In this case it is +** the responsibility of the caller to eventually free the error message buffer +** using sqlite3_free(). +*/ static int unionOpenDatabase(UnionTab *pTab, int iSrc, char **pzErr){ int rc = SQLITE_OK; UnionSrc *pSrc = &pTab->aSrc[iSrc]; assert( pTab->bSwarm && iSrcnSrc ); if( pSrc->db==0 ){ - while( rc==SQLITE_OK && pTab->pClosable && pTab->nOpen>=pTab->nMaxOpen ){ - rc = unionCloseDatabase(pTab, pzErr); - } + rc = unionCloseSources(pTab, pTab->nMaxOpen-1); if( rc==SQLITE_OK ){ - rc = sqlite3_open(pSrc->zFile, &pSrc->db); + rc = sqlite3_open_v2(pSrc->zFile, &pSrc->db, SQLITE_OPEN_READONLY, 0); if( rc!=SQLITE_OK ){ *pzErr = sqlite3_mprintf("%s", sqlite3_errmsg(pSrc->db)); }else{ @@ -538,8 +579,6 @@ static int unionFinalizeCsrStmt(UnionCsr *pCsr){ return rc; } - - /* ** xConnect/xCreate method. ** @@ -646,7 +685,7 @@ static int unionConnect( if( rc==SQLITE_OK ){ pTab->db = db; pTab->bSwarm = bSwarm; - pTab->nMaxOpen = SWARMVTAB_MAX_ATTACHED; + pTab->nMaxOpen = SWARMVTAB_MAX_OPEN; if( bSwarm ){ rc = unionOpenDatabase(pTab, 0, pzErr); }else{ @@ -685,7 +724,6 @@ static int unionConnect( return rc; } - /* ** xOpen */ @@ -708,9 +746,10 @@ static int unionClose(sqlite3_vtab_cursor *cur){ return SQLITE_OK; } - /* -** xNext +** This function does the work of the xNext() method. Except that, if it +** returns SQLITE_ROW, it should be called again within the same xNext() +** method call. See unionNext() for details. */ static int doUnionNext(UnionCsr *pCsr){ int rc = SQLITE_OK; @@ -746,6 +785,9 @@ static int doUnionNext(UnionCsr *pCsr){ return rc; } +/* +** xNext +*/ static int unionNext(sqlite3_vtab_cursor *cur){ int rc; do { @@ -888,7 +930,6 @@ static int unionFilter( } } - if( zSql==0 ){ return rc; }else{ diff --git a/manifest b/manifest index d6345db7a7..51d9536952 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Modify\sswarmvtab\sto\suse\sa\sseparate\sdatabase\sconnection\sfor\seach\sdatabase\sfile. -D 2017-08-03T20:13:00.017 +C Add\stest\scases\sand\sassociated\sfixes\sfor\sswarmvtab. +D 2017-08-04T16:16:32.840 F Makefile.in d9873c9925917cca9990ee24be17eb9613a668012c85a343aef7e5536ae266e8 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc 02b469e9dcd5b7ee63fc1fb05babc174260ee4cfa4e0ef2e48c3c6801567a016 @@ -281,7 +281,7 @@ F ext/misc/showauth.c 732578f0fe4ce42d577e1c86dc89dd14a006ab52 F ext/misc/spellfix.c a4723b6aff748a417b5091b68a46443265c40f0d F ext/misc/stmt.c 6f16443abb3551e3f5813bb13ba19a30e7032830015b0f92fe0c0453045c0a11 F ext/misc/totype.c 4a167594e791abeed95e0a8db028822b5e8fe512 -F ext/misc/unionvtab.c 96add94bbccd52227e1b8c73457aee563f6da099aa77060a6d95737c00fe3cfa +F ext/misc/unionvtab.c e388c29b4af5ad66fb463fd7db9242c89d03639f030e3e8829ad336b7002a48f F ext/misc/vfslog.c fe40fab5c077a40477f7e5eba994309ecac6cc95 F ext/misc/vfsstat.c bf10ef0bc51e1ad6756629e1edb142f7a8db1178 F ext/misc/vtshim.c 1976e6dd68dd0d64508c91a6dfab8e75f8aaf6cd @@ -1231,7 +1231,7 @@ F test/subselect.test 0966aa8e720224dbd6a5e769a3ec2a723e332303 F test/substr.test 18f57c4ca8a598805c4d64e304c418734d843c1a F test/subtype1.test 7fe09496352f97053af1437150751be2d0a0cae8 F test/superlock.test ec94f0556b6488d97f71c79f9061ae08d9ab8f12 -F test/swarmvtab.test c12cda41c77dc3cee58a0f73721817894c6202cc61ce44af2c5d2131e232fa11 +F test/swarmvtab.test fbb2415797477588337a54e3bc0ff8e1981d325d22b9e75a527438e79d926a6a F test/symlink.test c9ebe7330d228249e447038276bfc8a7b22f4849 F test/sync.test 2f84bdbc2b2df1fcb0220575b4b9f8cea94b7529 F test/sync2.test 6be8ed007fa063b147773c1982b5bdba97a32badc536bdc6077eff5cf8710ece @@ -1641,7 +1641,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 03d94388d62fd0f1fae377d273bbd5561208adc34bd97f7ce27783b30a369fd7 -R 625b97c97a5f8dd3636fd589eae8ca8e +P 1f05ad29c3a540408470da7f8111f1319f961539c1a96b1a81abf1423af90f15 +R 7e7ab9819e0aed7a3e8424d62e02f2a0 U dan -Z 0d29a0f09a9580a6cb1bdb3e0e661b27 +Z 2e976f6f51ee5c8476d79b5286404931 diff --git a/manifest.uuid b/manifest.uuid index 854ef35d5f..f000a4bbfd 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -1f05ad29c3a540408470da7f8111f1319f961539c1a96b1a81abf1423af90f15 \ No newline at end of file +7ae20eac83fc053dc1bbc42501dd41f77445a6b9a33cfa42b899fc7a18c637ab \ No newline at end of file diff --git a/test/swarmvtab.test b/test/swarmvtab.test index 7486088073..06cb328bc9 100644 --- a/test/swarmvtab.test +++ b/test/swarmvtab.test @@ -64,25 +64,55 @@ do_execsql_test 1.3 { proc do_compare_test {tn where} { set sql [subst { - SELECT ( - SELECT group_concat(a || ',' || b, ',') FROM t0 WHERE $where - ) IS ( - SELECT group_concat(a || ',' || b, ',') FROM s1 WHERE $where - ) + SELECT (SELECT group_concat(a || ',' || b, ',') FROM t0 WHERE $where) + IS + (SELECT group_concat(a || ',' || b, ',') FROM s1 WHERE $where) }] uplevel [list do_execsql_test $tn $sql 1] } -do_compare_test 1.4 "rowid = 55" -do_compare_test 1.5 "rowid BETWEEN 20 AND 100" -do_compare_test 1.6 "rowid > 350" -do_compare_test 1.7 "rowid >= 350" -do_compare_test 1.8 "rowid >= 200" -do_compare_test 1.9 "1" -do_compare_test 1.10 "rowid = 700" -do_compare_test 1.11 "rowid = -1" -do_compare_test 1.12 "rowid = 0" +do_compare_test 1.4.1 "rowid = 700" +do_compare_test 1.4.2 "rowid = -1" +do_compare_test 1.4.3 "rowid = 0" +do_compare_test 1.4.4 "rowid = 55" +do_compare_test 1.4.5 "rowid BETWEEN 20 AND 100" +do_compare_test 1.4.6 "rowid > 350" +do_compare_test 1.4.7 "rowid >= 350" +do_compare_test 1.4.8 "rowid >= 200" +do_compare_test 1.4.9 "1" + +# Multiple simultaneous cursors. +# +do_execsql_test 1.5.1.(5-seconds-or-so) { + SELECT count(*) FROM s1 a, s1 b WHERE b.rowid<=200; +} {80000} +do_execsql_test 1.5.2 { + SELECT count(*) FROM s1 a, s1 b, s1 c + WHERE a.rowid=b.rowid AND b.rowid=c.rowid; +} {400} + +# Empty source tables. +# +do_test 1.6.0 { + for {set i 0} {$i < 20} {incr i} { + sqlite3 db2 test.db$i + db2 eval " DELETE FROM t$i " + db2 close + } + db eval { DELETE FROM t0 WHERE rowid<=200 } +} {} + +do_compare_test 1.6.1 "rowid = 700" +do_compare_test 1.6.2 "rowid = -1" +do_compare_test 1.6.3 "rowid = 0" +do_compare_test 1.6.4 "rowid = 55" +do_compare_test 1.6.5 "rowid BETWEEN 20 AND 100" +do_compare_test 1.6.6 "rowid > 350" +do_compare_test 1.6.7 "rowid >= 350" +do_compare_test 1.6.8 "rowid >= 200" +do_compare_test 1.6.9 "1" +do_compare_test 1.6.10 "rowid >= 5" do_test 1.x { set sqlite_open_file_count @@ -90,6 +120,69 @@ do_test 1.x { do_test 1.y { db close } {} +# Delete all the database files created above. +# +for {set i 0} {$i < 40} {incr i} { forcedelete "test.db$i" } + +#------------------------------------------------------------------------- +# Test some error conditions: +# +# 2.1: Database file does not exist. +# 2.2: Table does not exist. +# 2.3: Table schema does not match. +# +reset_db +load_static_extension db unionvtab +do_test 2.0.1 { + db eval { + CREATE TABLE t0(a INTEGER PRIMARY KEY, b TEXT); + WITH s(i) AS ( SELECT 1 UNION ALL SELECT i+1 FROM s WHERE i<400) + INSERT INTO t0 SELECT i, hex(randomblob(50)) FROM s; + CREATE TABLE dir(f, t, imin, imax); + } + + for {set i 0} {$i < 40} {incr i} { + set iMin [expr $i*10 + 1] + set iMax [expr $iMin+9] + + forcedelete "test.db$i" + db eval [subst { + ATTACH 'test.db$i' AS aux; + CREATE TABLE aux.t$i (a INTEGER PRIMARY KEY, b TEXT); + INSERT INTO aux.t$i SELECT * FROM t0 WHERE a BETWEEN $iMin AND $iMax; + DETACH aux; + INSERT INTO dir VALUES('test.db$i', 't$i', $iMin, $iMax); + }] + } + execsql { + CREATE VIRTUAL TABLE temp.s1 USING swarmvtab('SELECT * FROM dir'); + } +} {} + +do_test 2.0.2 { + forcedelete test.db5 + + sqlite3 db2 test.db15 + db2 eval { DROP TABLE t15 } + db2 close + + sqlite3 db2 test.db25 + db2 eval { + DROP TABLE t25; + CREATE TABLE t25(x, y, z PRIMARY KEY); + } + db2 close +} {} + +do_catchsql_test 2.1 { + SELECT * FROM s1 WHERE rowid BETWEEN 1 AND 100; +} {1 {unable to open database file}} +do_catchsql_test 2.2 { + SELECT * FROM s1 WHERE rowid BETWEEN 101 AND 200; +} {1 {no such rowid table: t15}} +do_catchsql_test 2.3 { + SELECT * FROM s1 WHERE rowid BETWEEN 201 AND 300; +} {1 {source table schema mismatch}} finish_test