From: drh <> Date: Mon, 6 Mar 2023 23:38:44 +0000 (+0000) Subject: Improvements to query invariant testing such that it uses the new X-Git-Tag: version-3.42.0~285 X-Git-Url: http://git.ipfire.org/?a=commitdiff_plain;h=5b6ba9b250c4c31f8fb117e1f42086bfa5327ec7;p=thirdparty%2Fsqlite.git Improvements to query invariant testing such that it uses the new SQLITE_DBCONFIG_REVERSE_SCANORDER opcode to sqlite3_db_config() to make more accurate judgements about when a query is ambiguous, and hence when query invariant testing is approprate. FossilOrigin-Name: be9ab292cd14889b1c9648b47138260b33fe5be282ff2d90653b1387885a7d02 --- diff --git a/Makefile.in b/Makefile.in index 2a71bd2c5c..f278b053e6 100644 --- a/Makefile.in +++ b/Makefile.in @@ -602,17 +602,35 @@ SHELL_OPT += -DSQLITE_ENABLE_OFFSET_SQL_FUNC FUZZERSHELL_OPT = FUZZCHECK_OPT += -I$(TOP)/test FUZZCHECK_OPT += -I$(TOP)/ext/recover -FUZZCHECK_OPT += -DSQLITE_OMIT_LOAD_EXTENSION -FUZZCHECK_OPT += -DSQLITE_ENABLE_MEMSYS5 -DSQLITE_OSS_FUZZ -FUZZCHECK_OPT += -DSQLITE_MAX_MEMORY=50000000 -FUZZCHECK_OPT += -DSQLITE_PRINTF_PRECISION_LIMIT=1000 -FUZZCHECK_OPT += -DSQLITE_ENABLE_FTS4 -FUZZCHECK_OPT += -DSQLITE_ENABLE_FTS3_PARENTHESIS -FUZZCHECK_OPT += -DSQLITE_ENABLE_FTS5 -FUZZCHECK_OPT += -DSQLITE_ENABLE_RTREE -FUZZCHECK_OPT += -DSQLITE_ENABLE_GEOPOLY -FUZZCHECK_OPT += -DSQLITE_ENABLE_DBSTAT_VTAB -FUZZCHECK_OPT += -DSQLITE_ENABLE_BYTECODE_VTAB +FUZZCHECK_OPT += \ + -DSQLITE_OSS_FUZZ \ + -DSQLITE_ENABLE_BYTECODE_VTAB \ + -DSQLITE_ENABLE_DBPAGE_VTAB \ + -DSQLITE_ENABLE_DBSTAT_VTAB \ + -DSQLITE_ENABLE_BYTECODE_VTAB \ + -DSQLITE_ENABLE_DESERIALIZE \ + -DSQLITE_ENABLE_EXPLAIN_COMMENTS \ + -DSQLITE_ENABLE_FTS3_PARENTHESIS \ + -DSQLITE_ENABLE_FTS4 \ + -DSQLITE_ENABLE_FTS5 \ + -DSQLITE_ENABLE_GEOPOLY \ + -DSQLITE_ENABLE_MATH_FUNCTIONS \ + -DSQLITE_ENABLE_MEMSYS5 \ + -DSQLITE_ENABLE_NORMALIZE \ + -DSQLITE_ENABLE_OFFSET_SQL_FUNC \ + -DSQLITE_ENABLE_PREUPDATE_HOOK \ + -DSQLITE_ENABLE_RTREE \ + -DSQLITE_ENABLE_SESSION \ + -DSQLITE_ENABLE_STMTVTAB \ + -DSQLITE_ENABLE_UNKNOWN_SQL_FUNCTION \ + -DSQLITE_ENABLE_STAT4 \ + -DSQLITE_ENABLE_STMT_SCANSTATUS \ + -DSQLITE_MAX_MEMORY=50000000 \ + -DSQLITE_MAX_MMAP_SIZE=0 \ + -DSQLITE_OMIT_LOAD_EXTENSION \ + -DSQLITE_PRINTF_PRECISION_LIMIT=1000 \ + -DSQLITE_PRIVATE="" + FUZZCHECK_SRC += $(TOP)/test/fuzzcheck.c FUZZCHECK_SRC += $(TOP)/test/ossfuzz.c FUZZCHECK_SRC += $(TOP)/test/fuzzinvariants.c diff --git a/Makefile.msc b/Makefile.msc index 5528b8e722..a8b5d3083b 100644 --- a/Makefile.msc +++ b/Makefile.msc @@ -1671,6 +1671,33 @@ FUZZERSHELL_COMPILE_OPTS = FUZZCHECK_OPTS = $(FUZZCHECK_OPTS) -I$(TOP)\test -I$(TOP)\ext\recover FUZZCHECK_OPTS = $(FUZZCHECK_OPTS) -DSQLITE_ENABLE_MEMSYS5 FUZZCHECK_OPTS = $(FUZZCHECK_OPTS) -DSQLITE_OSS_FUZZ +FUZZCHECK_OPTS = $(FUZZCHECK_OPTS) -DSQLITE_ENABLE_BYTECODE_VTAB +FUZZCHECK_OPTS = $(FUZZCHECK_OPTS) -DSQLITE_ENABLE_DBPAGE_VTAB +FUZZCHECK_OPTS = $(FUZZCHECK_OPTS) -DSQLITE_ENABLE_DBSTAT_VTAB +FUZZCHECK_OPTS = $(FUZZCHECK_OPTS) -DSQLITE_ENABLE_BYTECODE_VTAB +FUZZCHECK_OPTS = $(FUZZCHECK_OPTS) -DSQLITE_ENABLE_DESERIALIZE +FUZZCHECK_OPTS = $(FUZZCHECK_OPTS) -DSQLITE_ENABLE_EXPLAIN_COMMENTS +FUZZCHECK_OPTS = $(FUZZCHECK_OPTS) -DSQLITE_ENABLE_FTS3_PARENTHESIS +FUZZCHECK_OPTS = $(FUZZCHECK_OPTS) -DSQLITE_ENABLE_FTS4 +FUZZCHECK_OPTS = $(FUZZCHECK_OPTS) -DSQLITE_ENABLE_FTS5 +FUZZCHECK_OPTS = $(FUZZCHECK_OPTS) -DSQLITE_ENABLE_GEOPOLY +FUZZCHECK_OPTS = $(FUZZCHECK_OPTS) -DSQLITE_ENABLE_MATH_FUNCTIONS +FUZZCHECK_OPTS = $(FUZZCHECK_OPTS) -DSQLITE_ENABLE_MEMSYS5 +FUZZCHECK_OPTS = $(FUZZCHECK_OPTS) -DSQLITE_ENABLE_NORMALIZE +FUZZCHECK_OPTS = $(FUZZCHECK_OPTS) -DSQLITE_ENABLE_OFFSET_SQL_FUNC +FUZZCHECK_OPTS = $(FUZZCHECK_OPTS) -DSQLITE_ENABLE_PREUPDATE_HOOK +FUZZCHECK_OPTS = $(FUZZCHECK_OPTS) -DSQLITE_ENABLE_RTREE +FUZZCHECK_OPTS = $(FUZZCHECK_OPTS) -DSQLITE_ENABLE_SESSION +FUZZCHECK_OPTS = $(FUZZCHECK_OPTS) -DSQLITE_ENABLE_STMTVTAB +FUZZCHECK_OPTS = $(FUZZCHECK_OPTS) -DSQLITE_ENABLE_UNKNOWN_SQL_FUNCTION +FUZZCHECK_OPTS = $(FUZZCHECK_OPTS) -DSQLITE_ENABLE_STAT4 +FUZZCHECK_OPTS = $(FUZZCHECK_OPTS) -DSQLITE_ENABLE_STMT_SCANSTATUS +FUZZCHECK_OPTS = $(FUZZCHECK_OPTS) -DSQLITE_MAX_MEMORY=50000000 +FUZZCHECK_OPTS = $(FUZZCHECK_OPTS) -DSQLITE_MAX_MMAP_SIZE=0 +FUZZCHECK_OPTS = $(FUZZCHECK_OPTS) -DSQLITE_OMIT_LOAD_EXTENSION +FUZZCHECK_OPTS = $(FUZZCHECK_OPTS) -DSQLITE_PRINTF_PRECISION_LIMIT=1000 +FUZZCHECK_OPTS = $(FUZZCHECK_OPTS) -DSQLITE_PRIVATE="" + FUZZCHECK_OPTS = $(FUZZCHECK_OPTS) -DSQLITE_MAX_MEMORY=50000000 FUZZCHECK_OPTS = $(FUZZCHECK_OPTS) -DSQLITE_PRINTF_PRECISION_LIMIT=1000 FUZZCHECK_OPTS = $(FUZZCHECK_OPTS) -DSQLITE_OMIT_LOAD_EXTENSION diff --git a/manifest b/manifest index d174bd6ab1..c7f872c26f 100644 --- a/manifest +++ b/manifest @@ -1,11 +1,11 @@ -C Add\sSQLITE_DBCONFIG_REVERSE_SCANORDER\sfor\sdirect\sC-language\saccess\sto\sthe\n"PRAGMA\sreverse_unordered_selects"\ssetting. -D 2023-03-06T19:04:39.903 +C Improvements\sto\squery\sinvariant\stesting\ssuch\sthat\sit\suses\sthe\snew\nSQLITE_DBCONFIG_REVERSE_SCANORDER\sopcode\sto\ssqlite3_db_config()\sto\smake\smore\naccurate\sjudgements\sabout\swhen\sa\squery\sis\sambiguous,\sand\shence\swhen\squery\ninvariant\stesting\sis\sapproprate. +D 2023-03-06T23:38:44.071 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 -F Makefile.in 594f07f8829020a45d825234edf14046544b5bf6bc0057a9d5b83b58686b1c46 +F Makefile.in 73076ae60a143476b6890b5c3b88848fce19ac8caefc3ff5ae97f72694dc946d F Makefile.linux-gcc f609543700659711fbd230eced1f01353117621dccae7b9fb70daa64236c5241 -F Makefile.msc b0026b61d1242add00c28dbe169271a763fffcb3af6762bcdbeb20c3efe6a69f +F Makefile.msc e3a7faf7da80b891edcc0378eb9b4b1274ae080ee1fd1337c649bfe9af9aa456 F README.md 8b8df9ca852aeac4864eb1e400002633ee6db84065bd01b78c33817f97d31f5e F VERSION 17f95ae2fdf21f0e9575eb0b0511ea63f15d71dfff431b21c2b4adbfa70cfbbf F aclocal.m4 a5c22d164aff7ed549d53a90fa56d56955281f50 @@ -1140,7 +1140,7 @@ F test/fuzz3.test 9c813e6613b837cb7a277b0383cd66bfa07042b4cf0317157c35852f30043c F test/fuzz4.test c229bcdb45518a89e1d208a21343e061503460ac69fae1539320a89f572eb634 F test/fuzz_common.tcl b7197de6ed1ee8250a4f82d67876f4561b42ee8cbbfc6160dcb66331bad3f830 F test/fuzz_malloc.test f348276e732e814802e39f042b1f6da6362a610af73a528d8f76898fde6b22f2 -F test/fuzzcheck.c 90d3435c5359029d79b0945110e9c83a529469c03e4380338578c47c5a9ea296 +F test/fuzzcheck.c a3d4967ba9a56b13e097f613409178cfb20796d291ad1d7d532c6677fd5909b9 F test/fuzzdata1.db 3e86d9cf5aea68ddb8e27c02d7dfdaa226347426c7eb814918e4d95475bf8517 F test/fuzzdata2.db 128b3feeb78918d075c9b14b48610145a0dd4c8d6f1ca7c2870c7e425f5bf31f F test/fuzzdata3.db c6586d3e3cef0fbc18108f9bb649aa77bfc38aba @@ -1152,7 +1152,7 @@ F test/fuzzdata8.db f6c2f2af4deaaae0ddb3310d509c2659990794aa653dc501b80a0534c349 F test/fuzzer1.test 3d4c4b7e547aba5e5511a2991e3e3d07166cfbb8 F test/fuzzer2.test a85ef814ce071293bce1ad8dffa217cbbaad4c14 F test/fuzzerfault.test f64c4aef4c9e9edf1d6dc0d3f1e65dcc81e67c996403c88d14f09b74807a42bc -F test/fuzzinvariants.c a153253600b2b33a7d5710d40e89b2ac1373a1912517867fb995a45b2d67dcb8 +F test/fuzzinvariants.c 3ec82ac4277a0c62918f39e7a35893d6fb5ed57c465a938ad18e8e7a69527615 F test/gcfault.test dd28c228a38976d6336a3fc42d7e5f1ad060cb8c F test/gencol1.test e40c94c0d3485c37c23ec8ff42dbb4464cef3f3e41d96ef1abf9c4f2b97936c9 F test/genesis.tcl 1e2e2e8e5cc4058549a154ff1892fe5c9de19f98 @@ -2048,8 +2048,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P af312b131457743d98b84137bd51d9ba60e0daf0bd8f5a66f05956ca35ab68fb -R ad7078f08d1f30e27457cbb0b25465a3 +P 83e84531b46814aea6dad1ce8283cb9f6b90ad52badb60b875ea8f66e4ac0925 +R f88f50a654bf2b41e280183e1005cba0 U drh -Z f646de84c6f7cf615b430f5fcccad8cd +Z 7f97178f4b5f9f04d04c9231829a35aa # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 238d5abcd3..3624529277 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -83e84531b46814aea6dad1ce8283cb9f6b90ad52badb60b875ea8f66e4ac0925 \ No newline at end of file +be9ab292cd14889b1c9648b47138260b33fe5be282ff2d90653b1387885a7d02 \ No newline at end of file diff --git a/test/fuzzcheck.c b/test/fuzzcheck.c index 65dd2fa722..9f509306b7 100644 --- a/test/fuzzcheck.c +++ b/test/fuzzcheck.c @@ -1001,12 +1001,14 @@ static int recoverSqlCb(void *pCtx, const char *zSql){ */ static int recoverDatabase(sqlite3 *db){ int rc; /* Return code from this routine */ + const char *zRecoveryDb = ""; /* Name of "recovery" database */ const char *zLAF = "lost_and_found"; /* Name of "lost_and_found" table */ int bFreelist = 1; /* True to scan the freelist */ int bRowids = 1; /* True to restore ROWID values */ - sqlite3_recover *p; /* The recovery object */ + sqlite3_recover *p = 0; /* The recovery object */ p = sqlite3_recover_init_sql(db, "main", recoverSqlCb, 0); + sqlite3_recover_config(p, 789, (void*)zRecoveryDb); sqlite3_recover_config(p, SQLITE_RECOVER_LOST_AND_FOUND, (void*)zLAF); sqlite3_recover_config(p, SQLITE_RECOVER_ROWIDS, (void*)&bRowids); sqlite3_recover_config(p, SQLITE_RECOVER_FREELIST_CORRUPT,(void*)&bFreelist); @@ -1038,7 +1040,7 @@ static int runDbSql(sqlite3 *db, const char *zSql, unsigned int *pBtsFlags){ printf("RUNNING-SQL: [%s]\n", zSql); fflush(stdout); } - (*pBtsFlags) &= ~BTS_BADPRAGMA; + (*pBtsFlags) &= BTS_BADPRAGMA; rc = sqlite3_prepare_v2(db, zSql, -1, &pStmt, 0); if( rc==SQLITE_OK ){ int nRow = 0; diff --git a/test/fuzzinvariants.c b/test/fuzzinvariants.c index 883f8cdfc9..0a5f607148 100644 --- a/test/fuzzinvariants.c +++ b/test/fuzzinvariants.c @@ -115,6 +115,8 @@ int fuzz_invariant( if( rc==SQLITE_DONE ){ /* No matching output row found */ sqlite3_stmt *pCk = 0; + int iOrigRSO; + /* This is not a fault if the database file is corrupt, because anything ** can happen with a corrupt database file */ @@ -136,28 +138,24 @@ int fuzz_invariant( } sqlite3_finalize(pCk); - if( sqlite3_strlike("%group%by%",sqlite3_sql(pStmt),0)==0 ){ - /* - ** If there is a GROUP BY clause, it might not cover every term in the - ** output. And then non-covered terms can take on a value from any - ** row in the result set. This can cause differing answers. - */ - goto not_a_fault; + /* + ** If inverting the scan order also results in a miss, assume that the + ** query is ambiguous and do not report a fault. + */ + sqlite3_db_config(db, SQLITE_DBCONFIG_REVERSE_SCANORDER, -1, &iOrigRSO); + sqlite3_db_config(db, SQLITE_DBCONFIG_REVERSE_SCANORDER, !iOrigRSO, 0); + sqlite3_prepare_v2(db, sqlite3_sql(pStmt), -1, &pCk, 0); + sqlite3_db_config(db, SQLITE_DBCONFIG_REVERSE_SCANORDER, iOrigRSO, 0); + while( (rc = sqlite3_step(pCk))==SQLITE_ROW ){ + for(i=0; i=nCol ) break; } - - if( sqlite3_strlike("%limit%)%order%by%", sqlite3_sql(pTestStmt),0)==0 ){ - /* crash-89bd6a6f8c6166e9a4c5f47b3e70b225f69b76c6 - ** Original statement is: - ** - ** SELECT a,b,c* FROM t1 LIMIT 1%5<4 - ** - ** When running: - ** - ** SELECT * FROM (...) ORDER BY 1 - ** - ** A different subset of the rows come out - */ - goto not_a_fault; + sqlite3_finalize(pCk); + if( rc==SQLITE_DONE ){ + sqlite3_finalize(pTestStmt); + return SQLITE_DONE; } /* The original sameValue() comparison assumed a collating sequence