From: drh Date: Tue, 29 Mar 2011 14:08:09 +0000 (+0000) Subject: Further improvements to the fuzzer. It still is not quite working. Pausing X-Git-Tag: version-3.7.6~60^2~4 X-Git-Url: http://git.ipfire.org/?a=commitdiff_plain;h=72384dc28f14b186d3211fc0181cda9ea3b7516f;p=thirdparty%2Fsqlite.git Further improvements to the fuzzer. It still is not quite working. Pausing to work on other things.... FossilOrigin-Name: 5f2f2fce40f43debeb0492c9b460b85c7dad2bde --- diff --git a/install-sh b/install-sh old mode 100644 new mode 100755 diff --git a/manifest b/manifest index a9cd7caec3..8148ffca72 100644 --- a/manifest +++ b/manifest @@ -1,8 +1,5 @@ ------BEGIN PGP SIGNED MESSAGE----- -Hash: SHA1 - -C Added\smost\sof\sthe\slogic.\s\sSimple\stest\sruns\swithout\ssegfaulting\sbut\sdoes\snot\ngive\sthe\scorrect\sanswer. -D 2011-03-26T19:04:47.346 +C Further\simprovements\sto\sthe\sfuzzer.\s\sIt\sstill\sis\snot\squite\sworking.\s\sPausing\nto\swork\son\sother\sthings.... +D 2011-03-29T14:08:09.188 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 6c96e694f446500449f683070b906de9fce17b88 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -102,7 +99,7 @@ F ext/rtree/rtree_util.tcl 06aab2ed5b826545bf215fff90ecb9255a8647ea F ext/rtree/sqlite3rtree.h 1af0899c63a688e272d69d8e746f24e76f10a3f0 F ext/rtree/tkt3363.test 142ab96eded44a3615ec79fba98c7bde7d0f96de F ext/rtree/viewrtree.tcl eea6224b3553599ae665b239bd827e182b466024 -F install-sh 9d4de14ab9fb0facae2f48780b874848cbf2f895 +F install-sh 9d4de14ab9fb0facae2f48780b874848cbf2f895 x F ltmain.sh 3ff0879076df340d2e23ae905484d8c15d5fdea8 F main.mk a767e12162f02719fa94697a6ff0c8b51bcd62a6 F mkdll.sh 7d09b23c05d56532e9d44a50868eb4b12ff4f74a @@ -205,7 +202,7 @@ F src/test_config.c 62f0f8f934b1d5c7e4cd4f506ae453a1117b47d7 F src/test_demovfs.c 0aed671636735116fc872c5b03706fd5612488b5 F src/test_devsym.c e7498904e72ba7491d142d5c83b476c4e76993bc F src/test_func.c cbdec5cededa0761daedde5baf06004a9bf416b5 -F src/test_fuzzer.c 133c830fdd4342b687910a04cf8e617e0f5f0e5f +F src/test_fuzzer.c 3e402dd5e6c4f096fdc92c52e775e8fad85bce9b F src/test_hexio.c 1237f000ec7a491009b1233f5c626ea71bce1ea2 F src/test_init.c 5d624ffd0409d424cf9adbfe1f056b200270077c F src/test_intarray.c d879bbf8e4ce085ab966d1f3c896a7c8b4f5fc99 @@ -479,7 +476,7 @@ F test/fuzz2.test 207d0f9d06db3eaf47a6b7bfc835b8e2fc397167 F test/fuzz3.test aec64345184d1662bd30e6a17851ff659d596dc5 F test/fuzz_common.tcl a87dfbb88c2a6b08a38e9a070dabd129e617b45b F test/fuzz_malloc.test dd7001ac86d09c154a7dff064f4739c60e2b312c -F test/fuzzer1.test e0fe96bb8d318250b35407954c7059eea8ea77b2 +F test/fuzzer1.test a5d60f618443b86b5f5a695a41d01b7d8697345d F test/hook.test f04c3412463f8ec117c1c704c74ca0f627ce733a F test/icu.test 70df4faca133254c042d02ae342c0a141f2663f4 F test/in.test 19b642bb134308980a92249750ea4ce3f6c75c2d @@ -608,7 +605,7 @@ F test/permutations.test 5b2a4cb756ffb2407cb4743163668d1d769febb6 F test/pragma.test fdfc09067ea104a0c247a1a79d8093b56656f850 F test/pragma2.test 5364893491b9231dd170e3459bfc2e2342658b47 F test/printf.test 05970cde31b1a9f54bd75af60597be75a5c54fea -F test/progress.test 5b075c3c790c7b2a61419bc199db87aaf48b8301 x +F test/progress.test 5b075c3c790c7b2a61419bc199db87aaf48b8301 F test/ptrchng.test ef1aa72d6cf35a2bbd0869a649b744e9d84977fc F test/quick.test 1681febc928d686362d50057c642f77a02c62e57 F test/quota.test ddafe133653093eb9a99ccd6264884ae43f9c9b8 @@ -896,7 +893,7 @@ F tool/genfkey.test 4196a8928b78f51d54ef58e99e99401ab2f0a7e5 F tool/lemon.c dfd81a51b6e27e469ba21d01a75ddf092d429027 F tool/lempar.c 01ca97f87610d1dac6d8cd96ab109ab1130e76dc F tool/mkkeywordhash.c d2e6b4a5965e23afb80fbe74bb54648cd371f309 -F tool/mkopts.tcl 66ac10d240cc6e86abd37dc908d50382f84ff46e x +F tool/mkopts.tcl 66ac10d240cc6e86abd37dc908d50382f84ff46e F tool/mkspeedsql.tcl a1a334d288f7adfe6e996f2e712becf076745c97 F tool/mksqlite3c.tcl cf44512a48112b1ba09590548660a5a6877afdb3 F tool/mksqlite3h.tcl d76c226a5e8e1f3b5f6593bcabe5e98b3b1ec9ff @@ -921,14 +918,7 @@ F tool/speedtest2.tcl ee2149167303ba8e95af97873c575c3e0fab58ff F tool/speedtest8.c 2902c46588c40b55661e471d7a86e4dd71a18224 F tool/speedtest8inst1.c 293327bc76823f473684d589a8160bde1f52c14e F tool/vdbe-compress.tcl d70ea6d8a19e3571d7ab8c9b75cba86d1173ff0f -P ea3a4ee136ff6699c3099178f0efaa8bb517715f -R 6a09f2dbc53bf6a269c56104914ae3e5 +P fb4c31eac8a7290f61c50a3552245660e1271871 +R 86b6649d084e41c015babc3a0e3fb294 U drh -Z c40c33ca74d9ed106c795140b6ad71ee ------BEGIN PGP SIGNATURE----- -Version: GnuPG v1.4.6 (GNU/Linux) - -iD8DBQFNjjjSoxKgR168RlERAjfMAKCJigUMIbnNV83nhVCWCpDK5WxPFQCeLIbS -+J80DLXre8S3k4SR8glB8Jc= -=nBDg ------END PGP SIGNATURE----- +Z 67d6460bdc70fd366585e4c8607ab8f4 diff --git a/manifest.uuid b/manifest.uuid index 54187bc30b..09a1e57cc7 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -fb4c31eac8a7290f61c50a3552245660e1271871 \ No newline at end of file +5f2f2fce40f43debeb0492c9b460b85c7dad2bde \ No newline at end of file diff --git a/src/test_fuzzer.c b/src/test_fuzzer.c index e4aca40fc8..82f35a2f51 100644 --- a/src/test_fuzzer.c +++ b/src/test_fuzzer.c @@ -17,6 +17,7 @@ #include #include #include +#include #ifndef SQLITE_OMIT_VIRTUALTABLE @@ -49,7 +50,15 @@ struct fuzzer_rule { }; /* -** A stem object is used to generate variants. +** A stem object is used to generate variants. It is also used to record +** previously generated outputs. +** +** Every stem is added to a hash table as it is output. Generation of +** duplicate stems is suppressed. +** +** Active stems (those that might generate new outputs) are kepts on a linked +** list sorted by increasing cost. The cost is the sum of rBaseCost and +** pRule->rCost. */ struct fuzzer_stem { char *zBasis; /* Word being fuzzed */ @@ -83,6 +92,7 @@ struct fuzzer_cursor { fuzzer_stem *pDone; /* Stems already processed to completion */ char *zBuf; /* Temporary use buffer */ int nBuf; /* Bytes allocated for zBuf */ + fuzzer_rule nullRule; /* Null rule used first */ fuzzer_stem *apHash[FUZZER_HASH]; /* Hash of previously generated terms */ }; @@ -171,7 +181,6 @@ static int fuzzerOpen(sqlite3_vtab *pVTab, sqlite3_vtab_cursor **ppCursor){ memset(pCur, 0, sizeof(*pCur)); pCur->pVtab = p; *ppCursor = &pCur->base; - p->nCursor++; if( p->nCursor==0 && p->pNewRule ){ unsigned int i; fuzzer_rule *pX; @@ -191,6 +200,7 @@ static int fuzzerOpen(sqlite3_vtab *pVTab, sqlite3_vtab_cursor **ppCursor){ } p->pRule = fuzzerMergeRules(p->pRule, pX); } + p->nCursor++; return SQLITE_OK; } @@ -221,35 +231,34 @@ static int fuzzerClose(sqlite3_vtab_cursor *cur){ fuzzerClearCursor(pCur, 0); sqlite3_free(pCur->zBuf); pCur->pVtab->nCursor--; + sqlite3_free(pCur); return SQLITE_OK; } /* ** Compute the current output term for a fuzzer_stem. */ -static int fuzzerComputeWord( - fuzzer_cursor *pCur, - fuzzer_stem *pStem +static int fuzzerRender( + fuzzer_stem *pStem, /* The stem to be rendered */ + char **pzBuf, /* Write results into this buffer. realloc if needed */ + int *pnBuf /* Size of the buffer */ ){ const fuzzer_rule *pRule = pStem->pRule; int n; + char *z; - n = pStem->nBasis; - if( pStem->n>=0 ) n += pRule->nTo - pRule->nFrom; - if( pCur->nBufzBuf = sqlite3_realloc(pCur->zBuf, n+100); - if( pCur->zBuf==0 ) return SQLITE_NOMEM; - pCur->nBuf = n+100; + n = pStem->nBasis + pRule->nTo - pRule->nFrom; + if( (*pnBuf)n; - if( n<0 ){ - memcpy(pCur->zBuf, pStem->zBasis, pStem->nBasis+1); - }else{ - memcpy(pCur->zBuf, pStem->zBasis, n); - memcpy(&pCur->zBuf[n], pRule->zTo, pRule->nTo); - memcpy(&pCur->zBuf[n+pRule->nTo], &pStem->zBasis[n+pRule->nFrom], - pStem->nBasis-n-pRule->nFrom+1); - } + z = *pzBuf; + memcpy(z, pStem->zBasis, n); + memcpy(&z[n], pRule->zTo, pRule->nTo); + memcpy(&z[n+pRule->nTo], &pStem->zBasis[n+pRule->nFrom], + pStem->nBasis-n-pRule->nFrom+1); return SQLITE_OK; } @@ -260,7 +269,7 @@ static int fuzzerComputeWord( static unsigned int fuzzerHash(const char *z){ unsigned int h = 0; while( *z ){ h = (h<<3) ^ (h>>29) ^ *(z++); } - return h%10007; + return h % FUZZER_HASH; } /* @@ -270,9 +279,30 @@ static fuzzer_cost fuzzerCost(fuzzer_stem *pStem){ return pStem->rBaseCost + pStem->pRule->rCost; } +/* +** Return 1 if the string to which the cursor is point has already +** been emitted. Return 0 if not. Return -1 on a memory allocation +** failures. +*/ +static int fuzzerSeen(fuzzer_cursor *pCur, fuzzer_stem *pStem){ + unsigned int h; + fuzzer_stem *pLookup; + + if( fuzzerRender(pStem, &pCur->zBuf, &pCur->nBuf)==SQLITE_NOMEM ){ + return -1; + } + h = fuzzerHash(pCur->zBuf); + pLookup = pCur->apHash[h]; + while( pLookup && strcmp(pLookup->zBasis, pCur->zBuf)!=0 ){ + pLookup = pLookup->pHash; + } + return pLookup!=0; +} + /* ** Advance a fuzzer_stem to its next value. Return 0 if there are -** no more values that can be generated by this fuzzer_stem. +** no more values that can be generated by this fuzzer_stem. Return +** -1 on a memory allocation failure. */ static int fuzzerAdvance(fuzzer_cursor *pCur, fuzzer_stem *pStem){ const fuzzer_rule *pRule; @@ -283,21 +313,14 @@ static int fuzzerAdvance(fuzzer_cursor *pCur, fuzzer_stem *pStem){ || memcmp(&pStem->zBasis[pStem->n], pRule->zFrom, pRule->nFrom)==0 ){ /* Found a rewrite case. Make sure it is not a duplicate */ - unsigned int h; - fuzzer_stem *pLookup; - - fuzzerComputeWord(pCur, pStem); - h = fuzzerHash(pCur->zBuf); - pLookup = pCur->apHash[h]; - while( pLookup && strcmp(pLookup->zBasis, pCur->zBuf)!=0 ){ - pLookup = pLookup->pHash; - } - if( pLookup==0 ) return 1; /* A new output is found. */ + int rc = fuzzerSeen(pCur, pStem); + if( rc<0 ) return -1; + if( rc==0 ) return 1; } } pStem->n = -1; pStem->pRule = pRule->pNext; - if( fuzzerCost(pStem)>pCur->rLimit ) pStem->pRule = 0; + if( pStem->pRule && fuzzerCost(pStem)>pCur->rLimit ) pStem->pRule = 0; } return 0; } @@ -310,6 +333,10 @@ static int fuzzerAdvance(fuzzer_cursor *pCur, fuzzer_stem *pStem){ static fuzzer_stem *fuzzerInsert(fuzzer_stem *pList, fuzzer_stem *pNew){ fuzzer_cost c1; + if( pList==0 ){ + pNew->pNext = 0; + return pNew; + } c1 = fuzzerCost(pNew); if( c1 <= fuzzerCost(pList) ){ pNew->pNext = pList; @@ -358,20 +385,30 @@ static fuzzer_stem *fuzzerNewStem( ** Advance a cursor to its next row of output */ static int fuzzerNext(sqlite3_vtab_cursor *cur){ - fuzzer_cursor *pCur = (fuzzer_cursor*)pCur; + fuzzer_cursor *pCur = (fuzzer_cursor*)cur; + int rc; fuzzer_stem *pStem, *pNew; /* Use the element the cursor is currently point to to create ** a new stem and insert the new stem into the priority queue. */ - fuzzerComputeWord(pCur, pCur->pStem); - pNew = fuzzerNewStem(pCur, pCur->zBuf, fuzzerCost(pCur->pStem)); - if( pNew ){ - if( fuzzerAdvance(pCur, pNew)==0 ){ - pNew->pNext = pCur->pDone; - pCur->pDone = pNew; + pStem = pCur->pStem; + if( fuzzerCost(pStem)>0 ){ + rc = fuzzerRender(pStem, &pCur->zBuf, &pCur->nBuf); + if( rc==SQLITE_NOMEM ) return SQLITE_NOMEM; + pNew = fuzzerNewStem(pCur, pCur->zBuf, fuzzerCost(pStem)); + if( pNew ){ + if( fuzzerAdvance(pCur, pNew)==0 ){ + pNew->pNext = pCur->pDone; + pCur->pDone = pNew; + }else{ + pCur->pStem = fuzzerInsert(pStem, pNew); + if( pCur->pStem==pNew ){ + return SQLITE_OK; + } + } }else{ - pCur->pStem = fuzzerInsert(pCur->pStem, pNew); + return SQLITE_NOMEM; } } @@ -381,7 +418,12 @@ static int fuzzerNext(sqlite3_vtab_cursor *cur){ while( (pStem = pCur->pStem)!=0 ){ if( fuzzerAdvance(pCur, pStem) ){ pCur->pStem = fuzzerInsert(pStem->pNext, pStem); - return SQLITE_OK; /* New word found */ + if( pCur->pStem!=pStem && (rc = fuzzerSeen(pCur, pStem))!=0 ){ + if( rc<0 ) return SQLITE_NOMEM; + continue; + }else{ + return SQLITE_OK; /* New word found */ + } } pCur->pStem = pStem->pNext; pStem->pNext = pCur->pDone; @@ -406,9 +448,10 @@ static int fuzzerFilter( ){ fuzzer_cursor *pCur = (fuzzer_cursor *)pVtabCursor; const char *zWord = 0; - pCur->rLimit = 2147483647; + fuzzer_stem *pStem; fuzzerClearCursor(pCur, 1); + pCur->rLimit = 2147483647; if( idxNum==1 ){ zWord = (const char*)sqlite3_value_text(argv[0]); }else if( idxNum==2 ){ @@ -418,8 +461,15 @@ static int fuzzerFilter( pCur->rLimit = (fuzzer_cost)sqlite3_value_int(argv[1]); } if( zWord==0 ) zWord = ""; - pCur->pStem = fuzzerNewStem(pCur, zWord, (fuzzer_cost)0); - if( pCur->pStem==0 ) return SQLITE_NOMEM; + pCur->pStem = pStem = fuzzerNewStem(pCur, zWord, (fuzzer_cost)0); + if( pStem==0 ) return SQLITE_NOMEM; + pCur->nullRule.pNext = pCur->pVtab->pRule; + pCur->nullRule.rCost = 0; + pCur->nullRule.nFrom = 0; + pCur->nullRule.nTo = 0; + pCur->nullRule.zFrom = ""; + pStem->pRule = &pCur->nullRule; + pStem->n = pStem->nBasis; return SQLITE_OK; } @@ -431,7 +481,7 @@ static int fuzzerColumn(sqlite3_vtab_cursor *cur, sqlite3_context *ctx, int i){ fuzzer_cursor *pCur = (fuzzer_cursor*)cur; if( i==0 ){ /* the "word" column */ - if( fuzzerComputeWord(pCur, pCur->pStem)==SQLITE_NOMEM ){ + if( fuzzerRender(pCur->pStem, &pCur->zBuf, &pCur->nBuf)==SQLITE_NOMEM ){ return SQLITE_NOMEM; } sqlite3_result_text(ctx, pCur->zBuf, -1, SQLITE_TRANSIENT); @@ -567,17 +617,16 @@ static int fuzzerUpdate( pVTab->zErrMsg = sqlite3_mprintf("cost must be positive"); return SQLITE_CONSTRAINT; } - nFrom = strlen(zFrom)+1; - nTo = strlen(zTo)+1; - if( nTo<4 ) nTo = 4; - pRule = sqlite3_malloc( sizeof(*pRule) + nFrom + nTo - 4 ); + nFrom = strlen(zFrom); + nTo = strlen(zTo); + pRule = sqlite3_malloc( sizeof(*pRule) + nFrom + nTo ); if( pRule==0 ){ return SQLITE_NOMEM; } - pRule->zFrom = &pRule->zTo[nTo]; + pRule->zFrom = &pRule->zTo[nTo+1]; pRule->nFrom = nFrom; - memcpy(pRule->zFrom, zFrom, nFrom); - memcpy(pRule->zTo, zTo, nTo); + memcpy(pRule->zFrom, zFrom, nFrom+1); + memcpy(pRule->zTo, zTo, nTo+1); pRule->nTo = nTo; pRule->rCost = rCost; pRule->pNext = p->pNewRule; diff --git a/test/fuzzer1.test b/test/fuzzer1.test index 1e33556aa3..aaceaf1ce1 100644 --- a/test/fuzzer1.test +++ b/test/fuzzer1.test @@ -32,8 +32,8 @@ do_test fuzzer1-1.1 { do_test fuzzer1-1.2 { db eval { INSERT INTO f1(cfrom, cto, cost) VALUES('e','a',1); - INSERT INTO f1(cfrom, cto, cost) VALUES('a','e',1); - INSERT INTO f1(cfrom, cto, cost) VALUES('e','o',2); + INSERT INTO f1(cfrom, cto, cost) VALUES('a','e',10); + INSERT INTO f1(cfrom, cto, cost) VALUES('e','o',100); } } {} diff --git a/test/progress.test b/test/progress.test old mode 100755 new mode 100644 diff --git a/tool/mkopts.tcl b/tool/mkopts.tcl old mode 100755 new mode 100644