From: dan Date: Tue, 17 Mar 2026 21:17:42 +0000 (+0000) Subject: Modify integrity-check so that it is only tolerant of very small distortions of real... X-Git-Tag: major-release~74^2~10 X-Git-Url: http://git.ipfire.org/gitweb.cgi?a=commitdiff_plain;h=362b9cecdbe24e66da8c5885a009457477d6aec1;p=thirdparty%2Fsqlite.git Modify integrity-check so that it is only tolerant of very small distortions of real values in indexed expression fields. FossilOrigin-Name: 82e6de835bde306778425d18ab1a15ae80fdf01af7f577ead89c09f9d53b5b2f --- diff --git a/manifest b/manifest index e57570c3c9..4092bacd96 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Improved\serror\shandling\sin\sOP_IFindKey\sand\sin\svdbeIsMatchingIndexKey(). -D 2026-03-17T18:55:24.081 +C Modify\sintegrity-check\sso\sthat\sit\sis\sonly\stolerant\sof\svery\ssmall\sdistortions\sof\sreal\svalues\sin\sindexed\sexpression\sfields. +D 2026-03-17T21:17:42.003 F .fossil-settings/binary-glob 61195414528fb3ea9693577e1980230d78a1f8b0a54c78cf1b9b24d0a409ed6a x F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea @@ -726,7 +726,7 @@ F src/parse.y 3b784d6083380a950e3b1b32ce5ddd303e8c7c209d8ab788df2c62aaf9ee8eb3 F src/pcache.c 588cc3c5ccaaadde689ed35ce5c5c891a1f7b1f4d1f56f6cf0143b74d8ee6484 F src/pcache.h 092b758d2c5e4dabb30eae46d8dfad77c0f70b16bf3ff1943f7a232b0fe0d4ba F src/pcache1.c 131ca0daf4e66b4608d2945ae76d6ed90de3f60539afbd5ef9ec65667a5f2fcd -F src/pragma.c aee946eefb15d3c587528391781c8cffe7b790aacd08929dcacd5f4638b10274 +F src/pragma.c 8b7aefee546ac7c5e599aee1ced7f40b1dba55f05e06b95dec9144ea7e541ce1 F src/prepare.c f6a6e28a281bd1d1da12f47d370a81af46159b40f73bf7fa0b276b664f9c8b7d F src/printf.c 9cff219dba73b1aa9a8113e83e962f03f7bea8b6eb51cefb25bc468d5a69fb2d F src/random.c 606b00941a1d7dd09c381d3279a058d771f406c5213c9932bbd93d5587be4b9c @@ -799,11 +799,11 @@ F src/upsert.c 215328c3f91623c520ec8672c44323553f12caeb4f01b1090ebdca99fdf7b4f1 F src/utf.c 7267c3fb9e2467020507601af3354c2446c61f444387e094c779dccd5ca62165 F src/util.c 0dbd633bdc509a1c967e4b49b1555820494d936131017634d7dec96c0b8343ce F src/vacuum.c d3d35d8ae893d419ade5fa196d761a83bddcbb62137a1a157ae751ef38b26e82 -F src/vdbe.c e0921e7046ef9f3c472b583ff8ee9a5db593d9be3570f22d44f69cb95a717dfa +F src/vdbe.c 034ccc09f315f6d30ed95b0e6af0a4279ed1e78579343e48cad9f12f2ed9f67e F src/vdbe.h 70e862ac8a11b590f8c1eaac17a0078429d42bc4ea3f757a9af0f451dd966a71 -F src/vdbeInt.h 9909bdaaa2ef3d47b05d93b3e22a4211903305f1ba0afb902c7448258c6418e2 +F src/vdbeInt.h f7157f110f88f1d9d8338c292faf23a9129f6712563ade2b408537c95e17bdef F src/vdbeapi.c 6cdcbe5c7afa754c998e73d2d5d2805556268362914b952811bdfb9c78a37cf1 -F src/vdbeaux.c 5b586e4b08c0ff2df2aa3b011afcb2382478444306940077105a10f8f816aa79 +F src/vdbeaux.c d6acd54e3dea2373a1d902a6f54718996fd765aa3ca2a4295d4026935337340c F src/vdbeblob.c b3f0640db9642fbdc88bd6ebcc83d6009514cafc98f062f675f2c8d505d82692 F src/vdbemem.c 317ec5e870ddb16951b606c9fe8be22baef22ecbe46f58fdefc259662238afb7 F src/vdbesort.c b69220f4ea9ffea5fdef34d968c60305444eea909252a81933b54c296d9cca70 @@ -1081,7 +1081,7 @@ F test/e_wal.test db7c33642711cf3c7959714b5f012aca08cacfa78da0382f95e849eb3ba66a F test/e_walauto.test 248af31e73c98df23476a22bdb815524c9dc3ba8 F test/e_walckpt.test 16e7d006e8687654ee59e7ad5a6d285ba23f0fe0eeb87f790afd6bc9cf1d1924 F test/e_walhook.test 01b494287ba9e60b70f6ebf3c6c62e0ffe01788e344a4846b08e5de0b344cb66 -F test/eiib1.test fac8ac065d8d62af280dbbbf07c2bc278f6c251f4823fa341f49564a56cfdcae +F test/eiib1.test 1b4e1f89c843e081b897219a1a11e7045e10170165b3e84d86ca9b8cae305357 F test/emptytable.test a38110becbdfa6325cd65cb588dca658cd885f62 F test/enc.test b5503a87b31cea8a5084c6e447383f9ca08933bd2f29d97b6b6201081b2343eb F test/enc2.test 872afe58db772e7dfa1ad8e0759f8cc820e9efc8172d460fae83023101c2e435 @@ -2194,8 +2194,8 @@ F tool/warnings-clang.sh bbf6a1e685e534c92ec2bfba5b1745f34fb6f0bc2a362850723a9ee F tool/warnings.sh d924598cf2f55a4ecbc2aeb055c10bd5f48114793e7ba25f9585435da29e7e98 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f F tool/winmain.c 00c8fb88e365c9017db14c73d3c78af62194d9644feaf60e220ab0f411f3604c -P 978cedcbd33165dadc7dffc118bde359eff6751ec9842230880cb827d9f7845f -R 531962050de8b772498e3016aa53cb5c -U drh -Z 4452c0c9088d2f943b3f97617ed9fb76 +P f7389cdb129d3386b7dfb8acacf84816cf10864c6800a9bd9a61c6364b850a31 +R 82803ca4f78db5dcdf077da2e4845bd5 +U dan +Z 1601c9d521a1531b56905fb2ee04daaf # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 40aec4c7fa..58b065a702 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -f7389cdb129d3386b7dfb8acacf84816cf10864c6800a9bd9a61c6364b850a31 +82e6de835bde306778425d18ab1a15ae80fdf01af7f577ead89c09f9d53b5b2f diff --git a/src/pragma.c b/src/pragma.c index ce3834a3a4..21799aaa14 100644 --- a/src/pragma.c +++ b/src/pragma.c @@ -2058,7 +2058,7 @@ void sqlite3Pragma( if( !isQuick ){ /* Omit the remaining tests for quick_check */ /* Validate index entries for the current row */ for(j=0, pIdx=pTab->pIndex; pIdx; pIdx=pIdx->pNext, j++){ - int jmp3, jmp4, jmp5, label6; + int jmp2, jmp3, jmp4, jmp5, label6; int kk; int ckUniq = sqlite3VdbeMakeLabel(pParse); if( pPk==pIdx ) continue; @@ -2069,9 +2069,20 @@ void sqlite3Pragma( /* Verify that an index entry exists for the current table row */ sqlite3VdbeAddOp4Int(v, OP_Found, iIdxCur+j, ckUniq, r1, pIdx->nColumn); VdbeCoverage(v); - sqlite3VdbeAddOp3(v, OP_IFindKey, iIdxCur+j, ckUniq, r1); + jmp2 = sqlite3VdbeAddOp3(v, OP_IFindKey, iIdxCur+j, ckUniq, r1); VdbeCoverage(v); sqlite3VdbeChangeP4(v, -1, (const char*)pIdx, P4_INDEX); + + sqlite3VdbeLoadString(v, 3, "WARNING: expression index "); + sqlite3VdbeLoadString(v, 4, pIdx->zName); + sqlite3VdbeAddOp3(v, OP_Concat, 4, 3, 3); + sqlite3VdbeLoadString(v, 4, " stores an imprecise value for row "); + sqlite3VdbeAddOp3(v, OP_Concat, 4, 3, 3); + sqlite3VdbeAddOp3(v, OP_Concat, 7, 3, 3); + integrityCheckResultRow(v); + sqlite3VdbeAddOp2(v, OP_Goto, 0, ckUniq); + + sqlite3VdbeJumpHere(v, jmp2); sqlite3VdbeLoadString(v, 3, "row "); sqlite3VdbeAddOp3(v, OP_Concat, 7, 3, 3); sqlite3VdbeLoadString(v, 4, " missing from index "); diff --git a/src/vdbe.c b/src/vdbe.c index 9617ef8f5a..32978479ad 100644 --- a/src/vdbe.c +++ b/src/vdbe.c @@ -6656,7 +6656,7 @@ case OP_IdxDelete: { rc = sqlite3BtreeIndexMoveto(pCrsr, &r, &res); if( rc ) goto abort_due_to_error; if( res!=0 ){ - rc = sqlite3VdbeFindIndexKey(pCrsr, pOp->p4.pIdx, 1, &r, &res); + rc = sqlite3VdbeFindIndexKey(pCrsr, pOp->p4.pIdx, &r, &res, 0); if( rc!=SQLITE_OK ) goto abort_due_to_error; if( res!=0 ){ if( !sqlite3WritableSchema(db) ){ @@ -7307,11 +7307,12 @@ case OP_IntegrityCk: { ** This opcode uses sqlite3VdbeFindIndexKey() to search around the current ** location for an index key for which all fields that are not indexed ** expressions or virtual columns match the expected values from the table. -** If one is found, jump to P2. Otherwise, fall through. +** If one is not found, jump to P2. Otherwise, fall through. */ case OP_IFindKey: { /* jump, in3 */ VdbeCursor *pC; int res; + int ulp; UnpackedRecord r; pC = p->apCsr[pOp->p1]; @@ -7324,12 +7325,10 @@ case OP_IFindKey: { /* jump, in3 */ r.nField = pOp->p4.pIdx->nColumn; r.pKeyInfo = pC->pKeyInfo; - rc = sqlite3VdbeFindIndexKey(pC->uc.pCursor, pOp->p4.pIdx, 0, &r, &res); + rc = sqlite3VdbeFindIndexKey(pC->uc.pCursor, pOp->p4.pIdx, &r, &res, 1); if( rc ) goto abort_due_to_error; - if( res==0 ){ - pC->nullRow = 0; - goto jump_to_p2; - } + if( res!=0 ) goto jump_to_p2; + pC->nullRow = 0; break; }; #endif /* SQLITE_OMIT_INTEGRITY_CHECK */ diff --git a/src/vdbeInt.h b/src/vdbeInt.h index 1e23f0e859..ac31d5afa8 100644 --- a/src/vdbeInt.h +++ b/src/vdbeInt.h @@ -687,7 +687,7 @@ void sqlite3VdbePreUpdateHook( Vdbe*,VdbeCursor*,int,const char*,Table*,i64,int,int); #endif int sqlite3VdbeTransferError(Vdbe *p); -int sqlite3VdbeFindIndexKey(BtCursor*, Index*, int, UnpackedRecord*, int*); +int sqlite3VdbeFindIndexKey(BtCursor*, Index*, UnpackedRecord*, int*, int); int sqlite3VdbeSorterInit(sqlite3 *, int, VdbeCursor *); void sqlite3VdbeSorterReset(sqlite3 *, VdbeSorter *); diff --git a/src/vdbeaux.c b/src/vdbeaux.c index db7f2c75d4..1549e55b1f 100644 --- a/src/vdbeaux.c +++ b/src/vdbeaux.c @@ -5397,15 +5397,59 @@ void sqlite3VdbeSetVarmask(Vdbe *v, int iVar){ } } + +/* +** Helper function for vdbeIsMatchingIndexKey(). Return true if column +** iCol should be ignored when comparing a record with a record from +** an index on disk. The field should be ignored if: +** +** * the corresponding bit in mask is set, and +** * either bIntegrity is false, or +** * the two Mem values are both real values that differ by +** BTREE_ULPDISTORTION or fewer ULPs. +*/ +static int vdbeSkipField( + Bitmask mask, /* Mask of indexed expression fields */ + int iCol, /* Column of index being considered */ + Mem *pMem1, /* Expected index value */ + Mem *pMem2, /* Actual indexed value */ + int bIntegrity +){ +#define BTREE_MANTISSA64 ((u64)0x0FFF << 52) +#define BTREE_ULPDISTORTION 2 + if( iCol>=BMS || (mask & MASKBIT(iCol))==0 ) return 0; + if( bIntegrity==0 ) return 1; + if( (pMem1->flags & MEM_Real) && (pMem2->flags & MEM_Real) ){ + u64 r1 = *(u64*)&pMem1->u.r; + u64 r2 = *(u64*)&pMem2->u.r; + if( (r1 & BTREE_MANTISSA64)==(r2 & BTREE_MANTISSA64) ){ + u64 diff; + r1 = r1 & ~BTREE_MANTISSA64; + r2 = r2 & ~BTREE_MANTISSA64; + diff = MIN(r1-r2, r2-r1); + if( diff<=BTREE_ULPDISTORTION ){ + return 1; + } + } + } + return 0; +} + /* ** This function compares the unpacked record with the current key that -** cursor pCur points to, ignoring any fields for which the corresponding -** bit in parameter "mask" is set. Return the usual less than zero, zero, or -** greater than zero if the remaining fields of the cursor cursor key are less -** than, equal to or greater than those in (*p). +** cursor pCur points to. If bInt is false, all fields for which the +** corresponding bit in parameter "mask" is set are ignored. Or, if +** bInt is true, then a difference of BTREE_ULPDISTORTION or fewer ULPs +** in real values is overlooked for fields with the corresponding bit +** set in mask. +** +** Return the usual less than zero, zero, or greater than zero if the +** remaining fields of the cursor cursor key are less than, equal to or +** greater than those in (*p). */ static int vdbeIsMatchingIndexKey( BtCursor *pCur, /* Cursor open on index */ + int bInt, Bitmask mask, UnpackedRecord *p, /* Index key being deleted */ int *piRes /* 0 for a match, non-zero for not a match */ @@ -5450,10 +5494,12 @@ static int vdbeIsMatchingIndexKey( nSerial = sqlite3VdbeSerialTypeLen(iSerial); if( (idxRec+nSerial)>nRec ){ rc = SQLITE_CORRUPT_BKPT; - }else if( ii>=BMS || (mask & MASKBIT(ii))==0 ){ + }else{ sqlite3VdbeSerialGet(&aRec[idxRec], iSerial, &mem); - res = sqlite3MemCompare(&mem, &p->aMem[ii], p->pKeyInfo->aColl[ii]); - if( res!=0 ) break; + if( vdbeSkipField(mask, ii, &p->aMem[ii], &mem, bInt)==0 ){ + res = sqlite3MemCompare(&mem, &p->aMem[ii], p->pKeyInfo->aColl[ii]); + if( res!=0 ) break; + } } idxRec += sqlite3VdbeSerialTypeLen(iSerial); } @@ -5467,16 +5513,14 @@ static int vdbeIsMatchingIndexKey( } /* -** This is called when the record in (*p) is to be deleted from the index -** opened by cursor pCur, but an exact match for the record was not found -** in the index. The result of searching for it was (*pRes) - if (*pRes) -** is -1, then the cursor points at a record that is smaller than (*p), -** if it is +1, then it points to a record greater than (*p). +** This is called when the record in (*p) should be found in the index +** opened by cursor pCur, but was not. This may happen as part of a DELETE +** operation or an integrity check. ** ** One reason that an exact match was not found may be the EIIB bug - that ** a text-to-float conversion may have caused a real value in record (*p) ** to be slightly different from its counterpart on disk. This function -** attempts to find the right record to delete. If it does find the right +** attempts to find the right index record. If it does find the right ** record, it leaves *pCur pointing to it and sets (*pRes) to 0 before ** returning. Otherwise, (*pRes) is set to non-zero and an SQLite error ** code returned. @@ -5484,15 +5528,21 @@ static int vdbeIsMatchingIndexKey( ** The algorithm used to find the correct record is: ** ** * Scan up to BTREE_FDK_RANGE entries either side of the current entry. +** If parameter bIntegrity is false, then all fields that are indexed +** expressions or virtual table columns are omitted from the comparison. +** If bIntegrity is true, then small differences in real values in +** such fields are overlooked, but they are not omitted from the comparison +** altogether. ** -** * If the above fails to find an entry to delete, search the entire index. +** * If the above fails to find an entry and bIntegrity is false, search +** the entire index. */ int sqlite3VdbeFindIndexKey( BtCursor *pCur, Index *pIdx, - int bExhaustive, UnpackedRecord *p, - int *pRes + int *pRes, + int bIntegrity ){ #define BTREE_FDK_RANGE 10 int nStep = 0; @@ -5540,7 +5590,7 @@ int sqlite3VdbeFindIndexKey( ** current cursor entry if (nStep>=0), or the entire index if (nStep<0). */ while( sqlite3BtreeCursorIsValidNN(pCur) ){ for(ii=0; rc==SQLITE_OK && (ii