When a floating-point RTREE is presented with large integer constraints -

integers that are too big to be represented exactly by a float - then take
extra steps to ensure that all possibly relevant entries in the RTREE are
returned, even in boundary cases.  Fix for the problem identified by
[forum:/forumpost/da70ee0d0d|forum post da70ee0d0d].

FossilOrigin-Name: bfd8d9100015f3e3fb011698963d670bd89b64ec8a8ab931e0c6c3076b029377

diff --git a/ext/rtree/rtree.c b/ext/rtree/rtree.c
index d24e9eb84cd1cb60f788a6dac2dd6e00b5a699ce..ec56d3261ff917ba8da62ab095ff84ea8261590a 100644 (file)
--- a/ext/rtree/rtree.c
+++ b/ext/rtree/rtree.c
@@ -1907,7 +1907,20 @@ static int rtreeFilter(
             p->pInfo->nCoord = pRtree->nDim2;
             p->pInfo->anQueue = pCsr->anQueue;
             p->pInfo->mxLevel = pRtree->iDepth + 1;
-          }else if( eType==SQLITE_INTEGER || eType==SQLITE_FLOAT ){
+          }else if( eType==SQLITE_INTEGER ){
+            sqlite3_int64 iVal = sqlite3_value_int64(argv[ii]);
+#ifdef SQLITE_RTREE_INT_ONLY
+            p->u.rValue = iVal;
+#else
+            p->u.rValue = (double)iVal;
+            if( iVal>=((sqlite3_int64)1)<<48
+             || -iVal>=((sqlite3_int64)1)<<48
+            ){
+              if( p->op==RTREE_LT ) p->op = RTREE_LE;
+              if( p->op==RTREE_GT ) p->op = RTREE_GE;
+            }
+#endif
+          }else if( eType==SQLITE_FLOAT ){
 #ifdef SQLITE_RTREE_INT_ONLY
             p->u.rValue = sqlite3_value_int64(argv[ii]);
 #else
@@ -2038,11 +2051,12 @@ static int rtreeBestIndex(sqlite3_vtab *tab, sqlite3_index_info *pIdxInfo){
         || p->op==SQLITE_INDEX_CONSTRAINT_MATCH)
     ){
       u8 op;
+      u8 doOmit = 1;
       switch( p->op ){
-        case SQLITE_INDEX_CONSTRAINT_EQ:    op = RTREE_EQ;    break;
-        case SQLITE_INDEX_CONSTRAINT_GT:    op = RTREE_GT;    break;
+        case SQLITE_INDEX_CONSTRAINT_EQ:    op = RTREE_EQ;    doOmit = 0; break;
+        case SQLITE_INDEX_CONSTRAINT_GT:    op = RTREE_GT;    doOmit = 0; break;
         case SQLITE_INDEX_CONSTRAINT_LE:    op = RTREE_LE;    break;
-        case SQLITE_INDEX_CONSTRAINT_LT:    op = RTREE_LT;    break;
+        case SQLITE_INDEX_CONSTRAINT_LT:    op = RTREE_LT;    doOmit = 0; break;
         case SQLITE_INDEX_CONSTRAINT_GE:    op = RTREE_GE;    break;
         case SQLITE_INDEX_CONSTRAINT_MATCH: op = RTREE_MATCH; break;
         default:                            op = 0;           break;
@@ -2051,7 +2065,7 @@ static int rtreeBestIndex(sqlite3_vtab *tab, sqlite3_index_info *pIdxInfo){
         zIdxStr[iIdx++] = op;
         zIdxStr[iIdx++] = (char)(p->iColumn - 1 + '0');
         pIdxInfo->aConstraintUsage[ii].argvIndex = (iIdx/2);
-        pIdxInfo->aConstraintUsage[ii].omit = (op!=RTREE_EQ);
+        pIdxInfo->aConstraintUsage[ii].omit = doOmit;
       }
     }
   }

diff --git a/ext/rtree/rtree1.test b/ext/rtree/rtree1.test
index 033bd7d9e75f4dd2549fcc4ac8bc88765156bc08..633d0a5d5fa1d9830ac39dfb4fc869c4e134b2a5 100644 (file)
--- a/ext/rtree/rtree1.test
+++ b/ext/rtree/rtree1.test
@@ -770,4 +770,18 @@ do_execsql_test 21.1 {
  SELECT x1=9223372036854775807 FROM t1;
 } {0}
 
+# 2023-05-22 https://sqlite.org/forum/forumpost/da70ee0d0d
+# Round-off error associated with using large integer constraints on
+# a rtree search.
+#
+reset_db
+do_execsql_test 22.0 {
+  CREATE VIRTUAL TABLE t1 USING rtree ( id, x0, x1 );  
+  INSERT INTO t1 VALUES (123, 9223372036854775799, 9223372036854775800);
+  SELECT id FROM t1 WHERE x0 > 9223372036854775807;
+} {123}
+do_execsql_test 22.1 {
+  SELECT id, x0 > 9223372036854775807 AS 'a0' FROM t1;
+} {123 1}
+
 finish_test

diff --git a/manifest b/manifest
index a59c9834cd3c869451ceaee1c3df7f21e41a8a7e..05b37d07e9523f2ece432cdbc6912d95a2ce263b 100644 (file)
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Fix\sa\sbuffer\soverrun\sthat\scould\soccur\sin\sfts5\swhen\sprocessing\scorrupt\srecords.
-D 2023-05-22T11:02:15.145
+C When\sa\sfloating-point\sRTREE\sis\spresented\swith\slarge\sinteger\sconstraints\s-\s\nintegers\sthat\sare\stoo\sbig\sto\sbe\srepresented\sexactly\sby\sa\sfloat\s-\sthen\stake\nextra\ssteps\sto\sensure\sthat\sall\spossibly\srelevant\sentries\sin\sthe\sRTREE\sare\nreturned,\seven\sin\sboundary\scases.\s\sFix\sfor\sthe\sproblem\sidentified\sby\n[forum:/forumpost/da70ee0d0d|forum\spost\sda70ee0d0d].
+D 2023-05-22T16:35:21.265
 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1
 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea
 F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724
@@ -403,9 +403,9 @@ F ext/repair/test/checkindex01.test b530f141413b587c9eb78ff734de6bb79bc3515c3350
 F ext/repair/test/test.tcl 686d76d888dffd021f64260abf29a55c57b2cedfa7fc69150b42b1d6119aac3c
 F ext/rtree/README 6315c0d73ebf0ec40dedb5aa0e942bc8b54e3761
 F ext/rtree/geopoly.c 971e0b5bd9adaf0811feb8c0842a310811159da10319eb0e74fdb42bf26b99ca
-F ext/rtree/rtree.c f58c50fc518623a5c6292b00b923a1e4de6df941638783a12c80b9639b0e40bb
+F ext/rtree/rtree.c 33d6642fcf956e62ca0e4aa06631c3529e3a9ac9bb4108542d81ac2bd0a8d24a
 F ext/rtree/rtree.h 4a690463901cb5e6127cf05eb8e642f127012fd5003830dbc974eca5802d9412
-F ext/rtree/rtree1.test 8cb77d2664be8b7c64c9ad5332ee564312a66a3649527e2bcf20194c4826b799
+F ext/rtree/rtree1.test 877d40b8b61b1f88cec9d4dc0ff8334f5b05299fac12a35141532e2881860e9d
 F ext/rtree/rtree2.test 9d9deddbb16fd0c30c36e6b4fdc3ee3132d765567f0f9432ee71e1303d32603d
 F ext/rtree/rtree3.test 272594f88c344e973864008bbe4c71fd3a41a264c097d568593ee7886d83d409
 F ext/rtree/rtree4.test 304de65d484540111b896827e4261815e5dca4ce28eeecd58be648cd73452c4b
@@ -2070,8 +2070,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P 3181c50540df0eff6cb5db79bb477c469bb7b73b0692260ba600db200fcef4ac
-R e5fab02aa334d098fe321a07525698af
-U dan
-Z ccaded5d38d053b56d3383a48fe3cfd9
+P 4891dbd938f4bfd345eaef01f2addea9512eaa98f860844c73abb907b6a8e0e8
+R ed1374de25fa501b0ccb77bcd8313770
+U drh
+Z e2f1ed41df27fe2b21731bc19aa8abb0
 # Remove this line to create a well-formed Fossil manifest.

diff --git a/manifest.uuid b/manifest.uuid
index f881045bf3ef570162f09f446dd291470c96dc2f..0e282d44786b370c0150abb05ee294a502a5c5df 100644 (file)
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-4891dbd938f4bfd345eaef01f2addea9512eaa98f860844c73abb907b6a8e0e8
\ No newline at end of file
+bfd8d9100015f3e3fb011698963d670bd89b64ec8a8ab931e0c6c3076b029377
\ No newline at end of file