Make sure key comprisons are done correctly if the index key contains NaN values...

FossilOrigin-Name: 7ed922d99ca9801dc1926df529b2fa93be374f587885282ab6e307ea6cf2ccee

diff --git a/manifest b/manifest
index ae72029b7d0f4e439d87fd09260e8cc2d920ee55..3795432334938ea2cc596a5f7bdde6fd3803d27f 100644 (file)
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C When\sinserting\sa\s3-byte\scell\sinto\sa\sbtree,\sensure\sthat\sthe\sextra\spadding\sbyte\sis\s0x00.\s\sThis\sis\snot\snecessary\sfor\ssecurity,\sas\sfar\sas\sI\scan\stell,\sbut\sit\sseems\slike\sa\sreasonable\sprecaution.
-D 2024-03-06T20:56:52.386
+C Make\ssure\skey\scomprisons\sare\sdone\scorrectly\sif\sthe\sindex\skey\scontains\sNaN\svalues\sthat\shave\snot\sbeen\sshifted\sinto\sNULLs.\sThat\scan\sonly\shappen\sdue\sto\sdatabase\scorruption,\sbut\swe\sneed\sto\sdeal\swith\sit\snevertheless.
+D 2024-03-06T20:58:08.260
 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1
 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea
 F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724
@@ -812,7 +812,7 @@ F src/vdbe.c b2a45392265cb83f60251406039bf5255462d4a6d8deb05b2eaccab5abb2e20b
 F src/vdbe.h 88e19a982df9027ec1c177c793d1a5d34dc23d8f06e3b2d997f43688b05ee0eb
 F src/vdbeInt.h 949669dfd8a41550d27dcb905b494f2ccde9a2e6c1b0b04daa1227e2e74c2b2c
 F src/vdbeapi.c 8f57d60c89da0b60e6d4e272358c511f6bae4e24330bdb11f8b42f986d1bf21b
-F src/vdbeaux.c c5a471b34e9c4cfc0295a3e10734fd197670ffaebcb742f284c8e17e8026ceea
+F src/vdbeaux.c 56900c9a41f23260c8346f212bd6005eb9171f9a2f70d0cfb1441a078a0e4b84
 F src/vdbeblob.c 13f9287b55b6356b4b1845410382d6bede203ceb29ef69388a4a3d007ffacbe5
 F src/vdbemem.c 0012d5f01cc866833847c2f3ae4c318ac53a1cb3d28acad9c35e688039464cf0
 F src/vdbesort.c 237840ca1947511fa59bd4e18b9eeae93f2af2468c34d2427b059f896230a547
@@ -2160,9 +2160,9 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P 22a33f133e3e833c960f46e1b4c74043010181ff7ba9e57f2d009b65d3275d82
-Q +5766f1279dab91e030d4dcf5133659e5cedf914a1628ccf00d67d8e50a9957fd
-R b691fdaab2b47111553543c1888af596
+P f3e841873dffaa8cd2f4f3cf260487ed402febb99b9ccad04e2a1d481e903aca
+Q +7e4c743f9e6ef33500795543e6db9a77c533025bf00c2ee97abd433a3871b5a1
+R 8fd592fe8ce103d2c4d8c550e6004690
 U mistachkin
-Z 6c5c8f9b4835564916ac3776cb0ec051
+Z 87b197a2d34aa64f4c763691cd702a68
 # Remove this line to create a well-formed Fossil manifest.

diff --git a/manifest.uuid b/manifest.uuid
index 6618d115d7af599b3c7e6007721e36779076f3a0..50a5e0932c6aed3dc4e87e79a4e850c54c2a8839 100644 (file)
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-f3e841873dffaa8cd2f4f3cf260487ed402febb99b9ccad04e2a1d481e903aca
\ No newline at end of file
+7ed922d99ca9801dc1926df529b2fa93be374f587885282ab6e307ea6cf2ccee
\ No newline at end of file

diff --git a/src/vdbeaux.c b/src/vdbeaux.c
index 420365e930922f1a32cdae913e681585a0829441..fe0dbd6b0ea9824c8067e698c4038eb1e4239e8f 100644 (file)
--- a/src/vdbeaux.c
+++ b/src/vdbeaux.c
@@ -4060,6 +4060,23 @@ static void serialGet(
     pMem->flags = IsNaN(x) ? MEM_Null : MEM_Real;
   }
 }
+static int serialGet7(
+  const unsigned char *buf,     /* Buffer to deserialize from */
+  Mem *pMem                     /* Memory cell to write value into */
+){
+  u64 x = FOUR_BYTE_UINT(buf);
+  u32 y = FOUR_BYTE_UINT(buf+4);
+  x = (x<<32) + y;
+  assert( sizeof(x)==8 && sizeof(pMem->u.r)==8 );
+  swapMixedEndianFloat(x);
+  memcpy(&pMem->u.r, &x, sizeof(x));
+  if( IsNaN(x) ){
+    pMem->flags = MEM_Null;
+    return 1;
+  }
+  pMem->flags = MEM_Real;
+  return 0;
+}
 void sqlite3VdbeSerialGet(
   const unsigned char *buf,     /* Buffer to deserialize from */
   u32 serial_type,              /* Serial type to deserialize */
@@ -4739,7 +4756,7 @@ int sqlite3VdbeRecordCompareWithSkip(
       }else if( serial_type==0 ){
         rc = -1;
       }else if( serial_type==7 ){
-        sqlite3VdbeSerialGet(&aKey1[d1], serial_type, &mem1);
+        serialGet7(&aKey1[d1], &mem1);
         rc = -sqlite3IntFloatCompare(pRhs->u.i, mem1.u.r);
       }else{
         i64 lhs = vdbeRecordDecodeInt(serial_type, &aKey1[d1]);
@@ -4764,14 +4781,18 @@ int sqlite3VdbeRecordCompareWithSkip(
       }else if( serial_type==0 ){
         rc = -1;
       }else{
-        sqlite3VdbeSerialGet(&aKey1[d1], serial_type, &mem1);
         if( serial_type==7 ){
-          if( mem1.u.r<pRhs->u.r ){
+          if( serialGet7(&aKey1[d1], &mem1) ){
+            rc = -1;  /* mem1 is a NaN */
+          }else if( mem1.u.r<pRhs->u.r ){
             rc = -1;
           }else if( mem1.u.r>pRhs->u.r ){
             rc = +1;
+          }else{
+            assert( rc==0 );
           }
         }else{
+          sqlite3VdbeSerialGet(&aKey1[d1], serial_type, &mem1);
           rc = sqlite3IntFloatCompare(mem1.u.i, pRhs->u.r);
         }
       }
@@ -4841,7 +4862,14 @@ int sqlite3VdbeRecordCompareWithSkip(
     /* RHS is null */
     else{
       serial_type = aKey1[idx1];
-      rc = (serial_type!=0 && serial_type!=10);
+      if( serial_type==0
+       || serial_type==10
+       || (serial_type==7 && serialGet7(&aKey1[d1], &mem1)!=0)
+      ){
+        assert( rc==0 );
+      }else{
+        rc = 1;
+      }
     }
 
     if( rc!=0 ){