From 31d4a3d27f363e5deaf44cc5b4e8f2c29f84c1d7 Mon Sep 17 00:00:00 2001
From: mistachkin <mistachkin@noemail.net>
Date: Wed, 6 Mar 2024 20:58:08 +0000
Subject: [PATCH] Make sure key comprisons are done correctly if the index key
 contains NaN values that have not been shifted into NULLs. That can only
 happen due to database corruption, but we need to deal with it nevertheless.

FossilOrigin-Name: 7ed922d99ca9801dc1926df529b2fa93be374f587885282ab6e307ea6cf2ccee
---
 manifest      | 14 +++++++-------
 manifest.uuid |  2 +-
 src/vdbeaux.c | 36 ++++++++++++++++++++++++++++++++----
 3 files changed, 40 insertions(+), 12 deletions(-)

diff --git a/manifest b/manifest
index ae72029b7d..3795432334 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C When\sinserting\sa\s3-byte\scell\sinto\sa\sbtree,\sensure\sthat\sthe\sextra\spadding\sbyte\sis\s0x00.\s\sThis\sis\snot\snecessary\sfor\ssecurity,\sas\sfar\sas\sI\scan\stell,\sbut\sit\sseems\slike\sa\sreasonable\sprecaution.
-D 2024-03-06T20:56:52.386
+C Make\ssure\skey\scomprisons\sare\sdone\scorrectly\sif\sthe\sindex\skey\scontains\sNaN\svalues\sthat\shave\snot\sbeen\sshifted\sinto\sNULLs.\sThat\scan\sonly\shappen\sdue\sto\sdatabase\scorruption,\sbut\swe\sneed\sto\sdeal\swith\sit\snevertheless.
+D 2024-03-06T20:58:08.260
 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1
 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea
 F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724
@@ -812,7 +812,7 @@ F src/vdbe.c b2a45392265cb83f60251406039bf5255462d4a6d8deb05b2eaccab5abb2e20b
 F src/vdbe.h 88e19a982df9027ec1c177c793d1a5d34dc23d8f06e3b2d997f43688b05ee0eb
 F src/vdbeInt.h 949669dfd8a41550d27dcb905b494f2ccde9a2e6c1b0b04daa1227e2e74c2b2c
 F src/vdbeapi.c 8f57d60c89da0b60e6d4e272358c511f6bae4e24330bdb11f8b42f986d1bf21b
-F src/vdbeaux.c c5a471b34e9c4cfc0295a3e10734fd197670ffaebcb742f284c8e17e8026ceea
+F src/vdbeaux.c 56900c9a41f23260c8346f212bd6005eb9171f9a2f70d0cfb1441a078a0e4b84
 F src/vdbeblob.c 13f9287b55b6356b4b1845410382d6bede203ceb29ef69388a4a3d007ffacbe5
 F src/vdbemem.c 0012d5f01cc866833847c2f3ae4c318ac53a1cb3d28acad9c35e688039464cf0
 F src/vdbesort.c 237840ca1947511fa59bd4e18b9eeae93f2af2468c34d2427b059f896230a547
@@ -2160,9 +2160,9 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P 22a33f133e3e833c960f46e1b4c74043010181ff7ba9e57f2d009b65d3275d82
-Q +5766f1279dab91e030d4dcf5133659e5cedf914a1628ccf00d67d8e50a9957fd
-R b691fdaab2b47111553543c1888af596
+P f3e841873dffaa8cd2f4f3cf260487ed402febb99b9ccad04e2a1d481e903aca
+Q +7e4c743f9e6ef33500795543e6db9a77c533025bf00c2ee97abd433a3871b5a1
+R 8fd592fe8ce103d2c4d8c550e6004690
 U mistachkin
-Z 6c5c8f9b4835564916ac3776cb0ec051
+Z 87b197a2d34aa64f4c763691cd702a68
 # Remove this line to create a well-formed Fossil manifest.
diff --git a/manifest.uuid b/manifest.uuid
index 6618d115d7..50a5e0932c 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-f3e841873dffaa8cd2f4f3cf260487ed402febb99b9ccad04e2a1d481e903aca
\ No newline at end of file
+7ed922d99ca9801dc1926df529b2fa93be374f587885282ab6e307ea6cf2ccee
\ No newline at end of file
diff --git a/src/vdbeaux.c b/src/vdbeaux.c
index 420365e930..fe0dbd6b0e 100644
--- a/src/vdbeaux.c
+++ b/src/vdbeaux.c
@@ -4060,6 +4060,23 @@ static void serialGet(
     pMem->flags = IsNaN(x) ? MEM_Null : MEM_Real;
   }
 }
+static int serialGet7(
+  const unsigned char *buf,     /* Buffer to deserialize from */
+  Mem *pMem                     /* Memory cell to write value into */
+){
+  u64 x = FOUR_BYTE_UINT(buf);
+  u32 y = FOUR_BYTE_UINT(buf+4);
+  x = (x<<32) + y;
+  assert( sizeof(x)==8 && sizeof(pMem->u.r)==8 );
+  swapMixedEndianFloat(x);
+  memcpy(&pMem->u.r, &x, sizeof(x));
+  if( IsNaN(x) ){
+    pMem->flags = MEM_Null;
+    return 1;
+  }
+  pMem->flags = MEM_Real;
+  return 0;
+}
 void sqlite3VdbeSerialGet(
   const unsigned char *buf,     /* Buffer to deserialize from */
   u32 serial_type,              /* Serial type to deserialize */
@@ -4739,7 +4756,7 @@ int sqlite3VdbeRecordCompareWithSkip(
       }else if( serial_type==0 ){
         rc = -1;
       }else if( serial_type==7 ){
-        sqlite3VdbeSerialGet(&aKey1[d1], serial_type, &mem1);
+        serialGet7(&aKey1[d1], &mem1);
         rc = -sqlite3IntFloatCompare(pRhs->u.i, mem1.u.r);
       }else{
         i64 lhs = vdbeRecordDecodeInt(serial_type, &aKey1[d1]);
@@ -4764,14 +4781,18 @@ int sqlite3VdbeRecordCompareWithSkip(
       }else if( serial_type==0 ){
         rc = -1;
       }else{
-        sqlite3VdbeSerialGet(&aKey1[d1], serial_type, &mem1);
         if( serial_type==7 ){
-          if( mem1.u.r<pRhs->u.r ){
+          if( serialGet7(&aKey1[d1], &mem1) ){
+            rc = -1;  /* mem1 is a NaN */
+          }else if( mem1.u.r<pRhs->u.r ){
             rc = -1;
           }else if( mem1.u.r>pRhs->u.r ){
             rc = +1;
+          }else{
+            assert( rc==0 );
           }
         }else{
+          sqlite3VdbeSerialGet(&aKey1[d1], serial_type, &mem1);
           rc = sqlite3IntFloatCompare(mem1.u.i, pRhs->u.r);
         }
       }
@@ -4841,7 +4862,14 @@ int sqlite3VdbeRecordCompareWithSkip(
     /* RHS is null */
     else{
       serial_type = aKey1[idx1];
-      rc = (serial_type!=0 && serial_type!=10);
+      if( serial_type==0
+       || serial_type==10
+       || (serial_type==7 && serialGet7(&aKey1[d1], &mem1)!=0)
+      ){
+        assert( rc==0 );
+      }else{
+        rc = 1;
+      }
     }
 
     if( rc!=0 ){
-- 
2.47.2