Experimental enhancement to automatically trim NULL values from the end of

records, for a reduced disk footprint.  This change also involves increasing
the P5 operand from 8 to 16 bits.

FossilOrigin-Name: 118ded403b95050b74ae2b03919c43d614094a32

diff --git a/manifest b/manifest
index edea9fbb44cb289db17e011a94ffd436be2e364a..86115ea620cdd7ccc877c48431cc457a5785eea0 100644 (file)
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Ensure\sthat\ssqlite3_blob_reopen()\scorrectly\shandles\sshort\srows.\s\nFix\sfor\sticket\s[e6e962d6b0f06f46e].
-D 2017-01-25T14:38:19.847
+C Experimental\senhancement\sto\sautomatically\strim\sNULL\svalues\sfrom\sthe\send\sof\nrecords,\sfor\sa\sreduced\sdisk\sfootprint.\s\sThis\schange\salso\sinvolves\sincreasing\nthe\sP5\soperand\sfrom\s8\sto\s16\sbits.
+D 2017-01-25T14:58:27.321
 F Makefile.in 5f415e7867296d678fed2e6779aea10c1318b4bc
 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434
 F Makefile.msc b8ca53350ae545e3562403d5da2a69cec79308da
@@ -352,7 +352,7 @@ F src/hash.c 63d0ee752a3b92d4695b2b1f5259c4621b2cfebd
 F src/hash.h ab34c5c54a9e9de2e790b24349ba5aab3dbb4fd4
 F src/hwtime.h 747c1bbe9df21a92e9c50f3bbec1de841dc5e5da
 F src/in-operator.md 10cd8f4bcd225a32518407c2fb2484089112fd71
-F src/insert.c 05e47e2de7b712a3a4148cd469e5f60873f5ef13
+F src/insert.c 07ccec2c1fb32dd2c5fbfd63426e98dc2a79a0fa
 F src/legacy.c 75d3023be8f0d2b99d60f905090341a03358c58e
 F src/loadext.c a68d8d1d14cf7488bb29dc5311cb1ce9a4404258
 F src/main.c e207b81542d13b9f13d61e78ca441f9781f055b0
@@ -395,7 +395,7 @@ F src/shell.c 59de9acab4423a536277653f2a9dcdd1307989f3
 F src/sqlite.h.in 1971ab9709e010d52a02a1a6276d5a2f9b947476
 F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8
 F src/sqlite3ext.h 8648034aa702469afb553231677306cc6492a1ae
-F src/sqliteInt.h 525c061ae9aafc8d4720a018d82f0936d9eee5ab
+F src/sqliteInt.h dc587b0727c2bc888d78d200d891620080567691
 F src/sqliteLimit.h c0373387c287c8d0932510b5547ecde31b5da247
 F src/status.c a9e66593dfb28a9e746cba7153f84d49c1ddc4b1
 F src/table.c b46ad567748f24a326d9de40e5b9659f96ffff34
@@ -457,11 +457,11 @@ F src/update.c b356b29d04c71f33c779f2cb557cf953819bdd7a
 F src/utf.c 699001c79f28e48e9bcdf8a463da029ea660540c
 F src/util.c a88b0466fddf445ce752226d4698ca3faada620a
 F src/vacuum.c 33c174b28886b2faf26e503b5a49a1c01a9b1c16
-F src/vdbe.c c7add5978cb84ae3a7bcb16f8b56cb3bbdf04b7e
-F src/vdbe.h b0866e4191f096f1c987a84b042c3599bdf5423b
+F src/vdbe.c b1f2448184fa58c66fc94591004b1258964ef9da
+F src/vdbe.h 59998ffd71d7caa8886bc78dafaf8caeccd4c13c
 F src/vdbeInt.h 281cb70332dc8b593b8c7afe776f3a2ba7d4255e
 F src/vdbeapi.c d6ebaa465f070eb1af8ba4e7b34583ece87bdd24
-F src/vdbeaux.c 35c9a9908174e5a26c96d15e1f98214814a39147
+F src/vdbeaux.c 8a2446741a2ec1072e744bc50f69b8b9e6c36592
 F src/vdbeblob.c 2b3d1ad915dbe5dc92c48759dc18fa8c697e78e5
 F src/vdbemem.c 3b5a9a5b375458d3e12a50ae1aaa41eeec2175fd
 F src/vdbesort.c eda25cb2d1727efca6f7862fea32b8aa33c0face
@@ -1547,8 +1547,10 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P 0803390c152141c9ab4e7a28406b2a5d72a5c2fa 57d8dad35c2a9ab635e954dce7f3986ae1ca8ed2
-R d15a5cee6a51c7118a4d78b7d0cbd2de
-T +closed 57d8dad35c2a9ab635e954dce7f3986ae1ca8ed2
+P 8cd1a4451cce1fe28f462800e2be1dee1735c0d0
+R 8038c1c21360d07be65589e30167ecdf
+T *branch * trim-nulls
+T *sym-trim-nulls *
+T -sym-trunk *
 U drh
-Z f9af43b146da8d8246d0b97014ae0d3a
+Z d170ee380cb17c0b674fe48dc7bc03d5

diff --git a/manifest.uuid b/manifest.uuid
index d15e64c7de36d8ec4f3eb7811e29e3feee30556a..6aea75dca3bf32d254151452ccd6284e1bc7f087 100644 (file)
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-8cd1a4451cce1fe28f462800e2be1dee1735c0d0
\ No newline at end of file
+118ded403b95050b74ae2b03919c43d614094a32
\ No newline at end of file

diff --git a/src/insert.c b/src/insert.c
index 93c22ae3f50c4545e313fd420fb822f42a5da75e..5bb8ba2ff9ed6793713972ea913f3287f5ce7a58 100644 (file)
--- a/src/insert.c
+++ b/src/insert.c
@@ -1668,6 +1668,23 @@ void sqlite3GenerateConstraintChecks(
   VdbeModuleComment((v, "END: GenCnstCks(%d)", seenReplace));
 }
 
+/*
+** Change the P5 operand on the last opcode (which should be an OP_MakeRecord)
+** to be the number of columns in table pTab that must not be NULL-trimmed.
+**
+** Or if no columns of pTab may be NULL-trimmed, leave P5 at zero.
+*/
+void sqlite3SetMakeRecordP5(Vdbe *v, Table *pTab){
+  u16 i;
+
+  /* Records with omitted columns are only allowed for schema format
+  ** version 2 and later (SQLite version 3.1.4, 2005-02-20). */
+  if( pTab->pSchema->file_format<2 ) return;
+
+  for(i=pTab->nCol; i>1 && pTab->aCol[i-1].pDflt==0; i--){}
+  sqlite3VdbeChangeP5(v, i);
+}
+
 /*
 ** This routine generates code to finish the INSERT or UPDATE operation
 ** that was started by a prior call to sqlite3GenerateConstraintChecks.
@@ -1727,6 +1744,7 @@ void sqlite3CompleteInsertion(
   regData = regNewData + 1;
   regRec = sqlite3GetTempReg(pParse);
   sqlite3VdbeAddOp3(v, OP_MakeRecord, regData, pTab->nCol, regRec);
+  sqlite3SetMakeRecordP5(v, pTab);
   if( !bAffinityDone ){
     sqlite3TableAffinity(v, pTab, 0);
     sqlite3ExprCacheAffinityChange(pParse, regData, pTab->nCol);

diff --git a/src/sqliteInt.h b/src/sqliteInt.h
index 6344cac5f1a78f11c7044dd490d2862c8f5ba0fa..ceae4dcb1f8d4568c795effea42ca911ca48a57c 100644 (file)
--- a/src/sqliteInt.h
+++ b/src/sqliteInt.h
@@ -3790,6 +3790,7 @@ int sqlite3GenerateIndexKey(Parse*, Index*, int, int, int, int*,Index*,int);
 void sqlite3ResolvePartIdxLabel(Parse*,int);
 void sqlite3GenerateConstraintChecks(Parse*,Table*,int*,int,int,int,int,
                                      u8,u8,int,int*,int*);
+void sqlite3SetMakeRecordP5(Vdbe*,Table*);
 void sqlite3CompleteInsertion(Parse*,Table*,int,int,int,int*,int,int,int);
 int sqlite3OpenTableAndIndices(Parse*, Table*, int, u8, int, u8*, int*, int*);
 void sqlite3BeginWriteOperation(Parse*, int, int);

diff --git a/src/vdbe.c b/src/vdbe.c
index cbb7867512f2091b0da1391b7e83aa640419f257..88010a9a6fdc2af24215ce05076c91b6e62317bd 100644 (file)
--- a/src/vdbe.c
+++ b/src/vdbe.c
@@ -2777,6 +2777,18 @@ case OP_MakeRecord: {
     }while( zAffinity[0] );
   }
 
+  /* NULLs can be safely trimmed from the end of the record, as long as
+  ** as the schema format is 2 or more and none of the omitted columns
+  ** have a non-NULL default value.  Also, the record must be left with
+  ** at least one field.  If P5>0 then it will be one more than the
+  ** index of the right-most column with a non-NULL default value */
+  if( pOp->p5 ){
+    while( (pLast->flags & MEM_Null)!=0 && nField>pOp->p5 ){
+      pLast--;
+      nField--;
+    }
+  }
+
   /* Loop through the elements that will make up the record to figure
   ** out how much space is required for the new record.
   */

diff --git a/src/vdbe.h b/src/vdbe.h
index feaf116807be0563a865c689d682de452118cf8c..a35f3be3443be2f2554a2cdb25384cfa56fecc13 100644 (file)
--- a/src/vdbe.h
+++ b/src/vdbe.h
@@ -41,8 +41,7 @@ typedef struct SubProgram SubProgram;
 struct VdbeOp {
   u8 opcode;          /* What operation to perform */
   signed char p4type; /* One of the P4_xxx constants for p4 */
-  u8 notUsed1;
-  u8 p5;              /* Fifth parameter is an unsigned character */
+  u16 p5;             /* Fifth parameter is an unsigned 16-bit integer */
   int p1;             /* First operand */
   int p2;             /* Second parameter (often the jump destination) */
   int p3;             /* The third parameter */
@@ -194,7 +193,7 @@ void sqlite3VdbeChangeOpcode(Vdbe*, u32 addr, u8);
 void sqlite3VdbeChangeP1(Vdbe*, u32 addr, int P1);
 void sqlite3VdbeChangeP2(Vdbe*, u32 addr, int P2);
 void sqlite3VdbeChangeP3(Vdbe*, u32 addr, int P3);
-void sqlite3VdbeChangeP5(Vdbe*, u8 P5);
+void sqlite3VdbeChangeP5(Vdbe*, u16 P5);
 void sqlite3VdbeJumpHere(Vdbe*, int addr);
 int sqlite3VdbeChangeToNoop(Vdbe*, int addr);
 int sqlite3VdbeDeletePriorOpcode(Vdbe*, u8 op);

diff --git a/src/vdbeaux.c b/src/vdbeaux.c
index 85d273f1ec8b84fc0d35c8974bb37530b8f15a63..cd423ed5b556ae2278acf041231a7c314ed37c34 100644 (file)
--- a/src/vdbeaux.c
+++ b/src/vdbeaux.c
@@ -783,7 +783,7 @@ void sqlite3VdbeChangeP2(Vdbe *p, u32 addr, int val){
 void sqlite3VdbeChangeP3(Vdbe *p, u32 addr, int val){
   sqlite3VdbeGetOp(p,addr)->p3 = val;
 }
-void sqlite3VdbeChangeP5(Vdbe *p, u8 p5){
+void sqlite3VdbeChangeP5(Vdbe *p, u16 p5){
   assert( p->nOp>0 || p->db->mallocFailed );
   if( p->nOp>0 ) p->aOp[p->nOp-1].p5 = p5;
 }