Merge latest trunk changes, including the changes to the fts5 locale=1 feature, into...

FossilOrigin-Name: d2f0d19936222911bc317efecc831007d3aba81f9b32877030ffb29d1728bbdc

diff --cc ext/fts5/fts5Int.h
Simple merge

diff --cc ext/fts5/fts5_config.c
Simple merge

diff --cc ext/fts5/fts5_main.c
index c5ff2fad759ffb1e787a5d7fbb82e1b3c57d5b5a,db413b572a16785d29ae3682f608b993d19a3831..f2749ffff1f8162b0b4db8e3e737f106d643626f
--- 1/ext/fts5/fts5_main.c
--- 2/ext/fts5/fts5_main.c
+++ b/ext/fts5/fts5_main.c
@@@ -2419,7 -2399,7 +2401,7 @@@ static int fts5ApiColumnSize(Fts5Contex
      if( pConfig->bColumnsize ){
        i64 iRowid = fts5CursorRowid(pCsr);
        rc = sqlite3Fts5StorageDocsize(pTab->pStorage, iRowid, pCsr->aColumnSize);
--    }else if( pConfig->zContent==0 ){
++    }else if( !pConfig->zContent || pConfig->eContent==FTS5_CONTENT_UNINDEXED ){
        int i;
        for(i=0; i<pConfig->nCol; i++){
          if( pConfig->abUnindexed[i]==0 ){

diff --cc ext/fts5/fts5_storage.c
index e2c47c0072b9118f903faef2a57451f5e209dc7d,e8649c703f581bc13f2d7baa09f5df14bcfd0d18..ee65b0f7ee47fa48efa48e9af68000f3c3722e29
--- 1/ext/fts5/fts5_storage.c
--- 2/ext/fts5/fts5_storage.c
+++ b/ext/fts5/fts5_storage.c
@@@ -141,24 -165,23 +141,36 @@@ static int fts5StorageGetStmt
          );
          break;
  
-       case FTS5_STMT_INSERT_CONTENT: 
-       case FTS5_STMT_REPLACE_CONTENT: {
-         int nCol = pC->nCol + 1;
+       case FTS5_STMT_INSERT_CONTENT: {
 -        int nCol = 0;
 -        char *zBind;
 +        char *zBind = 0;
          int i;
  
 -        nCol = 1 + pC->nCol;
 -        if( pC->bLocale ){
 -          for(i=0; i<pC->nCol; i++){
 -            if( pC->abUnindexed[i]==0 ) nCol++;
 +        assert( pC->eContent==FTS5_CONTENT_NORMAL 
 +             || pC->eContent==FTS5_CONTENT_UNINDEXED 
 +        );
 +
-         for(i=0; rc==SQLITE_OK && i<nCol; i++){
++        /* Add bindings for the "c*" columns - those that store the actual
++        ** table content. If eContent==NORMAL, then there is one binding
++        ** for each column. Or, if eContent==UNINDEXED, then there are only
++        ** bindings for the UNINDEXED columns. */
++        for(i=0; rc==SQLITE_OK && i<(pC->nCol+1); i++){
 +          if( !i || pC->eContent==FTS5_CONTENT_NORMAL || pC->abUnindexed[i-1] ){
 +            zBind = sqlite3Fts5Mprintf(&rc, "%z%s?%d", zBind, zBind?",":"",i+1);
            }
          }
  
 -        zBind = fts5BindingsList(&rc, nCol);
 -        if( zBind ){
 -          zSql = sqlite3_mprintf(azStmt[eStmt], pC->zDb, pC->zName, zBind);
 -          sqlite3_free(zBind);
++        /* Add bindings for any "l*" columns. Only non-UNINDEXED columns
++        ** require these.  */
++        if( pC->bLocale && pC->eContent==FTS5_CONTENT_NORMAL ){
++          for(i=0; rc==SQLITE_OK && i<pC->nCol; i++){
++            if( pC->abUnindexed[i]==0 ){
++              zBind = sqlite3Fts5Mprintf(&rc, "%z,?%d", zBind, pC->nCol+i+2);
++            }
++          }
+         }
++
 +        zSql = sqlite3Fts5Mprintf(&rc, azStmt[eStmt], pC->zDb, pC->zName,zBind);
 +        sqlite3_free(zBind);
          break;
        }
  
@@@ -344,11 -367,9 +356,11 @@@ int sqlite3Fts5StorageOpen
    p->pIndex = pIndex;
  
    if( bCreate ){
 -    if( pConfig->eContent==FTS5_CONTENT_NORMAL ){
 +    if( pConfig->eContent==FTS5_CONTENT_NORMAL 
 +     || pConfig->eContent==FTS5_CONTENT_UNINDEXED 
 +    ){
        int nDefn = 32 + pConfig->nCol*10;
-       char *zDefn = sqlite3_malloc64(32 + (sqlite3_int64)pConfig->nCol * 10);
+       char *zDefn = sqlite3_malloc64(32 + (sqlite3_int64)pConfig->nCol * 20);
        if( zDefn==0 ){
          rc = SQLITE_NOMEM;
        }else{
@@@ -357,13 -378,17 +369,21 @@@
          sqlite3_snprintf(nDefn, zDefn, "id INTEGER PRIMARY KEY");
          iOff = (int)strlen(zDefn);
          for(i=0; i<pConfig->nCol; i++){
 -          sqlite3_snprintf(nDefn-iOff, &zDefn[iOff], ", c%d", i);
 -          iOff += (int)strlen(&zDefn[iOff]);
 +          if( pConfig->eContent==FTS5_CONTENT_NORMAL 
 +           || pConfig->abUnindexed[i] 
 +          ){
 +            sqlite3_snprintf(nDefn-iOff, &zDefn[iOff], ", c%d", i);
 +            iOff += (int)strlen(&zDefn[iOff]);
 +          }
          }
+         if( pConfig->bLocale ){
+           for(i=0; i<pConfig->nCol; i++){
+             if( pConfig->abUnindexed[i]==0 ){
+               sqlite3_snprintf(nDefn-iOff, &zDefn[iOff], ", l%d", i);
+               iOff += (int)strlen(&zDefn[iOff]);
+             }
+           }
+         }
          rc = sqlite3Fts5CreateTable(pConfig, "content", zDefn, 0, pzErr);
        }
        sqlite3_free(zDefn);
@@@ -903,38 -942,49 +948,49 @@@ int sqlite3Fts5StorageContentInsert
    }else{
      sqlite3_stmt *pInsert = 0;    /* Statement to write %_content table */
      int i;                        /* Counter variable */
 -    int nIndexed = 0;             /* Number indexed columns seen */
      rc = fts5StorageGetStmt(p, FTS5_STMT_INSERT_CONTENT, &pInsert, 0);
-     for(i=1; rc==SQLITE_OK && i<=pConfig->nCol+1; i++){
-       if( i==1 
-        || pConfig->eContent==FTS5_CONTENT_NORMAL 
-        || pConfig->abUnindexed[i-2] 
-       ){
+     if( pInsert ) sqlite3_clear_bindings(pInsert);
+ 
+     /* Bind the rowid value */
+     sqlite3_bind_value(pInsert, 1, apVal[1]);
+ 
+     /* Loop through values for user-defined columns. i=2 is the leftmost
+     ** user-defined column. As is column 1 of pSavedRow.  */
+     for(i=2; rc==SQLITE_OK && i<=pConfig->nCol+1; i++){
+       int bUnindexed = pConfig->abUnindexed[i-2];
 -      sqlite3_value *pVal = apVal[i];
 -
 -      nIndexed += !bUnindexed;
 -      if( sqlite3_value_nochange(pVal) && p->pSavedRow ){
 -        /* This is an UPDATE statement, and column (i-2) was not modified.
 -        ** Retrieve the value from Fts5Storage.pSavedRow instead. */
 -        pVal = sqlite3_column_value(p->pSavedRow, i-1);
 -        if( pConfig->bLocale && bUnindexed==0 ){
 -          sqlite3_bind_value(pInsert, pConfig->nCol + 1 + nIndexed,
 -              sqlite3_column_value(p->pSavedRow, pConfig->nCol + i - 1)
 -          );
 -        }
 -      }else if( sqlite3Fts5IsLocaleValue(pConfig, pVal) ){
 -        const char *pText = 0;
 -        const char *pLoc = 0;
 -        int nText = 0;
 -        int nLoc = 0;
 -        assert( pConfig->bLocale );
++      if( pConfig->eContent==FTS5_CONTENT_NORMAL || bUnindexed ){
 +        sqlite3_value *pVal = apVal[i];
++
 +        if( sqlite3_value_nochange(pVal) && p->pSavedRow ){
-           /* This is an UPDATE statement, and column (i-2) was not modified.
-           ** Retrieve the value from Fts5Storage.pSavedRow instead. */
++          /* This is an UPDATE statement, and user-defined column (i-2) was not
++          ** modified.  Retrieve the value from Fts5Storage.pSavedRow.  */
 +          pVal = sqlite3_column_value(p->pSavedRow, i-1);
-         }else if( sqlite3_value_subtype(pVal)==FTS5_LOCALE_SUBTYPE ){
++          if( pConfig->bLocale && bUnindexed==0 ){
++            sqlite3_bind_value(pInsert, pConfig->nCol + i,
++                sqlite3_column_value(p->pSavedRow, pConfig->nCol + i - 1)
++            );
++          }
++        }else if( sqlite3Fts5IsLocaleValue(pConfig, pVal) ){
++          const char *pText = 0;
++          const char *pLoc = 0;
++          int nText = 0;
++          int nLoc = 0;
 +          assert( pConfig->bLocale );
-           assert( i>1 );
-           if( pConfig->abUnindexed[i-2] ){
-             /* At attempt to insert an fts5_locale() value into an UNINDEXED
-             ** column. Strip the locale away and just bind the text.  */
-             const char *pText = 0;
-             int nText = 0;
-             rc = sqlite3Fts5ExtractText(pConfig, pVal, 0, 0, &pText, &nText);
+ 
 -        rc = sqlite3Fts5DecodeLocaleValue(pVal, &pText, &nText, &pLoc, &nLoc);
 -        if( rc==SQLITE_OK ){
 -          sqlite3_bind_text(pInsert, i, pText, nText, SQLITE_TRANSIENT);
 -          if( bUnindexed==0 ){
 -            int iLoc = pConfig->nCol + 1 + nIndexed;
 -            sqlite3_bind_text(pInsert, iLoc, pLoc, nLoc, SQLITE_TRANSIENT);
++          rc = sqlite3Fts5DecodeLocaleValue(pVal, &pText, &nText, &pLoc, &nLoc);
++          if( rc==SQLITE_OK ){
 +            sqlite3_bind_text(pInsert, i, pText, nText, SQLITE_TRANSIENT);
-           }else{
-             const u8 *pBlob = (const u8*)sqlite3_value_blob(pVal);
-             int nBlob = sqlite3_value_bytes(pVal);
-             assert( nBlob>4 );
-             sqlite3_bind_blob(pInsert, i, pBlob+4, nBlob-4, SQLITE_TRANSIENT);
++            if( bUnindexed==0 ){
++              int iLoc = pConfig->nCol + i;
++              sqlite3_bind_text(pInsert, iLoc, pLoc, nLoc, SQLITE_TRANSIENT);
++            }
            }
++
 +          continue;
          }
-   
+ 
 -        continue;
 +        rc = sqlite3_bind_value(pInsert, i, pVal);
        }
 -
 -      rc = sqlite3_bind_value(pInsert, i, pVal);
      }
      if( rc==SQLITE_OK ){
        sqlite3_step(pInsert);

diff --cc manifest
index 1e012e151a7356d1f21f65221325e1ac1bd802b9,87fe20a98929e5462168c21c6888d8d5a56462d8..5812d09993ac49fcbb8b09222a620a325f85510a
--- 1/manifest
--- 2/manifest
+++ b/manifest
@@@ -1,5 -1,5 +1,5 @@@
- C Store\sthe\svalues\sof\sany\sUNINDEXED\scolumns\sof\sa\scontentless\sfts5\stable\spersistently\sin\sthe\sdatabase.\sWarning:\sThis\scurrently\screates\sa\s(technically)\sincompatible\sfile-format\sfor\scontentless\sfts5\stables\sthat\shave\sUNINDEXED\scolumns.
- D 2024-09-03T18:55:38.957
 -C Further\senhancement\sto\sPTRMAP\sdisplay\sin\sshowdb:\s\sShow\sthe\sdetails\sof\sinvalid\nentries\sthat\sare\swithin\sthe\srange\sof\sthe\sdatabase\sfile.\s\sContinue\sto\signore\ninvalid\sentries\sbeyond\sthe\send\sof\sthe\sdatabase\sfile.
 -D 2024-09-13T11:14:10.057
++C Merge\slatest\strunk\schanges,\sincluding\sthe\schanges\sto\sthe\sfts5\slocale=1\sfeature,\sinto\sthis\sbranch.
++D 2024-09-13T15:37:31.588
  F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1
  F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea
  F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724
@@@ -93,15 -93,15 +93,15 @@@ F ext/fts3/unicode/mkunicode.tcl 63db96
  F ext/fts3/unicode/parseunicode.tcl a981bd6466d12dd17967515801c3ff23f74a281be1a03cf1e6f52a6959fc77eb
  F ext/fts5/extract_api_docs.tcl 009cf59c77afa86d137b0cca3e3b1a5efbe2264faa2df233f9a7aa8563926d15
  F ext/fts5/fts5.h efaaac0df3d3bc740383044c144b582f47921aafa21d7b10eb98f42c24c740b0
- F ext/fts5/fts5Int.h d0c9bee4edfff9f0899648a9efc0afdebabbadb24403e2e50a1bb17aed6267d0
 -F ext/fts5/fts5Int.h 93aba03ca417f403b07b2ab6f50aa0e0c1b8b031917a9026b81520e7047a168e
++F ext/fts5/fts5Int.h 6ec0c8a49412e2d7433ebc416cc5a5fd8583dbda30ba46c8835027a4e5c9b41a
  F ext/fts5/fts5_aux.c 65a0468dd177d6093aa9ae1622e6d86b0136b8d267c62c0ad6493ad1e9a3d759
  F ext/fts5/fts5_buffer.c 0eec58bff585f1a44ea9147eae5da2447292080ea435957f7488c70673cb6f09
- F ext/fts5/fts5_config.c 8dba11620a7964e1e561aaae3d27fdd2d16b2769c979f157dd55a53903a9b734
 -F ext/fts5/fts5_config.c da21548ddbc1a457cb42545f527065221ede8ada6a734891b8c34317a7a9506b
++F ext/fts5/fts5_config.c 700d7accca99d931390c4fcb7c71b92bbcff9093aa9c2d566d7f16a3c9550a59
  F ext/fts5/fts5_expr.c 9a56f53700d1860f0ee2f373c2b9074eaf2a7aa0637d0e27a6476de26a3fee33
  F ext/fts5/fts5_hash.c adda4272be401566a6e0ba1acbe70ee5cb97fce944bc2e04dc707152a0ec91b1
  F ext/fts5/fts5_index.c 571483823193f09439356741669aa8c81da838ae6f5e1bfa7517f7ee2fb3addd
- F ext/fts5/fts5_main.c 402a65b6d922203a90e7d440f29d71f19f6863e6b663ca9f06c7321f291bda40
- F ext/fts5/fts5_storage.c ca3bdb799b9b039133e9b6e4881fc974ba34ae4ea57530210289f82313b409b5
 -F ext/fts5/fts5_main.c 4503498d3453e29a3cd89dacaba029011e89cb8c481a6241611d106e7a369bd4
 -F ext/fts5/fts5_storage.c 3332497823c3d171cf56379f2bd8c971ce15a19aadacff961106462022c92470
++F ext/fts5/fts5_main.c be195e918c40c304f81d899f8a9b00d47ed266583dd931e22730e83b011ddadc
++F ext/fts5/fts5_storage.c 1fbaf212042bdb363d74a48d3293b2c453a46880e86656df3ee19ade63472681
  F ext/fts5/fts5_tcl.c 4db9258a7882c5eac0da4433042132aaf15b87dd1e1636c7a6ca203abd2c8bfe
  F ext/fts5/fts5_test_mi.c 08c11ec968148d4cb4119d96d819f8c1f329812c568bac3684f5464be177d3ee
  F ext/fts5/fts5_test_tok.c 3cb0a9b508b30d17ef025ccddd26ae3dc8ddffbe76c057616e59a9aa85d36f3b
@@@ -436,6 -435,7 +436,7 @@@ F ext/misc/urifuncs.c f71360d14fa9e7626
  F ext/misc/uuid.c 5bb2264c1b64d163efa46509544fd7500cb8769cb7c16dd52052da8d961505cf
  F ext/misc/vfslog.c 3932ab932eeb2601dbc4447cb14d445aaa9fbe43b863ef5f014401c3420afd20
  F ext/misc/vfsstat.c a85df08654743922a19410d7b1e3111de41bb7cd07d20dd16eda4e2b808d269d
 -F ext/misc/vfstrace.c 03f90dd465968e01f5d1d3e79c36cbc53a5bfe1bd55d239435ce94df19d5b0ac
++F ext/misc/vfstrace.c 03f90dd465968e01f5d1d3e79c36cbc53a5bfe1bd55d239435ce94df19d5b0ac w src/test_vfstrace.c
  F ext/misc/vtablog.c 1100250ce8782db37c833e3a9a5c9a3ecf1af5e15b8325572b82e6e0a138ffb5
  F ext/misc/vtshim.c 1976e6dd68dd0d64508c91a6dfab8e75f8aaf6cd
  F ext/misc/wholenumber.c 0fa0c082676b7868bf2fa918e911133f2b349bcdceabd1198bba5f65b4fc0668
@@@ -2213,11 -2212,8 +2213,8 @@@ F vsixtest/vsixtest.tcl 6195aba1f12a5e1
  F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
  F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
  F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
- P 0ef65fd4ba17def4c13986365b3af300c4024725af4bc314845d1af8be568ab4
- R 87dfb79be977919afc56ac91ca0cf0b0
- T *branch * fts5-contentless-unindexed
- T *sym-fts5-contentless-unindexed *
- T -sym-trunk *
 -P a9f95fe5ce90ab9864165e603f3a34013c3c98d03f1db689996f4a32086e2ed6
 -R 5fde967f0ed3ce1cc53a6d5bcb8024a9
 -U drh
 -Z 8b78883255183c9952b265920384a36e
++P dcacb1a8ef359b4507b4733356d3150ba5dc105cc9867c103d16a0908a1a9f64 4cad385b90eaca2d90e3375e473472145af4134160b81097a8535d06638c2e4a
++R 6b427d068701369235fcf90674c4aaf6
 +U dan
- Z f39a0e93422c865c24dfd629a9a47017
++Z a31c0a78609956d9ed51a360c49325fd
  # Remove this line to create a well-formed Fossil manifest.

diff --cc manifest.uuid
index b545e371ff18b162565f7dfe7e49f9fdfc5e2e98,cf7caf5259fe6bee1719d38e1f78516fddf77673..e5ba7a67860f0711834cfe0d175cdbc64782dc23
--- 1/manifest.uuid
--- 2/manifest.uuid
+++ b/manifest.uuid
@@@ -1,1 -1,1 +1,1 @@@
- dcacb1a8ef359b4507b4733356d3150ba5dc105cc9867c103d16a0908a1a9f64
 -4cad385b90eaca2d90e3375e473472145af4134160b81097a8535d06638c2e4a
++d2f0d19936222911bc317efecc831007d3aba81f9b32877030ffb29d1728bbdc