From: dan <Dan Kennedy>
Date: Wed, 31 Jul 2024 20:49:00 +0000 (+0000)
Subject: Fix various problems with the code on this branch.
X-Git-Tag: version-3.47.0~220^2~24
X-Git-Url: http://git.ipfire.org/?a=commitdiff_plain;h=5bd8cc7dd53be6c9516c9060a4af9c8bfd6dec9e;p=thirdparty%2Fsqlite.git

Fix various problems with the code on this branch.

FossilOrigin-Name: 8bd4ae7e95c7b6ce34db5ea705dc136e742a22f333d0e7370b485ebd736b5ec2
---

diff --git a/ext/fts5/fts5Int.h b/ext/fts5/fts5Int.h
index 14140658b2..53e269df93 100644
--- a/ext/fts5/fts5Int.h
+++ b/ext/fts5/fts5Int.h
@@ -206,6 +206,14 @@ struct Fts5TokenizerConfig {
 **
 **       INSERT INTO tbl(tbl, rank) VALUES('prefix-index', $bPrefixIndex);
 **
+** bLocale:
+**   Set to true if locale=1 was specified when the table was created.
+**
+** eEnc:
+**   Set to either FTS5_ENCODING_UNKNOWN, ENCODING_UTF8, or ENCODING_UTF16,
+**   to indicate the encoding used by the database handle. This is initially
+**   set to UNKNOWN, then to one of the other two values the first time it
+**   is required.
 */
 struct Fts5Config {
   sqlite3 *db;                    /* Database handle */
@@ -270,6 +278,10 @@ struct Fts5Config {
 #define FTS5_PATTERN_LIKE     65  /* matches SQLITE_INDEX_CONSTRAINT_LIKE */
 #define FTS5_PATTERN_GLOB     66  /* matches SQLITE_INDEX_CONSTRAINT_GLOB */
 
+#define FTS5_ENCODING_UNKNOWN 0
+#define FTS5_ENCODING_UTF8    1
+#define FTS5_ENCODING_UTF16   2
+
 int sqlite3Fts5ConfigParse(
     Fts5Global*, sqlite3*, int, const char **, Fts5Config**, char**
 );
@@ -634,9 +646,9 @@ int sqlite3Fts5FlushToDisk(Fts5Table*);
 
 int sqlite3Fts5ExtractText(
   Fts5Config *pConfig,
-  int bContent,                   /* Loaded from content table */
   sqlite3_value *pVal,            /* Value to extract text from */
-  int *pbResetTokenizer,          /* OUT: True if xSetLocale(NULL) required */
+  int bContent,                   /* Loaded from content table */
+  int *pbResetTokenizer,          /* OUT: True if ClearLocale() required */
   const char **ppText,            /* OUT: Pointer to text buffer */
   int *pnText                     /* OUT: Size of (*ppText) in bytes */
 );
diff --git a/ext/fts5/fts5_aux.c b/ext/fts5/fts5_aux.c
index e2226e3f03..8695369f5e 100644
--- a/ext/fts5/fts5_aux.c
+++ b/ext/fts5/fts5_aux.c
@@ -239,6 +239,8 @@ static int fts5ConfigureTokenizer(
   const char *zLocale = 0;
   int nLocale = 0;
 
+  assert( pApi->iVersion>=4 );    /* Ensure xColumnLocale() is available */
+
   rc = pApi->xColumnLocale(pFts, iCol, &zLocale, &nLocale);
   if( rc==SQLITE_OK ){
     rc = pApi->xTokenizeSetLocale(pFts, zLocale, nLocale);
diff --git a/ext/fts5/fts5_config.c b/ext/fts5/fts5_config.c
index 3736f8685f..1312f459db 100644
--- a/ext/fts5/fts5_config.c
+++ b/ext/fts5/fts5_config.c
@@ -615,7 +615,9 @@ int sqlite3Fts5ConfigParse(
     sqlite3_free(zTwo);
   }
 
-  /* If this is not an FTS5_CONTENT_NORMAL table, set bLocale */
+  /* If this is not an FTS5_CONTENT_NORMAL table, set bLocale. There are
+  ** no restrictions on using fts5_locale() with external-content or
+  ** contentless tables. */
   if( pRet->eContent!=FTS5_CONTENT_NORMAL ){
     pRet->bLocale = 1;
   }
@@ -1028,7 +1030,7 @@ int sqlite3Fts5ConfigLoad(Fts5Config *pConfig, int iCookie){
     rc = SQLITE_ERROR;
     if( pConfig->pzErrmsg ){
       assert( 0==*pConfig->pzErrmsg );
-      *pConfig->pzErrmsg = sqlite3_mprintf("invalid fts5 file format "
+      sqlite3Fts5ConfigErrmsg(pConfig, "invalid fts5 file format "
           "(found %d, expected %d or %d) - run 'rebuild'",
           iVersion, FTS5_CURRENT_VERSION, FTS5_CURRENT_VERSION_SECUREDELETE
       );
@@ -1043,6 +1045,11 @@ int sqlite3Fts5ConfigLoad(Fts5Config *pConfig, int iCookie){
   return rc;
 }
 
+/*
+** Set (*pConfig->pzErrmsg) to point to an sqlite3_malloc()ed buffer 
+** containing the error message created using printf() style formatting
+** string zFmt and its trailing arguments.
+*/
 void sqlite3Fts5ConfigErrmsg(Fts5Config *pConfig, const char *zFmt, ...){
   va_list ap;                     /* ... printf arguments */
   char *zMsg = 0;
@@ -1050,6 +1057,7 @@ void sqlite3Fts5ConfigErrmsg(Fts5Config *pConfig, const char *zFmt, ...){
   va_start(ap, zFmt);
   zMsg = sqlite3_vmprintf(zFmt, ap);
   if( pConfig->pzErrmsg ){
+    assert( *pConfig->pzErrmsg==0 );
     *pConfig->pzErrmsg = zMsg;
   }else{
     sqlite3_free(zMsg);
diff --git a/ext/fts5/fts5_main.c b/ext/fts5/fts5_main.c
index 673a44d139..c9c01aafea 100644
--- a/ext/fts5/fts5_main.c
+++ b/ext/fts5/fts5_main.c
@@ -124,10 +124,6 @@ struct Fts5FullTable {
 #endif
 };
 
-#define FTS5_ENCODING_UNKNOWN 0
-#define FTS5_ENCODING_UTF8    1
-#define FTS5_ENCODING_UTF16   2
-
 struct Fts5MatchPhrase {
   Fts5Buffer *pPoslist;           /* Pointer to current poslist */
   int nTerm;                      /* Size of phrase in terms */
@@ -169,6 +165,12 @@ struct Fts5Sorter {
 **   If the cursor iterates in descending order of rowid, iFirstRowid
 **   is the upper limit (i.e. the "first" rowid visited) and iLastRowid
 **   the lower.
+**
+** pLocale, nLocale:
+**   These are set by API method xTokenizeSetLocale(). xTokenizeSetLocale()
+**   does not actually configure the tokenizer, it just stores the values
+**   it is passed in these variables. The fts5_tokenizer_v2.xSetLocale()
+**   method is called from within the xTokenize() API method if required.
 */
 struct Fts5Cursor {
   sqlite3_vtab_cursor base;       /* Base class used by SQLite core */
@@ -238,6 +240,9 @@ struct Fts5Cursor {
 #define BitFlagAllTest(x,y) (((x) & (y))==(y))
 #define BitFlagTest(x,y)    (((x) & (y))!=0)
 
+/*
+** The subtype values returned by fts5_locale() are tagged with.
+*/
 #define FTS5_LOCALE_SUBTYPE ((unsigned int)'L')
 
 
@@ -1003,7 +1008,7 @@ static int fts5PrepareStatement(
     rc = sqlite3_prepare_v3(pConfig->db, zSql, -1, 
                             SQLITE_PREPARE_PERSISTENT, &pRet, 0);
     if( rc!=SQLITE_OK ){
-      *pConfig->pzErrmsg = sqlite3_mprintf("%s", sqlite3_errmsg(pConfig->db));
+      sqlite3Fts5ConfigErrmsg(pConfig, "%s", sqlite3_errmsg(pConfig->db));
     }
     sqlite3_free(zSql);
   }
@@ -1238,7 +1243,11 @@ static void fts5SetVtabError(Fts5FullTable *p, const char *zFormat, ...){
   va_end(ap);
 }
 
-
+/*
+** Configure the tokenizer to use the locale specified by nLocale byte
+** buffer zLocale. Return SQLITE_OK if successful, or an SQLite error
+** code otherwise.
+*/
 static int fts5SetLocale(
   Fts5Config *pConfig, 
   const char *zLocale, 
@@ -1252,11 +1261,27 @@ static int fts5SetLocale(
   return rc;
 }
 
+/*
+** Reset the locale of the tokenizer to its default.
+*/
 void sqlite3Fts5ClearLocale(Fts5Config *pConfig){
   fts5SetLocale(pConfig, 0, 0);
 }
 
-static int fts5IsUtf16(Fts5Config *pConfig, int *pbIs){
+/*
+** This function is used to determine if the database handle uses utf-8 or
+** a utf-16 encoding. If it uses utf-8, then output parameter (*pbIs) is set
+** to 0. Otherwise, if it uses utf-16, then the output parameter is set
+** to 1.
+**
+** This function returns SQLITE_OK if successful, or an SQLite error code
+** otherwise. If an error code is returned, the final value of (*pbIs) is
+** undefined.
+*/
+static int fts5IsUtf16(
+  Fts5Config *pConfig,            /* Configuration object */
+  int *pbIs                       /* OUT: True if utf-16, false if utf-8 */
+){
   if( pConfig->eEnc==FTS5_ENCODING_UNKNOWN ){
     sqlite3_stmt *pPragma = 0;
     int rc = fts5PrepareStatement(&pPragma, pConfig,
@@ -1280,10 +1305,57 @@ static int fts5IsUtf16(Fts5Config *pConfig, int *pbIs){
   return SQLITE_OK;
 }
 
+/*
+** This function is used to extract utf-8 text from an sqlite3_value. This
+** is usually done in order to tokenize it. For example, when:
+**
+**     * a value is written to an fts5 table,
+**     * a value is deleted from an FTS5_CONTENT_NORMAL table,
+**     * a value containing a query expression is passed to xFilter()
+**
+** and so on.
+**
+** This function handles 3 cases:
+**
+**   1) Ordinary values. The text can be extracted from these using
+**      sqlite3_value_text().
+**
+**   2) Blobs tagged with sub-type FTS5_LOCALE_SUBTYPE, or those read from
+**      the content table of an FTS5_CONTENT_NORMAL table with locale=1
+**      set that do not begin with 0x00.
+**
+**      In these cases the value is a blob formatted by fts5_locale() that
+**      contains both a locale and a text value. The locale is first, as
+**      utf-8, followed by a single 0x00 byte, followed by the text value, 
+**      also as utf-8. There is no nul-terminator for the text value.
+**
+**   3) Blobs read from the content table of an FTS5_CONTENT_NORMAL table
+**      with locale=1 set that do begin with 0x00. These are used to
+**      store actual SQLITE_BLOB values written to the fts5 table by the
+**      user. They begin with 4 0x00 bytes, followed by the blob data as
+**      specified by the user.
+**
+** If successful, SQLITE_OK is returned and output parameters (*ppText)
+** and (*pnText) are set to point to a buffer containing the extracted utf-8 
+** text and its length in bytes, respectively. The buffer is not 
+** nul-terminated. It has the same lifetime as the sqlite3_value object
+** from which it is extracted.
+**
+** Parameter bContent must be true if the value was read from an indexed
+** column (i.e. not UNINDEXED) of the on disk content. In this case, if
+** the table is FTS5_CONTENT_NORMAL and locale=1 was specified, special blob
+** cases (2) and (3) above will apply.
+**
+** If pbResetTokenizer is not NULL and if case (2) is used, then the
+** tokenizer is configured to use the locale. In this case (*pbResetTokenizer)
+** is set to true before returning, to indicate that the caller must
+** call sqlite3Fts5ClearLocale() to reset the tokenizer after tokenizing
+** the text.
+*/
 int sqlite3Fts5ExtractText(
   Fts5Config *pConfig,
-  int bContent,
   sqlite3_value *pVal,            /* Value to extract text from */
+  int bContent,                   /* True if indexed table content */
   int *pbResetTokenizer,          /* OUT: True if xSetLocale(NULL) required */
   const char **ppText,            /* OUT: Pointer to text buffer */
   int *pnText                     /* OUT: Size of (*ppText) in bytes */
@@ -1309,11 +1381,23 @@ int sqlite3Fts5ExtractText(
     int nLocale = 0;
 
     if( nBlob>=4 && memcmp(pBlob, "\0\0\0\0", 4)==0 ){
-      int bIs16 = 0;
+      /* Extract the text from a blob stored in a locale=1 table. The
+      ** value consists of 4 0x00 bytes followed by the blob specified
+      ** by the user. The extracted text has to match the text extracted
+      ** when the blob was inserted - by calling sqlite3_value_text() on
+      ** the value without the 4 0x00 byte header.
+      **
+      ** The tricky bit here is that the exact text extracted from a blob
+      ** depends on the encoding of the database. To avoid reimplementing
+      ** SQLite's blob-to-text conversion code here, we call
+      ** sqlite3_value_text() on the blob with the header, then trim off the
+      ** leading utf-8 characters that the 4 byte header was converted to. In
+      ** practice this is 4 0x00 bytes for a utf-8 database, or 2 0x00 bytes
+      ** for a utf-16 database.  */
+      int bIs16 = 0;              /* True for utf-16 database */
       pText = (const char*)sqlite3_value_text(pVal);
       nText = sqlite3_value_bytes(pVal);
       rc = fts5IsUtf16(pConfig, &bIs16);
-      
       if( bIs16 ){
         pText += 2;
         nText -= 2;
@@ -1321,7 +1405,6 @@ int sqlite3Fts5ExtractText(
         pText += 4;
         nText -= 4;
       }
-
     }else{
       for(nLocale=0; nLocale<nBlob; nLocale++){
         if( pBlob[nLocale]==0x00 ) break;
@@ -1344,7 +1427,6 @@ int sqlite3Fts5ExtractText(
 
   *ppText = pText;
   *pnText = nText;
-
   return rc;
 }
 
@@ -1358,8 +1440,8 @@ int sqlite3Fts5ExtractText(
 ** configured to us the required locale.
 **
 ** If output variable (*pbFreeAndReset) is set to true, then the caller
-** is required to (a) call xSetLocale(NULL) to reset the tokenizer locale,
-** and (b) call sqlite3_free() to free (*pzText).
+** is required to (a) call sqlite3Fts5ClearLocale() to reset the tokenizer 
+** locale, and (b) call sqlite3_free() to free (*pzText).
 */
 static int fts5ExtractExprText(
   Fts5FullTable *pTab,
@@ -1373,7 +1455,7 @@ static int fts5ExtractExprText(
   int bReset = 0;
 
   *pbFreeAndReset = 0;
-  rc = sqlite3Fts5ExtractText(pTab->p.pConfig, 0, pVal, &bReset, &zText,&nText);
+  rc = sqlite3Fts5ExtractText(pTab->p.pConfig, pVal, 0, &bReset, &zText,&nText);
   if( rc==SQLITE_OK ){
     if( bReset ){
       *pzText = sqlite3Fts5Mprintf(&rc, "%.*s", nText, zText);
@@ -1930,16 +2012,12 @@ static int fts5UpdateMethod(
     else{
       int eType1 = sqlite3_value_numeric_type(apVal[1]);
 
-      if( (eType1!=SQLITE_INTEGER && eType1!=SQLITE_NULL)
-       || (eType0==SQLITE_INTEGER && eType1==SQLITE_NULL)
-      ){
-        rc = SQLITE_MISMATCH;
-      }
-
-      else if( eType0!=SQLITE_INTEGER ){
+      if( eType0!=SQLITE_INTEGER ){
         /* An INSERT statement. If the conflict-mode is REPLACE, first remove
         ** the current entry (if any). */
-        if( eConflict==SQLITE_REPLACE && eType1==SQLITE_INTEGER ){
+        if( eType1!=SQLITE_INTEGER && eType1!=SQLITE_NULL ){
+          rc = SQLITE_MISMATCH;
+        }else if( eConflict==SQLITE_REPLACE && eType1==SQLITE_INTEGER ){
           i64 iNew = sqlite3_value_int64(apVal[1]);  /* Rowid to delete */
           rc = sqlite3Fts5StorageDelete(pTab->pStorage, iNew, 0, 0);
           bUpdateOrDelete = 1;
@@ -1951,8 +2029,9 @@ static int fts5UpdateMethod(
       else{
         i64 iOld = sqlite3_value_int64(apVal[0]);  /* Old rowid */
         i64 iNew = sqlite3_value_int64(apVal[1]);  /* New rowid */
-        assert( eType1==SQLITE_INTEGER );
-        if( iOld!=iNew ){
+        if( eType1!=SQLITE_INTEGER ){
+          rc = SQLITE_MISMATCH;
+        }else if( iOld!=iNew ){
           if( eConflict==SQLITE_REPLACE ){
             rc = sqlite3Fts5StorageDelete(pTab->pStorage, iOld, 0, 1);
             if( rc==SQLITE_OK ){
@@ -2126,12 +2205,9 @@ static int fts5ApiColumnText(
     rc = fts5SeekCursor(pCsr, 0);
     if( rc==SQLITE_OK ){
       Fts5Config *pConfig = pTab->pConfig;
-      int bContent = (
-          pConfig->bLocale && pConfig->abUnindexed[iCol]==0 && 
-          pConfig->eContent==FTS5_CONTENT_NORMAL
-      );
+      int bContent = (pConfig->abUnindexed[iCol]==0);
       sqlite3_value *pVal = sqlite3_column_value(pCsr->pStmt, iCol+1);
-      sqlite3Fts5ExtractText(pConfig, bContent, pVal, 0, pz, pn);
+      sqlite3Fts5ExtractText(pConfig, pVal, bContent, 0, pz, pn);
     }
   }
   return rc;
@@ -2346,14 +2422,13 @@ static int fts5ApiColumnSize(Fts5Context *pCtx, int iCol, int *pnToken){
       rc = fts5SeekCursor(pCsr, 0);
       for(i=0; rc==SQLITE_OK && i<pConfig->nCol; i++){
         if( pConfig->abUnindexed[i]==0 ){
-          const int bContent = (pConfig->eContent==FTS5_CONTENT_NORMAL);
           const char *z = 0; 
           int n = 0;
           int bReset = 0;
           sqlite3_value *pVal = sqlite3_column_value(pCsr->pStmt, i+1);
 
           pCsr->aColumnSize[i] = 0;
-          rc = sqlite3Fts5ExtractText(pConfig, bContent, pVal, &bReset, &z, &n);
+          rc = sqlite3Fts5ExtractText(pConfig, pVal, 1, &bReset, &z, &n);
           if( rc==SQLITE_OK ){
             rc = sqlite3Fts5Tokenize(pConfig, FTS5_TOKENIZE_AUX, 
                 z, n, (void*)&pCsr->aColumnSize[i], fts5ColumnSizeCb
@@ -2602,6 +2677,9 @@ static int fts5ApiQueryPhrase(Fts5Context*, int, void*,
     int(*)(const Fts5ExtensionApi*, Fts5Context*, void*)
 );
 
+/*
+** The xColumnLocale() API.
+*/
 static int fts5ApiColumnLocale(
   Fts5Context *pCtx, 
   int iCol, 
@@ -2617,24 +2695,34 @@ static int fts5ApiColumnLocale(
 
   if( iCol<0 || iCol>=pConfig->nCol ){
     rc = SQLITE_RANGE;
-  }else{
-    int bNormal = (pConfig->eContent==FTS5_CONTENT_NORMAL);
-    if( pConfig->abUnindexed[iCol]==0
-     && pCsr->ePlan!=FTS5_PLAN_SPECIAL
-     && pConfig->eContent!=FTS5_CONTENT_NONE
-     && (bNormal==0 || pConfig->bLocale)
-    ){
-      rc = fts5SeekCursor(pCsr, 0);
-      if( rc==SQLITE_OK ){
-        sqlite3_value *pVal = sqlite3_column_value(pCsr->pStmt, iCol+1);
-        if( sqlite3_value_type(pVal)==SQLITE_BLOB
-         && (bNormal || sqlite3_value_subtype(pVal)==FTS5_LOCALE_SUBTYPE)
+  }else if(
+      pConfig->abUnindexed[iCol]==0
+   && pCsr->ePlan!=FTS5_PLAN_SPECIAL
+   && pConfig->eContent!=FTS5_CONTENT_NONE
+   && pConfig->bLocale
+  ){
+    rc = fts5SeekCursor(pCsr, 0);
+    if( rc==SQLITE_OK ){
+      /* Load the value into pVal. pVal is a locale/text pair iff:
+      **
+      **   1) It is an SQLITE_BLOB, and
+      **   2) Either the subtype is FTS5_LOCALE_SUBTYPE, or else the
+      **      value was loaded from an FTS5_CONTENT_NORMAL table, and
+      **   3) It does not begin with an 0x00 byte.
+      */ 
+      sqlite3_value *pVal = sqlite3_column_value(pCsr->pStmt, iCol+1);
+      if( sqlite3_value_type(pVal)==SQLITE_BLOB ){
+        if( pConfig->eContent==FTS5_CONTENT_NORMAL
+         || sqlite3_value_subtype(pVal)==FTS5_LOCALE_SUBTYPE
         ){
           const u8 *pBlob = (const u8*)sqlite3_value_blob(pVal);
           int nBlob = sqlite3_value_bytes(pVal);
           int nLocale = 0;
           for(nLocale=0; nLocale<nBlob && pBlob[nLocale]!=0x00; nLocale++);
-          if( nLocale!=0 && nLocale!=nBlob ){
+          if( nLocale==nBlob ){
+            rc = FTS5_CORRUPT;
+          }else if( nLocale!=0 ){
+            /* A locale/text pair */
             *pzLocale = (const char*)pBlob;
             *pnLocale = nLocale;
           }
@@ -2646,6 +2734,9 @@ static int fts5ApiColumnLocale(
   return rc;
 }
 
+/*
+** The xTokenizeSetLocale() API.
+*/
 static int fts5ApiTokenizeSetLocale(
   Fts5Context *pCtx, 
   const char *pLocale, 
@@ -2868,14 +2959,36 @@ static int fts5PoslistBlob(sqlite3_context *pCtx, Fts5Cursor *pCsr){
   return rc;
 }
 
+/*
+** Value pVal was read from column iCol of the FTS5 table. This function
+** returns it to the owner of pCtx via a call to an sqlite3_result_xxx()
+** function. This function deals with the same 3 cases as
+** sqlite3Fts5ExtractText():
+**
+**   1) Ordinary values. These can be returned using sqlite3_result_value().
+**
+**   2) Blobs with subtype FTS5_LOCALE_SUBTYPE, or read from an 
+**      FTS5_CONTENT_NORMAL table that do not begin with 0x00. These are
+**      locale/text pairs. In this case the text is extracted and returned
+**      via sqlite3_result_text().
+**
+**   3) Blobs with subtype FTS5_LOCALE_SUBTYPE, or read from an 
+**      FTS5_CONTENT_NORMAL table that begin with 4 0x00 bytes. These are
+**      blobs with a 4 byte header. In this case the user's blob is extracted
+**      and returned via sqlite3_result_blob().
+*/
 static void fts5ExtractValueFromColumn(
   sqlite3_context *pCtx,
   Fts5Config *pConfig, 
+  int iCol,
   sqlite3_value *pVal
 ){
-  if( sqlite3_value_type(pVal)==SQLITE_BLOB ){
+  if( pConfig->bLocale 
+   && sqlite3_value_type(pVal)==SQLITE_BLOB 
+   && pConfig->abUnindexed[iCol]==0
+  ){
     if( sqlite3_value_subtype(pVal)==FTS5_LOCALE_SUBTYPE 
-     || (pConfig->bLocale && pConfig->eContent==FTS5_CONTENT_NORMAL)
+     || pConfig->eContent==FTS5_CONTENT_NORMAL
     ){
       const u8 *pBlob = sqlite3_value_blob(pVal);
       int nBlob = sqlite3_value_bytes(pVal);
@@ -2953,7 +3066,7 @@ static int fts5ColumnMethod(
       rc = fts5SeekCursor(pCsr, 1);
       if( rc==SQLITE_OK ){
         sqlite3_value *pVal = sqlite3_column_value(pCsr->pStmt, iCol+1);
-        fts5ExtractValueFromColumn(pCtx, pConfig, pVal);
+        fts5ExtractValueFromColumn(pCtx, pConfig, iCol, pVal);
       }
       pConfig->pzErrmsg = 0;
     }
@@ -3322,7 +3435,17 @@ static void fts5SourceIdFunc(
 }
 
 /*
-** Implementation of fts5_locale() function.
+** Implementation of fts5_locale(LOCALE, TEXT) function.
+**
+** If parameter LOCALE is NULL, or a zero-length string, then a copy of
+** TEXT is returned. Otherwise, both LOCALE and TEXT are interpreted as
+** text, and the value returned is a blob consisting of:
+**
+**     * The LOCALE, as utf-8 text, followed by
+**     * 0x00, followed by
+**     * The TEXT, as utf-8 text.
+**
+** There is no final nul-terminator following the TEXT value.
 */
 static void fts5LocaleFunc(
   sqlite3_context *pCtx,          /* Function call context */
diff --git a/ext/fts5/fts5_storage.c b/ext/fts5/fts5_storage.c
index ea5b61a4b6..e55e2b833b 100644
--- a/ext/fts5/fts5_storage.c
+++ b/ext/fts5/fts5_storage.c
@@ -476,7 +476,7 @@ static int fts5StorageDeleteFromIndex(
         pVal = apVal[iCol-1];
       }
 
-      rc = sqlite3Fts5ExtractText(pConfig,pSeek!=0,pVal,&bReset,&pText,&nText);
+      rc = sqlite3Fts5ExtractText(pConfig, pVal, 1, &bReset, &pText, &nText);
       if( rc==SQLITE_OK ){
         ctx.szCol = 0;
         rc = sqlite3Fts5Tokenize(pConfig, FTS5_TOKENIZE_DOCUMENT, 
@@ -740,8 +740,8 @@ int sqlite3Fts5StorageRebuild(Fts5Storage *p){
         int bReset = 0;
         int nText = 0;
         const char *pText = 0;
-        rc = sqlite3Fts5ExtractText(pConfig, 1, 
-            sqlite3_column_value(pScan, ctx.iCol+1), &bReset, &pText, &nText
+        rc = sqlite3Fts5ExtractText(pConfig, 
+            sqlite3_column_value(pScan, ctx.iCol+1), 1, &bReset, &pText, &nText
         );
 
         if( rc==SQLITE_OK ){
@@ -851,11 +851,14 @@ int sqlite3Fts5StorageContentInsert(
           ** column. Strip the locale away and just bind the text.  */
           const char *pText = 0;
           int nText = 0;
-          rc = sqlite3Fts5ExtractText(pConfig, 0, pVal, 0, &pText, &nText);
+          rc = sqlite3Fts5ExtractText(pConfig, pVal, 0, 0, &pText, &nText);
           sqlite3_bind_text(pInsert, i, pText, nText, SQLITE_TRANSIENT);
           continue;
         }
-      }else if( pConfig->bLocale && sqlite3_value_type(pVal)==SQLITE_BLOB ){
+      }else if( pConfig->bLocale 
+       && sqlite3_value_type(pVal)==SQLITE_BLOB 
+       && i>=2 && pConfig->abUnindexed[i-2]==0
+      ){
         /* Inserting a blob into a normal content table with locale=1. */
         int n = sqlite3_value_bytes(pVal);
         u8 *pBlob = sqlite3Fts5MallocZero(&rc, n+4);
@@ -911,7 +914,7 @@ int sqlite3Fts5StorageIndexInsert(
         pVal = sqlite3_column_value(p->pSavedRow, ctx.iCol+1);
         bDisk = 1;
       }
-      rc = sqlite3Fts5ExtractText(pConfig, bDisk, pVal, &bReset,&pText,&nText);
+      rc = sqlite3Fts5ExtractText(pConfig, pVal, bDisk, &bReset,&pText,&nText);
       if( rc==SQLITE_OK ){
         if( bReset && pConfig->bLocale==0 ){
           rc = SQLITE_ERROR;
@@ -1100,8 +1103,8 @@ int sqlite3Fts5StorageIntegrity(Fts5Storage *p, int iArg){
             int nText = 0;
             int bReset = 0;
 
-            rc = sqlite3Fts5ExtractText(pConfig, 1,
-                sqlite3_column_value(pScan, i+1), &bReset, &pText, &nText
+            rc = sqlite3Fts5ExtractText(pConfig, 
+                sqlite3_column_value(pScan, i+1), 1, &bReset, &pText, &nText
             );
 
             if( rc==SQLITE_OK ){
diff --git a/ext/fts5/test/fts5blob.test b/ext/fts5/test/fts5blob.test
index 9c838b75a6..56a571b9c4 100644
--- a/ext/fts5/test/fts5blob.test
+++ b/ext/fts5/test/fts5blob.test
@@ -31,8 +31,6 @@ foreach {tn enc locale} {
 
   execsql "PRAGMA encoding = $enc"
 
-  if {$tn==3 || $tn==4} breakpoint
-
   execsql "
     CREATE VIRTUAL TABLE t1 USING fts5(x, y, locale=$locale);
   "
@@ -71,6 +69,20 @@ do_test 1.6.1 { set U(utf8,1) } $U(utf8,0)
 do_test 1.6.2 { set U(utf16,1) } $U(utf16,0)
 
 
+#--------------------------------------------------------------------------
+reset_db
+do_execsql_test 2.0 {
+  CREATE VIRTUAL TABLE t1 USING fts5(x, y UNINDEXED, locale=1);
+  INSERT INTO t1(rowid, x, y) VALUES(12, 'twelve', X'0000000041424320444546');
+}
+
+breakpoint
+do_execsql_test 2.1 {
+  select rowid, x, quote(y) FROM t1
+} {
+  12 twelve X'0000000041424320444546'
+}
+
 
 finish_test
 
diff --git a/manifest b/manifest
index a18841d7c4..9a70fc55a7 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Add\smissing\sdocumentation\sfor\snew\sfeatures\sto\sfts5.h.
-D 2024-07-31T15:46:41.319
+C Fix\svarious\sproblems\swith\sthe\scode\son\sthis\sbranch.
+D 2024-07-31T20:49:00.656
 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1
 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea
 F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724
@@ -93,15 +93,15 @@ F ext/fts3/unicode/mkunicode.tcl d5aebf022fa4577ee8cdf27468f0d847879993959101f6d
 F ext/fts3/unicode/parseunicode.tcl a981bd6466d12dd17967515801c3ff23f74a281be1a03cf1e6f52a6959fc77eb
 F ext/fts5/extract_api_docs.tcl bc3a0ca78be7d3df08e7602c00ca48021ebae40682d75eb001bfdf6e54ffb44e
 F ext/fts5/fts5.h f5451da088d0004c4be7314e2fdb41fda16ce682ce2aa3b54b9474ebe9a013d5
-F ext/fts5/fts5Int.h 833a2fe729f926ebcde47e21e495d141b99ede9a188fc577873f24bea0f0bfa2
-F ext/fts5/fts5_aux.c 652f839dc0c77431295f10b08f268631560bb5630e65fd701de7a58744428a82
+F ext/fts5/fts5Int.h 2eab38c52d12bfa0a1cedb742276035e05634c6365f8defb599f023bb8743559
+F ext/fts5/fts5_aux.c 598c80fc0faabab91c833cdda99f8e36387bd907f4acb0480a19b612a4add93e
 F ext/fts5/fts5_buffer.c 0eec58bff585f1a44ea9147eae5da2447292080ea435957f7488c70673cb6f09
-F ext/fts5/fts5_config.c 0c96490fbad746b3780174f38b2ee5e3d719f2f81ee6b58ca828772871e0f680
+F ext/fts5/fts5_config.c 31267fc68f9d16feaa5b5f7efb902a3410f0caa708cc2372c90a77d75890b2a8
 F ext/fts5/fts5_expr.c c7336d5f9ecc0e2b014d700be2bec0ea383b0e82c494a7c5c4ac622327c2bfad
 F ext/fts5/fts5_hash.c adda4272be401566a6e0ba1acbe70ee5cb97fce944bc2e04dc707152a0ec91b1
 F ext/fts5/fts5_index.c eb9a0dda3bc6ef969a6be8d2746af56856e67251810ddba08622b45be8477abe
-F ext/fts5/fts5_main.c d9a3fef86887e373027d48ab8216ab8caca95f2316e05d953d8885633162a9c5
-F ext/fts5/fts5_storage.c cc6173bb755d668573169c038034a9ec8deadd10b3c10c145adbdf04ab5f889c
+F ext/fts5/fts5_main.c bda8d421024191376343a571370a7b2f92fdf5303e32c6dc0d7456bef9ffedd8
+F ext/fts5/fts5_storage.c f94b924db1bc164af3feadcc3f08f0d8f5da5cd45e4909313637aeee85d0d13c
 F ext/fts5/fts5_tcl.c 93b705cb87633574983161edc5234f9b91ba03f9fecfbd2c5d401a1da6f93aa5
 F ext/fts5/fts5_test_mi.c 08c11ec968148d4cb4119d96d819f8c1f329812c568bac3684f5464be177d3ee
 F ext/fts5/fts5_test_tok.c 3cb0a9b508b30d17ef025ccddd26ae3dc8ddffbe76c057616e59a9aa85d36f3b
@@ -132,7 +132,7 @@ F ext/fts5/test/fts5auxdata.test 372549088ff792655f73e62b9dfaf4863ce74f5e604c06c
 F ext/fts5/test/fts5bigid.test 2860854c2561a57594192b00c33a29f91cb85e25f3d6c03b5c2b8f62708f39dd
 F ext/fts5/test/fts5bigpl.test 8f09858aab866c33593560e6480b2b6975ae7ff29ca32ad7b77e2da61402f8ef
 F ext/fts5/test/fts5bigtok.test 541119e616c637caea925a8c028c37c2c29e94383e00aa2f9198d530724b6e36
-F ext/fts5/test/fts5blob.test a16160688e181a212bcb4968325c57ef2864a0bcae0794d6a1e16185007f00b9
+F ext/fts5/test/fts5blob.test 6d1cf0c5ba2e6f8f9b4e915d3677c16ead3a79c22dcb386c0b21169a9349718d
 F ext/fts5/test/fts5cat.test daba0b80659460b0cb60bd1f40b402478a761fe7ea414c3c94c2be25568cc33a
 F ext/fts5/test/fts5circref.test f880dfd0d99f6fb73b88ccacb0927d18e833672fd906cc47d6b4e529419eaa62
 F ext/fts5/test/fts5colset.test 544f4998cdbfe06a3123887fc0221612e8aa8192cdaff152872f1aadb10e6897
@@ -2201,8 +2201,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P ff64939ec8399949393f1029fa6d514892fbf2bf1498545300cc5e120b9622a5
-R f7dbcf0c98ed562d42b896f7c143383b
+P fa0da3b28e411affd45a918d6b7faba49f744ca8b4adf3cd5ce6609bb630499e
+R 4281c2d6e2e80158745c4320fdfed09d
 U dan
-Z 344ab71a9372cfa91d638c7c0c606714
+Z 7ae6a6ddbc4056daf1c0903e2b7934cf
 # Remove this line to create a well-formed Fossil manifest.
diff --git a/manifest.uuid b/manifest.uuid
index a1e4d24ea7..dee0563fc6 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-fa0da3b28e411affd45a918d6b7faba49f744ca8b4adf3cd5ce6609bb630499e
+8bd4ae7e95c7b6ce34db5ea705dc136e742a22f333d0e7370b485ebd736b5ec2