Add the xTokenize extension API.

FossilOrigin-Name: 8c6b0aff3443fae4b7f0b9adcbf1514992b70653

diff --git a/ext/fts5/fts5.c b/ext/fts5/fts5.c
index 745a557128f58f63fc3679d40e98793db8baa003..2a0b428fd29faafe12fd395a56740e34ace3094b 100644 (file)
--- a/ext/fts5/fts5.c
+++ b/ext/fts5/fts5.c
@@ -630,8 +630,9 @@ static int fts5ApiTokenize(
   void *pUserData,
   int (*xToken)(void*, const char*, int, int, int, int)
 ){
-  assert( 0 );
-  return SQLITE_OK;
+  Fts5Cursor *pCsr = (Fts5Cursor*)pCtx;
+  Fts5Table *pTab = (Fts5Table*)(pCsr->base.pVtab);
+  return sqlite3Fts5Tokenize(pTab->pConfig, pText, nText, pUserData, xToken);
 }
 
 static int fts5ApiPhraseCount(Fts5Context *pCtx){

diff --git a/ext/fts5/fts5.h b/ext/fts5/fts5.h
index 6e5176adbba8120fbe5ae05e25e4e4900ba1bd58..572aef03c860cd3db469d184adbf482241c02ec6 100644 (file)
--- a/ext/fts5/fts5.h
+++ b/ext/fts5/fts5.h
@@ -68,6 +68,9 @@ typedef void (*fts5_extension_function)(
 **
 ** xPoslist:
 **   Iterate through instances of phrase iPhrase in the current row. 
+**
+** xTokenize:
+**   Tokenize text using the tokenizer belonging to the FTS5 table.
 */
 struct Fts5ExtensionApi {
   int iVersion;                   /* Currently always set to 1 */

diff --git a/ext/fts5/fts5_aux.c b/ext/fts5/fts5_aux.c
index deb2c8233e34c123de3e884dc0e354045bad36cb..25a1747fdb03309aa1b8bf6fc6e8302a2379906d 100644 (file)
--- a/ext/fts5/fts5_aux.c
+++ b/ext/fts5/fts5_aux.c
@@ -23,6 +23,24 @@ static void fts5SnippetFunction(
   assert( 0 );
 }
 
+static int fts5TestCallback(
+  void *pContext,                 /* Pointer to Fts5Buffer object */
+  const char *pToken,             /* Buffer containing token */
+  int nToken,                     /* Size of token in bytes */
+  int iStart,                     /* Start offset of token */
+  int iEnd,                       /* End offset of token */
+  int iPos                        /* Position offset of token */
+){
+  int rc = SQLITE_OK;
+  Fts5Buffer *pBuf = (Fts5Buffer*)pContext;
+  if( pBuf->n!=0 ){
+    sqlite3Fts5BufferAppendString(&rc, pBuf, " ");
+  }
+  sqlite3Fts5BufferAppendListElem(&rc, pBuf, pToken, nToken);
+  return rc;
+}
+
+
 static void fts5TestFunction(
   const Fts5ExtensionApi *pApi,   /* API offered by current FTS version */
   Fts5Context *pFts,              /* First arg to pass to pApi functions */
@@ -167,6 +185,33 @@ static void fts5TestFunction(
     sqlite3Fts5BufferAppendPrintf(&rc, &s, "%lld", iRowid);
   }
 
+  if( zReq==0 ){
+    sqlite3Fts5BufferAppendPrintf(&rc, &s, " tokenize ");
+  }
+  if( 0==zReq || 0==sqlite3_stricmp(zReq, "tokenize") ){
+    Fts5Buffer buf;
+    memset(&buf, 0, sizeof(buf));
+    for(i=0; rc==SQLITE_OK && i<nCol; i++){
+      const char *z;
+      int n;
+      rc = pApi->xColumnText(pFts, i, &z, &n);
+      if( rc==SQLITE_OK ){
+        Fts5Buffer buf1;
+        memset(&buf1, 0, sizeof(Fts5Buffer));
+        rc = pApi->xTokenize(pFts, z, n, (void*)&buf1, fts5TestCallback);
+        if( i!=0 ) sqlite3Fts5BufferAppendPrintf(&rc, &buf, " ");
+        sqlite3Fts5BufferAppendListElem(&rc, &buf, (const char*)buf1.p, buf1.n);
+        sqlite3_free(buf1.p);
+      }
+    }
+    if( zReq==0 ){
+      sqlite3Fts5BufferAppendListElem(&rc, &s, (const char*)buf.p, buf.n);
+    }else{
+      sqlite3Fts5BufferAppendString(&rc, &s, (const char*)buf.p);
+    }
+    sqlite3_free(buf.p);
+  }
+
   if( rc==SQLITE_OK ){
     sqlite3_result_text(pCtx, (const char*)s.p, -1, SQLITE_TRANSIENT);
   }else{

diff --git a/manifest b/manifest
index d18e5f644bdb80d1b5c6eb85e5a54b3ad48d74e5..c08a4a6ccc36db4fb51892b4babe5ba5a6b10b19 100644 (file)
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Fix\sthe\sxColumnSize()\sextension\sAPI.
-D 2014-07-21T11:44:47.659
+C Add\sthe\sxTokenize\sextension\sAPI.
+D 2014-07-21T14:22:38.753
 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f
 F Makefile.in b03432313a3aad96c706f8164fb9f5307eaf19f5
 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23
@@ -103,10 +103,10 @@ F ext/fts3/tool/fts3view.c 6cfc5b67a5f0e09c0d698f9fd012c784bfaa9197
 F ext/fts3/unicode/CaseFolding.txt 8c678ca52ecc95e16bc7afc2dbf6fc9ffa05db8c
 F ext/fts3/unicode/UnicodeData.txt cd07314edb62d49fde34debdaf92fa2aa69011e7
 F ext/fts3/unicode/mkunicode.tcl dc6f268eb526710e2c6e496c372471d773d0c368
-F ext/fts5/fts5.c 5bf93402f9bafa55181dfa70c3a785a41af31025
-F ext/fts5/fts5.h 84060c2fe91aa03a783cc993f313d8a5b22cbe11
+F ext/fts5/fts5.c fda4dd1d2c3e1c6bcb0bf673428a23e4752c21f5
+F ext/fts5/fts5.h c77b6a4a56d80f70fc4f0444030c88724397ed10
 F ext/fts5/fts5Int.h 12d03496152b716e63a5380e396b776fbefa2065
-F ext/fts5/fts5_aux.c 9f0487715cd9933f2268620b41fb47f78a389297
+F ext/fts5/fts5_aux.c 14961135231dd50e6c17894e649c3bbc8c042829
 F ext/fts5/fts5_buffer.c 00361d4a70040ebd2c32bc349ab708ff613a1749
 F ext/fts5/fts5_config.c 94f1b4cb4de6a7cd5780c14adb0198e289df8cef
 F ext/fts5/fts5_expr.c 288b3e016253eab69ea8cefbff346a4697b44291
@@ -599,7 +599,7 @@ F test/fts5aa.test 0f5d29bf0a86b9dff0906c9e166d624c591d3437
 F test/fts5ab.test dc04ed48cf93ca957d174406e6c192f2ff4f3397
 F test/fts5ac.test 9be418d037763f4cc5d86f4239db41fc86bb4f85
 F test/fts5ad.test 2ed38bbc865678cb2905247120d02ebba7f20e07
-F test/fts5ae.test b856782549ae0c56628e0980444fc38cf20b41a0
+F test/fts5ae.test fe9db78201bbb87c6f82b72a14b946d0f7fc3026
 F test/fts5ea.test ff43b40f8879ba50b82def70f2ab67c195d1a1d4
 F test/full.test 6b3c8fb43c6beab6b95438c1675374b95fab245d
 F test/func.test ae97561957aba6ca9e3a7b8a13aac41830d701ef
@@ -1195,7 +1195,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1
 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4
 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32
 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f
-P 1e9053abdaf5e128d44504ee00dfd909dc25f378
-R 962590916dedf53a395813418f70a86f
+P 19504c4108472d2ad1281221642b8bd06eb69f4e
+R 5c4f852f4d425543d226dd267d759872
 U dan
-Z 393e4d18f784466676fdeef71a737c83
+Z 17df0b0e1c67547ce560ffb2b8c93af3

diff --git a/manifest.uuid b/manifest.uuid
index 766db6dab7d291834b77466ff6a792e0c950a640..d01210d922e7d515f91e1bb6cd14fbf249ccd5d0 100644 (file)
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-19504c4108472d2ad1281221642b8bd06eb69f4e
\ No newline at end of file
+8c6b0aff3443fae4b7f0b9adcbf1514992b70653
\ No newline at end of file

diff --git a/test/fts5ae.test b/test/fts5ae.test
index 3fa99d3d5160b1bb9b90ccc1ce3a0160854a02ab..c1eabef3f8b6fa416b36a583ecf2270049e77b24 100644 (file)
--- a/test/fts5ae.test
+++ b/test/fts5ae.test
@@ -175,7 +175,22 @@ do_execsql_test 5.4 {
   1 {2 3}
 }
 
+#-------------------------------------------------------------------------
+# Test the xTokenize() API
+#
+reset_db
+do_execsql_test 6.1 {
+  CREATE VIRTUAL TABLE t6 USING fts5(x, y);
+  INSERT INTO t6 VALUES('There are more', 'things in heaven and earth');
+  INSERT INTO t6 VALUES(', Horatio, Than are', 'dreamt of in your philosophy.');
+}
 
+do_execsql_test 6.2 {
+  SELECT rowid, fts5_test(t6, 'tokenize') FROM t6 WHERE t6 MATCH 't*'
+} {
+  2 {{horatio than are} {dreamt of in your philosophy}}
+  1 {{there are more} {things in heaven and earth}}
+}
 
 finish_test