Add new fts5 API xQueryToken().

FossilOrigin-Name: 828566392b3ea8db603cb1ae5eccbc8ac035efaa284bc7c15ba89874f634aec9

diff --git a/ext/fts5/fts5.h b/ext/fts5/fts5.h
index 730a2fb5e371bb4b5c765616842554b02ec295cb..5a2008882fe6279ea11cec57f0f6adfdf3bbb0cb 100644 (file)
--- a/ext/fts5/fts5.h
+++ b/ext/fts5/fts5.h
@@ -263,7 +263,7 @@ struct Fts5PhraseIter {
 **   See xPhraseFirstColumn above.
 */
 struct Fts5ExtensionApi {
-  int iVersion;                   /* Currently always set to 2 */
+  int iVersion;                   /* Currently always set to 3 */
 
   void *(*xUserData)(Fts5Context*);
 

diff --git a/ext/fts5/fts5Int.h b/ext/fts5/fts5Int.h
index 4aa578559b8cd1dec2533d14693a8cf222eb4e05..df25fd9ffa6477b7e5127eeaa1fb1a7fe1a5aa3c 100644 (file)
--- a/ext/fts5/fts5Int.h
+++ b/ext/fts5/fts5Int.h
@@ -773,6 +773,8 @@ int sqlite3Fts5ExprClonePhrase(Fts5Expr*, int, Fts5Expr**);
 
 int sqlite3Fts5ExprPhraseCollist(Fts5Expr *, int, const u8 **, int *);
 
+int sqlite3Fts5ExprQueryToken(Fts5Expr*, int, int, const char**, int*);
+
 /*******************************************
 ** The fts5_expr.c API above this point is used by the other hand-written
 ** C code in this module. The interfaces below this point are called by

diff --git a/ext/fts5/fts5_expr.c b/ext/fts5/fts5_expr.c
index a2c6320719af4675d7c6ec07e8d9d9a5f3cd5c1f..80387a90fdc56a053bcde8b9a5377fc22e1e9899 100644 (file)
--- a/ext/fts5/fts5_expr.c
+++ b/ext/fts5/fts5_expr.c
@@ -3145,3 +3145,29 @@ int sqlite3Fts5ExprPhraseCollist(
 
   return rc;
 }
+
+/*
+** Does the work of the fts5_api.xQueryToken() API method.
+*/
+int sqlite3Fts5ExprQueryToken(
+  Fts5Expr *pExpr, 
+  int iPhrase, 
+  int iToken, 
+  const char **ppOut, 
+  int *pnOut
+){
+  Fts5ExprPhrase *pPhrase = 0;
+
+  if( iPhrase<0 || iPhrase>=pExpr->nPhrase ){
+    return SQLITE_RANGE;
+  }
+  pPhrase = pExpr->apExprPhrase[iPhrase];
+  if( iToken<0 || iToken>=pPhrase->nTerm ){
+    return SQLITE_RANGE;
+  }
+
+  *ppOut = pPhrase->aTerm[iToken].pTerm;
+  *pnOut = pPhrase->aTerm[iToken].nFullTerm;
+  return SQLITE_OK;
+}
+

diff --git a/ext/fts5/fts5_main.c b/ext/fts5/fts5_main.c
index 6e86ca5951172a931b6e21155c5bb24fef89ba6c..7ddc7b6fcf3dbc9e40d716361990ff0e40e5d9c0 100644 (file)
--- a/ext/fts5/fts5_main.c
+++ b/ext/fts5/fts5_main.c
@@ -2323,13 +2323,24 @@ static int fts5ApiPhraseFirstColumn(
   return rc;
 }
 
+static int fts5ApiQueryToken(
+  Fts5Context* pCtx, 
+  int iPhrase, 
+  int iToken, 
+  const char **ppOut, 
+  int *pnOut
+){
+  Fts5Cursor *pCsr = (Fts5Cursor*)pCtx;
+  return sqlite3Fts5ExprQueryToken(pCsr->pExpr, iPhrase, iToken, ppOut, pnOut);
+}
+
 
 static int fts5ApiQueryPhrase(Fts5Context*, int, void*, 
     int(*)(const Fts5ExtensionApi*, Fts5Context*, void*)
 );
 
 static const Fts5ExtensionApi sFts5Api = {
-  2,                            /* iVersion */
+  3,                            /* iVersion */
   fts5ApiUserData,
   fts5ApiColumnCount,
   fts5ApiRowCount,
@@ -2349,6 +2360,9 @@ static const Fts5ExtensionApi sFts5Api = {
   fts5ApiPhraseNext,
   fts5ApiPhraseFirstColumn,
   fts5ApiPhraseNextColumn,
+  fts5ApiQueryToken,
+  0,
+  0
 };
 
 /*

diff --git a/ext/fts5/fts5_tcl.c b/ext/fts5/fts5_tcl.c
index fb4bea8e9e05417288f8a86216c4e269f5944fc1..a9390adc9ecd464cf28dfdc829ae179458975421 100644 (file)
--- a/ext/fts5/fts5_tcl.c
+++ b/ext/fts5/fts5_tcl.c
@@ -244,6 +244,8 @@ static int SQLITE_TCLAPI xF5tApi(
     { "xGetAuxdataInt",    1, "CLEAR" },              /* 15 */
     { "xPhraseForeach",    4, "IPHRASE COLVAR OFFVAR SCRIPT" }, /* 16 */
     { "xPhraseColumnForeach", 3, "IPHRASE COLVAR SCRIPT" }, /* 17 */
+
+    { "xQueryToken",       2, "IPHRASE ITERM" },      /* 19 */
     { 0, 0, 0}
   };
 
@@ -500,6 +502,22 @@ static int SQLITE_TCLAPI xF5tApi(
       break;
     }
 
+    CASE(18, "xQueryToken") {
+      const char *pTerm = 0;
+      int nTerm = 0;
+      int iPhrase = 0;
+      int iTerm = 0;
+
+      if( Tcl_GetIntFromObj(interp, objv[2], &iPhrase) ) return TCL_ERROR;
+      if( Tcl_GetIntFromObj(interp, objv[3], &iTerm) ) return TCL_ERROR;
+      rc = p->pApi->xQueryToken(p->pFts, iPhrase, iTerm, &pTerm, &nTerm);
+      if( rc==SQLITE_OK ){
+        Tcl_SetObjResult(interp, Tcl_NewStringObj(pTerm, nTerm));
+      }
+
+      break;
+    }
+
     default: 
       assert( 0 );
       break;

diff --git a/ext/fts5/test/fts5origintext.test b/ext/fts5/test/fts5origintext.test
index 3fd5f17e7bd0536d1ca967a76c478507950e33a3..2c9a4ba5a37e2bdcdd9a179080ed795e4f5da4b5 100644 (file)
--- a/ext/fts5/test/fts5origintext.test
+++ b/ext/fts5/test/fts5origintext.test
@@ -157,5 +157,48 @@ do_execsql_test 3.1.3 { SELECT rowid FROM ft2('HELLO') } {1 2 3}
 
 do_execsql_test 3.1.4 { SELECT rowid FROM ft2('hello*') } {1 2 3 10}
 
+#-------------------------------------------------------------------------
+#
+reset_db
+sqlite3_fts5_register_origintext db
+proc querytoken {cmd iPhrase iToken} { 
+  set txt [$cmd xQueryToken $iPhrase $iToken]
+  string map [list "\0" "."] $txt
+}
+sqlite3_fts5_create_function db querytoken querytoken
+
+do_execsql_test 3.0 {
+  CREATE VIRTUAL TABLE ft USING fts5(
+      x, tokenize='origintext unicode61', tokendata=1
+  );
+  INSERT INTO ft VALUES('one two three four');
+}
+
+do_execsql_test 3.1 {
+  SELECT rowid, querytoken(ft, 0, 0) FROM ft('TwO')
+} {1 two.TwO}
+do_execsql_test 3.2 {
+  SELECT rowid, querytoken(ft, 0, 0) FROM ft('one TWO ThreE')
+} {1 one}
+do_execsql_test 3.3 {
+  SELECT rowid, querytoken(ft, 1, 0) FROM ft('one TWO ThreE')
+} {1 two.TWO}
+do_execsql_test 3.4 {
+  SELECT rowid, querytoken(ft, 0, 2) FROM ft('"one TWO ThreE"')
+} {1 three.ThreE}
+
+do_catchsql_test 3.5 {
+  SELECT rowid, querytoken(ft, 0, 3) FROM ft('"one TWO ThreE"')
+} {1 SQLITE_RANGE}
+do_catchsql_test 3.6 {
+  SELECT rowid, querytoken(ft, 1, 0) FROM ft('"one TWO ThreE"')
+} {1 SQLITE_RANGE}
+do_catchsql_test 3.7 {
+  SELECT rowid, querytoken(ft, -1, 0) FROM ft('"one TWO ThreE"')
+} {1 SQLITE_RANGE}
+
+
+
+
 finish_test
 

diff --git a/manifest b/manifest
index baff022d139e9315780df8451c96ead56c91c8dd..4a539764cfa0388c10f9d599718569c47307a6f1 100644 (file)
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Add\sdeclarations\sfor\snew\sAPI\sfunctions.
-D 2023-11-08T14:55:20.854
+C Add\snew\sfts5\sAPI\sxQueryToken().
+D 2023-11-13T14:29:12.382
 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1
 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea
 F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724
@@ -87,17 +87,17 @@ F ext/fts3/unicode/UnicodeData.txt cd07314edb62d49fde34debdaf92fa2aa69011e7
 F ext/fts3/unicode/mkunicode.tcl d5aebf022fa4577ee8cdf27468f0d847879993959101f6dbd6348ef0cfc324a7
 F ext/fts3/unicode/parseunicode.tcl a981bd6466d12dd17967515801c3ff23f74a281be1a03cf1e6f52a6959fc77eb
 F ext/fts5/extract_api_docs.tcl a36e54ec777172ddd3f9a88daf593b00848368e0
-F ext/fts5/fts5.h 68256dd94eaba3e874762a63578922fd62a5ca87c76a28f7636effee4d9bd781
-F ext/fts5/fts5Int.h a21eb1cf036ac9eb943e45ed307762901ea86f0159bf0848baa2079a112ddc2f
+F ext/fts5/fts5.h e27cdb10e38d87cb041dcb56cef97addf7d902aeab07e84e7102f5fc65d3357c
+F ext/fts5/fts5Int.h 19b198459a2791415919428d44ebf4c830b59b2da6f27f8faaffe39a876b7ecf
 F ext/fts5/fts5_aux.c ee770eec0af8646db9e18fc01a0dad7345b5f5e8cbba236704cfae2d777022ad
 F ext/fts5/fts5_buffer.c 3001fbabb585d6de52947b44b455235072b741038391f830d6b729225eeaf6a5
 F ext/fts5/fts5_config.c 8072a207034b51ae9b7694121d1b5715c794e94b275e088f70ae532378ca5cdf
-F ext/fts5/fts5_expr.c fd091d0558fda2517602ed5886ec615ce3e1bd76fb0bb0e5d1aa85ba8db287a8
+F ext/fts5/fts5_expr.c 69c81af515ce1cedccf093c7c76f8b3b4f24bafbfb1d03a431af9f5c69a81834
 F ext/fts5/fts5_hash.c adda4272be401566a6e0ba1acbe70ee5cb97fce944bc2e04dc707152a0ec91b1
 F ext/fts5/fts5_index.c 145723e22ffee28dbe2a24933e74ad998d32419223c0ddb8506a1f0c39b952c4
-F ext/fts5/fts5_main.c a07ed863b8bd9e6fefb62db2fd40a3518eb30a5f7dcfda5be915dd2db45efa2f
+F ext/fts5/fts5_main.c ddac85dbd28167af81f64e568bfe9020bf9708d650de207d1465ed19938316d1
 F ext/fts5/fts5_storage.c 5d10b9bdcce5b90656cad13c7d12ad4148677d4b9e3fca0481fca56d6601426d
-F ext/fts5/fts5_tcl.c 0d2bb0ff7bf6ee136015be118167f0bd956ddd05a8f02c68bd34299b50648f9f
+F ext/fts5/fts5_tcl.c 71641a0c5693c64acfad9d10e64475ec92d9f464d06ba7fd350552de373586d8
 F ext/fts5/fts5_test_mi.c 08c11ec968148d4cb4119d96d819f8c1f329812c568bac3684f5464be177d3ee
 F ext/fts5/fts5_test_tok.c 3cb0a9b508b30d17ef025ccddd26ae3dc8ddffbe76c057616e59a9aa85d36f3b
 F ext/fts5/fts5_tokenize.c 83cfcede3898001cab84432a36ce1503e3080cf9b1c682b022ec82e267ea4c13
@@ -188,7 +188,7 @@ F ext/fts5/test/fts5onepass.test f9b7d9b2c334900c6542a869760290e2ab5382af8fbd618
 F ext/fts5/test/fts5optimize.test 36a752d24c818792032e4ff502936fc9cc5ef938721696396fdc79214b2717f1
 F ext/fts5/test/fts5optimize2.test 93e742c36b487d8874621360af5b1ce4d39b04fb9e71ce9bc34015c5fc811785
 F ext/fts5/test/fts5optimize3.test bf9c91bb927d0fb2b9a06318a217a0419183ac5913842e062c7e0b98ea5d0fca
-F ext/fts5/test/fts5origintext.test 646df137f1aa5b3d7032374ebe82bfdbe88d9f825d73ce8d44bead480317a9c5
+F ext/fts5/test/fts5origintext.test 8296984d268d1d20f85c9de316f422ffb6ebc12020d3f8a0a18144d6ca7b347f
 F ext/fts5/test/fts5phrase.test 13e5d8e9083077b3d9c74315b3c92ec723cc6eb37c8155e0bfe1bba00559f07b
 F ext/fts5/test/fts5plan.test b65cfcca9ddd6fdaa118c61e17aeec8e8433bc5b6bb307abd116514f79c49c5a
 F ext/fts5/test/fts5porter.test 8d08010c28527db66bc3feebd2b8767504aaeb9b101a986342fa7833d49d0d15
@@ -2143,8 +2143,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P 3a869cf1f84b0e9bdcc4de53685430ab41eafacbba1ca7b87e727aa98811c6c5
-R 9e06a9e0b44056d476c02db915884372
+P b8a48cc18c94d15017f898c820fdd784efbaac20d7a45c4d97269333e8f2ec60
+R 177f59b3c48b3865f7f595e0ec855966
 U dan
-Z 6c0dbde9f9a0f05c4a27f8bd01924beb
+Z ac818dc8fd8dc9007ff77eae486bea2e
 # Remove this line to create a well-formed Fossil manifest.

diff --git a/manifest.uuid b/manifest.uuid
index cf6088cc7ca4e28a6cc0515817b9bf5eb9f8582f..f6858a05cd6ffd4a4014671611f80a50e78961cf 100644 (file)
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-b8a48cc18c94d15017f898c820fdd784efbaac20d7a45c4d97269333e8f2ec60
\ No newline at end of file
+828566392b3ea8db603cb1ae5eccbc8ac035efaa284bc7c15ba89874f634aec9
\ No newline at end of file