it is registered as tokenizer <tokenizer-name> and a copy of it
returned. If only one argument is passed, a pointer to the tokenizer
implementation currently registered as <tokenizer-name> is returned,
- encoded as a blob. Or, if no such tokenizer exists, an SQL NULL value
- is returned.
+ encoded as a blob. Or, if no such tokenizer exists, an SQL exception
+ (error) is raised.
SECURITY: If the fts2 extension is used in an environment where potentially
malicious users may execute arbitrary SQL (i.e. gears), they should be
The following two code samples illustrate the way C code should invoke
the fts2_tokenizer() scalar function:
- int registerTokenizer(sqlite3 *db, char *zName, sqlite3_tokenizer_module *p){
- sqlite3_stmt *pStmt;
- const char zSql[] = "SELECT fts2_tokenizer(?, ?)";
-
- rc = sqlite3_prepare_v2(db, zSql, &pStmt);
- if( rc!=SQLITE_OK ){
- return rc;
- }
-
- sqlite3_bind_text(pStmt, 1, zName, -1, SQLITE_STATIC);
- sqlite3_bind_blob(pStmt, 2, &p, sizeof(p), SQLITE_STATIC);
- sqlite3_step(pStmt);
-
- return sqlite3_finalize(pStmt);
- }
-
- int queryTokenizer(sqlite3 *db, char *zName, sqlite3_tokenizer_module **pp){
- sqlite3_stmt *pStmt;
- const char zSql[] = "SELECT fts2_tokenizer(?)";
-
- *pp = 0;
- rc = sqlite3_prepare_v2(db, zSql, &pStmt);
- if( rc!=SQLITE_OK ){
- return rc;
- }
-
- sqlite3_bind_text(pStmt, 1, zName, -1, SQLITE_STATIC);
- if( SQLITE_ROW==sqlite3_step(pStmt) ){
- if( sqlite3_column_type(pStmt, 0)==SQLITE_BLOB ){
- memcpy(pp, sqlite3_column_blob(pStmt, 0), sizeof(*pp));
+ int registerTokenizer(
+ sqlite3 *db,
+ char *zName,
+ const sqlite3_tokenizer_module *p
+ ){
+ int rc;
+ sqlite3_stmt *pStmt;
+ const char zSql[] = "SELECT fts2_tokenizer(?, ?)";
+
+ rc = sqlite3_prepare_v2(db, zSql, -1, &pStmt, 0);
+ if( rc!=SQLITE_OK ){
+ return rc;
+ }
+
+ sqlite3_bind_text(pStmt, 1, zName, -1, SQLITE_STATIC);
+ sqlite3_bind_blob(pStmt, 2, &p, sizeof(p), SQLITE_STATIC);
+ sqlite3_step(pStmt);
+
+ return sqlite3_finalize(pStmt);
+ }
+
+ int queryTokenizer(
+ sqlite3 *db,
+ char *zName,
+ const sqlite3_tokenizer_module **pp
+ ){
+ int rc;
+ sqlite3_stmt *pStmt;
+ const char zSql[] = "SELECT fts2_tokenizer(?)";
+
+ *pp = 0;
+ rc = sqlite3_prepare_v2(db, zSql, -1, &pStmt, 0);
+ if( rc!=SQLITE_OK ){
+ return rc;
+ }
+
+ sqlite3_bind_text(pStmt, 1, zName, -1, SQLITE_STATIC);
+ if( SQLITE_ROW==sqlite3_step(pStmt) ){
+ if( sqlite3_column_type(pStmt, 0)==SQLITE_BLOB ){
+ memcpy(pp, sqlite3_column_blob(pStmt, 0), sizeof(*pp));
+ }
+ }
+
+ return sqlite3_finalize(pStmt);
}
- }
-
- return sqlite3_finalize(pStmt);
- }
#ifdef SQLITE_TEST
#include <tcl.h>
+#include <string.h>
/*
** Implementation of a special SQL scalar function for testing tokenizers
}
Tcl_DecrRefCount(pRet);
}
+
+static
+int registerTokenizer(
+ sqlite3 *db,
+ char *zName,
+ const sqlite3_tokenizer_module *p
+){
+ int rc;
+ sqlite3_stmt *pStmt;
+ const char zSql[] = "SELECT fts2_tokenizer(?, ?)";
+
+ rc = sqlite3_prepare_v2(db, zSql, -1, &pStmt, 0);
+ if( rc!=SQLITE_OK ){
+ return rc;
+ }
+
+ sqlite3_bind_text(pStmt, 1, zName, -1, SQLITE_STATIC);
+ sqlite3_bind_blob(pStmt, 2, &p, sizeof(p), SQLITE_STATIC);
+ sqlite3_step(pStmt);
+
+ return sqlite3_finalize(pStmt);
+}
+
+static
+int queryTokenizer(
+ sqlite3 *db,
+ char *zName,
+ const sqlite3_tokenizer_module **pp
+){
+ int rc;
+ sqlite3_stmt *pStmt;
+ const char zSql[] = "SELECT fts2_tokenizer(?)";
+
+ *pp = 0;
+ rc = sqlite3_prepare_v2(db, zSql, -1, &pStmt, 0);
+ if( rc!=SQLITE_OK ){
+ return rc;
+ }
+
+ sqlite3_bind_text(pStmt, 1, zName, -1, SQLITE_STATIC);
+ if( SQLITE_ROW==sqlite3_step(pStmt) ){
+ if( sqlite3_column_type(pStmt, 0)==SQLITE_BLOB ){
+ memcpy(pp, sqlite3_column_blob(pStmt, 0), sizeof(*pp));
+ }
+ }
+
+ return sqlite3_finalize(pStmt);
+}
+
+void sqlite3Fts2SimpleTokenizerModule(sqlite3_tokenizer_module const**ppModule);
+
+/*
+** Implementation of the scalar function fts2_tokenizer_internal_test().
+** This function is used for testing only, it is not included in the
+** build unless SQLITE_TEST is defined.
+**
+** The purpose of this is to test that the fts2_tokenizer() function
+** can be used as designed by the C-code in the queryTokenizer and
+** registerTokenizer() functions above. These two functions are repeated
+** in the README.tokenizer file as an example, so it is important to
+** test them.
+**
+** To run the tests, evaluate the fts2_tokenizer_internal_test() scalar
+** function with no arguments. An assert() will fail if a problem is
+** detected. i.e.:
+**
+** SELECT fts2_tokenizer_internal_test();
+**
+*/
+static void intTestFunc(
+ sqlite3_context *context,
+ int argc,
+ sqlite3_value **argv
+){
+ int rc;
+ const sqlite3_tokenizer_module *p1;
+ const sqlite3_tokenizer_module *p2;
+ sqlite3 *db = (sqlite3 *)sqlite3_user_data(context);
+
+ /* Test the query function */
+ sqlite3Fts2SimpleTokenizerModule(&p1);
+ rc = queryTokenizer(db, "simple", &p2);
+ assert( rc==SQLITE_OK );
+ assert( p1==p2 );
+ rc = queryTokenizer(db, "nosuchtokenizer", &p2);
+ assert( rc==SQLITE_ERROR );
+ assert( p2==0 );
+ assert( 0==strcmp(sqlite3_errmsg(db), "unknown tokenizer: nosuchtokenizer") );
+
+ /* Test the storage function */
+ rc = registerTokenizer(db, "nosuchtokenizer", p1);
+ assert( rc==SQLITE_OK );
+ rc = queryTokenizer(db, "nosuchtokenizer", &p2);
+ assert( rc==SQLITE_OK );
+ assert( p2==p1 );
+
+ sqlite3_result_text(context, "ok", -1, SQLITE_STATIC);
+}
+
#endif
/*
){
int rc;
void *p = (void *)pHash;
+ void *pdb = (void *)db;
const int any = SQLITE_ANY;
char *zTest = 0;
+ char *zTest2 = 0;
#ifdef SQLITE_TEST
zTest = sqlite3_mprintf("%s_test", zName);
- if( !zTest ){
- return SQLITE_NOMEM;
+ zTest2 = sqlite3_mprintf("%s_internal_test", zName);
+ if( !zTest || !zTest2 ){
+ rc = SQLITE_NOMEM;
}
#endif
- if( (rc = sqlite3_create_function(db, zName, 1, any, p, scalarFunc, 0, 0))
+ if( rc!=SQLITE_OK
+ || (rc = sqlite3_create_function(db, zName, 1, any, p, scalarFunc, 0, 0))
|| (rc = sqlite3_create_function(db, zName, 2, any, p, scalarFunc, 0, 0))
#ifdef SQLITE_TEST
|| (rc = sqlite3_create_function(db, zTest, 2, any, p, testFunc, 0, 0))
|| (rc = sqlite3_create_function(db, zTest, 3, any, p, testFunc, 0, 0))
+ || (rc = sqlite3_create_function(db, zTest2, 0, any, pdb, intTestFunc, 0, 0))
#endif
);
sqlite3_free(zTest);
+ sqlite3_free(zTest2);
return rc;
}
-C Add\ssome\stests\sfor\sthe\sfts2\sicu\stokenizer.\s(CVS\s4117)
-D 2007-06-25T11:24:39
+C Add\sa\stest\sthat\scalls\sfts2_tokenizer()\swith\san\sargument\sset\svia\sC\scode.\s(CVS\s4118)
+D 2007-06-25T12:05:40
F Makefile.in 7f7485a4cc039476a42e534b3f26ec90e2f9753e
F Makefile.linux-gcc 2d8574d1ba75f129aba2019f0b959db380a90935
F README 9c4e2d6706bdcc3efdd773ce752a8cdab4f90028
F ext/fts1/fulltext.h 08525a47852d1d62a0be81d3fc3fe2d23b094efd
F ext/fts1/simple_tokenizer.c 1844d72f7194c3fd3d7e4173053911bf0661b70d
F ext/fts1/tokenizer.h 0c53421b832366d20d720d21ea3e1f6e66a36ef9
-F ext/fts2/README.tokenizers f358364121285c402d7b38fd44ba87b40903859b
+F ext/fts2/README.tokenizers 2ff290e0a130f6e7611f2e608cb3b5aaea721abc
F ext/fts2/README.txt 8c18f41574404623b76917b9da66fcb0ab38328d
F ext/fts2/fts2.c 841766f2f14d68e623404f9531d98afa0f7cbf05
F ext/fts2/fts2.h 591916a822cfb6426518fdbf6069359119bc46eb
F ext/fts2/fts2_hash.h e283308156018329f042816eb09334df714e105e
F ext/fts2/fts2_icu.c 45b54d1e075020b35db20f69d829f95ca0651111
F ext/fts2/fts2_porter.c 991a45463553c7318063fe7773368a6c0f39e35d
-F ext/fts2/fts2_tokenizer.c 40aa54fab0305b53f883fe4a394552f71dda29ee
+F ext/fts2/fts2_tokenizer.c 9200889237906d5607496dbd4c8b8c53a617ca71
F ext/fts2/fts2_tokenizer.h 6d151c51382e8f6cf689c616bb697fe780478089
F ext/fts2/fts2_tokenizer1.c 5c979fe8815f95396beb22b627571da895a025af
F ext/fts2/mkfts2amal.tcl 2a9ec76b0760fe7f3669dca5bc0d60728bc1c977
F test/fts2m.test 4b30142ead6f3ed076e880a2a464064c5ad58c51
F test/fts2n.test a70357e72742681eaebfdbe9007b87ff3b771638
F test/fts2o.test 05ce2ac9111c29998418a584de02136a0ded471b
-F test/fts2token.test 5f349000669bc10bb392c87443ca5d3a587c9df0
+F test/fts2token.test d8070b241a15ff13592a9ae4a8b7c171af6f445a
F test/func.test 605989453d1b42cec1d05c17aa232dc98e3e04e6
F test/fuzz.test 62fc19dd36a427777fd671b569df07166548628a
F test/fuzz2.test ea38692ce2da99ad79fe0be5eb1a452c1c4d37bb
F www/vdbe.tcl 87a31ace769f20d3627a64fa1fade7fed47b90d0
F www/version3.tcl 890248cf7b70e60c383b0e84d77d5132b3ead42b
F www/whentouse.tcl fc46eae081251c3c181bd79c5faef8195d7991a5
-P 5a9eee86587219a68655d548864d129edec969ae
-R 61c2bfb00d5da35c1ad182696b085ade
+P b79ced3e0a26b0db13613073c847c2d2ba7e174e
+R bd43a4e803b49844e3492af252e992ad
U danielk1977
-Z 97c1d47f7790b3efb6130173bff1e39a
+Z c0d7af5f5f203b9a840aeab00383f3fd
-b79ced3e0a26b0db13613073c847c2d2ba7e174e
\ No newline at end of file
+fbcf2d75cd2b88d175c122477aa483f0771870e5
\ No newline at end of file
# of this script is testing the pluggable tokeniser feature of the
# FTS2 module.
#
-# $Id: fts2token.test,v 1.2 2007/06/25 11:24:39 danielk1977 Exp $
+# $Id: fts2token.test,v 1.3 2007/06/25 12:05:40 danielk1977 Exp $
#
set testdir [file dirname $argv0]
do_icu_test fts2token-4.8 en_US $input $output
}
+do_test fts2token-internal {
+ execsql { SELECT fts2_tokenizer_internal_test() }
+} {ok}
+
finish_test
projects / thirdparty / sqlite.git / commitdiff
