#include "fts5Int.h"
#include <math.h>
-typedef struct SnippetPhrase SnippetPhrase;
-typedef struct SnippetIter SnippetIter;
+typedef struct SnipPhrase SnipPhrase;
+typedef struct SnipIter SnipIter;
typedef struct SnippetCtx SnippetCtx;
-struct SnippetPhrase {
+struct SnipPhrase {
u64 mask; /* Current mask */
int nToken; /* Tokens in this phrase */
int i; /* Current offset in phrase poslist */
i64 iPos; /* Next position in phrase (-ve -> EOF) */
};
-struct SnippetIter {
+struct SnipIter {
i64 iLast; /* Last token position of current snippet */
int nScore; /* Score of current snippet */
const Fts5ExtensionApi *pApi;
Fts5Context *pFts;
- u64 szmask; /* Mask used to on SnippetPhrase.mask */
+ u64 szmask; /* Mask used to on SnipPhrase.mask */
int nPhrase; /* Number of phrases */
- SnippetPhrase aPhrase[0]; /* Array of size nPhrase */
+ SnipPhrase aPhrase[0]; /* Array of size nPhrase */
};
struct SnippetCtx {
/*
** Set pIter->nScore to the score for the current entry.
*/
-static void fts5SnippetCalculateScore(SnippetIter *pIter){
+static void fts5SnippetCalculateScore(SnipIter *pIter){
int i;
int nScore = 0;
assert( pIter->iLast>=0 );
for(i=0; i<pIter->nPhrase; i++){
- SnippetPhrase *p = &pIter->aPhrase[i];
+ SnipPhrase *p = &pIter->aPhrase[i];
u64 mask = p->mask;
if( mask ){
u64 j;
/*
** Allocate a new snippet iter.
*/
-static int fts5SnippetIterNew(
+static int fts5SnipIterNew(
const Fts5ExtensionApi *pApi, /* API offered by current FTS version */
Fts5Context *pFts, /* First arg to pass to pApi functions */
int nToken, /* Number of tokens in snippets */
- SnippetIter **ppIter /* OUT: New object */
+ SnipIter **ppIter /* OUT: New object */
){
int i; /* Counter variable */
- SnippetIter *pIter; /* New iterator object */
+ SnipIter *pIter; /* New iterator object */
int nByte; /* Bytes of space to allocate */
int nPhrase; /* Number of phrases in query */
*ppIter = 0;
nPhrase = pApi->xPhraseCount(pFts);
- nByte = sizeof(SnippetIter) + nPhrase * sizeof(SnippetPhrase);
- pIter = (SnippetIter*)sqlite3_malloc(nByte);
+ nByte = sizeof(SnipIter) + nPhrase * sizeof(SnipPhrase);
+ pIter = (SnipIter*)sqlite3_malloc(nByte);
if( pIter==0 ) return SQLITE_NOMEM;
memset(pIter, 0, nByte);
/*
** Set the iterator to point to the first candidate snippet.
*/
-static void fts5SnippetIterFirst(SnippetIter *pIter){
+static void fts5SnipIterFirst(SnipIter *pIter){
const Fts5ExtensionApi *pApi = pIter->pApi;
Fts5Context *pFts = pIter->pFts;
int i; /* Used to iterate through phrases */
- SnippetPhrase *pMin = 0; /* Phrase with first match */
+ SnipPhrase *pMin = 0; /* Phrase with first match */
- memset(pIter->aPhrase, 0, sizeof(SnippetPhrase) * pIter->nPhrase);
+ memset(pIter->aPhrase, 0, sizeof(SnipPhrase) * pIter->nPhrase);
for(i=0; i<pIter->nPhrase; i++){
- SnippetPhrase *p = &pIter->aPhrase[i];
+ SnipPhrase *p = &pIter->aPhrase[i];
p->nToken = pApi->xPhraseSize(pFts, i);
pApi->xPoslist(pFts, i, &p->i, &p->iPos);
if( p->iPos>=0 && (pMin==0 || p->iPos<pMin->iPos) ){
/*
** Advance the snippet iterator to the next candidate snippet.
*/
-static void fts5SnippetIterNext(SnippetIter *pIter){
+static void fts5SnipIterNext(SnipIter *pIter){
const Fts5ExtensionApi *pApi = pIter->pApi;
Fts5Context *pFts = pIter->pFts;
int nPhrase = pIter->nPhrase;
int i; /* Used to iterate through phrases */
- SnippetPhrase *pMin = 0;
+ SnipPhrase *pMin = 0;
for(i=0; i<nPhrase; i++){
- SnippetPhrase *p = &pIter->aPhrase[i];
+ SnipPhrase *p = &pIter->aPhrase[i];
if( p->iPos>=0 && (pMin==0 || p->iPos<pMin->iPos) ) pMin = p;
}
if( pMin==0 ){
- /* pMin==0 indicates that the SnippetIter is at EOF. */
+ /* pMin==0 indicates that the SnipIter is at EOF. */
pIter->iLast = -1;
}else{
i64 nShift = pMin->iPos - pIter->iLast;
assert( nShift>=0 );
for(i=0; i<nPhrase; i++){
- SnippetPhrase *p = &pIter->aPhrase[i];
+ SnipPhrase *p = &pIter->aPhrase[i];
if( nShift>=63 ){
p->mask = 0;
}else{
}
}
-static void fts5SnippetIterFree(SnippetIter *pIter){
+static void fts5SnipIterFree(SnipIter *pIter){
if( pIter ){
sqlite3_free(pIter);
}
static int fts5SnippetText(
const Fts5ExtensionApi *pApi, /* API offered by current FTS version */
Fts5Context *pFts, /* First arg to pass to pApi functions */
- SnippetIter *pIter, /* Snippet to write to buffer */
+ SnipIter *pIter, /* Snippet to write to buffer */
int nToken, /* Size of desired snippet in tokens */
const char *zStart,
const char *zFinal,
/* Check if this is the first token of any phrase match. */
int ip;
for(ip=0; ip<pIter->nPhrase; ip++){
- SnippetPhrase *pPhrase = &pIter->aPhrase[ip];
+ SnipPhrase *pPhrase = &pIter->aPhrase[ip];
u64 m = (1 << (iLast - i - pPhrase->nToken + 1));
if( i<=iLast && (pPhrase->mask & m) ){
int nToken = -15;
int nAbs;
int rc;
- SnippetIter *pIter = 0;
+ SnipIter *pIter = 0;
if( nVal>=1 ) zStart = (const char*)sqlite3_value_text(apVal[0]);
if( nVal>=2 ) zFinal = (const char*)sqlite3_value_text(apVal[1]);
}
nAbs = nToken * (nToken<0 ? -1 : 1);
- rc = fts5SnippetIterNew(pApi, pFts, nAbs, &pIter);
+ rc = fts5SnipIterNew(pApi, pFts, nAbs, &pIter);
if( rc==SQLITE_OK ){
Fts5Buffer buf; /* Result buffer */
int nBestScore = 0; /* Score of best snippet found */
- for(fts5SnippetIterFirst(pIter);
+ for(fts5SnipIterFirst(pIter);
pIter->iLast>=0;
- fts5SnippetIterNext(pIter)
+ fts5SnipIterNext(pIter)
){
if( pIter->nScore>nBestScore ) nBestScore = pIter->nScore;
}
- for(fts5SnippetIterFirst(pIter);
+ for(fts5SnipIterFirst(pIter);
pIter->iLast>=0;
- fts5SnippetIterNext(pIter)
+ fts5SnipIterNext(pIter)
){
if( pIter->nScore==nBestScore ) break;
}
sqlite3_free(buf.p);
}
- fts5SnippetIterFree(pIter);
+ fts5SnipIterFree(pIter);
if( rc!=SQLITE_OK ){
sqlite3_result_error_code(pCtx, rc);
}
return sqlite3_mprintf("%s", z);
}
-void sqlite3Fts3SimpleTokenizerModule(sqlite3_tokenizer_module**);
+void sqlite3Fts3SimpleTokenizerModule(sqlite3_tokenizer_module const**);
/*
** Allocate an instance of the default tokenizer ("simple") at
** code if an error occurs.
*/
static int fts5ConfigDefaultTokenizer(Fts5Config *pConfig){
- sqlite3_tokenizer_module *pMod; /* Tokenizer module "simple" */
+ const sqlite3_tokenizer_module *pMod; /* Tokenizer module "simple" */
sqlite3_tokenizer *pTokenizer; /* Tokenizer instance */
int rc; /* Return code */
** It is the responsibility of the caller to eventually free the returned
** buffer using sqlite3_free(). If an OOM error occurs, NULL is returned.
*/
-static char *fts5Strdup(const char *pIn, int nIn){
+static char *fts5Strndup(const char *pIn, int nIn){
char *zRet = (char*)sqlite3_malloc(nIn+1);
if( zRet ){
memcpy(zRet, pIn, nIn);
}
static int fts5ParseStringFromToken(Fts5Token *pToken, char **pz){
- *pz = fts5Strdup(pToken->p, pToken->n);
+ *pz = fts5Strndup(pToken->p, pToken->n);
if( *pz==0 ) return SQLITE_NOMEM;
return SQLITE_OK;
}
pTerm = &pPhrase->aTerm[pPhrase->nTerm++];
memset(pTerm, 0, sizeof(Fts5ExprTerm));
- pTerm->zTerm = fts5Strdup(pToken, nToken);
+ pTerm->zTerm = fts5Strndup(pToken, nToken);
return pTerm->zTerm ? SQLITE_OK : SQLITE_NOMEM;
}
$(TOP)/ext/rtree/rtree.h \
$(TOP)/ext/rtree/rtree.c
+SRC += \
+ $(TOP)/ext/fts5/fts5.h \
+ $(TOP)/ext/fts5/fts5Int.h \
+ $(TOP)/ext/fts5/fts5_aux.c \
+ $(TOP)/ext/fts5/fts5_buffer.c \
+ $(TOP)/ext/fts5/fts5.c \
+ $(TOP)/ext/fts5/fts5_config.c \
+ $(TOP)/ext/fts5/fts5_expr.c \
+ $(TOP)/ext/fts5/fts5_index.c \
+ fts5parse.c \
+ $(TOP)/ext/fts5/fts5_storage.c
+
# Generated source code files
#
speedtest1$(EXE): $(TOP)/test/speedtest1.c sqlite3.o
$(TCC) -I. -o speedtest1$(EXE) $(TOP)/test/speedtest1.c sqlite3.o $(THREADLIB)
+loadfts: $(TOP)/tool/loadfts.c libsqlite3.a
+ $(TCC) $(TOP)/tool/loadfts.c libsqlite3.a -o loadfts $(THREADLIB)
+
# This target will fail if the SQLite amalgamation contains any exported
# symbols that do not begin with "sqlite3_". It is run as part of the
# releasetest.tcl script.
-C Add\stests\sand\sfixes\sfor\sbm25()\sfunction.
-D 2014-07-26T18:38:51.294
+C Add\sthe\s"loadfts"\sprogram,\sfor\sperformance\stesting\sthe\sloading\sof\sdata\sinto\sfts3/fts4/fts5\stables.
+D 2014-07-28T20:14:02.001
F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f
F Makefile.in b03432313a3aad96c706f8164fb9f5307eaf19f5
F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23
F ext/fts5/fts5.c 1496aff16dd9b0a013d14b6c8cf5b7df8c170abe
F ext/fts5/fts5.h 8ace10d5b249a3baa983c79e7a1306d2a79cfd6a
F ext/fts5/fts5Int.h 92fb9c4f759674ef569aebc338f363e167a8933c
-F ext/fts5/fts5_aux.c 78adc5db0ff4d6834df220ba6b3caa351d98b971
+F ext/fts5/fts5_aux.c 243156c197384e17983d6a3ed149fa2270b5bb85
F ext/fts5/fts5_buffer.c 248c61ac9fec001602efc72a45704f3b8d367c00
-F ext/fts5/fts5_config.c 94f1b4cb4de6a7cd5780c14adb0198e289df8cef
-F ext/fts5/fts5_expr.c 65c1918002f2ec1755e4c0c28bf007659409fbd8
+F ext/fts5/fts5_config.c 2138741013e189724b5d40ea7af0f48952a44916
+F ext/fts5/fts5_expr.c e426baa54b9473cb31b8d891d7d1b923bfb5d017
F ext/fts5/fts5_index.c 68d2d41b5c6d2f8838c3d6ebdc8b242718b8e997
F ext/fts5/fts5_storage.c 2866e7e1de9dc851756c3a9c76b6e1d75e0facb7
F ext/fts5/fts5parse.y 777da8e5819f75c217982c79c29d014c293acac9
F install-sh 9d4de14ab9fb0facae2f48780b874848cbf2f895 x
F ltmain.sh 3ff0879076df340d2e23ae905484d8c15d5fdea8
F magic.txt 8273bf49ba3b0c8559cb2774495390c31fd61c60
-F main.mk cffc02a30f1af82d35410674f70a0286587add81
+F main.mk 8118631727a27fa88eb38a07ac3b86ecb86e9eb0
F mkopcodec.awk c2ff431854d702cdd2d779c9c0d1f58fa16fa4ea
F mkopcodeh.awk c6b3fa301db6ef7ac916b14c60868aeaec1337b5
F mkso.sh fd21c06b063bb16a5d25deea1752c2da6ac3ed83
F tool/getlock.c f4c39b651370156cae979501a7b156bdba50e7ce
F tool/lemon.c 3ff0fec22f92dfb54e62eeb48772eddffdbeb0d6
F tool/lempar.c 01ca97f87610d1dac6d8cd96ab109ab1130e76dc
+F tool/loadfts.c 3bdd46090112c84df44a4fbae740af3836108b3f
F tool/logest.c eef612f8adf4d0993dafed0416064cf50d5d33c6
F tool/mkautoconfamal.sh f8d8dbf7d62f409ebed5134998bf5b51d7266383
F tool/mkkeywordhash.c dfff09dbbfaf950e89af294f48f902181b144670
F tool/mkpragmatab.tcl 78a77b2c554d534c6f2dc903130186ed15715460
F tool/mkspeedsql.tcl a1a334d288f7adfe6e996f2e712becf076745c97
F tool/mksqlite3c-noext.tcl 1712d3d71256ca1f297046619c89e77a4d7c8f6d
-F tool/mksqlite3c.tcl ba274df71f5e6534b0a913c7c48eabfcbd0934b6
+F tool/mksqlite3c.tcl becaa9d5617dfe137e73dddda9dab8f58bc71e8c
F tool/mksqlite3h.tcl ba24038056f51fde07c0079c41885ab85e2cff12
F tool/mksqlite3internalh.tcl b6514145a7d5321b47e64e19b8116cc44f973eb1
F tool/mkvsix.tcl 52a4c613707ac34ae9c226e5ccc69cb948556105
F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4
F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32
F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f
-P c4d50428ab97f77e6721c4f8d03eaaf3ea91f3eb
-R 3301ccb2b839356242606883792ca77e
+P 71d32f53e81921e43c933cc968cb1c18d83fe1e0
+R 378763b2640fc19d9f72a0522c9f77b1
U dan
-Z 456b4a2f1abc554b124e25c35490489e
+Z 3cf4ed481646bab9077300595c244e00
-71d32f53e81921e43c933cc968cb1c18d83fe1e0
\ No newline at end of file
+770b9540c19ad1e3d24adff382332bf032065efd
\ No newline at end of file
--- /dev/null
+/*
+** 2013-06-10
+**
+** The author disclaims copyright to this source code. In place of
+** a legal notice, here is a blessing:
+**
+** May you do good and not evil.
+** May you find forgiveness for yourself and forgive others.
+** May you share freely, never taking more than you give.
+**
+*************************************************************************
+*/
+
+#include <stdio.h>
+#include <stdlib.h>
+#include <ctype.h>
+#include <assert.h>
+#include <string.h>
+#include <errno.h>
+#include <dirent.h>
+#include "sqlite3.h"
+
+/*
+** Implementation of the "readtext(X)" SQL function. The entire content
+** of the file named X is read and returned as a TEXT value. It is assumed
+** the file contains UTF-8 text. NULL is returned if the file does not
+** exist or is unreadable.
+*/
+static void readfileFunc(
+ sqlite3_context *context,
+ int argc,
+ sqlite3_value **argv
+){
+ const char *zName;
+ FILE *in;
+ long nIn;
+ void *pBuf;
+
+ zName = (const char*)sqlite3_value_text(argv[0]);
+ if( zName==0 ) return;
+ in = fopen(zName, "rb");
+ if( in==0 ) return;
+ fseek(in, 0, SEEK_END);
+ nIn = ftell(in);
+ rewind(in);
+ pBuf = sqlite3_malloc( nIn );
+ if( pBuf && 1==fread(pBuf, nIn, 1, in) ){
+ sqlite3_result_text(context, pBuf, nIn, sqlite3_free);
+ }else{
+ sqlite3_free(pBuf);
+ }
+ fclose(in);
+}
+
+/*
+** Print usage text for this program and exit.
+*/
+static void showHelp(const char *zArgv0){
+ printf("\n"
+"Usage: %s SWITCHES... DB\n"
+"\n"
+" This program opens the database named on the command line and attempts to\n"
+" create an FTS table named \"fts\" with a single column. If successful, it\n"
+" recursively traverses the directory named by the -dir option and inserts\n"
+" the contents of each file into the fts table. All files are assumed to\n"
+" contain UTF-8 text.\n"
+"\n"
+"Switches are:\n"
+" -fts [345] FTS version to use (default=5)\n"
+" -idx [01] Create a mapping from filename to rowid (default=0)\n"
+" -dir <path> Root of directory tree to load data from (default=.)\n"
+, zArgv0
+);
+ exit(1);
+}
+
+/*
+** Exit with a message based on the argument and the current value of errno.
+*/
+static void error_out(const char *zText){
+ fprintf(stderr, "%s: %s\n", zText, strerror(errno));
+ exit(-1);
+}
+
+/*
+** Exit with a message based on the first argument and the error message
+** currently stored in database handle db.
+*/
+static void sqlite_error_out(const char *zText, sqlite3 *db){
+ fprintf(stderr, "%s: %s\n", zText, sqlite3_errmsg(db));
+ exit(-1);
+}
+
+/*
+** Context object for visit_file().
+*/
+typedef struct VisitContext VisitContext;
+struct VisitContext {
+ sqlite3 *db; /* Database handle */
+ sqlite3_stmt *pInsert; /* INSERT INTO fts VALUES(readtext(:1)) */
+};
+
+/*
+** Callback used with traverse(). The first argument points to an object
+** of type VisitContext. This function inserts the contents of the text
+** file zPath into the FTS table.
+*/
+void visit_file(void *pCtx, const char *zPath){
+ int rc;
+ VisitContext *p = (VisitContext*)pCtx;
+ /* printf("%s\n", zPath); */
+ sqlite3_bind_text(p->pInsert, 1, zPath, -1, SQLITE_STATIC);
+ sqlite3_step(p->pInsert);
+ rc = sqlite3_reset(p->pInsert);
+ if( rc!=SQLITE_OK ) sqlite_error_out("insert", p->db);
+}
+
+/*
+** Recursively traverse directory zDir. For each file that is not a
+** directory, invoke the supplied callback with its path.
+*/
+static void traverse(
+ const char *zDir, /* Directory to traverse */
+ void *pCtx, /* First argument passed to callback */
+ void (*xCallback)(void*, const char *zPath)
+){
+ DIR *d;
+ struct dirent *e;
+
+ d = opendir(zDir);
+ if( d==0 ) error_out("opendir()");
+
+ for(e=readdir(d); e; e=readdir(d)){
+ if( strcmp(e->d_name, ".")==0 || strcmp(e->d_name, "..")==0 ) continue;
+ char *zPath = sqlite3_mprintf("%s/%s", zDir, e->d_name);
+ if (e->d_type & DT_DIR) {
+ traverse(zPath, pCtx, xCallback);
+ }else{
+ xCallback(pCtx, zPath);
+ }
+ sqlite3_free(zPath);
+ }
+
+ closedir(d);
+}
+
+int main(int argc, char **argv){
+ int iFts = 5; /* Value of -fts option */
+ int bMap = 0; /* True to create mapping table */
+ const char *zDir = "."; /* Directory to scan */
+ int i;
+ int rc;
+ sqlite3 *db;
+ char *zSql;
+ VisitContext sCtx;
+
+ if( argc % 2 ) showHelp(argv[0]);
+
+ for(i=1; i<(argc-1); i+=2){
+ char *zOpt = argv[i];
+ char *zArg = argv[i+1];
+ if( strcmp(zOpt, "-fts")==0 ){
+ iFts = atoi(zArg);
+ if( iFts!=3 && iFts!=4 && iFts!= 5) showHelp(argv[0]);
+ }
+ else if( strcmp(zOpt, "-idx")==0 ){
+ bMap = atoi(zArg);
+ if( bMap!=0 && bMap!=1 ) showHelp(argv[0]);
+ }
+ else if( strcmp(zOpt, "-dir")==0 ){
+ zDir = zArg;
+ }
+ }
+
+ /* Open the database file */
+ rc = sqlite3_open(argv[argc-1], &db);
+ if( rc!=SQLITE_OK ) sqlite_error_out("sqlite3_open()", db);
+
+ rc = sqlite3_create_function(db, "readtext", 1, SQLITE_UTF8, 0,
+ readfileFunc, 0, 0);
+ if( rc!=SQLITE_OK ) sqlite_error_out("sqlite3_create_function()", db);
+
+ /* Create the FTS table */
+ zSql = sqlite3_mprintf("CREATE VIRTUAL TABLE fts USING fts%d(content)", iFts);
+ rc = sqlite3_exec(db, zSql, 0, 0, 0);
+ if( rc!=SQLITE_OK ) sqlite_error_out("sqlite3_exec(1)", db);
+ sqlite3_free(zSql);
+
+ /* Compile the INSERT statement to write data to the FTS table. */
+ memset(&sCtx, 0, sizeof(VisitContext));
+ sCtx.db = db;
+ rc = sqlite3_prepare_v2(db,
+ "INSERT INTO fts VALUES(readtext(?))", -1, &sCtx.pInsert, 0
+ );
+ if( rc!=SQLITE_OK ) sqlite_error_out("sqlite3_prepare_v2(1)", db);
+
+ /* Load all files in the directory hierarchy into the FTS table. */
+ traverse(zDir, (void*)&sCtx, visit_file);
+
+ /* Clean up and exit. */
+ sqlite3_finalize(sCtx.pInsert);
+ sqlite3_close(db);
+ return 0;
+}
fts3Int.h
fts3_hash.h
fts3_tokenizer.h
+ fts5.h
+ fts5Int.h
hash.h
hwtime.h
keywordhash.h
fts3_unicode.c
fts3_unicode2.c
+ fts5_aux.c
+ fts5_buffer.c
+ fts5.c
+ fts5_config.c
+ fts5_expr.c
+ fts5_index.c
+ fts5parse.c
+ fts5_storage.c
+
rtree.c
icu.c
fts3_icu.c
projects / thirdparty / sqlite.git / commitdiff
