From b2822a2b5e6e5aa3fac48549064fd6a64caa4f16 Mon Sep 17 00:00:00 2001 From: shess Date: Tue, 22 Jul 2008 22:57:54 +0000 Subject: [PATCH] Changes fts2 to use only sqlite3_malloc() and not system malloc. Backports (4554) and (4555) from fts3. (CVS 5454) FossilOrigin-Name: ecf2dec66cb979cb7d8db3b7ce5c64cab57fe2bb --- ext/fts2/fts2.c | 114 +++++++++++++++++++------------------ ext/fts2/fts2_hash.c | 1 + ext/fts2/fts2_porter.c | 14 ++--- ext/fts2/fts2_tokenizer1.c | 15 ++--- manifest | 19 ++++--- manifest.uuid | 2 +- test/fts2.test | 68 ++++++++++++++++++++++ 7 files changed, 155 insertions(+), 78 deletions(-) create mode 100644 test/fts2.test diff --git a/ext/fts2/fts2.c b/ext/fts2/fts2.c index a806719611..2a995b0214 100644 --- a/ext/fts2/fts2.c +++ b/ext/fts2/fts2.c @@ -470,13 +470,13 @@ static void dataBufferInit(DataBuffer *pBuffer, int nCapacity){ assert( nCapacity>=0 ); pBuffer->nData = 0; pBuffer->nCapacity = nCapacity; - pBuffer->pData = nCapacity==0 ? NULL : malloc(nCapacity); + pBuffer->pData = nCapacity==0 ? NULL : sqlite3_malloc(nCapacity); } static void dataBufferReset(DataBuffer *pBuffer){ pBuffer->nData = 0; } static void dataBufferDestroy(DataBuffer *pBuffer){ - if( pBuffer->pData!=NULL ) free(pBuffer->pData); + if( pBuffer->pData!=NULL ) sqlite3_free(pBuffer->pData); SCRAMBLE(pBuffer); } static void dataBufferExpand(DataBuffer *pBuffer, int nAddCapacity){ @@ -487,7 +487,7 @@ static void dataBufferExpand(DataBuffer *pBuffer, int nAddCapacity){ */ if( pBuffer->nData+nAddCapacity>pBuffer->nCapacity ){ pBuffer->nCapacity = pBuffer->nData+nAddCapacity; - pBuffer->pData = realloc(pBuffer->pData, pBuffer->nCapacity); + pBuffer->pData = sqlite3_realloc(pBuffer->pData, pBuffer->nCapacity); } } static void dataBufferAppend(DataBuffer *pBuffer, @@ -1082,7 +1082,7 @@ static void dlcAddPos(DLCollector *pCollector, int iColumn, int iPos, } static DLCollector *dlcNew(sqlite_int64 iDocid, DocListType iType){ - DLCollector *pCollector = malloc(sizeof(DLCollector)); + DLCollector *pCollector = sqlite3_malloc(sizeof(DLCollector)); dataBufferInit(&pCollector->b, 0); dlwInit(&pCollector->dlw, iType, &pCollector->b); plwInit(&pCollector->plw, &pCollector->dlw, iDocid); @@ -1093,7 +1093,7 @@ static void dlcDelete(DLCollector *pCollector){ dlwDestroy(&pCollector->dlw); dataBufferDestroy(&pCollector->b); SCRAMBLE(pCollector); - free(pCollector); + sqlite3_free(pCollector); } @@ -1608,7 +1608,7 @@ static void docListExceptMerge( } static char *string_dup_n(const char *s, int n){ - char *str = malloc(n + 1); + char *str = sqlite3_malloc(n + 1); memcpy(str, s, n); str[n] = '\0'; return str; @@ -1641,7 +1641,7 @@ static char *string_format(const char *zFormat, } len += 1; /* for null terminator */ - r = result = malloc(len); + r = result = sqlite3_malloc(len); for(p = zFormat; *p; ++p){ if( *p=='%' ){ memcpy(r, zDb, nDb); @@ -1664,7 +1664,7 @@ static int sql_exec(sqlite3 *db, const char *zDb, const char *zName, int rc; TRACE(("FTS2 sql: %s\n", zCommand)); rc = sqlite3_exec(db, zCommand, NULL, 0, NULL); - free(zCommand); + sqlite3_free(zCommand); return rc; } @@ -1674,7 +1674,7 @@ static int sql_prepare(sqlite3 *db, const char *zDb, const char *zName, int rc; TRACE(("FTS2 prepare: %s\n", zCommand)); rc = sqlite3_prepare_v2(db, zCommand, -1, ppStmt, NULL); - free(zCommand); + sqlite3_free(zCommand); return rc; } @@ -1938,7 +1938,7 @@ static int sql_get_statement(fulltext_vtab *v, fulltext_statement iStmt, } rc = sql_prepare(v->db, v->zDb, v->zName, &v->pFulltextStatements[iStmt], zStmt); - if( zStmt != fulltext_zStatement[iStmt]) free((void *) zStmt); + if( zStmt != fulltext_zStatement[iStmt]) sqlite3_free((void *) zStmt); if( rc!=SQLITE_OK ) return rc; } else { int rc = sqlite3_reset(v->pFulltextStatements[iStmt]); @@ -2023,9 +2023,9 @@ static void freeStringArray(int nString, const char **pString){ int i; for (i=0 ; i < nString ; ++i) { - if( pString[i]!=NULL ) free((void *) pString[i]); + if( pString[i]!=NULL ) sqlite3_free((void *) pString[i]); } - free((void *) pString); + sqlite3_free((void *) pString); } /* select * from %_content where rowid = [iRow] @@ -2052,7 +2052,7 @@ static int content_select(fulltext_vtab *v, sqlite_int64 iRow, rc = sqlite3_step(s); if( rc!=SQLITE_ROW ) return rc; - values = (const char **) malloc(v->nColumn * sizeof(const char *)); + values = (const char **) sqlite3_malloc(v->nColumn * sizeof(const char *)); for(i=0; inColumn; ++i){ if( sqlite3_column_type(s, i)==SQLITE_NULL ){ values[i] = NULL; @@ -2294,12 +2294,12 @@ static void fulltext_vtab_destroy(fulltext_vtab *v){ clearPendingTerms(v); - free(v->azColumn); + sqlite3_free(v->azColumn); for(i = 0; i < v->nColumn; ++i) { sqlite3_free(v->azContentColumn[i]); } - free(v->azContentColumn); - free(v); + sqlite3_free(v->azContentColumn); + sqlite3_free(v); } /* @@ -2410,7 +2410,7 @@ typedef struct Token { */ static char **tokenizeString(const char *z, int *pnToken){ int nToken = 0; - Token *aToken = malloc( strlen(z) * sizeof(aToken[0]) ); + Token *aToken = sqlite3_malloc( strlen(z) * sizeof(aToken[0]) ); int n = 1; int e, i; int totalSize = 0; @@ -2426,7 +2426,7 @@ static char **tokenizeString(const char *z, int *pnToken){ } z += n; } - azToken = (char**)malloc( nToken*sizeof(char*) + totalSize ); + azToken = (char**)sqlite3_malloc( nToken*sizeof(char*) + totalSize ); zCopy = (char*)&azToken[nToken]; nToken--; for(i=0; iazColumn); - free(p->azContentColumn); - free(p->azTokenizer); + sqlite3_free(p->azColumn); + sqlite3_free(p->azContentColumn); + sqlite3_free(p->azTokenizer); } /* Parse a CREATE VIRTUAL TABLE statement, which looks like this: @@ -2612,7 +2612,7 @@ static int parseSpec(TableSpec *pSpec, int argc, const char *const*argv, for(i=n=0; iazContentColumn = malloc( pSpec->nColumn * sizeof(char *) ); + pSpec->azContentColumn = sqlite3_malloc( pSpec->nColumn * sizeof(char *) ); if( pSpec->azContentColumn==0 ){ clearTableSpec(pSpec); return SQLITE_NOMEM; @@ -2726,7 +2726,7 @@ static int constructVtab( char const *zTok; /* Name of tokenizer to use for this fts table */ int nTok; /* Length of zTok, including nul terminator */ - v = (fulltext_vtab *) malloc(sizeof(fulltext_vtab)); + v = (fulltext_vtab *) sqlite3_malloc(sizeof(fulltext_vtab)); if( v==0 ) return SQLITE_NOMEM; CLEAR(v); /* sqlite will initialize v->base */ @@ -2913,12 +2913,16 @@ static int fulltextDestroy(sqlite3_vtab *pVTab){ static int fulltextOpen(sqlite3_vtab *pVTab, sqlite3_vtab_cursor **ppCursor){ fulltext_cursor *c; - c = (fulltext_cursor *) calloc(sizeof(fulltext_cursor), 1); - /* sqlite will initialize c->base */ - *ppCursor = &c->base; - TRACE(("FTS2 Open %p: %p\n", pVTab, c)); - - return SQLITE_OK; + c = (fulltext_cursor *) sqlite3_malloc(sizeof(fulltext_cursor)); + if( c ){ + memset(c, 0, sizeof(fulltext_cursor)); + /* sqlite will initialize c->base */ + *ppCursor = &c->base; + TRACE(("FTS2 Open %p: %p\n", pVTab, c)); + return SQLITE_OK; + }else{ + return SQLITE_NOMEM; + } } @@ -2927,9 +2931,9 @@ static int fulltextOpen(sqlite3_vtab *pVTab, sqlite3_vtab_cursor **ppCursor){ static void queryClear(Query *q){ int i; for(i = 0; i < q->nTerms; ++i){ - free(q->pTerms[i].pTerm); + sqlite3_free(q->pTerms[i].pTerm); } - free(q->pTerms); + sqlite3_free(q->pTerms); CLEAR(q); } @@ -2937,9 +2941,9 @@ static void queryClear(Query *q){ ** Snippet */ static void snippetClear(Snippet *p){ - free(p->aMatch); - free(p->zOffset); - free(p->zSnippet); + sqlite3_free(p->aMatch); + sqlite3_free(p->zOffset); + sqlite3_free(p->zSnippet); CLEAR(p); } /* @@ -2954,7 +2958,7 @@ static void snippetAppendMatch( struct snippetMatch *pMatch; if( p->nMatch+1>=p->nAlloc ){ p->nAlloc = p->nAlloc*2 + 10; - p->aMatch = realloc(p->aMatch, p->nAlloc*sizeof(p->aMatch[0]) ); + p->aMatch = sqlite3_realloc(p->aMatch, p->nAlloc*sizeof(p->aMatch[0]) ); if( p->aMatch==0 ){ p->nMatch = 0; p->nAlloc = 0; @@ -3181,7 +3185,7 @@ static void snippetText( int iMatch; - free(pCursor->snippet.zSnippet); + sqlite3_free(pCursor->snippet.zSnippet); pCursor->snippet.zSnippet = 0; aMatch = pCursor->snippet.aMatch; nMatch = pCursor->snippet.nMatch; @@ -3283,7 +3287,7 @@ static int fulltextClose(sqlite3_vtab_cursor *pCursor){ snippetClear(&c->snippet); if( c->result.nData!=0 ) dlrDestroy(&c->reader); dataBufferDestroy(&c->result); - free(c); + sqlite3_free(c); return SQLITE_OK; } @@ -3388,14 +3392,14 @@ static int docListOfTerm( static void queryAdd(Query *q, const char *pTerm, int nTerm){ QueryTerm *t; ++q->nTerms; - q->pTerms = realloc(q->pTerms, q->nTerms * sizeof(q->pTerms[0])); + q->pTerms = sqlite3_realloc(q->pTerms, q->nTerms * sizeof(q->pTerms[0])); if( q->pTerms==0 ){ q->nTerms = 0; return; } t = &q->pTerms[q->nTerms - 1]; CLEAR(t); - t->pTerm = malloc(nTerm+1); + t->pTerm = sqlite3_malloc(nTerm+1); memcpy(t->pTerm, pTerm, nTerm); t->pTerm[nTerm] = 0; t->nTerm = nTerm; @@ -3959,18 +3963,20 @@ typedef struct InteriorBlock { static InteriorBlock *interiorBlockNew(int iHeight, sqlite_int64 iChildBlock, const char *pTerm, int nTerm){ - InteriorBlock *block = calloc(1, sizeof(InteriorBlock)); + InteriorBlock *block = sqlite3_malloc(sizeof(InteriorBlock)); char c[VARINT_MAX+VARINT_MAX]; int n; - dataBufferInit(&block->term, 0); - dataBufferReplace(&block->term, pTerm, nTerm); - - n = putVarint(c, iHeight); - n += putVarint(c+n, iChildBlock); - dataBufferInit(&block->data, INTERIOR_MAX); - dataBufferReplace(&block->data, c, n); + if( block ){ + memset(block, 0, sizeof(*block)); + dataBufferInit(&block->term, 0); + dataBufferReplace(&block->term, pTerm, nTerm); + n = putVarint(c, iHeight); + n += putVarint(c+n, iChildBlock); + dataBufferInit(&block->data, INTERIOR_MAX); + dataBufferReplace(&block->data, c, n); + } return block; } @@ -4135,11 +4141,11 @@ static int interiorWriterDestroy(InteriorWriter *pWriter){ block = block->next; dataBufferDestroy(&b->term); dataBufferDestroy(&b->data); - free(b); + sqlite3_free(b); } if( pWriter->parentWriter!=NULL ){ interiorWriterDestroy(pWriter->parentWriter); - free(pWriter->parentWriter); + sqlite3_free(pWriter->parentWriter); } dataBufferDestroy(&pWriter->term); SCRAMBLE(pWriter); @@ -4173,7 +4179,7 @@ static int interiorWriterRootInfo(fulltext_vtab *v, InteriorWriter *pWriter, if( rc!=SQLITE_OK ) return rc; *piEndBlockid = iBlockid; - pWriter->parentWriter = malloc(sizeof(*pWriter->parentWriter)); + pWriter->parentWriter = sqlite3_malloc(sizeof(*pWriter->parentWriter)); interiorWriterInit(pWriter->iHeight+1, block->term.pData, block->term.nData, iBlockid, pWriter->parentWriter); @@ -5563,7 +5569,7 @@ static int writeZeroSegment(fulltext_vtab *v, fts2Hash *pTerms){ if( rc!=SQLITE_OK ) return rc; n = fts2HashCount(pTerms); - pData = malloc(n*sizeof(TermData)); + pData = sqlite3_malloc(n*sizeof(TermData)); for(i = 0, e = fts2HashFirst(pTerms); e; i++, e = fts2HashNext(e)){ assert( i #include +#include "sqlite3.h" #include "fts2_hash.h" /* diff --git a/ext/fts2/fts2_porter.c b/ext/fts2/fts2_porter.c index dab1849531..97a95c872d 100644 --- a/ext/fts2/fts2_porter.c +++ b/ext/fts2/fts2_porter.c @@ -66,9 +66,9 @@ static int porterCreate( sqlite3_tokenizer **ppTokenizer ){ porter_tokenizer *t; - t = (porter_tokenizer *) calloc(sizeof(*t), 1); + t = (porter_tokenizer *) sqlite3_malloc(sizeof(*t)); if( t==NULL ) return SQLITE_NOMEM; - + memset(t, 0, sizeof(*t)); *ppTokenizer = &t->base; return SQLITE_OK; } @@ -77,7 +77,7 @@ static int porterCreate( ** Destroy a tokenizer */ static int porterDestroy(sqlite3_tokenizer *pTokenizer){ - free(pTokenizer); + sqlite3_free(pTokenizer); return SQLITE_OK; } @@ -94,7 +94,7 @@ static int porterOpen( ){ porter_tokenizer_cursor *c; - c = (porter_tokenizer_cursor *) malloc(sizeof(*c)); + c = (porter_tokenizer_cursor *) sqlite3_malloc(sizeof(*c)); if( c==NULL ) return SQLITE_NOMEM; c->zInput = zInput; @@ -120,8 +120,8 @@ static int porterOpen( */ static int porterClose(sqlite3_tokenizer_cursor *pCursor){ porter_tokenizer_cursor *c = (porter_tokenizer_cursor *) pCursor; - free(c->zToken); - free(c); + sqlite3_free(c->zToken); + sqlite3_free(c); return SQLITE_OK; } /* @@ -603,7 +603,7 @@ static int porterNext( int n = c->iOffset-iStartOffset; if( n>c->nAllocated ){ c->nAllocated = n+20; - c->zToken = realloc(c->zToken, c->nAllocated); + c->zToken = sqlite3_realloc(c->zToken, c->nAllocated); if( c->zToken==NULL ) return SQLITE_NOMEM; } porter_stemmer(&z[iStartOffset], n, c->zToken, pnBytes); diff --git a/ext/fts2/fts2_tokenizer1.c b/ext/fts2/fts2_tokenizer1.c index 540ba27def..f2ba49e573 100644 --- a/ext/fts2/fts2_tokenizer1.c +++ b/ext/fts2/fts2_tokenizer1.c @@ -65,8 +65,9 @@ static int simpleCreate( ){ simple_tokenizer *t; - t = (simple_tokenizer *) calloc(sizeof(*t), 1); + t = (simple_tokenizer *) sqlite3_malloc(sizeof(*t)); if( t==NULL ) return SQLITE_NOMEM; + memset(t, 0, sizeof(*t)); /* TODO(shess) Delimiters need to remain the same from run to run, ** else we need to reindex. One solution would be a meta-table to @@ -79,7 +80,7 @@ static int simpleCreate( unsigned char ch = argv[1][i]; /* We explicitly don't support UTF-8 delimiters for now. */ if( ch>=0x80 ){ - free(t); + sqlite3_free(t); return SQLITE_ERROR; } t->delim[ch] = 1; @@ -100,7 +101,7 @@ static int simpleCreate( ** Destroy a tokenizer */ static int simpleDestroy(sqlite3_tokenizer *pTokenizer){ - free(pTokenizer); + sqlite3_free(pTokenizer); return SQLITE_OK; } @@ -117,7 +118,7 @@ static int simpleOpen( ){ simple_tokenizer_cursor *c; - c = (simple_tokenizer_cursor *) malloc(sizeof(*c)); + c = (simple_tokenizer_cursor *) sqlite3_malloc(sizeof(*c)); if( c==NULL ) return SQLITE_NOMEM; c->pInput = pInput; @@ -143,8 +144,8 @@ static int simpleOpen( */ static int simpleClose(sqlite3_tokenizer_cursor *pCursor){ simple_tokenizer_cursor *c = (simple_tokenizer_cursor *) pCursor; - free(c->pToken); - free(c); + sqlite3_free(c->pToken); + sqlite3_free(c); return SQLITE_OK; } @@ -182,7 +183,7 @@ static int simpleNext( int i, n = c->iOffset-iStartOffset; if( n>c->nTokenAllocated ){ c->nTokenAllocated = n+20; - c->pToken = realloc(c->pToken, c->nTokenAllocated); + c->pToken = sqlite3_realloc(c->pToken, c->nTokenAllocated); if( c->pToken==NULL ) return SQLITE_NOMEM; } for(i=0; i=0} continue + if {[llength $INCLUDE]>0 && [lsearch -exact $INCLUDE $tail]<0} continue + source $testfile + catch {db close} + if {$sqlite_open_file_count>0} { + puts "$tail did not close all files: $sqlite_open_file_count" + incr nErr + lappend ::failList $tail + set sqlite_open_file_count 0 + } +} + +set sqlite_open_file_count 0 +really_finish_test -- 2.47.2