From: drh Date: Fri, 12 Aug 2005 22:56:09 +0000 (+0000) Subject: Optimize LIKE and GLOB operators in the WHERE clause. Code passes all X-Git-Tag: version-3.6.10~3578 X-Git-Url: http://git.ipfire.org/?a=commitdiff_plain;h=d2687b77317721f6e2844abd8046aed492d40469;p=thirdparty%2Fsqlite.git Optimize LIKE and GLOB operators in the WHERE clause. Code passes all regression tests but still needs additional tests. (CVS 2581) FossilOrigin-Name: 3edbe8d6217fd1180883e6b9f1e5b9011a39f80d --- diff --git a/manifest b/manifest index b67e077981..b9e32e2317 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Improve\sthe\serror\smessage\sassociated\swith\sSQLITE_FULL.\s\sTicket\s#1353.\nAlso\sremove\serror\smessages\sfor\sobsolete\serror\scodes\sSQLITE_INTERNAL,\nSQLITE_NOTFOUND,\sand\sSQLITE_TOOBIG.\s(CVS\s2580) -D 2005-08-11T02:10:19 +C Optimize\sLIKE\sand\sGLOB\soperators\sin\sthe\sWHERE\sclause.\s\sCode\spasses\sall\nregression\stests\sbut\sstill\sneeds\sadditional\stests.\s(CVS\s2581) +D 2005-08-12T22:56:09 F Makefile.in 22ea9c0fe748f591712d8fe3c6d972c6c173a165 F Makefile.linux-gcc 06be33b2a9ad4f005a5f42b22c4a19dab3cbb5c7 F README 9c4e2d6706bdcc3efdd773ce752a8cdab4f90028 @@ -39,7 +39,7 @@ F src/callback.c 0910b611e0c158f107ee3ff86f8a371654971e2b F src/date.c 7444b0900a28da77e57e3337a636873cff0ae940 F src/delete.c be1fc25c9e109cd8cbab42a43ee696263da7c04b F src/experimental.c 50c1e3b34f752f4ac10c36f287db095c2b61766d -F src/expr.c c71265b78d383382fa8798ac9436246e3eb6d16c +F src/expr.c 9c319b9f9fbb79f06adbcd0ed2deab012a00ba37 F src/func.c 2be0799df0c05066a29e589485ebee0b3f756a15 F src/hash.c 2b1b13f7400e179631c83a1be0c664608c8f021f F src/hash.h 1b0c445e1c89ff2aaad9b4605ba61375af001e84 @@ -65,7 +65,7 @@ F src/random.c 90adff4e73a3b249eb4f1fc2a6ff9cf78c7233a4 F src/select.c c611471052773b94af771693686bd5bcdbbb0dba F src/shell.c 86c16f0d534aa51cc82cf9f66903d4eb681580e7 F src/sqlite.h.in a3b75a6b2e66865fba4ec1b698d00c7d95fe27a2 -F src/sqliteInt.h 4cacefaca973cbf8e6a82910c704bf9b4a32fdf1 +F src/sqliteInt.h 67693a8ed955e96e0f87eb4a2c9793125f27f20c F src/table.c 25b3ff2b39b7d87e8d4a5da0713d68dfc06cbee9 F src/tclsqlite.c 96feead1f1d301efa1d3ac6b89ecea7592ab18f9 F src/test1.c 8a2b5ccc4be7450d41100778f346cbcf540febdd @@ -80,13 +80,13 @@ F src/utf.c bda5eb85039ef16f2d17004c1e18c96e1ab0a80c F src/util.c 668d31be592753e5b8ea00e69ea8d3eedb29fa22 F src/vacuum.c 829d9e1a6d7c094b80e0899686670932eafd768c F src/vdbe.c 69f33e22c7d0a64b23fbb69e6da95a1bb6869032 -F src/vdbe.h 75e466d84d362b0c4498978a9d6b1e6bd32ecf3b +F src/vdbe.h 68e80f65658e7fd85561721f617bdebf8bb84b59 F src/vdbeInt.h 9be9a6c43d38124bd03cc5cf05715605b1789fd9 F src/vdbeapi.c 7f392f0792d1258c958083d7de9eae7c3530c9a6 -F src/vdbeaux.c 3732a86566a6be4da4c606e9334baf3fd98667af +F src/vdbeaux.c d53139d819b887dac608ac4ae9a501baee3cd311 F src/vdbefifo.c b8805850afe13b43f1de78d58088cb5d66f88e1e F src/vdbemem.c da8e8d6f29dd1323f782f000d7cd120027c9ff03 -F src/where.c 72f68f996a1fd0139ef0ca77f407406c97b15d0a +F src/where.c ac754c021f716e17337f45ffdc2436c6d5109fd3 F tclinstaller.tcl 046e3624671962dc50f0481d7c25b38ef803eb42 F test/all.test 7f0988442ab811dfa41793b5b550f5828ce316f3 F test/alter.test 9d6837a3d946b73df692b7cef2a7644d2e2f6bc6 @@ -291,7 +291,7 @@ F www/tclsqlite.tcl 3df553505b6efcad08f91e9b975deb2e6c9bb955 F www/vdbe.tcl 87a31ace769f20d3627a64fa1fade7fed47b90d0 F www/version3.tcl a99cf5f6d8bd4d5537584a2b342f0fb9fa601d8b F www/whentouse.tcl 528299b8316726dbcc5548e9aa0648c8b1bd055b -P 8e007f8c06748c5f02068c1f244bc8e3026e16fd -R 2ec1633eaf76b1bbd075deefedf07962 +P fa7403c7d9948cc4a6c6ed00a614e3d6a3682e78 +R f95355ea47c912177ab06d841df32b8a U drh -Z ae91ac04dccfd6c4c10cb1686551ca42 +Z ca309b724231e8b642eb0d9abe08ca05 diff --git a/manifest.uuid b/manifest.uuid index b6447d70cf..d88d2b9427 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -fa7403c7d9948cc4a6c6ed00a614e3d6a3682e78 \ No newline at end of file +3edbe8d6217fd1180883e6b9f1e5b9011a39f80d \ No newline at end of file diff --git a/src/expr.c b/src/expr.c index a5e4c7d197..597a2a776c 100644 --- a/src/expr.c +++ b/src/expr.c @@ -12,7 +12,7 @@ ** This file contains routines used for analyzing expressions and ** for generating VDBE code that evaluates expressions in SQLite. ** -** $Id: expr.c,v 1.214 2005/07/29 15:10:18 drh Exp $ +** $Id: expr.c,v 1.215 2005/08/12 22:56:09 drh Exp $ */ #include "sqliteInt.h" #include @@ -380,6 +380,25 @@ void sqlite3ExprDelete(Expr *p){ sqliteFree(p); } +/* +** The Expr.token field might be a string literal that is quoted. +** If so, remove the quotation marks. +*/ +void sqlite3DequoteExpr(Expr *p){ + if( ExprHasAnyProperty(p, EP_Dequoted) ){ + return; + } + ExprSetProperty(p, EP_Dequoted); + if( p->token.dyn==0 ){ + if( p->op==TK_BLOB ){ + p->token.n--; + p->token.z++; + } + sqlite3TokenCopy(&p->token, &p->token); + } + sqlite3Dequote((char*)p->token.z); +} + /* ** The following group of routines make deep copies of expressions, @@ -1442,8 +1461,8 @@ void sqlite3ExprCode(Parse *pParse, Expr *pExpr){ case TK_STRING: { assert( TK_FLOAT==OP_Real ); assert( TK_STRING==OP_String8 ); + sqlite3DequoteExpr(pExpr); sqlite3VdbeOp3(v, op, 0, 0, pExpr->token.z, pExpr->token.n); - sqlite3VdbeDequoteP3(v, -1); break; } case TK_NULL: { @@ -1453,8 +1472,7 @@ void sqlite3ExprCode(Parse *pParse, Expr *pExpr){ #ifndef SQLITE_OMIT_BLOB_LITERAL case TK_BLOB: { assert( TK_BLOB==OP_HexBlob ); - sqlite3VdbeOp3(v, op, 0, 0, pExpr->token.z+1, pExpr->token.n-1); - sqlite3VdbeDequoteP3(v, -1); + sqlite3VdbeOp3(v, op, 0, 0, pExpr->token.z+2, pExpr->token.n-3); break; } #endif @@ -1716,9 +1734,10 @@ void sqlite3ExprCode(Parse *pParse, Expr *pExpr){ assert( pExpr->iColumn==OE_Rollback || pExpr->iColumn == OE_Abort || pExpr->iColumn == OE_Fail ); + sqlite3DequoteExpr(pExpr); sqlite3VdbeOp3(v, OP_Halt, SQLITE_CONSTRAINT, pExpr->iColumn, pExpr->token.z, pExpr->token.n); - sqlite3VdbeDequoteP3(v, -1); +// sqlite3VdbeDequoteP3(v, -1); } else { assert( pExpr->iColumn == OE_Ignore ); sqlite3VdbeAddOp(v, OP_ContextPop, 0, 0); diff --git a/src/sqliteInt.h b/src/sqliteInt.h index e80af77ea7..9cfce8a75b 100644 --- a/src/sqliteInt.h +++ b/src/sqliteInt.h @@ -11,7 +11,7 @@ ************************************************************************* ** Internal interface definitions for SQLite. ** -** @(#) $Id: sqliteInt.h,v 1.400 2005/07/23 22:59:56 drh Exp $ +** @(#) $Id: sqliteInt.h,v 1.401 2005/08/12 22:56:09 drh Exp $ */ #ifndef _SQLITEINT_H_ #define _SQLITEINT_H_ @@ -846,6 +846,7 @@ struct Expr { #define EP_Error 0x08 /* Expression contains one or more errors */ #define EP_Not 0x10 /* Operator preceeded by NOT */ #define EP_VarSelect 0x20 /* pSelect is correlated, not constant */ +#define EP_Dequoted 0x40 /* True if the string has been dequoted */ /* ** These macros can be used to test, set, or clear bits in the @@ -1348,6 +1349,7 @@ void *sqlite3TextToPtr(const char*); void sqlite3SetString(char **, ...); void sqlite3ErrorMsg(Parse*, const char*, ...); void sqlite3Dequote(char*); +void sqlite3DequoteExpr(Expr*); int sqlite3KeywordCode(const char*, int); int sqlite3RunParser(Parse*, const char*, char **); void sqlite3FinishCoding(Parse*); diff --git a/src/vdbe.h b/src/vdbe.h index 31bf8021f1..4c93807a39 100644 --- a/src/vdbe.h +++ b/src/vdbe.h @@ -15,7 +15,7 @@ ** or VDBE. The VDBE implements an abstract machine that runs a ** simple program to access and modify the underlying database. ** -** $Id: vdbe.h,v 1.95 2005/05/19 08:43:00 danielk1977 Exp $ +** $Id: vdbe.h,v 1.96 2005/08/12 22:56:09 drh Exp $ */ #ifndef _SQLITE_VDBE_H_ #define _SQLITE_VDBE_H_ @@ -105,7 +105,6 @@ int sqlite3VdbeAddOpList(Vdbe*, int nOp, VdbeOpList const *aOp); void sqlite3VdbeChangeP1(Vdbe*, int addr, int P1); void sqlite3VdbeChangeP2(Vdbe*, int addr, int P2); void sqlite3VdbeChangeP3(Vdbe*, int addr, const char *zP1, int N); -void sqlite3VdbeDequoteP3(Vdbe*, int addr); int sqlite3VdbeFindOp(Vdbe*, int, int, int); VdbeOp *sqlite3VdbeGetOp(Vdbe*, int); int sqlite3VdbeMakeLabel(Vdbe*); diff --git a/src/vdbeaux.c b/src/vdbeaux.c index c251fc938d..b3fa8c36ae 100644 --- a/src/vdbeaux.c +++ b/src/vdbeaux.c @@ -442,33 +442,6 @@ void sqlite3VdbeComment(Vdbe *p, const char *zFormat, ...){ } #endif -/* -** If the P3 operand to the specified instruction appears -** to be a quoted string token, then this procedure removes -** the quotes. -** -** The quoting operator can be either a grave ascent (ASCII 0x27) -** or a double quote character (ASCII 0x22). Two quotes in a row -** resolve to be a single actual quote character within the string. -*/ -void sqlite3VdbeDequoteP3(Vdbe *p, int addr){ - Op *pOp; - assert( p->magic==VDBE_MAGIC_INIT ); - if( p->aOp==0 ) return; - if( addr<0 || addr>=p->nOp ){ - addr = p->nOp - 1; - if( addr<0 ) return; - } - pOp = &p->aOp[addr]; - if( pOp->p3==0 || pOp->p3[0]==0 ) return; - if( pOp->p3type==P3_STATIC ){ - pOp->p3 = sqliteStrDup(pOp->p3); - pOp->p3type = P3_DYNAMIC; - } - assert( pOp->p3type==P3_DYNAMIC ); - sqlite3Dequote(pOp->p3); -} - /* ** Search the current program starting at instruction addr for the given ** opcode and P2 value. Return the address plus 1 if found and 0 if not diff --git a/src/where.c b/src/where.c index 4a8ee2461e..4f5da89313 100644 --- a/src/where.c +++ b/src/where.c @@ -16,7 +16,7 @@ ** so is applicable. Because this module is responsible for selecting ** indices, you might also think of this module as the "query optimizer". ** -** $Id: where.c,v 1.159 2005/08/02 17:48:22 drh Exp $ +** $Id: where.c,v 1.160 2005/08/12 22:56:09 drh Exp $ */ #include "sqliteInt.h" @@ -457,6 +457,69 @@ static void exprAnalyzeAll( } } +#ifndef SQLITE_OMIT_LIKE_OPTIMIZATION +/* +** Check to see if the given expression is a LIKE or GLOB operator that +** can be optimized using inequality constraints. Return TRUE if it is +** so and false if not. +** +** In order for the operator to be optimizible, the RHS must be a string +** literal that does not begin with a wildcard. +*/ +static int isLikeOrGlob( + Expr *pExpr, /* Test this expression */ + int *pnPattern, /* Number of non-wildcard prefix characters */ + int *pisComplete /* True if the only wildcard is % in the last character */ +){ + const char *z; + Expr *pRight, *pLeft; + int c, cnt; + char wc1, wc2, wc3; + if( pExpr->op!=TK_FUNCTION ){ + return 0; + } + if( pExpr->pList->nExpr!=2 ){ + return 0; + } + if( pExpr->token.n!=4 ){ + return 0; + } + z = pExpr->token.z; + if( sqlite3StrNICmp(z, "glob", 4)==0 ){ + wc1 = '*'; + wc2 = '?'; + wc3 = '['; + } +#ifdef SQLITE_CASE_SENSITIVE_LIKE + else if( sqlite3StrNICmp(z, "like", 4)==0 ){ + wc1 = '%'; + wc2 = '_'; + wc3 = '_'; + } +#endif + else{ + return 0; + } + pRight = pExpr->pList->a[0].pExpr; + if( pRight->op!=TK_STRING ){ + return 0; + } + pLeft = pExpr->pList->a[1].pExpr; + if( pLeft->op!=TK_COLUMN ){ + return 0; + } + sqlite3DequoteExpr(pRight); + z = pRight->token.z; + for(cnt=0; (c=z[cnt])!=0 && c!=wc1 && c!=wc2 && c!=wc3; cnt++){} + if( cnt==0 || 255==(u8)z[cnt] ){ + return 0; + } + *pisComplete = z[cnt]==wc1 && z[cnt+1]==0; + *pnPattern = cnt; + return 1; +} +#endif /* SQLITE_OMIT_LIKE_OPTIMIZATION */ + /* ** The input to this routine is an WhereTerm structure with only the ** "pExpr" field filled in. The job of this routine is to analyze the @@ -478,6 +541,8 @@ static void exprAnalyze( Bitmask prereqLeft; Bitmask prereqAll; int idxRight; + int nPattern; + int isComplete; prereqLeft = exprTableUsage(pMaskSet, pExpr->pLeft); pTerm->prereqRight = exprTableUsage(pMaskSet, pExpr->pRight); @@ -518,6 +583,7 @@ static void exprAnalyze( } } +#ifndef SQLITE_OMIT_BETWEEN_OPTIMIZATION /* If a term is the BETWEEN operator, create two new virtual terms ** that define the range that the BETWEEN implements. */ @@ -539,7 +605,9 @@ static void exprAnalyze( } pTerm->nChild = 2; } +#endif /* SQLITE_OMIT_BETWEEN_OPTIMIZATION */ +#ifndef SQLITE_OMIT_OR_OPTIMIZATION /* Attempt to convert OR-connected terms into an IN operator so that ** they can make use of indices. */ @@ -597,6 +665,44 @@ static void exprAnalyze( or_not_possible: whereClauseClear(&sOr); } +#endif /* SQLITE_OMIT_OR_OPTIMIZATION */ + +#ifndef SQLITE_OMIT_LIKE_OPTIMIZATION + /* Add constraints to reduce the search space on a LIKE or GLOB + ** operator. + */ + if( isLikeOrGlob(pExpr, &nPattern, &isComplete) ){ + Expr *pLeft, *pRight; + Expr *pStr1, *pStr2; + Expr *pNewExpr1, *pNewExpr2; + WhereTerm *pNewTerm1, *pNewTerm2; + pLeft = pExpr->pList->a[1].pExpr; + pRight = pExpr->pList->a[0].pExpr; + pStr1 = sqlite3Expr(TK_STRING, 0, 0, 0); + if( pStr1 ){ + sqlite3TokenCopy(&pStr1->token, &pRight->token); + pStr1->token.n = nPattern; + } + pStr2 = sqlite3ExprDup(pStr1); + if( pStr2 ){ + assert( pStr2->token.dyn ); + ++*(u8*)&pStr2->token.z[nPattern-1]; + } + pNewExpr1 = sqlite3Expr(TK_GE, sqlite3ExprDup(pLeft), pStr1, 0); + pNewTerm1 = whereClauseInsert(pTerm->pWC, pNewExpr1, + TERM_VIRTUAL|TERM_DYNAMIC); + exprAnalyze(pSrc, pMaskSet, pNewTerm1); + pNewExpr2 = sqlite3Expr(TK_LT, sqlite3ExprDup(pLeft), pStr2, 0); + pNewTerm2 = whereClauseInsert(pTerm->pWC, pNewExpr2, + TERM_VIRTUAL|TERM_DYNAMIC); + exprAnalyze(pSrc, pMaskSet, pNewTerm2); + if( isComplete ){ + pNewTerm2->iParent = pTerm->idx; + pNewTerm1->iParent = pTerm->idx; + pTerm->nChild = 2; + } + } +#endif /* SQLITE_OMIT_LIKE_OPTIMIZATION */ }