From: drh Date: Mon, 31 Dec 2012 19:18:38 +0000 (+0000) Subject: Add the test_regexp.c module containing a cross-platform implementation X-Git-Tag: version-3.7.16~91 X-Git-Url: http://git.ipfire.org/?a=commitdiff_plain;h=14172743a22fce42e77e4ce69dee89511709ebba;p=thirdparty%2Fsqlite.git Add the test_regexp.c module containing a cross-platform implementation of the REGEXP operator. FossilOrigin-Name: 46c8c01b751c1ea7fc02cc35e3b5bb99dbe46c4b --- diff --git a/Makefile.in b/Makefile.in index 8d9d3c9694..98a3edb091 100644 --- a/Makefile.in +++ b/Makefile.in @@ -370,6 +370,7 @@ TESTSRC = \ $(TOP)/src/test_osinst.c \ $(TOP)/src/test_pcache.c \ $(TOP)/src/test_quota.c \ + $(TOP)/src/test_regexp.c \ $(TOP)/src/test_rtree.c \ $(TOP)/src/test_schema.c \ $(TOP)/src/test_server.c \ diff --git a/Makefile.msc b/Makefile.msc index d12e5d0ec1..631e9edcbf 100644 --- a/Makefile.msc +++ b/Makefile.msc @@ -691,6 +691,7 @@ TESTSRC = \ $(TOP)\src\test_osinst.c \ $(TOP)\src\test_pcache.c \ $(TOP)\src\test_quota.c \ + $(TOP)\src\test_regexp.c \ $(TOP)\src\test_rtree.c \ $(TOP)\src\test_schema.c \ $(TOP)\src\test_server.c \ diff --git a/main.mk b/main.mk index cbae400a50..fdabcb4636 100644 --- a/main.mk +++ b/main.mk @@ -253,6 +253,7 @@ TESTSRC = \ $(TOP)/src/test_osinst.c \ $(TOP)/src/test_pcache.c \ $(TOP)/src/test_quota.c \ + $(TOP)/src/test_regexp.c \ $(TOP)/src/test_rtree.c \ $(TOP)/src/test_schema.c \ $(TOP)/src/test_server.c \ diff --git a/manifest b/manifest index 56b649c34b..69c3503ae5 100644 --- a/manifest +++ b/manifest @@ -1,9 +1,9 @@ -C Ensure\sthe\sdatabase\ssize\sfield\sin\sthe\sdb\sheader\sof\sa\sbackup\sdatabase\sis\sset\scorrectly.\sFix\sfor\s[0cfd98ee201]. -D 2012-12-21T16:15:35.911 +C Add\sthe\stest_regexp.c\smodule\scontaining\sa\scross-platform\simplementation\s\nof\sthe\sREGEXP\soperator. +D 2012-12-31T19:18:38.256 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f -F Makefile.in 690d441a758cbffd13e814dc2724a721a6ebd400 +F Makefile.in a48faa9e7dd7d556d84f5456eabe5825dd8a6282 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 -F Makefile.msc 5a3b6f34d263b01f8b798c291fac1529fd650308 +F Makefile.msc 2b8371775ea8df029d1acf0c3d4c3782d3bd5711 F Makefile.vxworks b18ad88e9a8c6a001f5cf4a389116a4f1a7ab45f F README cd04a36fbc7ea56932a4052d7d0b7f09f27c33d6 F VERSION 6d4f66eaebabc42ef8c2a4d2d0caf4ce7ee81137 @@ -103,7 +103,7 @@ F ext/rtree/tkt3363.test 142ab96eded44a3615ec79fba98c7bde7d0f96de F ext/rtree/viewrtree.tcl eea6224b3553599ae665b239bd827e182b466024 F install-sh 9d4de14ab9fb0facae2f48780b874848cbf2f895 x F ltmain.sh 3ff0879076df340d2e23ae905484d8c15d5fdea8 -F main.mk a0d170ae1a8a8683688e281194e09d47a68eaa3f +F main.mk 718265bbf49a846c6898b4da09593eef4068fa39 F mkdll.sh 7d09b23c05d56532e9d44a50868eb4b12ff4f74a F mkextu.sh 416f9b7089d80e5590a29692c9d9280a10dbad9f F mkextw.sh 4123480947681d9b434a5e7b1ee08135abe409ac @@ -175,7 +175,7 @@ F src/random.c cd4a67b3953b88019f8cd4ccd81394a8ddfaba50 F src/resolve.c 52331299f4095397d6d00715b70cd153baa11931 F src/rowset.c 64655f1a627c9c212d9ab497899e7424a34222e0 F src/select.c 5eab6941c0ac97355817f846b77cd20bfdf5a82e -F src/shell.c e392dd1ccbb77cc1d75a8367a89b473c24bea019 +F src/shell.c e6525781d27a84f1b74586831b6ad8472a8c8dc6 F src/sqlite.h.in 39cc33bb08897c748fe3383c29ccf56585704177 F src/sqlite3.rc fea433eb0a59f4c9393c8e6d76a6e2596b1fe0c0 F src/sqlite3ext.h 6904f4aadf976f95241311fbffb00823075d9477 @@ -183,7 +183,7 @@ F src/sqliteInt.h 2e5d50f26abf7cbc6162117735379d412f4091da F src/sqliteLimit.h 164b0e6749d31e0daa1a4589a169d31c0dec7b3d F src/status.c bedc37ec1a6bb9399944024d63f4c769971955a9 F src/table.c 2cd62736f845d82200acfa1287e33feb3c15d62e -F src/tclsqlite.c 515abd8e33e82aa330eeb54675185a7e1e5b6778 +F src/tclsqlite.c 5203bb7b71a302bea8896f176cd178e38d7803a4 F src/test1.c f62769c989146149590662ab02de4a813813a9c5 F src/test2.c 4178056dd1e7d70f954ad8a1e3edb71a2a784daf F src/test3.c 3c3c2407fa6ec7a19e24ae23f7cb439d0275a60d @@ -217,6 +217,7 @@ F src/test_osinst.c 90a845c8183013d80eccb1f29e8805608516edba F src/test_pcache.c a5cd24730cb43c5b18629043314548c9169abb00 F src/test_quota.c 0e0e2e3bf6766b101ecccd8c042b66e44e9be8f5 F src/test_quota.h 8761e463b25e75ebc078bd67d70e39b9c817a0cb +F src/test_regexp.c c24ae2a0de64eb9dfa1dd77b77448b1d794cd395 F src/test_rtree.c aba603c949766c4193f1068b91c787f57274e0d9 F src/test_schema.c 8c06ef9ddb240c7a0fcd31bc221a6a2aade58bf0 F src/test_server.c 2f99eb2837dfa06a4aacf24af24c6affdf66a84f @@ -666,6 +667,7 @@ F test/quote.test 215897dbe8de1a6f701265836d6601cc6ed103e6 F test/randexpr1.tcl 40dec52119ed3a2b8b2a773bce24b63a3a746459 F test/randexpr1.test eda062a97e60f9c38ae8d806b03b0ddf23d796df F test/rdonly.test c267d050a1d9a6a321de502b737daf28821a518d +F test/regexp1.test 38da302b75504dd8b960c8f06968ddf8039777ad F test/reindex.test 44edd3966b474468b823d481eafef0c305022254 F test/releasetest.mk 2eced2f9ae701fd0a29e714a241760503ccba25a F test/releasetest.tcl 06d289d8255794073a58d2850742f627924545ce @@ -985,7 +987,7 @@ F test/win32lock.test b2a539e85ae6b2d78475e016a9636b4451dc7fb9 F test/zeroblob.test caaecfb4f908f7bc086ed238668049f96774d688 F test/zerodamage.test e7f77fded01dfcdf92ac2c5400f1e35d7a21463c F tool/build-all-msvc.bat 74fb6e5cca66ebdb6c9bbafb2f8b802f08146d38 x -F tool/build-shell.sh b64a481901fc9ffe5ca8812a2a9255b6cfb77381 +F tool/build-shell.sh 562df23cfdd25822b909b382afd5f99d968437fe F tool/checkSpacing.c 810e51703529a204fc4e1eb060e9ab663e3c06d2 F tool/diffdb.c 7524b1b5df217c20cd0431f6789851a4e0cb191b F tool/extract.c 054069d81b095fbdc189a6f5d4466e40380505e2 @@ -1028,7 +1030,7 @@ F tool/vdbe-compress.tcl f12c884766bd14277f4fcedcae07078011717381 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 F tool/win/sqlite.vsix 97894c2790eda7b5bce3cc79cb2a8ec2fde9b3ac -P e408dc9080594dc464b8763dece6b365772c6105 -R 029b6fc924e25c670b20c53f3ff270ca -U dan -Z feee379cb7d3362de52284de8aba8c99 +P ff6857b6ed6a46671006b75157d8cf853a816ef9 +R 2e582c43743655b03da399d168172dad +U drh +Z c7b82d9d758f8f0ccceb37ea12601a6d diff --git a/manifest.uuid b/manifest.uuid index 75e67ccb96..74ce482207 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -ff6857b6ed6a46671006b75157d8cf853a816ef9 \ No newline at end of file +46c8c01b751c1ea7fc02cc35e3b5bb99dbe46c4b \ No newline at end of file diff --git a/src/shell.c b/src/shell.c index 7dd741b2d7..26de51c4f6 100644 --- a/src/shell.c +++ b/src/shell.c @@ -1479,6 +1479,12 @@ static void open_db(struct callback_data *p){ } #ifndef SQLITE_OMIT_LOAD_EXTENSION sqlite3_enable_load_extension(p->db, 1); +#endif +#ifdef SQLITE_ENABLE_REGEXP + { + extern sqlite3_add_regexp_func(sqlite3*); + sqlite3_add_regexp_func(db); + } #endif } } diff --git a/src/tclsqlite.c b/src/tclsqlite.c index 57dab85d4b..267f759701 100644 --- a/src/tclsqlite.c +++ b/src/tclsqlite.c @@ -3684,6 +3684,7 @@ static void init_all(Tcl_Interp *interp){ extern int SqlitetestSyscall_Init(Tcl_Interp*); extern int Sqlitetestfuzzer_Init(Tcl_Interp*); extern int Sqlitetestwholenumber_Init(Tcl_Interp*); + extern int Sqlitetestregexp_Init(Tcl_Interp*); #if defined(SQLITE_ENABLE_FTS3) || defined(SQLITE_ENABLE_FTS4) extern int Sqlitetestfts3_Init(Tcl_Interp *interp); @@ -3727,6 +3728,7 @@ static void init_all(Tcl_Interp *interp){ SqlitetestSyscall_Init(interp); Sqlitetestfuzzer_Init(interp); Sqlitetestwholenumber_Init(interp); + Sqlitetestregexp_Init(interp); #if defined(SQLITE_ENABLE_FTS3) || defined(SQLITE_ENABLE_FTS4) Sqlitetestfts3_Init(interp); diff --git a/src/test_regexp.c b/src/test_regexp.c new file mode 100644 index 0000000000..353955cee5 --- /dev/null +++ b/src/test_regexp.c @@ -0,0 +1,721 @@ +/* +** 2012-11-13 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +****************************************************************************** +** +** The code in this file implements a compact but reasonably +** efficient regular-expression matcher for posix extended regular +** expressions against UTF8 text. The following syntax is supported: +** +** X* zero or more occurrences of X +** X+ one or more occurrences of X +** X? zero or one occurrences of X +** X{p,q} between p and q occurrences of X +** (X) match X +** X|Y X or Y +** ^X X occurring at the beginning of the string +** X$ X occurring at the end of the string +** . Match any single character +** \c Character c where c is one of \{}()[]|*+?. +** \c C-language escapes for c in afnrtv. ex: \t or \n +** \uXXXX Where XXXX is exactly 4 hex digits, unicode value XXXX +** \xXXX Where XXX is any number of hex digits, unicode value XXX +** [abc] Any single character from the set abc +** [^abc] Any single character not in the set abc +** [a-z] Any single character in the range a-z +** [^a-z] Any single character not in the range a-z +** \b Word boundary +** \w Word character. [A-Za-z0-9_] +** \W Non-word character +** \d Digit +** \D Non-digit +** \s Whitespace character +** \S Non-whitespace character +** +** A nondeterministic finite automaton (NFA) is used for matching, so the +** performance is bounded by O(N*M) where N is the size of the regular +** expression and M is the size of the input string. The matcher never +** exhibits exponential behavior. Note that the X{p,q} operator expands +** to p copies of X following by q-p copies of X? and that the size of the +** regular expression in the O(N*M) performance bound is computed after +** this expansion. +*/ +#include +#include +#include "sqlite3.h" + +/* The end-of-input character */ +#define RE_EOF 0 /* End of input */ + +/* The NFA is implemented as sequence of opcodes taken from the following +** set. Each opcode has a single integer argument. +*/ +#define RE_OP_MATCH 1 /* Match the one character in the argument */ +#define RE_OP_ANY 2 /* Match any one character. (Implements ".") */ +#define RE_OP_ANYSTAR 3 /* Special optimized version of .* */ +#define RE_OP_FORK 4 /* Continue to both next and opcode at iArg */ +#define RE_OP_GOTO 5 /* Jump to opcode at iArg */ +#define RE_OP_ACCEPT 6 /* Halt and indicate a successful match */ +#define RE_OP_CC_INC 7 /* Beginning of a [...] character class */ +#define RE_OP_CC_EXC 8 /* Beginning of a [^...] character class */ +#define RE_OP_CC_VALUE 9 /* Single value in a character class */ +#define RE_OP_CC_RANGE 10 /* Range of values in a character class */ +#define RE_OP_WORD 11 /* Perl word character [A-Za-z0-9_] */ +#define RE_OP_NOTWORD 12 /* Not a perl word character */ +#define RE_OP_DIGIT 13 /* digit: [0-9] */ +#define RE_OP_NOTDIGIT 14 /* Not a digit */ +#define RE_OP_SPACE 15 /* space: [ \t\n\r\v\f] */ +#define RE_OP_NOTSPACE 16 /* Not a digit */ +#define RE_OP_BOUNDARY 17 /* Boundary between word and non-word */ + +/* Each opcode is a "state" in the NFA */ +typedef unsigned short ReStateNumber; + +/* Because this is an NFA and not a DFA, multiple states can be active at +** once. An instance of the following object records all active states in +** the NFA. The implementation is optimized for the common case where the +** number of actives states is small. +*/ +typedef struct ReStateSet { + unsigned nState; /* Number of current states */ + ReStateNumber *aState; /* Current states */ +} ReStateSet; + +/* A compiled NFA (or an NFA that is in the process of being compiled) is +** an instance of the following object. +*/ +typedef struct ReCompiled { + const unsigned char *zIn; /* Regular expression text */ + const char *zErr; /* Error message to return */ + char *aOp; /* Operators for the virtual machine */ + int *aArg; /* Arguments to each operator */ + char zInit[12]; /* Initial text to match */ + int nInit; /* Number of characters in zInit */ + unsigned nState; /* Number of entries in aOp[] and aArg[] */ + unsigned nAlloc; /* Slots allocated for aOp[] and aArg[] */ +} ReCompiled; + +/* Add a state to the given state set if it is not already there */ +static void re_add_state(ReStateSet *pSet, int newState){ + unsigned i; + for(i=0; inState; i++) if( pSet->aState[i]==newState ) return; + pSet->aState[pSet->nState++] = newState; +} + +/* Extract the next unicode character from *pzIn and return it. Advance +** *pzIn to the first byte past the end of the character returned. To +** be clear: this routine converts utf8 to unicode. This routine is +** optimized for the common case where the next character is a single byte. +*/ +static unsigned re_next_char(const unsigned char **pzIn){ + unsigned c = **pzIn; + if( c>0 ) (*pzIn)++; + if( c>0x80 ){ + if( (c&0xe0)==0xc0 && ((*pzIn)[0]&0xc0)==0x80 ){ + c = (c&0x1f)<<6 | ((*pzIn)[0]&0x3f); + (*pzIn)++; + if( c<0x80 ) c = 0xfffd; + }else if( (c&0xf0)==0xe0 && ((*pzIn)[0]&0xc0)==0x80 + && ((*pzIn)[1]&0xc0)==0x80 ){ + c = (c&0x0f)<<12 | (((*pzIn)[0]&0x3f)<<6) | ((*pzIn)[1]&0x3f); + *pzIn += 2; + if( c<0x3ff || (c>=0xd800 && c<=0xdfff) ) c = 0xfffd; + }else if( (c&0xf8)==0xf0 && ((*pzIn)[0]&0xc0)==0x80 + && ((*pzIn)[1]&0xc0)==0x80 && ((*pzIn)[2]&0xc0)==0x80 ){ + c = (c&0x07)<<18 | (((*pzIn)[0]&0x3f)<<12) | (((*pzIn)[1]&0x3f)<<6) + | ((*pzIn)[2]&0x3f); + *pzIn += 3; + if( c<0xffff ) c = 0xfffd; + }else{ + c = 0xfffd; + } + } + return c; +} + +/* Return true if c is a perl "word" character: [A-Za-z0-9_] */ +static int re_word_char(int c){ + return (c>='0' && c<='9') || (c>='a' && c<='z') + || (c>='A' && c<='Z') || c=='_'; +} + +/* Return true if c is a "digit" character: [0-9] */ +static int re_digit_char(int c){ + return (c>='0' && c<='9'); +} + +/* Return true if c is a perl "space" character: [ \t\r\n\v\f] */ +static int re_space_char(int c){ + return c==' ' || c=='\t' || c=='\n' || c=='\v' || c=='\f'; +} + +/* Run a compiled regular expression on the zero-terminated input +** string zIn[]. Return true on a match and false if there is no match. +*/ +static int re_exec(ReCompiled *pRe, const unsigned char *zIn){ + ReStateSet aStateSet[2], *pThis, *pNext; + ReStateNumber aSpace[100]; + ReStateNumber *pToFree; + unsigned int i = 0; + unsigned int iSwap = 0; + int c = RE_EOF+1; + int cPrev = 0; + int rc = 0; + + if( pRe->nInit ){ + unsigned char x = pRe->zInit[0]; + while( zIn[0] && (zIn[0]!=x || memcmp(zIn, pRe->zInit, pRe->nInit)!=0) ){ + zIn++; + } + if( zIn[0]==0 ) return 0; + } + if( pRe->nState<=(sizeof(aSpace)/(sizeof(aSpace[0])*2)) ){ + pToFree = 0; + aStateSet[0].aState = aSpace; + }else{ + pToFree = malloc( sizeof(ReStateNumber)*2*pRe->nState ); + if( pToFree==0 ) return -1; + aStateSet[0].aState = pToFree; + } + aStateSet[1].aState = &aStateSet[0].aState[pRe->nState]; + pNext = &aStateSet[1]; + pNext->nState = 0; + re_add_state(pNext, 0); + while( c!=RE_EOF && pNext->nState>0 ){ + cPrev = c; + c = re_next_char(&zIn); + pThis = pNext; + pNext = &aStateSet[iSwap]; + iSwap = 1 - iSwap; + pNext->nState = 0; + for(i=0; inState; i++){ + int x = pThis->aState[i]; + switch( pRe->aOp[x] ){ + case RE_OP_MATCH: { + if( pRe->aArg[x]==c ) re_add_state(pNext, x+1); + break; + } + case RE_OP_ANY: { + re_add_state(pNext, x+1); + break; + } + case RE_OP_WORD: { + if( re_word_char(c) ) re_add_state(pNext, x+1); + break; + } + case RE_OP_NOTWORD: { + if( !re_word_char(c) ) re_add_state(pNext, x+1); + break; + } + case RE_OP_DIGIT: { + if( re_digit_char(c) ) re_add_state(pNext, x+1); + break; + } + case RE_OP_NOTDIGIT: { + if( !re_digit_char(c) ) re_add_state(pNext, x+1); + break; + } + case RE_OP_SPACE: { + if( re_space_char(c) ) re_add_state(pNext, x+1); + break; + } + case RE_OP_NOTSPACE: { + if( !re_space_char(c) ) re_add_state(pNext, x+1); + break; + } + case RE_OP_BOUNDARY: { + if( re_word_char(c)!=re_word_char(cPrev) ) re_add_state(pThis, x+1); + break; + } + case RE_OP_ANYSTAR: { + re_add_state(pNext, x); + re_add_state(pThis, x+1); + break; + } + case RE_OP_FORK: { + re_add_state(pThis, x+pRe->aArg[x]); + re_add_state(pThis, x+1); + break; + } + case RE_OP_GOTO: { + re_add_state(pThis, x+pRe->aArg[x]); + break; + } + case RE_OP_ACCEPT: { + rc = 1; + goto re_exec_end; + } + case RE_OP_CC_INC: + case RE_OP_CC_EXC: { + int j = 1; + int n = pRe->aArg[x]; + int hit = 0; + for(j=1; j>0 && jaOp[x+j]==RE_OP_CC_VALUE ){ + if( pRe->aArg[x+j]==c ){ + hit = 1; + j = -1; + } + }else{ + if( pRe->aArg[x+j]<=c && pRe->aArg[x+j+1]>=c ){ + hit = 1; + j = -1; + }else{ + j++; + } + } + } + if( pRe->aOp[x]==RE_OP_CC_EXC ) hit = !hit; + if( hit ) re_add_state(pNext, x+n); + break; + } + } + } + } + for(i=0; inState; i++){ + if( pRe->aOp[pNext->aState[i]]==RE_OP_ACCEPT ){ rc = 1; break; } + } +re_exec_end: + free(pToFree); + return rc; +} + +/* Resize the opcode and argument arrays for an RE under construction. +*/ +static int re_resize(ReCompiled *p, int N){ + char *aOp; + int *aArg; + aOp = realloc(p->aOp, N*sizeof(p->aOp[0])); + if( aOp==0 ) return 1; + p->aOp = aOp; + aArg = realloc(p->aArg, N*sizeof(p->aArg[0])); + if( aArg==0 ) return 1; + p->aArg = aArg; + p->nAlloc = N; + return 0; +} + +/* Insert a new opcode and argument into an RE under construction. The +** insertion point is just prior to existing opcode iBefore. +*/ +static int re_insert(ReCompiled *p, int iBefore, int op, int arg){ + int i; + if( p->nAlloc<=p->nState && re_resize(p, p->nAlloc*2) ) return 0; + for(i=p->nState; i>iBefore; i--){ + p->aOp[i] = p->aOp[i-1]; + p->aArg[i] = p->aArg[i-1]; + } + p->nState++; + p->aOp[iBefore] = op; + p->aArg[iBefore] = arg; + return iBefore; +} + +/* Append a new opcode and argument to the end of the RE under construction. +*/ +static int re_append(ReCompiled *p, int op, int arg){ + return re_insert(p, p->nState, op, arg); +} + +/* Make a copy of N opcodes starting at iStart onto the end of the RE +** under construction. +*/ +static void re_copy(ReCompiled *p, int iStart, int N){ + if( p->nState+N>=p->nAlloc && re_resize(p, p->nAlloc*2+N) ) return; + memcpy(&p->aOp[p->nState], &p->aOp[iStart], N*sizeof(p->aOp[0])); + memcpy(&p->aArg[p->nState], &p->aArg[iStart], N*sizeof(p->aArg[0])); + p->nState += N; +} + +/* Return true if c is a hexadecimal digit character: [0-9a-fA-F] +** If c is a hex digit, also set *pV = (*pV)*16 + valueof(c). If +** c is not a hex digit *pV is unchanged. +*/ +static int re_hex(int c, int *pV){ + if( c>='0' && c<='9' ){ + c -= '0'; + }else if( c>='a' && c<='f' ){ + c -= 'a' + 10; + }else if( c>='A' && c<='F' ){ + c -= 'A' + 10; + }else{ + return 0; + } + *pV = (*pV)*16 + c; + return 1; +} + +/* A backslash character has been seen, read the next character and +** return its intepretation. +*/ +static unsigned re_esc_char(ReCompiled *p){ + static const char zEsc[] = "afnrtv\\()*.+?[$^{|"; + static const char zTrans[] = "\a\f\n\r\t\v"; + int i, v = 0; + char c = p->zIn[0]; + if( c=='u' ){ + v = 0; + if( re_hex(p->zIn[1],&v) + && re_hex(p->zIn[2],&v) + && re_hex(p->zIn[3],&v) + && re_hex(p->zIn[4],&v) + ){ + p->zIn += 5; + return v; + } + } + if( c=='x' ){ + v = 0; + for(i=1; re_hex(p->zIn[i], &v); i++){} + if( i>1 ){ + p->zIn += i; + return v; + } + } + for(i=0; zEsc[i] && zEsc[i]!=c; i++){} + if( zEsc[i] ){ + if( c<6 ) c = zTrans[i]; + p->zIn++; + }else{ + p->zErr = "unknown \\ escape"; + } + return c; +} + +/* Forward declaration */ +static const char *re_subcompile_string(ReCompiled*); + +/* Compile RE text into a sequence of opcodes. Continue up to the +** first unmatched ")" character, then return. If an error is found, +** return a pointer to the error message string. +*/ +static const char *re_subcompile_re(ReCompiled *p){ + const char *zErr; + int iStart, iEnd, iGoto; + iStart = p->nState; + zErr = re_subcompile_string(p); + if( zErr ) return zErr; + while( p->zIn[0]=='|' ){ + iEnd = p->nState; + re_insert(p, iStart, RE_OP_FORK, iEnd + 2 - iStart); + iGoto = re_append(p, RE_OP_GOTO, 0); + p->zIn++; + zErr = re_subcompile_string(p); + if( zErr ) return zErr; + p->aArg[iGoto] = p->nState - iGoto; + } + return 0; +} + +/* Compile an element of regular expression text (anything that can be +** an operand to the "|" operator). Return NULL on success or a pointer +** to the error message if there is a problem. +*/ +static const char *re_subcompile_string(ReCompiled *p){ + int iPrev = -1; + int iStart; + unsigned c; + const char *zErr; + while( (c = re_next_char(&p->zIn))!=0 ){ + iStart = p->nState; + switch( c ){ + case '|': + case '$': + case ')': { + p->zIn--; + return 0; + } + case '(': { + zErr = re_subcompile_re(p); + if( zErr ) return zErr; + if( p->zIn[0]!=')' ) return "unmatched '('"; + p->zIn++; + break; + } + case '.': { + if( p->zIn[0]=='*' ){ + re_append(p, RE_OP_ANYSTAR, 0); + p->zIn++; + }else{ + re_append(p, RE_OP_ANY, 0); + } + break; + } + case '*': { + if( iPrev<0 ) return "'*' without operand"; + re_insert(p, iPrev, RE_OP_GOTO, p->nState - iPrev + 1); + re_append(p, RE_OP_FORK, iPrev - p->nState + 1); + break; + } + case '+': { + if( iPrev<0 ) return "'+' without operand"; + re_append(p, RE_OP_FORK, iPrev - p->nState); + break; + } + case '?': { + if( iPrev<0 ) return "'?' without operand"; + re_insert(p, iPrev, RE_OP_FORK, p->nState - iPrev+1); + break; + } + case '{': { + int m = 0, n = 0; + int sz, j; + if( iPrev<0 ) return "'{m,n}' without operand"; + while( (c=p->zIn[0])>='0' && c<='9' ){ m = m*10 + c - '0'; p->zIn++; } + n = m; + if( c==',' ){ + p->zIn++; + n = 0; + while( (c=p->zIn[0])>='0' && c<='9' ){ n = n*10 + c - '0'; p->zIn++; } + } + if( c!='}' ) return "unmatched '{'"; + if( n>0 && nzIn++; + sz = p->nState - iPrev; + if( m==0 ){ + if( n==0 ) return "both m and n are zero in '{m,n}'"; + re_insert(p, iPrev, RE_OP_FORK, sz+1); + n--; + }else{ + for(j=1; j0 ){ + re_append(p, RE_OP_FORK, -sz); + } + break; + } + case '[': { + int iFirst = p->nState; + if( p->zIn[0]=='^' ){ + re_append(p, RE_OP_CC_EXC, 0); + p->zIn++; + }else{ + re_append(p, RE_OP_CC_INC, 0); + } + while( (c = re_next_char(&p->zIn))!=0 ){ + if( c=='[' && p->zIn[0]==':' ){ + return "POSIX character classes not supported"; + } + if( c=='\\' ) c = re_esc_char(p); + if( p->zIn[0]=='-' && p->zIn[1] ){ + re_append(p, RE_OP_CC_RANGE, c); + p->zIn++; + c = re_next_char(&p->zIn); + if( c=='\\' ) c = re_esc_char(p); + re_append(p, RE_OP_CC_RANGE, c); + }else{ + re_append(p, RE_OP_CC_VALUE, c); + } + if( p->zIn[0]==']' ){ p->zIn++; break; } + } + if( c==0 ) return "unclosed '['"; + p->aArg[iFirst] = p->nState - iFirst; + break; + } + case '\\': { + int specialOp = 0; + switch( p->zIn[0] ){ + case 'b': specialOp = RE_OP_BOUNDARY; break; + case 'd': specialOp = RE_OP_DIGIT; break; + case 'D': specialOp = RE_OP_NOTDIGIT; break; + case 's': specialOp = RE_OP_SPACE; break; + case 'S': specialOp = RE_OP_NOTSPACE; break; + case 'w': specialOp = RE_OP_WORD; break; + case 'W': specialOp = RE_OP_NOTWORD; break; + } + if( specialOp ){ + p->zIn++; + re_append(p, specialOp, 0); + }else{ + c = re_esc_char(p); + re_append(p, RE_OP_MATCH, c); + } + break; + } + default: { + re_append(p, RE_OP_MATCH, c); + break; + } + } + iPrev = iStart; + } + return 0; +} + +/* Free and reclaim all the memory used by a previously compiled +** regular expression. Applications should invoke this routine once +** for every call to re_compile() to avoid memory leaks. +*/ +static void re_free(ReCompiled *pRe){ + if( pRe ){ + free(pRe->aOp); + free(pRe->aArg); + } +} + +/* +** Compile a textual regular expression in zIn[] into a compiled regular +** expression suitable for us by re_exec() and return a pointer to the +** compiled regular expression in *ppRe. Return NULL on success or an +** error message if something goes wrong. +*/ +static const char *re_compile(ReCompiled **ppRe, const char *zIn){ + ReCompiled *pRe; + const char *zErr; + int i, j; + + *ppRe = 0; + pRe = malloc( sizeof(*pRe) ); + if( pRe==0 ){ + return "out of memory"; + } + memset(pRe, 0, sizeof(*pRe)); + if( re_resize(pRe, 30) ){ + re_free(pRe); + return "out of memory"; + } + if( zIn[0]=='^' ){ + zIn++; + }else{ + re_append(pRe, RE_OP_ANYSTAR, 0); + } + pRe->zIn = (unsigned char*)zIn; + zErr = re_subcompile_re(pRe); + if( zErr ){ + re_free(pRe); + return zErr; + } + if( pRe->zIn[0]=='$' && pRe->zIn[1]==0 ){ + re_append(pRe, RE_OP_MATCH, RE_EOF); + re_append(pRe, RE_OP_ACCEPT, 0); + *ppRe = pRe; + }else if( pRe->zIn[0]==0 ){ + re_append(pRe, RE_OP_ACCEPT, 0); + *ppRe = pRe; + }else{ + re_free(pRe); + return "unrecognized character"; + } + if( pRe->aOp[0]==RE_OP_ANYSTAR ){ + for(j=0, i=1; jzInit)-2 && pRe->aOp[i]==RE_OP_MATCH; i++){ + unsigned x = pRe->aArg[i]; + if( x<=127 ){ + pRe->zInit[j++] = x; + }else if( x<=0xfff ){ + pRe->zInit[j++] = 0xc0 | (x>>6); + pRe->zInit[j++] = 0x80 | (x&0x3f); + }else if( x<=0xffff ){ + pRe->zInit[j++] = 0xd0 | (x>>12); + pRe->zInit[j++] = 0x80 | ((x>>6)&0x3f); + pRe->zInit[j++] = 0x80 | ((x>>6)&0x3f); + }else{ + break; + } + } + pRe->nInit = j; + } + return pRe->zErr; +} + +/* +** Implementation of the regexp() SQL function. This function implements +** the build-in REGEXP operator. The first argument to the function is the +** pattern and the second argument is the string. So, the SQL statements: +** +** A REGEXP B +** +** is implemented as regexp(B,A). +*/ +static void re_sql_func( + sqlite3_context *context, + int argc, + sqlite3_value **argv +){ + ReCompiled *pRe; /* Compiled regular expression */ + const char *zPattern; /* The regular expression */ + const unsigned char *zStr;/* String being searched */ + const char *zErr; /* Compile error message */ + + pRe = sqlite3_get_auxdata(context, 0); + if( pRe==0 ){ + zPattern = (const char*)sqlite3_value_text(argv[0]); + if( zPattern==0 ) return; + zErr = re_compile(&pRe, zPattern); + if( zErr ){ + sqlite3_result_error(context, zErr, -1); + return; + } + if( pRe==0 ){ + sqlite3_result_error_nomem(context); + return; + } + sqlite3_set_auxdata(context, 0, pRe, (void(*)(void*))re_free); + } + zStr = (const unsigned char*)sqlite3_value_text(argv[1]); + if( zStr!=0 ){ + sqlite3_result_int(context, re_exec(pRe, zStr)); + } +} + +/* +** Invoke this routine in order to install the REGEXP function in an +** SQLite database connection. +** +** Use: +** +** sqlite3_auto_extension(sqlite3_add_regexp_func); +** +** to cause this extension to be automatically loaded into each new +** database connection. +*/ +int sqlite3_add_regexp_func(sqlite3 *db){ + return sqlite3_create_function(db, "regexp", 2, SQLITE_UTF8, 0, + re_sql_func, 0, 0); +} + + +/***************************** Test Code ***********************************/ +#ifdef SQLITE_TEST +#include +extern int getDbPointer(Tcl_Interp *interp, const char *zA, sqlite3 **ppDb); + +/* Implementation of the TCL command: +** +** sqlite3_add_regexp_func $DB +*/ +static int tclSqlite3AddRegexpFunc( + void * clientData, + Tcl_Interp *interp, + int objc, + Tcl_Obj *CONST objv[] +){ + sqlite3 *db; + if( objc!=2 ){ + Tcl_WrongNumArgs(interp, 1, objv, "DB"); + return TCL_ERROR; + } + if( getDbPointer(interp, Tcl_GetString(objv[1]), &db) ) return TCL_ERROR; + sqlite3_add_regexp_func(db); + return TCL_OK; +} + +/* Register the sqlite3_add_regexp_func TCL command with the TCL interpreter. +*/ +int Sqlitetestregexp_Init(Tcl_Interp *interp){ + Tcl_CreateObjCommand(interp, "sqlite3_add_regexp_func", + tclSqlite3AddRegexpFunc, 0, 0); + return TCL_OK; +} +#endif /* SQLITE_TEST */ +/**************************** End Of Test Code *******************************/ diff --git a/test/regexp1.test b/test/regexp1.test new file mode 100644 index 0000000000..db019a542d --- /dev/null +++ b/test/regexp1.test @@ -0,0 +1,84 @@ +# 2012 December 31 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# +# This file implements test for the REGEXP operator in test_regexp.c. +# + +set testdir [file dirname $argv0] +source $testdir/tester.tcl + +do_test regexp1-1.1 { + sqlite3_add_regexp_func db + db eval { + CREATE TABLE t1(x INTEGER PRIMARY KEY, y TEXT); + INSERT INTO t1 VALUES(1, 'For since by man came death,'); + INSERT INTO t1 VALUES(2, 'by man came also the resurrection of the dead.'); + INSERT INTO t1 VALUES(3, 'For as in Adam all die,'); + INSERT INTO t1 VALUES(4, 'even so in Christ shall all be made alive.'); + + SELECT x FROM t1 WHERE y REGEXP '^For ' ORDER BY x; + } +} {1 3} + +do_execsql_test regexp1-1.2 { + SELECT x FROM t1 WHERE y REGEXP 'by|in' ORDER BY x; +} {1 2 3 4} +do_execsql_test regexp1-1.3 { + SELECT x FROM t1 WHERE y REGEXP 'by|Christ' ORDER BY x; +} {1 2 4} +do_execsql_test regexp1-1.4 { + SELECT x FROM t1 WHERE y REGEXP 'shal+ al+' ORDER BY x; +} {4} +do_execsql_test regexp1-1.5 { + SELECT x FROM t1 WHERE y REGEXP 'shall x*y*z*all' ORDER BY x; +} {4} +do_execsql_test regexp1-1.6 { + SELECT x FROM t1 WHERE y REGEXP 'shallx?y? ?z?all' ORDER BY x; +} {4} +do_execsql_test regexp1-1.7 { + SELECT x FROM t1 WHERE y REGEXP 'r{2}' ORDER BY x; +} {2} +do_execsql_test regexp1-1.8 { + SELECT x FROM t1 WHERE y REGEXP 'r{3}' ORDER BY x; +} {} +do_execsql_test regexp1-1.9 { + SELECT x FROM t1 WHERE y REGEXP 'r{1}' ORDER BY x; +} {1 2 3 4} +do_execsql_test regexp1-1.10 { + SELECT x FROM t1 WHERE y REGEXP 'ur{2,10}e' ORDER BY x; +} {2} +do_execsql_test regexp1-1.11 { + SELECT x FROM t1 WHERE y REGEXP '[Aa]dam' ORDER BY x; +} {3} +do_execsql_test regexp1-1.12 { + SELECT x FROM t1 WHERE y REGEXP '[^Aa]dam' ORDER BY x; +} {} +do_execsql_test regexp1-1.13 { + SELECT x FROM t1 WHERE y REGEXP '[^b-zB-Z]dam' ORDER BY x; +} {3} +do_execsql_test regexp1-1.14 { + SELECT x FROM t1 WHERE y REGEXP 'alive' ORDER BY x; +} {4} +do_execsql_test regexp1-1.15 { + SELECT x FROM t1 WHERE y REGEXP '^alive' ORDER BY x; +} {} +do_execsql_test regexp1-1.16 { + SELECT x FROM t1 WHERE y REGEXP 'alive$' ORDER BY x; +} {} +do_execsql_test regexp1-1.17 { + SELECT x FROM t1 WHERE y REGEXP 'alive.$' ORDER BY x; +} {4} +do_execsql_test regexp1-1.18 { + SELECT x FROM t1 WHERE y REGEXP 'alive\.$' ORDER BY x; +} {4} + + +finish_test diff --git a/tool/build-shell.sh b/tool/build-shell.sh index 8e62a71746..665480540c 100644 --- a/tool/build-shell.sh +++ b/tool/build-shell.sh @@ -15,7 +15,9 @@ gcc -o sqlite3 -g -Os -I. \ -DSQLITE_ENABLE_STAT3 \ -DSQLITE_ENABLE_FTS4 \ -DSQLITE_ENABLE_RTREE \ + -DSQLITE_ENABLE_REGEXP \ -DHAVE_READLINE \ -DHAVE_USLEEP=1 \ ../sqlite/src/shell.c ../sqlite/src/test_vfstrace.c \ + ../sqlite/src/test_regexp.c \ sqlite3.c -ldl -lreadline -lncurses