From: dan <dan@noemail.net>
Date: Mon, 13 Jan 2014 15:12:23 +0000 (+0000)
Subject: Add code to handle non-recursive CTEs in the same way as SQL views.
X-Git-Tag: version-3.8.3~45^2~20
X-Git-Url: http://git.ipfire.org/cgi-bin/gitweb.cgi?a=commitdiff_plain;h=4e9119d9e8bc3ed4c75b2c77f9ec80c6cf5f5463;p=thirdparty%2Fsqlite.git

Add code to handle non-recursive CTEs in the same way as SQL views.

FossilOrigin-Name: a26f399ba485e8127c276c5f103ec6c555e11734
---

diff --git a/manifest b/manifest
index c9d03126cd..010cb6fcfd 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Update\sthe\sparser\sso\sthat\ssub-queries\sand\sCTEs\smay\shave\sWITH\sclauses.
-D 2014-01-11T19:19:36.147
+C Add\scode\sto\shandle\snon-recursive\sCTEs\sin\sthe\ssame\sway\sas\sSQL\sviews.
+D 2014-01-13T15:12:23.149
 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f
 F Makefile.in 2ef13430cd359f7b361bb863504e227b25cc7f81
 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23
@@ -169,13 +169,13 @@ F src/btmutex.c 976f45a12e37293e32cae0281b15a21d48a8aaa7
 F src/btree.c 11e29ef8cf16a42925fde036bcffbeffd9cc82df
 F src/btree.h a61ddebc78c66795a2b93181321a116746302cc9
 F src/btreeInt.h f038e818bfadf75afbd09819ed93c26a333d39e0
-F src/build.c 1da29e4e93d9774599f1f791d011ba46d841000c
+F src/build.c 1efdc65020e1f566383b66b30ccf730b4ef6f926
 F src/callback.c 174e3c8656bc29f91d710ab61550d16eea34be98
 F src/complete.c dc1d136c0feee03c2f7550bafc0d29075e36deac
 F src/ctime.c 77779efbe78dd678d84bfb4fc2e87b6b6ad8dccd
 F src/date.c 593c744b2623971e45affd0bde347631bdfa4625
 F src/delete.c 91e1321021db5dc266360531b8b6550009d771ff
-F src/expr.c 4115ad67088cdd55f4fa0ef3ddd22cb8da8f9c94
+F src/expr.c de86bf987c532ec9d63fb1bf8a6eb6ec2cf5ba69
 F src/fault.c 160a0c015b6c2629d3899ed2daf63d75754a32bb
 F src/fkey.c 2ab0f5384b70594468ef3ac5c7ed8ca24bfd17d5
 F src/func.c 6325ac2ec10833ccf4d5c36d323709221d37ea19
@@ -183,7 +183,7 @@ F src/global.c 1d7bb7ea8254ae6a68ed9bfaf65fcb3d1690b486
 F src/hash.c ac3470bbf1ca4ae4e306a8ecb0fdf1731810ffe4
 F src/hash.h 8890a25af81fb85a9ad7790d32eedab4b994da22
 F src/hwtime.h d32741c8f4df852c7d959236615444e2b1063b08
-F src/insert.c 5ddb48c7f1cb399993744f23f03948989ce1790e
+F src/insert.c cb4c8ad02b6feb95d34614d94a3c68e0116fbf07
 F src/journal.c b4124532212b6952f42eb2c12fa3c25701d8ba8d
 F src/legacy.c 0df0b1550b9cc1f58229644735e317ac89131f12
 F src/lempar.c cdf0a000315332fc9b50b62f3b5e22e080a0952b
@@ -209,7 +209,7 @@ F src/os_unix.c 3a4dcb554d3c915075766162f28c3fd4cdb75968
 F src/os_win.c 16eac0961603182ffc10c02b39fe830126538e07
 F src/pager.c efa923693e958696eee69b205a20bfbc402c8480
 F src/pager.h ffd5607f7b3e4590b415b007a4382f693334d428
-F src/parse.y 1e3fd22fc82930d0cce01f2ab90616100c59ee0c
+F src/parse.y b433588d7993d6475d9944be0fda2fb3df3cb138
 F src/pcache.c f8043b433a57aba85384a531e3937a804432a346
 F src/pcache.h a5e4f5d9f5d592051d91212c5949517971ae6222
 F src/pcache1.c 57fee9a9a617218f5037afbbe49b09da65bde56b
@@ -219,12 +219,12 @@ F src/printf.c 85d07756e45d7496d19439dcae3e6e9e0090f269
 F src/random.c d10c1f85b6709ca97278428fd5db5bbb9c74eece
 F src/resolve.c 7eda9097b29fcf3d2b42fdc17d1de672134e09b6
 F src/rowset.c 64655f1a627c9c212d9ab497899e7424a34222e0
-F src/select.c 819bb090c9a348d17f69f136cad2bfa9ee9cbb41
+F src/select.c 9dc9177bfb9278a603e3835e828ff60ec68fba6f
 F src/shell.c a3541193d5fce37e91dad8ef46a9505aa7c9b344
 F src/sqlite.h.in d94a8b89522f526ba711182ee161e06f8669bcc9
 F src/sqlite3.rc 11094cc6a157a028b301a9f06b3d03089ea37c3e
 F src/sqlite3ext.h 886f5a34de171002ad46fae8c36a7d8051c190fc
-F src/sqliteInt.h 2710b3a6c66edda9f193453268f6fe878ff5a0ca
+F src/sqliteInt.h a350bcf6d62204b6b4720ce1922c95a575a0b5ad
 F src/sqliteLimit.h 164b0e6749d31e0daa1a4589a169d31c0dec7b3d
 F src/status.c 7ac05a5c7017d0b9f0b4bcd701228b784f987158
 F src/table.c 2cd62736f845d82200acfa1287e33feb3c15d62e
@@ -274,7 +274,7 @@ F src/test_thread.c 1e133a40b50e9c035b00174035b846e7eef481cb
 F src/test_vfs.c e72f555ef7a59080f898fcf1a233deb9eb704ea9
 F src/test_vfstrace.c 3a0ab304682fecbceb689e7d9b904211fde11d78
 F src/test_wsd.c 41cadfd9d97fe8e3e4e44f61a4a8ccd6f7ca8fe9
-F src/tokenize.c 5d04a1b7d1fe7e18556a869788f5d3e132a586b6
+F src/tokenize.c 7dc42e9beb8c3263b79d10c195b3f5264b5f874a
 F src/trigger.c 5c1c0b899ac0ce284763dcb8fdbaa38ecf15ef98
 F src/update.c c2706a6eb232a96345c35b7e1e75a188e26812bb
 F src/utf.c 6fc6c88d50448c469c5c196acf21617a24f90269
@@ -728,7 +728,7 @@ F test/pager1.test 1acbdb14c5952a72dd43129cabdbf69aaa3ed1fa
 F test/pager2.test 67b8f40ae98112bcdba1f2b2d03ea83266418c71
 F test/pager3.test 3856d9c80839be0668efee1b74811b1b7f7fc95f
 F test/pager4.test b40ecb4cc7dff957ee7916e41ab21d1ed702a642
-F test/pagerfault.test 7285379906ab2f1108b8e82bbdf2d386cc8ff3ff
+F test/pagerfault.test cee8488a935e42a4a2e241e0317dc2814c997783
 F test/pagerfault2.test caf4c7facb914fd3b03a17b31ae2b180c8d6ca1f
 F test/pagerfault3.test 1003fcda009bf48a8e22a516e193b6ef0dd1bbd8
 F test/pageropt.test 6b8f6a123a5572c195ad4ae40f2987007923bbd6
@@ -1091,7 +1091,8 @@ F test/wild001.test bca33f499866f04c24510d74baf1e578d4e44b1c
 F test/win32heap.test ea19770974795cff26e11575e12d422dbd16893c
 F test/win32lock.test 7a6bd73a5dcdee39b5bb93e92395e1773a194361
 F test/win32longpath.test 169c75a3b2e43481f4a62122510210c67b08f26d
-F test/with1.test e8146198319c3627bb21ef085c58a827f35686a7
+F test/with1.test fb8409a35b1314be6e73a87597322f3369b59b2b
+F test/withM.test 036349a9ec081fee8b1f72e71ad8a5a5b80f1674
 F test/without_rowid1.test aaa26da19d543cd8d3d2d0e686dfa255556c15c8
 F test/without_rowid2.test af260339f79d13cb220288b67cd287fbcf81ad99
 F test/without_rowid3.test eac3d5c8a1924725b58503a368f2cbd24fd6c8a0
@@ -1149,7 +1150,7 @@ F tool/vdbe-compress.tcl 0cf56e9263a152b84da86e75a5c0cdcdb7a47891
 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4
 F tool/warnings.sh d1a6de74685f360ab718efda6265994b99bbea01
 F tool/win/sqlite.vsix 030f3eeaf2cb811a3692ab9c14d021a75ce41fff
-P da98b7205eb3d7ec2ddbf8a8e24eee0b2ff499a5
-R bbb9d541af414d7b8bb3f418deb50875
+P 704d3931b855562a619769955969d439c42ca406
+R fb5ad7a801e83802b92e3793bba4fd6c
 U dan
-Z a973a84816d55810384771a899b1092b
+Z 663a5542bd9dc2bb50ed6e3a07d040fb
diff --git a/manifest.uuid b/manifest.uuid
index f6a009da18..267d8a5499 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-704d3931b855562a619769955969d439c42ca406
\ No newline at end of file
+a26f399ba485e8127c276c5f103ec6c555e11734
\ No newline at end of file
diff --git a/src/build.c b/src/build.c
index 23628f73be..ca184766f9 100644
--- a/src/build.c
+++ b/src/build.c
@@ -4208,18 +4208,61 @@ With *sqlite3WithAdd(
   Parse *pParse,          /* Parsing context */
   With *pWith,            /* Existing WITH clause, or NULL */
   Token *pName,           /* Name of the common-table */
-  IdList *pArglist,       /* Optional column name list for the table */
+  ExprList *pArglist,     /* Optional column name list for the table */
   Select *pQuery          /* Query used to initialize the table */
 ){
-  sqlite3IdListDelete(pParse->db, pArglist);
-  sqlite3SelectDelete(pParse->db, pQuery);
-  return 0;
+  sqlite3 *db = pParse->db;
+  With *pNew;
+  char *zName;
+
+  /* Check that the CTE name is unique within this WITH clause. If
+  ** not, store an error in the Parse structure. */
+  zName = sqlite3NameFromToken(pParse->db, pName);
+  if( zName && pWith ){
+    int i;
+    for(i=0; i<pWith->nCte; i++){
+      if( sqlite3StrICmp(zName, pWith->a[i].zName)==0 ){
+        sqlite3ErrorMsg(pParse, "duplicate cte name: %s", zName);
+      }
+    }
+  }
+
+  if( pWith ){
+    int nByte = sizeof(*pWith) + (sizeof(pWith->a[1]) * pWith->nCte);
+    pNew = sqlite3DbRealloc(db, pWith, nByte);
+  }else{
+    pNew = sqlite3DbMallocZero(db, sizeof(*pWith));
+  }
+  assert( zName!=0 || pNew==0 );
+
+  if( pNew==0 ){
+    sqlite3WithDelete(db, pWith);
+    sqlite3ExprListDelete(db, pArglist);
+    sqlite3SelectDelete(db, pQuery);
+    sqlite3DbFree(db, zName);
+  }else{
+    pNew->a[pNew->nCte].pSelect = pQuery;
+    pNew->a[pNew->nCte].pCols = pArglist;
+    pNew->a[pNew->nCte].zName = zName;
+    pNew->nCte++;
+  }
+
+  return pNew;
 }
 
 /*
 ** Free the contents of the With object passed as the second argument.
 */
 void sqlite3WithDelete(sqlite3 *db, With *pWith){
-  /* TBD */
+  if( pWith ){
+    int i;
+    for(i=0; i<pWith->nCte; i++){
+      struct Cte *pCte = &pWith->a[i];
+      sqlite3ExprListDelete(db, pCte->pCols);
+      sqlite3SelectDelete(db, pCte->pSelect);
+      sqlite3DbFree(db, pCte->zName);
+    }
+    sqlite3DbFree(db, pWith);
+  }
 }
 #endif /* !defined(SQLITE_OMIT_CTE) */
diff --git a/src/expr.c b/src/expr.c
index aad6cd1cda..d0ea280029 100644
--- a/src/expr.c
+++ b/src/expr.c
@@ -895,6 +895,24 @@ static Expr *exprDup(sqlite3 *db, Expr *p, int flags, u8 **pzBuffer){
   return pNew;
 }
 
+With *withDup(sqlite3 *db, With *p){
+  With *pRet = 0;
+  if( p ){
+    int nByte = sizeof(*p) + sizeof(p->a[0]) * (p->nCte-1);
+    pRet = sqlite3DbMallocZero(db, nByte);
+    if( pRet ){
+      int i;
+      pRet->nCte = p->nCte;
+      for(i=0; i<p->nCte; i++){
+        pRet->a[i].pSelect = sqlite3SelectDup(db, p->a[i].pSelect, 0);
+        pRet->a[i].pCols = sqlite3ExprListDup(db, p->a[i].pCols, 0);
+        pRet->a[i].zName = sqlite3DbStrDup(db, p->a[i].zName);
+      }
+    }
+  }
+  return pRet;
+}
+
 /*
 ** The following group of routines make deep copies of expressions,
 ** expression lists, ID lists, and select statements.  The copies can
@@ -1036,6 +1054,7 @@ Select *sqlite3SelectDup(sqlite3 *db, Select *p, int flags){
   pNew->addrOpenEphm[0] = -1;
   pNew->addrOpenEphm[1] = -1;
   pNew->addrOpenEphm[2] = -1;
+  pNew->pWith = withDup(db, p->pWith);
   return pNew;
 }
 #else
diff --git a/src/insert.c b/src/insert.c
index 038d08a40d..b4bd6b7160 100644
--- a/src/insert.c
+++ b/src/insert.c
@@ -667,7 +667,8 @@ void sqlite3Insert(
   **
   ** This is the 2nd template.
   */
-  if( pColumn==0 && xferOptimization(pParse, pTab, pSelect, onError, iDb) ){
+  if( pColumn==0 && pParse->pWith==0 
+   && xferOptimization(pParse, pTab, pSelect, onError, iDb) ){
     assert( !pTrigger );
     assert( pList==0 );
     goto insert_end;
diff --git a/src/parse.y b/src/parse.y
index c3cb2d62aa..1c6465e933 100644
--- a/src/parse.y
+++ b/src/parse.y
@@ -412,7 +412,7 @@ cmd ::= select(X).  {
 %type oneselect {Select*}
 %destructor oneselect {sqlite3SelectDelete(pParse->db, $$);}
 
-select(A) ::= with selectnowith(X). {A = X;}
+select(A) ::= with(W) selectnowith(X). { if( X ) X->pWith = W; A = X; }
 
 selectnowith(A) ::= oneselect(X).                      {A = X;}
 %ifndef SQLITE_OMIT_COMPOUND_SELECT
@@ -652,15 +652,17 @@ limit_opt(A) ::= LIMIT expr(X) COMMA expr(Y).
 /////////////////////////// The DELETE statement /////////////////////////////
 //
 %ifdef SQLITE_ENABLE_UPDATE_DELETE_LIMIT
-cmd ::= with DELETE FROM fullname(X) indexed_opt(I) where_opt(W) 
+cmd ::= with(C) DELETE FROM fullname(X) indexed_opt(I) where_opt(W) 
         orderby_opt(O) limit_opt(L). {
+  sqlite3WithPush(pParse,C);
   sqlite3SrcListIndexedBy(pParse, X, &I);
   W = sqlite3LimitWhere(pParse, X, W, O, L.pLimit, L.pOffset, "DELETE");
   sqlite3DeleteFrom(pParse,X,W);
 }
 %endif
 %ifndef SQLITE_ENABLE_UPDATE_DELETE_LIMIT
-cmd ::= with DELETE FROM fullname(X) indexed_opt(I) where_opt(W). {
+cmd ::= with(C) DELETE FROM fullname(X) indexed_opt(I) where_opt(W). {
+  sqlite3WithPush(pParse,C);
   sqlite3SrcListIndexedBy(pParse, X, &I);
   sqlite3DeleteFrom(pParse,X,W);
 }
@@ -684,8 +686,9 @@ cmd ::= with UPDATE orconf(R) fullname(X) indexed_opt(I) SET setlist(Y)
 }
 %endif
 %ifndef SQLITE_ENABLE_UPDATE_DELETE_LIMIT
-cmd ::= with UPDATE orconf(R) fullname(X) indexed_opt(I) SET setlist(Y)
+cmd ::= with(C) UPDATE orconf(R) fullname(X) indexed_opt(I) SET setlist(Y)
         where_opt(W).  {
+  sqlite3WithPush(pParse, C);
   sqlite3SrcListIndexedBy(pParse, X, &I);
   sqlite3ExprListCheckLength(pParse,Y,"set list"); 
   sqlite3Update(pParse,X,Y,W,R);
@@ -706,10 +709,15 @@ setlist(A) ::= nm(X) EQ expr(Y). {
 
 ////////////////////////// The INSERT command /////////////////////////////////
 //
-cmd ::= with insert_cmd(R) INTO fullname(X) inscollist_opt(F) select(S).
-            {sqlite3Insert(pParse, X, S, F, R);}
-cmd ::= with insert_cmd(R) INTO fullname(X) inscollist_opt(F) DEFAULT VALUES.
-            {sqlite3Insert(pParse, X, 0, F, R);}
+cmd ::= with(W) insert_cmd(R) INTO fullname(X) inscollist_opt(F) select(S). {
+  sqlite3WithPush(pParse, W);
+  sqlite3Insert(pParse, X, S, F, R);
+}
+cmd ::= with(W) insert_cmd(R) INTO fullname(X) inscollist_opt(F) DEFAULT VALUES.
+{
+  sqlite3WithPush(pParse, W);
+  sqlite3Insert(pParse, X, 0, F, R);
+}
 
 %type insert_cmd {u8}
 insert_cmd(A) ::= INSERT orconf(R).   {A = R;}
@@ -1372,16 +1380,17 @@ anylist ::= anylist ANY.
 %type with {With*}
 %type wqlist {With*}
 %destructor with {sqlite3WithDelete(pParse->db, $$);}
+%destructor wqlist {sqlite3WithDelete(pParse->db, $$);}
 
 with(A) ::= . {A = 0;}
 %ifndef SQLITE_OMIT_CTE
 with(A) ::= WITH wqlist(W).              { A = W; }
 with(A) ::= WITH RECURSIVE wqlist(W).    { A = W; }
 
-wqlist(A) ::= nm(X) inscollist_opt(Y) AS LP select(Z) RP. {
+wqlist(A) ::= nm(X) idxlist_opt(Y) AS LP select(Z) RP. {
   A = sqlite3WithAdd(pParse, 0, &X, Y, Z);
 }
-wqlist(A) ::= wqlist(W) COMMA nm(X) inscollist_opt(Y) AS LP select(Z) RP. {
+wqlist(A) ::= wqlist(W) COMMA nm(X) idxlist_opt(Y) AS LP select(Z) RP. {
   A = sqlite3WithAdd(pParse, W, &X, Y, Z);
 }
 %endif  SQLITE_OMIT_CTE
diff --git a/src/select.c b/src/select.c
index d075116749..b054489b00 100644
--- a/src/select.c
+++ b/src/select.c
@@ -29,6 +29,7 @@ static void clearSelect(sqlite3 *db, Select *p){
   sqlite3SelectDelete(db, p->pPrior);
   sqlite3ExprDelete(db, p->pLimit);
   sqlite3ExprDelete(db, p->pOffset);
+  sqlite3WithDelete(db, p->pWith);
 }
 
 /*
@@ -3393,6 +3394,62 @@ static int convertCompoundSelectToSubquery(Walker *pWalker, Select *p){
   return WRC_Continue;
 }
 
+static struct Cte *searchWith(Parse *pParse, struct SrcList_item *p){
+  if( p->zDatabase==0 ){
+    char *zName = p->zName;
+    With *pWith;
+
+    for(pWith=pParse->pWith; pWith; pWith=pWith->pOuter){
+      int i;
+      for(i=0; i<pWith->nCte; i++){
+        if( sqlite3StrICmp(zName, pWith->a[i].zName)==0 ){
+          return &pWith->a[i];
+        }
+      }
+    }
+  }
+  return 0;
+}
+
+void sqlite3WithPush(Parse *pParse, With *pWith){
+  if( pWith ){
+    pWith->pOuter = pParse->pWith;
+    pParse->pWith = pWith;
+  }
+}
+static void withPop(Parse *pParse, With *pWith){
+  if( pWith ){
+    assert( pParse->pWith==pWith );
+    pParse->pWith = pWith->pOuter;
+  }
+}
+
+static int ctePush(Parse *pParse, struct Cte *pCte){
+  if( pCte ){
+    struct Cte *p;
+    for(p=pParse->pCte; p; p=p->pOuterCte){
+      if( p==pCte ){
+        sqlite3ErrorMsg(
+            pParse, "illegal recursive defininition in cte: %s", pCte->zName
+        );
+        return SQLITE_ERROR;
+      }
+    }
+    
+    pCte->pOuterCte = pParse->pCte;
+    pParse->pCte = pCte;
+  }
+  return SQLITE_OK;
+}
+
+static void ctePop(Parse *pParse, struct Cte *pCte){
+  if( pCte ){
+    assert( pParse->pCte==pCte );
+    pParse->pCte = pCte->pOuterCte;
+  }
+}
+
+
 /*
 ** This routine is a Walker callback for "expanding" a SELECT statement.
 ** "Expanding" means to do the following:
@@ -3447,6 +3504,7 @@ static int selectExpander(Walker *pWalker, Select *p){
   ** then create a transient table structure to describe the subquery.
   */
   for(i=0, pFrom=pTabList->a; i<pTabList->nSrc; i++, pFrom++){
+    struct Cte *pCte = 0;
     Table *pTab;
     if( pFrom->pTab!=0 ){
       /* This statement has already been prepared.  There is no need
@@ -3454,19 +3512,34 @@ static int selectExpander(Walker *pWalker, Select *p){
       assert( i==0 );
       return WRC_Prune;
     }
-    if( pFrom->zName==0 ){
+#ifndef SQLITE_OMIT_CTE
+    pCte = searchWith(pParse, pFrom);
+    if( pCte ){
+      pFrom->pSelect = sqlite3SelectDup(db, pCte->pSelect, 0);
+      if( pFrom->pSelect==0 ) return WRC_Abort;
+    }
+#endif
+    if( pFrom->zName==0 || pCte ){
 #ifndef SQLITE_OMIT_SUBQUERY
+      ExprList *pEList;
       Select *pSel = pFrom->pSelect;
       /* A sub-query in the FROM clause of a SELECT */
       assert( pSel!=0 );
       assert( pFrom->pTab==0 );
+      if( ctePush(pParse, pCte) ) return WRC_Abort;
       sqlite3WalkSelect(pWalker, pSel);
+      ctePop(pParse, pCte);
       pFrom->pTab = pTab = sqlite3DbMallocZero(db, sizeof(Table));
       if( pTab==0 ) return WRC_Abort;
       pTab->nRef = 1;
       pTab->zName = sqlite3MPrintf(db, "sqlite_sq_%p", (void*)pTab);
       while( pSel->pPrior ){ pSel = pSel->pPrior; }
-      selectColumnsFromExprList(pParse, pSel->pEList, &pTab->nCol, &pTab->aCol);
+      if( pCte && pCte->pCols ){
+        pEList = pCte->pCols;
+      }else{
+        pEList = pSel->pEList;
+      }
+      selectColumnsFromExprList(pParse, pEList, &pTab->nCol, &pTab->aCol);
       pTab->iPKey = -1;
       pTab->nRowEst = 1048576;
       pTab->tabFlags |= TF_Ephemeral;
@@ -3679,6 +3752,14 @@ static int selectExpander(Walker *pWalker, Select *p){
   return WRC_Continue;
 }
 
+static int selectExpanderWith(Walker *pWalker, Select *p){
+  int res;
+  sqlite3WithPush(pWalker->pParse, p->pWith);
+  res = selectExpander(pWalker, p);
+  withPop(pWalker->pParse, p->pWith);
+  return res;
+}
+
 /*
 ** No-op routine for the parse-tree walker.
 **
@@ -3715,7 +3796,7 @@ static void sqlite3SelectExpand(Parse *pParse, Select *pSelect){
     w.xSelectCallback = convertCompoundSelectToSubquery;
     sqlite3WalkSelect(&w, pSelect);
   }
-  w.xSelectCallback = selectExpander;
+  w.xSelectCallback = selectExpanderWith;
   sqlite3WalkSelect(&w, pSelect);
 }
 
diff --git a/src/sqliteInt.h b/src/sqliteInt.h
index 8846ffb04c..6bd65592b7 100644
--- a/src/sqliteInt.h
+++ b/src/sqliteInt.h
@@ -2144,6 +2144,7 @@ struct Select {
   Select *pRightmost;    /* Right-most select in a compound select statement */
   Expr *pLimit;          /* LIMIT expression. NULL means not used. */
   Expr *pOffset;         /* OFFSET expression. NULL means not used. */
+  With *pWith;           /* WITH clause attached to this select. Or NULL. */
 };
 
 /*
@@ -2365,6 +2366,8 @@ struct Parse {
 #endif
   Table *pZombieTab;        /* List of Table objects to delete after code gen */
   TriggerPrg *pTriggerPrg;  /* Linked list of coded triggers */
+  With *pWith;              /* Current WITH clause, or NULL */
+  struct Cte *pCte;         /* Current CTE, or NULL */
 };
 
 /*
@@ -2638,10 +2641,12 @@ int sqlite3WalkSelectFrom(Walker*, Select*);
 */
 struct With {
   int nCte;                       /* Number of CTEs */
+  With *pOuter;                   /* Containing WITH clause, or NULL */
   struct Cte {
-    const char *zName;            /* Name of this CTE */
-    IdList *pCol;                 /* List of explicit column names, or NULL */
+    char *zName;                  /* Name of this CTE */
+    ExprList *pCols;              /* List of explicit column names, or NULL */
     Select *pSelect;              /* The contents of the CTE */
+    struct Cte *pOuterCte;
   } a[1];
 };
 
@@ -3344,8 +3349,11 @@ const char *sqlite3JournalModename(int);
   int sqlite3WalDefaultHook(void*,sqlite3*,const char*,int);
 #endif
 #ifndef SQLITE_OMIT_CTE
-  With *sqlite3WithAdd(Parse*,With*,Token*,IdList*,Select*);
+  With *sqlite3WithAdd(Parse*,With*,Token*,ExprList*,Select*);
   void sqlite3WithDelete(sqlite3*,With*);
+  void sqlite3WithPush(Parse*, With*);
+#else
+#define sqlite3WithPush(x,y)
 #endif
 
 /* Declarations for functions in fkey.c. All of these are replaced by
diff --git a/src/tokenize.c b/src/tokenize.c
index 6e796ef82e..3fd4f78f7a 100644
--- a/src/tokenize.c
+++ b/src/tokenize.c
@@ -494,6 +494,8 @@ abort_parse:
     sqlite3DeleteTable(db, pParse->pNewTable);
   }
 
+  assert( pParse->pWith==0 || pParse->pWith->pOuter==0 );
+  sqlite3WithDelete(db, pParse->pWith);
   sqlite3DeleteTrigger(db, pParse->pNewTrigger);
   for(i=pParse->nzVar-1; i>=0; i--) sqlite3DbFree(db, pParse->azVar[i]);
   sqlite3DbFree(db, pParse->azVar);
diff --git a/test/pagerfault.test b/test/pagerfault.test
index 796f531c37..1e57c93903 100644
--- a/test/pagerfault.test
+++ b/test/pagerfault.test
@@ -63,6 +63,7 @@ do_faultsim_test pagerfault-1 -prep {
     error "Database content appears incorrect"
   }
 }
+finish_test
 
 #-------------------------------------------------------------------------
 # Test fault-injection while rolling back a hot-journal file with a 
diff --git a/test/with1.test b/test/with1.test
index 80e879ada0..0b7810ebd3 100644
--- a/test/with1.test
+++ b/test/with1.test
@@ -37,4 +37,107 @@ do_execsql_test 1.4 {
   WITH x(a) AS ( SELECT * FROM t1) UPDATE t1 SET x = y;
 } {}
  
+#--------------------------------------------------------------------------
+
+do_execsql_test 2.1 {
+  DROP TABLE IF EXISTS t1;
+  CREATE TABLE t1(x);
+  INSERT INTO t1 VALUES(1);
+  INSERT INTO t1 VALUES(2);
+  WITH tmp AS ( SELECT * FROM t1 ) SELECT x FROM tmp;
+} {1 2}
+
+do_execsql_test 2.2 {
+  WITH tmp(a) AS ( SELECT * FROM t1 ) SELECT a FROM tmp;
+} {1 2}
+
+do_execsql_test 2.3 {
+  SELECT * FROM (
+    WITH tmp(a) AS ( SELECT * FROM t1 ) SELECT a FROM tmp
+  );
+} {1 2}
+
+do_execsql_test 2.4 {
+  WITH tmp1(a) AS ( SELECT * FROM t1 ),
+       tmp2(x) AS ( SELECT * FROM tmp1)
+  SELECT * FROM tmp2;
+} {1 2}
+
+do_execsql_test 2.5 {
+  WITH tmp2(x) AS ( SELECT * FROM tmp1),
+       tmp1(a) AS ( SELECT * FROM t1 )
+  SELECT * FROM tmp2;
+} {1 2}
+
+#-------------------------------------------------------------------------
+do_catchsql_test 3.1 {
+  WITH tmp2(x) AS ( SELECT * FROM tmp1),
+       tmp1(a) AS ( SELECT * FROM tmp2 )
+  SELECT * FROM tmp1;
+} {1 {illegal recursive defininition in cte: tmp1}}
+
+do_catchsql_test 3.2 {
+  CREATE TABLE t2(x INTEGER);
+  WITH tmp(a) AS (SELECT * FROM t1),
+       tmp(a) AS (SELECT * FROM t1)
+  SELECT * FROM tmp;
+} {1 {duplicate cte name: tmp}}
+
+do_execsql_test 3.3 {
+  CREATE TABLE t3(x);
+  CREATE TABLE t4(x);
+
+  INSERT INTO t3 VALUES('T3');
+  INSERT INTO t4 VALUES('T4');
+
+  WITH t3(a) AS (SELECT * FROM t4)
+  SELECT * FROM t3;
+} {T4}
+
+do_execsql_test 3.4 {
+  WITH tmp  AS ( SELECT * FROM t3 ),
+       tmp2 AS ( WITH tmp AS ( SELECT * FROM t4 ) SELECT * FROM tmp )
+  SELECT * FROM tmp2;
+} {T4}
+
+do_execsql_test 3.5 {
+  WITH tmp  AS ( SELECT * FROM t3 ),
+       tmp2 AS ( WITH xxxx AS ( SELECT * FROM t4 ) SELECT * FROM tmp )
+  SELECT * FROM tmp2;
+} {T3}
+
+do_catchsql_test 3.6 {
+  WITH tmp AS ( SELECT * FROM t3 ),
+  SELECT * FROM tmp;
+} {1 {near "SELECT": syntax error}}
+
+#-------------------------------------------------------------------------
+do_execsql_test 4.1 {
+  DROP TABLE IF EXISTS t1;
+  CREATE TABLE t1(x);
+  INSERT INTO t1 VALUES(1);
+  INSERT INTO t1 VALUES(2);
+  INSERT INTO t1 VALUES(3);
+  INSERT INTO t1 VALUES(4);
+
+  WITH dset AS ( SELECT 2 UNION ALL SELECT 4 )
+  DELETE FROM t1 WHERE x IN dset;
+  SELECT * FROM t1;
+} {1 3}
+
+do_execsql_test 4.2 {
+  WITH iset AS ( SELECT 2 UNION ALL SELECT 4 )
+  INSERT INTO t1 SELECT * FROM iset;
+  SELECT * FROM t1;
+} {1 3 2 4}
+
+do_execsql_test 4.3 {
+  WITH uset(a, b) AS ( SELECT 2, 8 UNION ALL SELECT 4, 9 )
+  UPDATE t1 SET x = COALESCE( (SELECT b FROM uset WHERE a=x), x );
+  SELECT * FROM t1;
+} {1 3 8 9}
+
 finish_test
+
+
+
diff --git a/test/withM.test b/test/withM.test
new file mode 100644
index 0000000000..de46c85228
--- /dev/null
+++ b/test/withM.test
@@ -0,0 +1,40 @@
+# 2014 January 11
+#
+# The author disclaims copyright to this source code.  In place of
+# a legal notice, here is a blessing:
+#
+#    May you do good and not evil.
+#    May you find forgiveness for yourself and forgive others.
+#    May you share freely, never taking more than you give.
+#
+#***********************************************************************
+# This file implements regression tests for SQLite library.  The
+# focus of this file is testing the WITH clause.
+#
+
+set testdir [file dirname $argv0]
+source $testdir/tester.tcl
+source $testdir/malloc_common.tcl
+set ::testprefix withM
+
+do_execsql_test 1.0 {
+  CREATE TABLE t1(x INTEGER, y INTEGER);
+  INSERT INTO t1 VALUES(123, 456);
+}
+
+do_faultsim_test withM-1 -prep {
+  sqlite3 db test.db
+} -body {
+  execsql { 
+    WITH tmp AS ( SELECT * FROM t1 )
+    SELECT * FROM tmp;
+  }
+} -test {
+  faultsim_test_result {0 {123 456}}
+  db close
+}
+
+finish_test
+
+
+