From: larrybr <larrybr@noemail.net>
Date: Fri, 10 Sep 2021 00:58:46 +0000 (+0000)
Subject: Speed-up CLI's processing for huge block comments, for any content
X-Git-Tag: version-3.37.0~212^2~4
X-Git-Url: http://git.ipfire.org/?a=commitdiff_plain;h=8bc4cbce6b454a174ee0cfc955da66125f31f789;p=thirdparty%2Fsqlite.git

Speed-up CLI's processing for huge block comments, for any content

FossilOrigin-Name: c4568f9f458bad16bd4481f1f6f5e7b6f39f5ccf23f2b05890078577ce2ab9a2
---

diff --git a/manifest b/manifest
index 34f8d53906..79ef6d71e9 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Remove\sa\sNEVER()\sin\sbtree\sthat\sis\ssometimes\sreachable.\ndbsqlfuzz\sb9140023005430654c8fe544cf0a082ef8d561c1.
-D 2021-09-09T19:19:02.452
+C Speed-up\sCLI's\sprocessing\sfor\shuge\sblock\scomments,\sfor\sany\scontent
+D 2021-09-10T00:58:46.038
 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1
 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea
 F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724
@@ -545,7 +545,7 @@ F src/random.c 097dc8b31b8fba5a9aca1697aeb9fd82078ec91be734c16bffda620ced7ab83c
 F src/resolve.c 42b94d37a54200707a95566eff4f7e8a380e32d080016b699f23bd79a73a5028
 F src/rowset.c ba9515a922af32abe1f7d39406b9d35730ed65efab9443dc5702693b60854c92
 F src/select.c b2c48dfc02b486fd2da2be1605503615958ba1997d40d994c2946975d0150a31
-F src/shell.c.in af6c3a4942c17a5e28cd47cc111d2f6d6946388c1ff0f5a402208e3f28b666bf
+F src/shell.c.in a9db850f1254f281a59042d180d2c6c8f3208cc8455e140600c3d95481c6c168
 F src/sqlite.h.in 4e977a5e2ed1a9e8987ff65a2cab5f99a4298ebf040ea5ff636e1753339ff45a
 F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8
 F src/sqlite3ext.h e97f4e9b509408fea4c4e9bef5a41608dfac343b4d3c7a990dedde1e19af9510
@@ -1922,7 +1922,10 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P 3ebfe7128a20b270de65ebf4620f62e34ea6cc46b472cc52aed96af504eb9637
-R 18891cf2bf7f46de337f5e14da9a6bc6
-U drh
-Z 418794be93dd5023a367639347086d0f
+P 6844ede29e1dac93a392dfb1e7e676bb9d0a2e7bbec0a4a5804ffc2025a99b66
+R 5477d1fa459b11493c202e9f0cfe1148
+T *branch * speedy_cli
+T *sym-speedy_cli *
+T -sym-trunk *
+U larrybr
+Z b7d7b6be2bca4895f81d545876286c4b
diff --git a/manifest.uuid b/manifest.uuid
index 377fbd2779..13f0ec0131 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-6844ede29e1dac93a392dfb1e7e676bb9d0a2e7bbec0a4a5804ffc2025a99b66
\ No newline at end of file
+c4568f9f458bad16bd4481f1f6f5e7b6f39f5ccf23f2b05890078577ce2ab9a2
\ No newline at end of file
diff --git a/src/shell.c.in b/src/shell.c.in
index 438119c08f..dd261613d5 100644
--- a/src/shell.c.in
+++ b/src/shell.c.in
@@ -10598,38 +10598,109 @@ meta_command_exit:
   return rc;
 }
 
-/*
-** Return TRUE if a semicolon occurs anywhere in the first N characters
-** of string z[].
-*/
-static int line_contains_semicolon(const char *z, int N){
-  int i;
-  for(i=0; i<N; i++){  if( z[i]==';' ) return 1; }
-  return 0;
-}
-
-/*
-** Test to see if a line consists entirely of whitespace.
-*/
-static int _all_whitespace(const char *z){
-  for(; *z; z++){
-    if( IsSpace(z[0]) ) continue;
-    if( *z=='/' && z[1]=='*' ){
-      z += 2;
-      while( *z && (*z!='*' || z[1]!='/') ){ z++; }
-      if( *z==0 ) return 0;
-      z++;
-      continue;
+/* Line scan result and intermediate states (supporting scan resumption)
+*/
+typedef enum {
+  QSS_InPlain = 0, QSS_InString, QSS_InDquote,
+  QSS_InBlockComment, QSS_EndingSemi,
+  QSS_NoDark = 0,  QSS_HasDark = 1<<3, 
+  QSS_ScanMask = 0x7, QSS_DarkMask = 1<<3
+} QuickScanState;
+#define QSS_STATE(qss) ((qss) & QSS_ScanMask)
+#define QSS_SETV(qss, newst) (newst | ((qss) & QSS_DarkMask))
+#define QSS_PLAINWHITE(qss) ((qss)==QSS_NoDark)
+#define QSS_PLAINDARK(qss) ((qss)==QSS_HasDark)
+
+/*
+** Scan line for classification to guide shell's handling.
+** The scan is resumable for subsequent lines when prior
+** return values are passed as the 2nd argument.
+*/
+static QuickScanState quickscan(char *zLine, QuickScanState qss){
+  char cin;
+  switch( QSS_STATE(qss) ){
+  case QSS_InPlain:
+  InPlainSet:
+    qss = QSS_SETV(qss, QSS_InPlain);
+  InPlain:
+    while (cin = *zLine++){
+      if( IsSpace(cin) )
+        continue;
+      switch (cin){
+      case '-':
+        if( *zLine=='-' ){
+          while(*++zLine!=0)
+            ;
+          return qss;
+        }
+        break;
+      case ';':
+        qss = QSS_SETV(qss, QSS_EndingSemi);
+        goto EndingSemi;
+      case '/':
+        if( *zLine=='*' ){
+          ++zLine;
+          qss = QSS_SETV(qss, QSS_InBlockComment);
+          goto InBlockComment;
+        }
+        break;
+      case '\'':
+        qss = QSS_SETV(qss, QSS_InString);
+        goto InString;
+      case '"':
+        qss = QSS_SETV(qss, QSS_InDquote);
+        goto InDquote;
+      default:
+        break;
+      }
+      qss |= QSS_HasDark;
     }
-    if( *z=='-' && z[1]=='-' ){
-      z += 2;
-      while( *z && *z!='\n' ){ z++; }
-      if( *z==0 ) return 1;
-      continue;
+    break;
+  case QSS_InString:
+  InString:
+    while (cin = *zLine++){
+      if( cin=='\'' ){
+        if( *zLine!='\'' ){
+          goto InPlainSet;
+        }else{
+          ++zLine;
+        }
+      }
     }
-    return 0;
+    break;
+  case QSS_InDquote:
+  InDquote:
+    while (cin = *zLine++){
+      if( cin=='"' ){
+        if( *zLine!='"' ){
+          goto InPlainSet;
+        }else{
+          ++zLine;
+        }
+      }
+    }
+    break;
+  case QSS_InBlockComment:
+  InBlockComment:
+    while (cin = *zLine++){
+      if( cin=='*' && *zLine=='/' ){
+        ++zLine;
+        goto InPlainSet;
+      }
+    }
+    break;
+  case QSS_EndingSemi:
+  EndingSemi:
+    while (cin = *zLine++){
+      if( !IsSpace(cin) && cin!=';' ){
+        --zLine;
+        goto InPlain;
+      }
+    }
+    break;
+  default: assert(0);
   }
-  return 1;
+  return qss;
 }
 
 /*
@@ -10637,16 +10708,15 @@ static int _all_whitespace(const char *z){
 ** than a semi-colon.  The SQL Server style "go" command is understood
 ** as is the Oracle "/".
 */
-static int line_is_command_terminator(const char *zLine){
+static int line_is_command_terminator(char *zLine){
   while( IsSpace(zLine[0]) ){ zLine++; };
-  if( zLine[0]=='/' && _all_whitespace(&zLine[1]) ){
-    return 1;  /* Oracle */
-  }
-  if( ToLower(zLine[0])=='g' && ToLower(zLine[1])=='o'
-         && _all_whitespace(&zLine[2]) ){
-    return 1;  /* SQL Server */
-  }
-  return 0;
+  if( zLine[0]=='/' )
+    zLine += 1; /* Oracle */
+  else if ( ToLower(zLine[0])=='g' && ToLower(zLine[1])=='o' )
+    zLine += 2; /* SQL Server */
+  else
+    return 0;
+  return quickscan(zLine,QSS_NoDark)==QSS_NoDark;
 }
 
 /*
@@ -10729,7 +10799,7 @@ static int process_input(ShellState *p){
   int rc;                   /* Error code */
   int errCnt = 0;           /* Number of errors seen */
   int startline = 0;        /* Line number for start of current input */
-  int bAllWhite = 1;        /* Accumulated line empty or all whitespace */
+  QuickScanState qss = QSS_InPlain; /* Accumulated line status (so far) */
 
   p->lineno = 0;
   while( errCnt==0 || !bail_on_error || (p->in==0 && stdin_is_interactive) ){
@@ -10745,14 +10815,17 @@ static int process_input(ShellState *p){
       seenInterrupt = 0;
     }
     p->lineno++;
-    if( _all_whitespace(zLine) ){
-      if( nSql==0 ){
-        if( ShellHasFlag(p, SHFLG_Echo) )
-          printf("%s\n", zLine);
-        continue;
-      }
-    }else{
-      bAllWhite = 0;
+    if( QSS_STATE(qss)==QSS_InPlain
+        && line_is_command_terminator(zLine)
+        && line_is_complete(zSql, nSql) ){
+      memcpy(zLine,";",2);
+    }
+    qss = quickscan(zLine, qss);
+    if( QSS_PLAINWHITE(qss) && nSql==0 ){
+      if( ShellHasFlag(p, SHFLG_Echo) )
+        printf("%s\n", zLine);
+      /* Just swallow leading whitespace */
+      continue;
     }
     if( zLine && (zLine[0]=='.' || zLine[0]=='#') && nSql==0 ){
       if( ShellHasFlag(p, SHFLG_Echo) ) printf("%s\n", zLine);
@@ -10766,9 +10839,6 @@ static int process_input(ShellState *p){
       }
       continue;
     }
-    if( line_is_command_terminator(zLine) && line_is_complete(zSql, nSql) ){
-      memcpy(zLine,";",2);
-    }
     nLine = strlen30(zLine);
     if( nSql+nLine+2>=nAlloc ){
       /* Grow buffer by half-again increments when big. */
@@ -10789,8 +10859,7 @@ static int process_input(ShellState *p){
       memcpy(zSql+nSql, zLine, nLine+1);
       nSql += nLine;
     }
-    if( nSql && line_contains_semicolon(&zSql[nSqlPrior], nSql-nSqlPrior)
-                && sqlite3_complete(zSql) ){
+    if( nSql && QSS_STATE(qss)==QSS_EndingSemi && sqlite3_complete(zSql) ){
       errCnt += runOneSqlLine(p, zSql, p->in, startline);
       nSql = 0;
       if( p->outCount ){
@@ -10800,12 +10869,12 @@ static int process_input(ShellState *p){
         clearTempFile(p);
       }
       p->bSafeMode = p->bSafeModePersist;
-    }else if( nSql && bAllWhite ){
+    }else if( nSql && QSS_PLAINWHITE(qss) ){
       if( ShellHasFlag(p, SHFLG_Echo) ) printf("%s\n", zSql);
       nSql = 0;
     }
   }
-  if( nSql && !bAllWhite ){
+  if( nSql && QSS_PLAINDARK(qss) ){
     errCnt += runOneSqlLine(p, zSql, p->in, startline);
   }
   free(zSql);