From: drh <drh@noemail.net>
Date: Fri, 9 Feb 2018 23:25:14 +0000 (+0000)
Subject: Improve the performance of the built-in REPLACE() function in cases where
X-Git-Tag: version-3.23.0~141
X-Git-Url: http://git.ipfire.org/?a=commitdiff_plain;h=f313952097725eaf25f80b3431035d19bb1c5c72;p=thirdparty%2Fsqlite.git

Improve the performance of the built-in REPLACE() function in cases where
it does many substitutions that make the string larger.  OSSFuzz is reporting
intermittant timeouts when running a test where it does a REPLACE() on a
930KB random blob. Perhaps this enhancement will fix that.

FossilOrigin-Name: fab2c2b07b5d3cd851db3e6f5c8a44155e32b0df22905ea33412b153b825a928
---

diff --git a/manifest b/manifest
index 6cda78885d..e41ea214cc 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Add\sthe\szorder.c\sextension\simplementing\szorder()\sand\sunzorder()\sSQL\sfunctions.
-D 2018-02-09T20:49:15.905
+C Improve\sthe\sperformance\sof\sthe\sbuilt-in\sREPLACE()\sfunction\sin\scases\swhere\nit\sdoes\smany\ssubstitutions\sthat\smake\sthe\sstring\slarger.\s\sOSSFuzz\sis\sreporting\nintermittant\stimeouts\swhen\srunning\sa\stest\swhere\sit\sdoes\sa\sREPLACE()\son\sa\s\n930KB\srandom\sblob.\sPerhaps\sthis\senhancement\swill\sfix\sthat.
+D 2018-02-09T23:25:14.764
 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1
 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea
 F Makefile.in 7a3f714b4fcf793108042b7b0a5c720b0b310ec84314d61ba7f3f49f27e550ea
@@ -444,7 +444,7 @@ F src/delete.c 20c8788451dc737a967c87ea53ad43544d617f5b57d32ccce8bd52a0daf9e89b
 F src/expr.c 9e06de431c09f144438aa6895ea4d4290fa3c6875bfcc3ba331012ca78deadf0
 F src/fault.c 460f3e55994363812d9d60844b2a6de88826e007
 F src/fkey.c d617daf66b5515e2b42c1405b2b4984c30ca50fb705ab164271a9bf66c69e331
-F src/func.c 9970db37cc004136996a5c9b966b86f06583bcf3f275449b977fbb06d75e7300
+F src/func.c 385b9b01851f55c6547f2592ac378572298eb979b02516387ec6f21379d85507
 F src/global.c ac3094f1dc59fbeb919aef7cc0cc827a8459d1fb1adb7972ef75bd9e0c10b75b
 F src/hash.c a12580e143f10301ed5166ea4964ae2853d3905a511d4e0c44497245c7ce1f7a
 F src/hash.h ab34c5c54a9e9de2e790b24349ba5aab3dbb4fd4
@@ -922,7 +922,7 @@ F test/fts4onepass.test 7319d61a2ed1325fc54afd0c060a0513b462303a
 F test/fts4opt.test fd6a11684b965e1999564ae763797b7fb9e34c96
 F test/fts4unicode.test ceca76422abc251818cb25dabe33d3c3970da5f7c90e1540f190824e6b3a7c95
 F test/full.test 6b3c8fb43c6beab6b95438c1675374b95fab245d
-F test/func.test ae97561957aba6ca9e3a7b8a13aac41830d701ef
+F test/func.test 09dda479bcfc568f99f3070413e9672a8eeedc1be9c5d819bf55d4788c2583b7
 F test/func2.test 772d66227e4e6684b86053302e2d74a2500e1e0f
 F test/func3.test d202a7606d23f90988a664e88e268aed1087c11c
 F test/func4.test 6beacdfcb0e18c358e6c2dcacf1b65d1fa80955f
@@ -1705,7 +1705,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P b685d3231097fb90e7d61d9ac01cc560e8bf2671d49390ae7af5bfdbd6d04f11
-R 574de61b6452baa1e71eeeef413a6e94
+P a57a77dc0cc9fbaa9d5b134422f7a8cc8d4c2851ed3c2bdd449800c6a5d2aae0
+R adb83e680df996ddba44e43de1a05c6b
 U drh
-Z 5935918b7bd1bd1e30b211bf68154f29
+Z 6ead8882aba40c327286a8b049f64ee5
diff --git a/manifest.uuid b/manifest.uuid
index c0133280c0..fcc9af0088 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-a57a77dc0cc9fbaa9d5b134422f7a8cc8d4c2851ed3c2bdd449800c6a5d2aae0
\ No newline at end of file
+fab2c2b07b5d3cd851db3e6f5c8a44155e32b0df22905ea33412b153b825a928
\ No newline at end of file
diff --git a/src/func.c b/src/func.c
index 4e80535e81..399f6dba35 100644
--- a/src/func.c
+++ b/src/func.c
@@ -1199,6 +1199,8 @@ static void replaceFunc(
   i64 nOut;                /* Maximum size of zOut */
   int loopLimit;           /* Last zStr[] that might match zPattern[] */
   int i, j;                /* Loop counters */
+  unsigned cntExpand;      /* Number zOut expansions */
+  sqlite3 *db = sqlite3_context_db_handle(context);
 
   assert( argc==3 );
   UNUSED_PARAMETER(argc);
@@ -1230,33 +1232,40 @@ static void replaceFunc(
     return;
   }
   loopLimit = nStr - nPattern;  
+  cntExpand = 0;
   for(i=j=0; i<=loopLimit; i++){
     if( zStr[i]!=zPattern[0] || memcmp(&zStr[i], zPattern, nPattern) ){
       zOut[j++] = zStr[i];
     }else{
-      u8 *zOld;
-      sqlite3 *db = sqlite3_context_db_handle(context);
-      nOut += nRep - nPattern;
-      testcase( nOut-1==db->aLimit[SQLITE_LIMIT_LENGTH] );
-      testcase( nOut-2==db->aLimit[SQLITE_LIMIT_LENGTH] );
-      if( nOut-1>db->aLimit[SQLITE_LIMIT_LENGTH] ){
-        sqlite3_result_error_toobig(context);
-        sqlite3_free(zOut);
-        return;
-      }
-      zOld = zOut;
-      zOut = sqlite3_realloc64(zOut, (int)nOut);
-      if( zOut==0 ){
-        sqlite3_result_error_nomem(context);
-        sqlite3_free(zOld);
-        return;
+      if( nRep>nPattern ){
+        nOut += nRep - nPattern;
+        if( nOut-1>db->aLimit[SQLITE_LIMIT_LENGTH] ){
+          sqlite3_result_error_toobig(context);
+          sqlite3_free(zOut);
+          return;
+        }
+        testcase( nOut-1==db->aLimit[SQLITE_LIMIT_LENGTH] );
+        testcase( nOut-2==db->aLimit[SQLITE_LIMIT_LENGTH] );
+        cntExpand++;
+        if( (cntExpand&(cntExpand-1))==0 ){
+          /* Grow the size of the output buffer only on substitutions
+          ** whose index is a power of two: 1, 2, 4, 8, 16, 32, ... */
+          u8 *zOld;
+          zOld = zOut;
+          zOut = sqlite3_realloc64(zOut, (int)nOut + (nOut - nStr - 1));
+          if( zOut==0 ){
+            sqlite3_result_error_nomem(context);
+            sqlite3_free(zOld);
+            return;
+          }
+        }
       }
       memcpy(&zOut[j], zRep, nRep);
       j += nRep;
       i += nPattern-1;
     }
   }
-  assert( j+nStr-i+1==nOut );
+  assert( j+nStr-i+1<=nOut );
   memcpy(&zOut[j], &zStr[i], nStr-i);
   j += nStr - i;
   assert( j<=nOut );
diff --git a/test/func.test b/test/func.test
index 98ae8ddeb5..23a3ae4392 100644
--- a/test/func.test
+++ b/test/func.test
@@ -507,6 +507,17 @@ if {$encoding=="UTF-16le"} {
     execsql {SELECT hex(replace('aabcdefg','a','aaa'))}
   } {616161616161626364656667}
 }
+do_execsql_test func-9.14 {
+  WITH RECURSIVE c(x) AS (
+     VALUES(1)
+     UNION ALL
+     SELECT x+1 FROM c WHERE x<1040
+  )
+  SELECT 
+    count(*),
+    sum(length(replace(printf('abc%.*cxyz',x,'m'),'m','nnnn'))-(6+x*4))
+  FROM c;
+} {1040 0}
   
 # Use the "sqlite_register_test_function" TCL command which is part of
 # the text fixture in order to verify correct operation of some of