From: Anders Björklund <anders.f.bjorklund@gmail.com>
Date: Mon, 23 May 2022 19:40:32 +0000 (+0200)
Subject: fix: Fix and optimize Redis upload script (#1081)
X-Git-Tag: v4.7~221
X-Git-Url: http://git.ipfire.org/gitweb.cgi?a=commitdiff_plain;h=d09df3b94a1e32a1b4993d1b3a7a1f081726208a;p=thirdparty%2Fccache.git

fix: Fix and optimize Redis upload script (#1081)
---

diff --git a/misc/upload-redis b/misc/upload-redis
index 1fed79d61..87a34b361 100755
--- a/misc/upload-redis
+++ b/misc/upload-redis
@@ -34,6 +34,7 @@ for dirpath, dirnames, filenames in os.walk(ccache):
 filelist.sort()
 files = result = manifest = objects = 0
 size = 0
+batchsize = 0
 columns = os.get_terminal_size()[0]
 width = min(columns - 22, 100)
 bar = progress.bar.Bar(
@@ -57,11 +58,16 @@ for mtime, dirpath, filename, filesize in filelist:
             val = open(os.path.join(dirpath, filename), "rb").read() or None
             if val:
                 # print("%s: %s %d" % (key, ext, len(val)))
-                pipe.set(key, val)
+                pipe.setnx(key, val)
                 objects = objects + 1
         files = files + 1
         size = size + filesize
+        batchsize = batchsize + filesize
+        if batchsize > 64 * 1024 * 1024:
+            pipe.execute()
+            batchsize = 0
     bar.next()
+pipe.execute()
 bar.finish()
 print(
     "%d files, %d result (%d manifest) = %d objects (%s)"