fix: Fix and optimize Redis upload script (#1081)

diff --git a/misc/upload-redis b/misc/upload-redis
index 1fed79d619a44b61a246ad345633413a2a227039..87a34b3615ac412cac4ff72182885a6077e9c039 100755 (executable)
--- a/misc/upload-redis
+++ b/misc/upload-redis
@@ -34,6 +34,7 @@ for dirpath, dirnames, filenames in os.walk(ccache):
 filelist.sort()
 files = result = manifest = objects = 0
 size = 0
+batchsize = 0
 columns = os.get_terminal_size()[0]
 width = min(columns - 22, 100)
 bar = progress.bar.Bar(
@@ -57,11 +58,16 @@ for mtime, dirpath, filename, filesize in filelist:
             val = open(os.path.join(dirpath, filename), "rb").read() or None
             if val:
                 # print("%s: %s %d" % (key, ext, len(val)))
-                pipe.set(key, val)
+                pipe.setnx(key, val)
                 objects = objects + 1
         files = files + 1
         size = size + filesize
+        batchsize = batchsize + filesize
+        if batchsize > 64 * 1024 * 1024:
+            pipe.execute()
+            batchsize = 0
     bar.next()
+pipe.execute()
 bar.finish()
 print(
     "%d files, %d result (%d manifest) = %d objects (%s)"