chore: Clean up Redis helper scripts a bit (#1254)

- Support any version of manifest and result.
- Skip over tmp dir and allow avoiding setnx.

diff --git a/misc/download-redis b/misc/download-redis
index 0352b476e54c4dd1c7b90f867738aac8fb024c52..1bbebd72133099ab68dc66161fb9859a42d6c4bb 100755 (executable)
--- a/misc/download-redis
+++ b/misc/download-redis
@@ -1,7 +1,7 @@
 #!/usr/bin/env python3
 
-# This script downloads the contents of the local cache from a Redis remote
-# storage.
+# This script downloads the contents of the local cache
+# from a Redis remote storage.
 
 import redis
 import os
@@ -53,10 +53,10 @@ for key in context.scan_iter():
         continue
     if val[0:2] == b"\xcc\xac":  # magic
         objects += 1
-        if val[2] == 0 and val[3] == 0:
+        if val[3] == 0:
             ext = "R"
             result += 1
-        elif val[2] == 0 and val[3] == 1:
+        elif val[3] == 1:
             ext = "M"
             manifest += 1
         else:

diff --git a/misc/upload-redis b/misc/upload-redis
index b3595c5afc2b823982ac4130a9f4f8f42386657a..a1f9cb08544060a2724ee73262f3e5c73a73d285 100755 (executable)
--- a/misc/upload-redis
+++ b/misc/upload-redis
@@ -1,6 +1,7 @@
 #!/usr/bin/env python3
 
-# This script uploads the contents of the local cache to a Redis remote storage.
+# This script uploads the contents of the local cache
+# to a Redis remote storage.
 
 import redis
 import os
@@ -16,20 +17,24 @@ elif config.startswith("/"):
     host, port, sock = None, None, config
 else:
     host, port, sock = config, 6379, None
+
 username = os.getenv("REDIS_USERNAME")
 password = os.getenv("REDIS_PASSWORD")
 context = redis.Redis(host=host, port=port, unix_socket_path=sock, password=password)
 pipe = context.pipeline(transaction=False)
 
+use_setnx = True
+
 ccache = os.getenv("CCACHE_DIR", os.path.expanduser("~/.cache/ccache"))
 filelist = []
-for dirpath, dirnames, filenames in os.walk(ccache):
+for dirpath, dirnames, filenames in os.walk(ccache, topdown=True):
     # sort by modification time, most recently used last
     for filename in filenames:
         if filename.endswith(".lock"):
             continue
         stat = os.stat(os.path.join(dirpath, filename))
         filelist.append((stat.st_mtime, dirpath, filename, stat.st_size))
+    dirnames[:] = [d for d in dirnames if d != "tmp"]
 filelist.sort()
 
 files = result = manifest = objects = 0
@@ -42,12 +47,7 @@ bar = progress.bar.Bar(
 )
 for mtime, dirpath, filename, filesize in filelist:
     dirname = dirpath.replace(ccache + os.path.sep, "")
-    if dirname == "tmp":
-        continue
-    elif filename == "CACHEDIR.TAG" or filename == "stats":
-        # ignore these
-        files += 1
-    else:
+    if filename != "ccache.conf" and filename != "CACHEDIR.TAG" and filename != "stats":
         (base, ext) = filename[:-1], filename[-1:]
         if ext == "R" or ext == "M":
             if ext == "R":
@@ -58,7 +58,10 @@ for mtime, dirpath, filename, filesize in filelist:
             val = open(os.path.join(dirpath, filename), "rb").read() or None
             if val:
                 # print("%s: %s %d" % (key, ext, len(val)))
-                pipe.setnx(key, val)
+                if use_setnx:
+                    pipe.setnx(key, val)
+                else:
+                    pipe.set(key, val)
                 objects += 1
         files += 1
         size += filesize