download: Improve cache handling

This adds a new --force-cache parameter which will force use of the
cache even for expired images.

An expired image is now only flushed from the cache once a new one is
successfuly downloaded (to avoid destroying the local cache when the
host doesn't have internet connectivity).

The ID of the build in cache is also tracked so that we don't
re-download something we already have (should only happen if we don't
have a new build published by the time the previous one expires).

Signed-off-by: Stéphane Graber <stgraber@ubuntu.com>
Acked-by: Serge E. Hallyn <serge.hallyn@ubuntu.com>

diff --git a/templates/lxc-download.in b/templates/lxc-download.in
index 70a23896d65c9eb56d8a1c37a932a31acc92b6cf..c3180411cc026a7234d758da51e416d5267c7069 100644 (file)
--- a/templates/lxc-download.in
+++ b/templates/lxc-download.in
@@ -42,6 +42,7 @@ DOWNLOAD_SHOW_HTTP_WARNING="true"
 DOWNLOAD_SHOW_GPG_WARNING="true"
 DOWNLOAD_COMPAT_LEVEL=1
 DOWNLOAD_LIST_IMAGES="false"
+DOWNLOAD_BUILD=
 
 LXC_NAME=
 LXC_PATH=
@@ -159,6 +160,7 @@ Optional arguments:
 [ --keyserver <keyserver> ]: GPG keyserver to use
 [ --no-validate ]: Disable GPG validation (not recommended)
 [ --flush-cache ]: Flush the local copy (if present)
+[ --force-cache ]; Force the use of the local copy even if expired
 
 LXC internal arguments (do not pass manually!):
 [ --name <name> ]: The container name
@@ -170,7 +172,8 @@ EOF
 }
 
 options=$(getopt -o d:r:a:hl -l dist:,release:,arch:,help,list,variant:,\
-server:,keyid:,no-validate,flush-cache,name:,path:,rootfs:,mapped-uid: -- "$@")
+server:,keyid:,no-validate,flush-cache,force-cache:,name:,path:,\
+rootfs:,mapped-uid: -- "$@")
 
 if [ $? -ne 0 ]; then
     usage
@@ -190,6 +193,7 @@ while :; do
         --keyid)            DOWNLOAD_KEYID=$2; shift 2;;
         --no-validate)      DOWNLOAD_VALIDATE="false"; shift 1;;
         --flush-cache)      DOWNLOAD_FLUSH_CACHE="true"; shift 1;;
+        --force-cache)      DOWNLOAD_FORCE_CACHE="true"; shift 1;;
         --name)             LXC_NAME=$2; shift 2;;
         --path)             LXC_PATH=$2; shift 2;;
         --rootfs)           LXC_ROOTFS=$2; shift 2;;
@@ -300,13 +304,14 @@ if [ -d "$LXC_CACHE_PATH" ]; then
     if [ "$DOWNLOAD_FLUSH_CACHE" = "true" ]; then
         echo "Flushing the cache..."
         rm -Rf $LXC_CACHE_PATH
+    elif [ "$DOWNLOAD_FORCE_CACHE" = "true" ]; then
+        DOWNLOAD_USE_CACHE="true"
     else
         DOWNLOAD_USE_CACHE="true"
         if [ -e "$(relevant_file expiry)" ]; then
             if [ "$(cat $(relevant_file expiry))" -lt $(date +%s) ]; then
                 echo "The cached copy has expired, re-downloading..."
                 DOWNLOAD_USE_CACHE="false"
-                rm -Rf $LXC_CACHE_PATH
             fi
         fi
     fi
@@ -348,6 +353,7 @@ if [ "$DOWNLOAD_USE_CACHE" = "false" ]; then
             continue
         fi
 
+        DOWNLOAD_BUILD=$5
         DOWNLOAD_URL=$6
         break
     done < ${DOWNLOAD_TEMP}/index
@@ -357,32 +363,43 @@ if [ "$DOWNLOAD_USE_CACHE" = "false" ]; then
         exit 1
     fi
 
-    # Download the actual files
-    echo "Downloading the rootfs"
-    download_file $DOWNLOAD_URL/rootfs.tar.xz \
-        ${DOWNLOAD_TEMP}/rootfs.tar.xz normal
-    download_sig  $DOWNLOAD_URL/rootfs.tar.xz.asc \
-         ${DOWNLOAD_TEMP}/rootfs.tar.xz.asc normal
-    gpg_validate ${DOWNLOAD_TEMP}/rootfs.tar.xz.asc
-
-    echo "Downloading the metadata"
-    download_file $DOWNLOAD_URL/meta.tar.xz \
-        ${DOWNLOAD_TEMP}/meta.tar.xz normal
-    download_sig  $DOWNLOAD_URL/meta.tar.xz.asc \
-        ${DOWNLOAD_TEMP}/meta.tar.xz.asc normal
-    gpg_validate ${DOWNLOAD_TEMP}/meta.tar.xz.asc
-
-    mkdir -p $LXC_CACHE_PATH
-    mv ${DOWNLOAD_TEMP}/rootfs.tar.xz $LXC_CACHE_PATH
-    if ! tar Jxf ${DOWNLOAD_TEMP}/meta.tar.xz -C $LXC_CACHE_PATH; then
-        echo "ERROR: Invalid rootfs tarball." 2>&1
-        exit 1
-    fi
+    if [ -d "$LXC_CACHE_PATH" ] && [ -f "$LXC_CACHE_PATH/build_id" ] && \
+       [ "$(cat $LXC_CACHE_PATH/build_id)" = "$DOWNLOAD_BUILD" ]; then
+        echo "The cache is already up to date."
+        echo "Using image from local cache"
+    else
+        # Download the actual files
+        echo "Downloading the rootfs"
+        download_file $DOWNLOAD_URL/rootfs.tar.xz \
+            ${DOWNLOAD_TEMP}/rootfs.tar.xz normal
+        download_sig  $DOWNLOAD_URL/rootfs.tar.xz.asc \
+             ${DOWNLOAD_TEMP}/rootfs.tar.xz.asc normal
+        gpg_validate ${DOWNLOAD_TEMP}/rootfs.tar.xz.asc
+
+        echo "Downloading the metadata"
+        download_file $DOWNLOAD_URL/meta.tar.xz \
+            ${DOWNLOAD_TEMP}/meta.tar.xz normal
+        download_sig  $DOWNLOAD_URL/meta.tar.xz.asc \
+            ${DOWNLOAD_TEMP}/meta.tar.xz.asc normal
+        gpg_validate ${DOWNLOAD_TEMP}/meta.tar.xz.asc
+
+        if [ -d $LXC_CACHE_PATH ]; then
+            rm -Rf $LXC_CACHE_PATH
+        fi
+        mkdir -p $LXC_CACHE_PATH
+        mv ${DOWNLOAD_TEMP}/rootfs.tar.xz $LXC_CACHE_PATH
+        if ! tar Jxf ${DOWNLOAD_TEMP}/meta.tar.xz -C $LXC_CACHE_PATH; then
+            echo "ERROR: Invalid rootfs tarball." 2>&1
+            exit 1
+        fi
 
-    if [ -n "$LXC_MAPPED_UID" ] && [ "$LXC_MAPPED_UID" != "-1" ]; then
-        chown $LXC_MAPPED_UID -Rf $LXC_CACHE_BASE >/dev/null 2>&1 || true
+        echo $DOWNLOAD_BUILD > $LXC_CACHE_PATH/build_id
+
+        if [ -n "$LXC_MAPPED_UID" ] && [ "$LXC_MAPPED_UID" != "-1" ]; then
+            chown $LXC_MAPPED_UID -Rf $LXC_CACHE_BASE >/dev/null 2>&1 || true
+        fi
+        echo "The image cache is now ready"
     fi
-    echo "The image cache is now ready"
 else
     echo "Using image from local cache"
 fi