From 329631dee9f0adfb512f48aac61bbfdfe8f09a7a Mon Sep 17 00:00:00 2001 From: Justin Luth Date: Sat, 30 Dec 2017 22:12:01 +0300 Subject: [PATCH] Fix bug 11558 updxlrator: use mirror mode for SHA1, filenames Most Microsoft updates now contain an SHA1 hash in the filename. Since these files are uniquely identifiable, use mirror mode (which creates a hash of just the filename instead of the entire URL) to cache them. (But first check the URL cache to see if it has been downloaded as a URL already.) This is a HUGELY needed fix. Windows 10 updates are 5+ GB per month, and we lose several days of bandwidth downloading duplicates from different mirrors. Sometimes a single client will request the same patch from multiple mirrors. That's bad. This patch will save a ton of bandwidth, and lots of disk space. The patch limits the SHA1 test to microsoft only, but it could be easily expanded to other vendors if there is a need. Signed-off-by: Justin Luth Signed-off-by: Michael Tremer --- config/updxlrator/updxlrator | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/config/updxlrator/updxlrator b/config/updxlrator/updxlrator index b728902f6d..bc98786385 100644 --- a/config/updxlrator/updxlrator +++ b/config/updxlrator/updxlrator @@ -86,6 +86,8 @@ while (<>) { && ($source_url !~ m@\&@) ) { + # NOTE: check_cache will change to $mirror instead of $unique if the filename contains an SHA1 hash + # and the URL is not found in cache! $xlrator_url = &check_cache($source_url,$hostaddr,$username,"Microsoft",$unique); } @@ -400,6 +402,17 @@ sub check_cache &debuglog("Retrieving file from cache ($updsource)"); &setcachestatus("$updcachedir/$vendorid/$uuid/access.log",time); $cacheurl="http://$netsettings{'GREEN_ADDRESS'}:$http_port/updatecache/$vendorid/$uuid/$updfile"; + } + elsif ( + ($cfmirror == $unique) && + ($vendorid == "microsoft") && + ($source_url =~ m@.*[0-9a-f]{40}\.[^\.]+@i) + ) + { + # Most Microsoft updates now have an SHA1 hash in the name. These should be treated as unique files. + # Since it wasn't found in the URL cache, switch to mirror mode and try again using just the filename. + &debuglog("SHA1: $vendorid $uuid not cached. Reprocessing as mirror $sourceurl"); + $cacheurl = &check_cache($source_url,$hostaddr,$username,$vendorid,$mirror); } else { -- 2.39.2