From: Michael Tremer <michael.tremer@ipfire.org>
Date: Sun, 11 Dec 2016 22:04:26 +0000 (+0100)
Subject: http: Download to a temporary file
X-Git-Tag: 0.9.28~1285^2~1371
X-Git-Url: http://git.ipfire.org/cgi-bin/gitweb.cgi?a=commitdiff_plain;h=6ea6183574744fd880b23bc4ee5c80f65e003ed4;p=pakfire.git

http: Download to a temporary file

This patch makes the HTTP client download data
into memory first and writes it to a temporary
file after 10M of data was downloaded.

This makes downloads faster and reduces IO load
but does not give us any limitation in case bigger
files are downloaded.

We get rid of some file locking which is always
difficult to entertain.

Signed-off-by: Michael Tremer <michael.tremer@ipfire.org>
---

diff --git a/src/pakfire/http.py b/src/pakfire/http.py
index 9c25016be..ec9aa3d7a 100644
--- a/src/pakfire/http.py
+++ b/src/pakfire/http.py
@@ -20,10 +20,11 @@
 ###############################################################################
 
 import base64
-import fcntl
 import json
 import logging
+import shutil
 import ssl
+import tempfile
 import time
 import urllib.parse
 import urllib.request
@@ -38,6 +39,9 @@ from . import errors
 log = logging.getLogger("pakfire.http")
 log.propagate = 1
 
+# Maximum size of temporary files that is being kept in memory
+TMP_MAX_SIZE = 10485760 # 10M
+
 class Client(object):
 	"""
 		Implements a basic HTTP client which is used to download
@@ -287,15 +291,14 @@ class Client(object):
 		if self.mirrors and not self.mirror:
 			self._next_mirror()
 
+		# Create a temporary file where the downloaded data is stored
+		# This is a spooled file which will be kept in memory and only
+		# be written to disk when max_size is exceeded.
+		f = tempfile.SpooledTemporaryFile(max_size=TMP_MAX_SIZE)
+
 		try:
 			with self._make_progressbar(message) as p:
-				with open(filename, "wb") as f:
-					# Exclusively lock the file for download
-					try:
-						fcntl.flock(f, fcntl.LOCK_EX)
-					except OSError as e:
-						raise DownloadError(_("Could not lock target file")) from e
-
+				with f:
 					while True:
 						# Prepare HTTP request
 						r = self._make_request(url, mirror=self.mirror, **kwargs)
@@ -342,6 +345,10 @@ class Client(object):
 							# Otherwise raise this error
 							raise e
 
+						# Downloaded succeeded, writing data to filesystem
+						with open(filename, "wb") as output:
+							shutil.copyobj(f, output)
+
 		finally:
 			# Re-add any skipped mirrors again so that the next
 			# request will be tried on all mirrors, too.