From 2f64fe6858c09905ec2ccf0266d811d7c414d3ab Mon Sep 17 00:00:00 2001 From: Michael Tremer Date: Sat, 21 Oct 2017 18:32:17 +0100 Subject: [PATCH] Refactor uploads Signed-off-by: Michael Tremer --- src/buildservice/uploads.py | 90 +++++++++++++++++++++---------------- src/hub/handlers.py | 16 ++++--- src/web/handlers.py | 4 +- 3 files changed, 63 insertions(+), 47 deletions(-) diff --git a/src/buildservice/uploads.py b/src/buildservice/uploads.py index 6e48dc8..2e1331a 100644 --- a/src/buildservice/uploads.py +++ b/src/buildservice/uploads.py @@ -16,46 +16,40 @@ from . import misc from . import packages from .constants import * +from .decorators import * class Uploads(base.Object): - def get_by_uuid(self, _uuid): - upload = self.db.get("SELECT id FROM uploads WHERE uuid = %s", _uuid) - - return Upload(self.pakfire, upload.id) + def _get_upload(self, query, *args): + res = self.db.get(query, *args) - def get_all(self): - uploads = self.db.query("SELECT id FROM uploads ORDER BY time_started DESC") + if res: + return Upload(self.backend, res.id, data=res) - return [Upload(self.pakfire, u.id) for u in uploads] + def _get_uploads(self, query, *args): + res = self.db.query(query, *args) - def cleanup(self): - for upload in self.get_all(): - upload.cleanup() + for row in res: + yield Upload(self.backend, row.id, data=row) + def __iter__(self): + uploads = self._get_uploads("SELECT * FROM uploads ORDER BY time_started DESC") -class Upload(base.Object): - def __init__(self, pakfire, id): - base.Object.__init__(self, pakfire) + return iter(sorted(uploads)) - self.id = id - self.data = self.db.get("SELECT * FROM uploads WHERE id = %s", self.id) + def get_by_uuid(self, _uuid): + return self._get_upload("SELECT * FROM uploads WHERE uuid = %s", uuid) - @classmethod - def create(cls, pakfire, filename, size, hash, builder=None, user=None): + def create(filename, size, hash, builder=None, user=None): assert builder or user - id = pakfire.db.execute("INSERT INTO uploads(uuid, filename, size, hash) \ - VALUES(%s, %s, %s, %s)", "%s" % uuid.uuid4(), filename, size, hash) + upload = self._get_upload("INSERT INTO uploads(uuid, filename, size, hash) \ + VALUES(%s, %s, %s, %s) RETURNING *", "%s" % uuid.uuid4(), filename, size, hash) if builder: - pakfire.db.execute("UPDATE uploads SET builder_id = %s WHERE id = %s", - builder.id, id) + upload.builder = builder elif user: - pakfire.db.execute("UPDATE uploads SET user_id = %s WHERE id = %s", - user.id, id) - - upload = cls(pakfire, id) + upload.user = user # Create space to where we save the data. dirname = os.path.dirname(upload.path) @@ -68,6 +62,14 @@ class Upload(base.Object): return upload + def cleanup(self): + for upload in self.get_all(): + upload.cleanup() + + +class Upload(base.DataObject): + table = "uploads" + @property def uuid(self): return self.data.uuid @@ -92,15 +94,29 @@ class Upload(base.Object): def progress(self): return self.data.progress / self.size - @property - def builder(self): + # Builder + + def get_builder(self): if self.data.builder_id: - return self.pakfire.builders.get_by_id(self.data.builder_id) + return self.backend.builders.get_by_id(self.data.builder_id) - @property - def user(self): + def set_builder(self, builder): + self._set_attribute("builder", builder.id) + self.builder = builder + + builder = lazy_property(get_builder, set_builder) + + # User + + def get_user(self): if self.data.user_id: - return self.pakfire.users.get_by_id(self.data.user_id) + return self.backend.users.get_by_id(self.data.user_id) + + def set_user(self, user): + self._set_attribute("user", user.id) + self.user = user + + user = lazy_property(get_user, set_user) def append(self, data): # Check if the filesize was exceeded. @@ -110,12 +126,10 @@ class Upload(base.Object): logging.debug("Writing %s bytes to %s" % (len(data), self.path)) - f = open(self.path, "ab") - f.write(data) - f.close() + with open(self.path, "ab") as f: + f.write(data) - self.db.execute("UPDATE uploads SET progress = %s WHERE id = %s", - size, self.id) + self._set_attribute("progress", size) def validate(self): size = os.path.getsize(self.path) @@ -141,8 +155,8 @@ class Upload(base.Object): if not self.validate(): return False - self.db.execute("UPDATE uploads SET finished = 'Y', time_finished = NOW() \ - WHERE id = %s", self.id) + self._set_attribute("finished", True) + self._set_attribute("time_finished", datetime.datetime.utcnow()) return True diff --git a/src/hub/handlers.py b/src/hub/handlers.py index c51d9ad..37fb60e 100644 --- a/src/hub/handlers.py +++ b/src/hub/handlers.py @@ -151,10 +151,11 @@ class UploadsCreateHandler(BaseHandler): filesize = self.get_argument_int("filesize") filehash = self.get_argument("hash") - upload = uploads.Upload.create(self.backend, filename, filesize, - filehash, user=self.user, builder=self.builder) + with self.db.transaction(): + upload = self.backend.uploads.create(filename, filesize, + filehash, user=self.user, builder=self.builder) - self.finish(upload.uuid) + self.finish(upload.uuid) class UploadsSendChunkHandler(BaseHandler): @@ -181,7 +182,8 @@ class UploadsSendChunkHandler(BaseHandler): raise tornado.web.HTTPError(400, "Checksum mismatch") # Append the data to file. - upload.append(data) + with self.db.transaction(): + upload.append(data) class UploadsFinishedHandler(BaseHandler): @@ -207,7 +209,8 @@ class UploadsFinishedHandler(BaseHandler): # In case the download was corrupted or incomplete, we delete it # and tell the client to start over. - upload.remove() + with self.db.transaction(): + upload.remove() self.finish("ERROR: CORRUPTED OR INCOMPLETE FILE") @@ -223,7 +226,8 @@ class UploadsDestroyHandler(BaseHandler): raise tornado.web.HTTPError(403, "Removing an other host's file.") # Remove the upload from the database and trash the data. - upload.remove() + with self.db.transaction(): + upload.remove() # Builds diff --git a/src/web/handlers.py b/src/web/handlers.py index e83a82e..ae103a4 100644 --- a/src/web/handlers.py +++ b/src/web/handlers.py @@ -62,9 +62,7 @@ class UploadsHandler(BaseHandler): if not self.current_user.is_admin(): raise tornado.web.HTTPError(403) - uploads = self.pakfire.uploads.get_all() - - self.render("uploads-list.html", uploads=uploads) + self.render("uploads-list.html", uploads=self.backend.uploads) class DocsIndexHandler(BaseHandler): -- 2.39.2