src/buildservice/uploads.py

   1 #!/usr/bin/python
   2
   3 from __future__ import division
   4
   5 import datetime
   6 import hashlib
   7 import logging
   8 import os
   9 import shutil
  10 import uuid
  11
  12 import pakfire.packages
  13
  14 from . import base
  15 from . import misc
  16 from . import packages
  17
  18 from .constants import *
  19 from .decorators import *
  20
  21 class Uploads(base.Object):
  22         def _get_upload(self, query, *args):
  23                 res = self.db.get(query, *args)
  24
  25                 if res:
  26                         return Upload(self.backend, res.id, data=res)
  27
  28         def _get_uploads(self, query, *args):
  29                 res = self.db.query(query, *args)
  30
  31                 for row in res:
  32                         yield Upload(self.backend, row.id, data=row)
  33
  34         def __iter__(self):
  35                 uploads = self._get_uploads("SELECT * FROM uploads ORDER BY time_started DESC")
  36
  37                 return iter(sorted(uploads))
  38
  39         def get_by_uuid(self, _uuid):
  40                 return self._get_upload("SELECT * FROM uploads WHERE uuid = %s", uuid)
  41
  42         def create(filename, size, hash, builder=None, user=None):
  43                 assert builder or user
  44
  45                 upload = self._get_upload("INSERT INTO uploads(uuid, filename, size, hash) \
  46                         VALUES(%s, %s, %s, %s) RETURNING *", "%s" % uuid.uuid4(), filename, size, hash)
  47
  48                 if builder:
  49                         upload.builder = builder
  50
  51                 elif user:
  52                         upload.user = user
  53
  54                 # Create space to where we save the data.
  55                 dirname = os.path.dirname(upload.path)
  56                 if not os.path.exists(dirname):
  57                         os.makedirs(dirname)
  58
  59                 # Create empty file.
  60                 f = open(upload.path, "w")
  61                 f.close()
  62
  63                 return upload
  64
  65         def cleanup(self):
  66                 for upload in self.get_all():
  67                         upload.cleanup()
  68
  69
  70 class Upload(base.DataObject):
  71         table = "uploads"
  72
  73         @property
  74         def uuid(self):
  75                 return self.data.uuid
  76
  77         @property
  78         def hash(self):
  79                 return self.data.hash
  80
  81         @property
  82         def filename(self):
  83                 return self.data.filename
  84
  85         @property
  86         def path(self):
  87                 return os.path.join(UPLOADS_DIR, self.uuid, self.filename)
  88
  89         @property
  90         def size(self):
  91                 return self.data.size
  92
  93         @property
  94         def progress(self):
  95                 return self.data.progress / self.size
  96
  97         # Builder
  98
  99         def get_builder(self):
 100                 if self.data.builder_id:
 101                         return self.backend.builders.get_by_id(self.data.builder_id)
 102
 103         def set_builder(self, builder):
 104                 self._set_attribute("builder", builder.id)
 105                 self.builder = builder
 106
 107         builder = lazy_property(get_builder, set_builder)
 108
 109         # User
 110
 111         def get_user(self):
 112                 if self.data.user_id:
 113                         return self.backend.users.get_by_id(self.data.user_id)
 114
 115         def set_user(self, user):
 116                 self._set_attribute("user", user.id)
 117                 self.user = user
 118
 119         user = lazy_property(get_user, set_user)
 120
 121         def append(self, data):
 122                 # Check if the filesize was exceeded.
 123                 size = os.path.getsize(self.path) + len(data)
 124                 if size > self.data.size:
 125                         raise Exception, "Given filesize was exceeded for upload %s" % self.uuid
 126
 127                 logging.debug("Writing %s bytes to %s" % (len(data), self.path))
 128
 129                 with open(self.path, "ab") as f:
 130                         f.write(data)
 131
 132                 self._set_attribute("progress", size)
 133
 134         def validate(self):
 135                 size = os.path.getsize(self.path)
 136                 if not size == self.data.size:
 137                         logging.error("Filesize is not okay: %s" % (self.uuid))
 138                         return False
 139
 140                 # Calculate a hash to validate the upload.
 141                 hash = misc.calc_hash1(self.path)
 142
 143                 if not self.hash == hash:
 144                         logging.error("Hash did not match: %s != %s" % (self.hash, hash))
 145                         return False
 146
 147                 return True
 148
 149         def finished(self):
 150                 """
 151                         Update the status of the upload in the database to "finished".
 152                 """
 153                 # Check if the file was completely uploaded and the hash is correct.
 154                 # If not, the upload has failed.
 155                 if not self.validate():
 156                         return False
 157
 158                 self._set_attribute("finished", True)
 159                 self._set_attribute("time_finished", datetime.datetime.utcnow())
 160
 161                 return True
 162
 163         def remove(self):
 164                 # Remove the uploaded data.
 165                 path = os.path.dirname(self.path)
 166                 if os.path.exists(path):
 167                         shutil.rmtree(path, ignore_errors=True)
 168
 169                 # Delete the upload from the database.
 170                 self.db.execute("DELETE FROM uploads WHERE id = %s", self.id)
 171
 172         @property
 173         def time_started(self):
 174                 return self.data.time_started
 175
 176         @property
 177         def time_running(self):
 178                 # Get the seconds since we are running.
 179                 try:
 180                         time_running = datetime.datetime.utcnow() - self.time_started
 181                         time_running = time_running.total_seconds()
 182                 except:
 183                         time_running = 0
 184
 185                 return time_running
 186
 187         @property
 188         def speed(self):
 189                 if not self.time_running:
 190                         return 0
 191
 192                 return self.data.progress / self.time_running
 193
 194         def cleanup(self):
 195                 # Remove uploads that are older than 2 hours.
 196                 if self.time_running >= 3600 * 2:
 197                         self.remove()