src/buildservice/uploads.py

   1 #!/usr/bin/python
   2
   3 from __future__ import division
   4
   5 import datetime
   6 import hashlib
   7 import logging
   8 import os
   9 import shutil
  10
  11 import pakfire.packages
  12
  13 from . import base
  14 from . import misc
  15 from . import packages
  16 from . import users
  17
  18 from .constants import *
  19 from .decorators import *
  20
  21 class Uploads(base.Object):
  22         def _get_upload(self, query, *args):
  23                 res = self.db.get(query, *args)
  24
  25                 if res:
  26                         return Upload(self.backend, res.id, data=res)
  27
  28         def _get_uploads(self, query, *args):
  29                 res = self.db.query(query, *args)
  30
  31                 for row in res:
  32                         yield Upload(self.backend, row.id, data=row)
  33
  34         def __iter__(self):
  35                 uploads = self._get_uploads("SELECT * FROM uploads ORDER BY time_started DESC")
  36
  37                 return iter(uploads)
  38
  39         def get_by_uuid(self, uuid):
  40                 return self._get_upload("SELECT * FROM uploads WHERE uuid = %s", uuid)
  41
  42         def create(self, filename, size, hash, builder=None, user=None):
  43                 assert builder or user
  44
  45                 # Create a random ID for this upload
  46                 uuid = users.generate_random_string(64)
  47
  48                 upload = self._get_upload("INSERT INTO uploads(uuid, filename, size, hash) \
  49                         VALUES(%s, %s, %s, %s) RETURNING *", uuid, filename, size, hash)
  50
  51                 if builder:
  52                         upload.builder = builder
  53
  54                 elif user:
  55                         upload.user = user
  56
  57                 # Create space to where we save the data.
  58                 dirname = os.path.dirname(upload.path)
  59                 if not os.path.exists(dirname):
  60                         os.makedirs(dirname)
  61
  62                 # Create empty file.
  63                 f = open(upload.path, "w")
  64                 f.close()
  65
  66                 return upload
  67
  68         def cleanup(self):
  69                 for upload in self.get_all():
  70                         upload.cleanup()
  71
  72
  73 class Upload(base.DataObject):
  74         table = "uploads"
  75
  76         @property
  77         def uuid(self):
  78                 return self.data.uuid
  79
  80         @property
  81         def hash(self):
  82                 return self.data.hash
  83
  84         @property
  85         def filename(self):
  86                 return self.data.filename
  87
  88         @property
  89         def path(self):
  90                 return os.path.join(UPLOADS_DIR, self.uuid, self.filename)
  91
  92         @property
  93         def size(self):
  94                 return self.data.size
  95
  96         @property
  97         def progress(self):
  98                 return self.data.progress / self.size
  99
 100         # Builder
 101
 102         def get_builder(self):
 103                 if self.data.builder_id:
 104                         return self.backend.builders.get_by_id(self.data.builder_id)
 105
 106         def set_builder(self, builder):
 107                 self._set_attribute("builder_id", builder.id)
 108
 109         builder = lazy_property(get_builder, set_builder)
 110
 111         # User
 112
 113         def get_user(self):
 114                 if self.data.user_id:
 115                         return self.backend.users.get_by_id(self.data.user_id)
 116
 117         def set_user(self, user):
 118                 self._set_attribute("user_id", user.id)
 119
 120         user = lazy_property(get_user, set_user)
 121
 122         def append(self, data):
 123                 # Check if the filesize was exceeded.
 124                 size = os.path.getsize(self.path) + len(data)
 125                 if size > self.data.size:
 126                         raise Exception, "Given filesize was exceeded for upload %s" % self.uuid
 127
 128                 logging.debug("Writing %s bytes to %s" % (len(data), self.path))
 129
 130                 with open(self.path, "ab") as f:
 131                         f.write(data)
 132
 133                 self._set_attribute("progress", size)
 134
 135         def validate(self):
 136                 size = os.path.getsize(self.path)
 137                 if not size == self.data.size:
 138                         logging.error("Filesize is not okay: %s" % (self.uuid))
 139                         return False
 140
 141                 # Calculate a hash to validate the upload.
 142                 hash = misc.calc_hash1(self.path)
 143
 144                 if not self.hash == hash:
 145                         logging.error("Hash did not match: %s != %s" % (self.hash, hash))
 146                         return False
 147
 148                 return True
 149
 150         def finished(self):
 151                 """
 152                         Update the status of the upload in the database to "finished".
 153                 """
 154                 # Check if the file was completely uploaded and the hash is correct.
 155                 # If not, the upload has failed.
 156                 if not self.validate():
 157                         return False
 158
 159                 self._set_attribute("finished", True)
 160                 self._set_attribute("time_finished", datetime.datetime.utcnow())
 161
 162                 return True
 163
 164         def remove(self):
 165                 # Remove the uploaded data.
 166                 path = os.path.dirname(self.path)
 167                 if os.path.exists(path):
 168                         shutil.rmtree(path, ignore_errors=True)
 169
 170                 # Delete the upload from the database.
 171                 self.db.execute("DELETE FROM uploads WHERE id = %s", self.id)
 172
 173         @property
 174         def time_started(self):
 175                 return self.data.time_started
 176
 177         @property
 178         def time_running(self):
 179                 # Get the seconds since we are running.
 180                 try:
 181                         time_running = datetime.datetime.utcnow() - self.time_started
 182                         time_running = time_running.total_seconds()
 183                 except:
 184                         time_running = 0
 185
 186                 return time_running
 187
 188         @property
 189         def speed(self):
 190                 if not self.time_running:
 191                         return 0
 192
 193                 return self.data.progress / self.time_running
 194
 195         def cleanup(self):
 196                 # Remove uploads that are older than 2 hours.
 197                 if self.time_running >= 3600 * 2:
 198                         self.remove()