]>
Commit | Line | Data |
---|---|---|
9137135a MT |
1 | #!/usr/bin/python |
2 | ||
f6e6ff79 MT |
3 | from __future__ import division |
4 | ||
5 | import datetime | |
9137135a MT |
6 | import hashlib |
7 | import logging | |
8 | import os | |
f6e6ff79 | 9 | import shutil |
9137135a | 10 | |
f6e6ff79 MT |
11 | import pakfire.packages |
12 | ||
2c909128 MT |
13 | from . import base |
14 | from . import misc | |
15 | from . import packages | |
1b4386e0 | 16 | from . import users |
9137135a | 17 | |
2c909128 | 18 | from .constants import * |
2f64fe68 | 19 | from .decorators import * |
9137135a MT |
20 | |
21 | class Uploads(base.Object): | |
2f64fe68 MT |
22 | def _get_upload(self, query, *args): |
23 | res = self.db.get(query, *args) | |
9137135a | 24 | |
2f64fe68 MT |
25 | if res: |
26 | return Upload(self.backend, res.id, data=res) | |
9137135a | 27 | |
2f64fe68 MT |
28 | def _get_uploads(self, query, *args): |
29 | res = self.db.query(query, *args) | |
9137135a | 30 | |
2f64fe68 MT |
31 | for row in res: |
32 | yield Upload(self.backend, row.id, data=row) | |
9137135a | 33 | |
2f64fe68 MT |
34 | def __iter__(self): |
35 | uploads = self._get_uploads("SELECT * FROM uploads ORDER BY time_started DESC") | |
9137135a | 36 | |
57767c6e | 37 | return iter(uploads) |
9137135a | 38 | |
1b4386e0 | 39 | def get_by_uuid(self, uuid): |
2f64fe68 | 40 | return self._get_upload("SELECT * FROM uploads WHERE uuid = %s", uuid) |
9137135a | 41 | |
57767c6e | 42 | def create(self, filename, size, hash, builder=None, user=None): |
f6e6ff79 | 43 | assert builder or user |
9137135a | 44 | |
1b4386e0 MT |
45 | # Create a random ID for this upload |
46 | uuid = users.generate_random_string(64) | |
47 | ||
2f64fe68 | 48 | upload = self._get_upload("INSERT INTO uploads(uuid, filename, size, hash) \ |
1b4386e0 | 49 | VALUES(%s, %s, %s, %s) RETURNING *", uuid, filename, size, hash) |
f6e6ff79 MT |
50 | |
51 | if builder: | |
2f64fe68 | 52 | upload.builder = builder |
f6e6ff79 MT |
53 | |
54 | elif user: | |
2f64fe68 | 55 | upload.user = user |
9137135a MT |
56 | |
57 | # Create space to where we save the data. | |
58 | dirname = os.path.dirname(upload.path) | |
59 | if not os.path.exists(dirname): | |
60 | os.makedirs(dirname) | |
61 | ||
62 | # Create empty file. | |
63 | f = open(upload.path, "w") | |
64 | f.close() | |
65 | ||
66 | return upload | |
67 | ||
2f64fe68 MT |
68 | def cleanup(self): |
69 | for upload in self.get_all(): | |
70 | upload.cleanup() | |
71 | ||
72 | ||
73 | class Upload(base.DataObject): | |
74 | table = "uploads" | |
75 | ||
9137135a MT |
76 | @property |
77 | def uuid(self): | |
78 | return self.data.uuid | |
79 | ||
80 | @property | |
81 | def hash(self): | |
82 | return self.data.hash | |
83 | ||
84 | @property | |
85 | def filename(self): | |
86 | return self.data.filename | |
87 | ||
88 | @property | |
89 | def path(self): | |
90 | return os.path.join(UPLOADS_DIR, self.uuid, self.filename) | |
91 | ||
f6e6ff79 MT |
92 | @property |
93 | def size(self): | |
94 | return self.data.size | |
95 | ||
96 | @property | |
97 | def progress(self): | |
98 | return self.data.progress / self.size | |
99 | ||
2f64fe68 MT |
100 | # Builder |
101 | ||
102 | def get_builder(self): | |
f6e6ff79 | 103 | if self.data.builder_id: |
2f64fe68 | 104 | return self.backend.builders.get_by_id(self.data.builder_id) |
f6e6ff79 | 105 | |
2f64fe68 | 106 | def set_builder(self, builder): |
57767c6e | 107 | self._set_attribute("builder_id", builder.id) |
2f64fe68 MT |
108 | |
109 | builder = lazy_property(get_builder, set_builder) | |
110 | ||
111 | # User | |
112 | ||
113 | def get_user(self): | |
f6e6ff79 | 114 | if self.data.user_id: |
2f64fe68 MT |
115 | return self.backend.users.get_by_id(self.data.user_id) |
116 | ||
117 | def set_user(self, user): | |
57767c6e | 118 | self._set_attribute("user_id", user.id) |
2f64fe68 MT |
119 | |
120 | user = lazy_property(get_user, set_user) | |
9137135a MT |
121 | |
122 | def append(self, data): | |
f6e6ff79 MT |
123 | # Check if the filesize was exceeded. |
124 | size = os.path.getsize(self.path) + len(data) | |
125 | if size > self.data.size: | |
126 | raise Exception, "Given filesize was exceeded for upload %s" % self.uuid | |
127 | ||
9137135a MT |
128 | logging.debug("Writing %s bytes to %s" % (len(data), self.path)) |
129 | ||
2f64fe68 MT |
130 | with open(self.path, "ab") as f: |
131 | f.write(data) | |
9137135a | 132 | |
2f64fe68 | 133 | self._set_attribute("progress", size) |
f6e6ff79 | 134 | |
9137135a | 135 | def validate(self): |
f6e6ff79 MT |
136 | size = os.path.getsize(self.path) |
137 | if not size == self.data.size: | |
138 | logging.error("Filesize is not okay: %s" % (self.uuid)) | |
139 | return False | |
140 | ||
9137135a MT |
141 | # Calculate a hash to validate the upload. |
142 | hash = misc.calc_hash1(self.path) | |
143 | ||
f6e6ff79 | 144 | if not self.hash == hash: |
9137135a | 145 | logging.error("Hash did not match: %s != %s" % (self.hash, hash)) |
f6e6ff79 MT |
146 | return False |
147 | ||
148 | return True | |
9137135a | 149 | |
f6e6ff79 MT |
150 | def finished(self): |
151 | """ | |
152 | Update the status of the upload in the database to "finished". | |
153 | """ | |
154 | # Check if the file was completely uploaded and the hash is correct. | |
155 | # If not, the upload has failed. | |
156 | if not self.validate(): | |
157 | return False | |
158 | ||
2f64fe68 MT |
159 | self._set_attribute("finished", True) |
160 | self._set_attribute("time_finished", datetime.datetime.utcnow()) | |
f6e6ff79 MT |
161 | |
162 | return True | |
9137135a MT |
163 | |
164 | def remove(self): | |
165 | # Remove the uploaded data. | |
f6e6ff79 MT |
166 | path = os.path.dirname(self.path) |
167 | if os.path.exists(path): | |
168 | shutil.rmtree(path, ignore_errors=True) | |
9137135a MT |
169 | |
170 | # Delete the upload from the database. | |
171 | self.db.execute("DELETE FROM uploads WHERE id = %s", self.id) | |
172 | ||
f6e6ff79 MT |
173 | @property |
174 | def time_started(self): | |
175 | return self.data.time_started | |
9137135a | 176 | |
f6e6ff79 MT |
177 | @property |
178 | def time_running(self): | |
9137135a MT |
179 | # Get the seconds since we are running. |
180 | try: | |
f6e6ff79 | 181 | time_running = datetime.datetime.utcnow() - self.time_started |
9137135a MT |
182 | time_running = time_running.total_seconds() |
183 | except: | |
184 | time_running = 0 | |
185 | ||
f6e6ff79 MT |
186 | return time_running |
187 | ||
188 | @property | |
189 | def speed(self): | |
190 | if not self.time_running: | |
191 | return 0 | |
192 | ||
193 | return self.data.progress / self.time_running | |
194 | ||
195 | def cleanup(self): | |
196 | # Remove uploads that are older than 2 hours. | |
197 | if self.time_running >= 3600 * 2: | |
9137135a | 198 | self.remove() |