]>
Commit | Line | Data |
---|---|---|
9137135a MT |
1 | #!/usr/bin/python |
2 | ||
9729c5b3 | 3 | import logging |
f6e6ff79 MT |
4 | import os.path |
5 | ||
9729c5b3 MT |
6 | log = logging.getLogger("repositories") |
7 | log.propagate = 1 | |
8 | ||
2c909128 MT |
9 | from . import base |
10 | from . import logs | |
9137135a | 11 | |
9729c5b3 | 12 | from .constants import * |
e459cbba MT |
13 | from .decorators import * |
14 | ||
9137135a | 15 | class Repositories(base.Object): |
e459cbba MT |
16 | def _get_repository(self, query, *args): |
17 | res = self.db.get(query, *args) | |
9137135a | 18 | |
e459cbba MT |
19 | if res: |
20 | return Repository(self.backend, res.id, data=res) | |
9137135a | 21 | |
e459cbba MT |
22 | def _get_repositories(self, query, *args): |
23 | res = self.db.query(query, *args) | |
24 | ||
25 | for row in res: | |
26 | yield Repository(self.backend, row.id, data=row) | |
27 | ||
28 | def __iter__(self): | |
29 | repositories = self._get_repositories("SELECT * FROM repositories \ | |
8cb14e63 | 30 | WHERE deleted IS FALSE ORDER BY distro_id, name") |
9137135a | 31 | |
e459cbba MT |
32 | return iter(repositories) |
33 | ||
d629da45 MT |
34 | def create(self, distro, name, description): |
35 | return self._get_repository("INSERT INTO repositories(distro_id, name, description) \ | |
36 | VALUES(%s, %s, %s) RETURNING *", distro.id, name, description) | |
37 | ||
e459cbba MT |
38 | def get_by_id(self, repo_id): |
39 | return self._get_repository("SELECT * FROM repositories \ | |
40 | WHERE id = %s", repo_id) | |
9137135a | 41 | |
f6e6ff79 MT |
42 | def get_history(self, limit=None, offset=None, build=None, repo=None, user=None): |
43 | query = "SELECT * FROM repositories_history" | |
44 | args = [] | |
9137135a | 45 | |
f6e6ff79 | 46 | query += " ORDER BY time DESC" |
9137135a | 47 | |
f6e6ff79 MT |
48 | if limit: |
49 | if offset: | |
50 | query += " LIMIT %s,%s" | |
51 | args += [offset, limit,] | |
52 | else: | |
53 | query += " LIMIT %s" | |
54 | args += [limit,] | |
9137135a | 55 | |
f6e6ff79 MT |
56 | entries = [] |
57 | for entry in self.db.query(query, *args): | |
58 | entry = logs.RepositoryLogEntry(self.pakfire, entry) | |
59 | entries.append(entry) | |
9137135a | 60 | |
f6e6ff79 | 61 | return entries |
9137135a | 62 | |
9729c5b3 MT |
63 | def remaster(self): |
64 | """ | |
65 | Remasters all repositories | |
66 | """ | |
67 | for repo in self: | |
68 | # Skip all repositories that don't need an update | |
69 | if not repo.needs_update: | |
70 | log.debug("Repository %s does not need an update" % repo) | |
71 | continue | |
72 | ||
73 | with self.db.transaction(): | |
74 | repo.remaster() | |
75 | ||
9137135a | 76 | |
d629da45 MT |
77 | class Repository(base.DataObject): |
78 | table = "repositories" | |
f6e6ff79 | 79 | |
d629da45 MT |
80 | def __eq__(self, other): |
81 | if isinstance(other, self.__class__): | |
82 | return self.id == other.id | |
f6e6ff79 | 83 | |
d629da45 MT |
84 | def __lt__(self, other): |
85 | if isinstance(other, self.__class__): | |
86 | return self.parent_id == other.id | |
9137135a | 87 | |
d629da45 MT |
88 | def __iter__(self): |
89 | builds = self.backend.builds._get_builds("SELECT builds.* FROM repositories_builds \ | |
90 | LEFT JOIN builds ON repositories_builds.build_id = builds.id \ | |
78f9cb76 | 91 | WHERE repositories_builds.repo_id = %s", self.id) |
9137135a | 92 | |
d629da45 | 93 | return iter(builds) |
f6e6ff79 | 94 | |
d629da45 MT |
95 | def __len__(self): |
96 | res = self.db.get("SELECT COUNT(*) AS len FROM repositories_builds \ | |
97 | WHERE repo_id = %s", self.id) | |
9137135a | 98 | |
d629da45 | 99 | return res.len |
9137135a | 100 | |
d629da45 | 101 | @lazy_property |
9137135a | 102 | def next(self): |
d629da45 MT |
103 | return self.backend.repos._get_repository("SELECT * FROM repositories \ |
104 | WHERE parent_id = %s", self.id) | |
9137135a | 105 | |
d629da45 | 106 | @lazy_property |
f6e6ff79 | 107 | def parent(self): |
d629da45 MT |
108 | if self.data.parent_id: |
109 | return self.backend.repos._get_repository("SELECT * FROM repositories \ | |
110 | WHERE id = %s", self.data.parent_id) | |
9137135a | 111 | |
e459cbba | 112 | @lazy_property |
9137135a | 113 | def distro(self): |
e459cbba | 114 | return self.backend.distros.get_by_id(self.data.distro_id) |
9137135a | 115 | |
5bc58fba MT |
116 | def set_priority(self, priority): |
117 | self._set_attribute("priority", priority) | |
118 | ||
119 | priority = property(lambda s: s.data.priority, set_priority) | |
120 | ||
a7a18be1 MT |
121 | def get_user(self): |
122 | if self.data.user_id: | |
123 | return self.backend.users.get_by_id(self.data.user_id) | |
124 | ||
125 | def set_user(self, user): | |
126 | self._set_attribute("user_id", user.id) | |
127 | ||
128 | user = property(get_user, set_user) | |
129 | ||
9137135a MT |
130 | @property |
131 | def info(self): | |
132 | return { | |
133 | "id" : self.id, | |
134 | "distro" : self.distro.info, | |
135 | "name" : self.name, | |
136 | "arches" : self.arches, | |
137 | } | |
138 | ||
139 | @property | |
f6e6ff79 MT |
140 | def url(self): |
141 | url = os.path.join( | |
142 | self.settings.get("repository_baseurl", "http://pakfire.ipfire.org/repositories/"), | |
143 | self.distro.identifier, | |
144 | self.identifier, | |
145 | "%{arch}" | |
146 | ) | |
147 | ||
148 | return url | |
149 | ||
150 | @property | |
151 | def mirrorlist(self): | |
152 | url = os.path.join( | |
153 | self.settings.get("mirrorlist_baseurl", "https://pakfire.ipfire.org/"), | |
154 | "distro", self.distro.identifier, | |
155 | "repo", self.identifier, | |
156 | "mirrorlist?arch=%{arch}" | |
157 | ) | |
158 | ||
159 | return url | |
160 | ||
161 | def get_conf(self): | |
f6e6ff79 MT |
162 | lines = [ |
163 | "[repo:%s]" % self.identifier, | |
164 | "description = %s - %s" % (self.distro.name, self.summary), | |
165 | "enabled = 1", | |
166 | "baseurl = %s" % self.url, | |
f6e6ff79 MT |
167 | ] |
168 | ||
ce88a1bb MT |
169 | if self.mirrored: |
170 | lines.append("mirrors = %s" % self.mirrorlist) | |
171 | ||
5bc58fba MT |
172 | if self.priority: |
173 | lines.append("priority = %s" % self.priority) | |
f6e6ff79 MT |
174 | |
175 | return "\n".join(lines) | |
9137135a MT |
176 | |
177 | @property | |
178 | def name(self): | |
179 | return self.data.name | |
180 | ||
181 | @property | |
f6e6ff79 MT |
182 | def identifier(self): |
183 | return self.name.lower() | |
9137135a MT |
184 | |
185 | @property | |
f6e6ff79 MT |
186 | def type(self): |
187 | return self.data.type | |
9137135a MT |
188 | |
189 | @property | |
f6e6ff79 MT |
190 | def summary(self): |
191 | lines = self.description.splitlines() | |
9137135a | 192 | |
f6e6ff79 MT |
193 | if lines: |
194 | return lines[0] | |
9137135a | 195 | |
f6e6ff79 | 196 | return "N/A" |
9137135a | 197 | |
f6e6ff79 MT |
198 | @property |
199 | def description(self): | |
200 | return self.data.description or "" | |
9137135a | 201 | |
f6e6ff79 MT |
202 | @property |
203 | def parent_id(self): | |
204 | return self.data.parent_id | |
9137135a | 205 | |
d629da45 | 206 | @lazy_property |
f6e6ff79 MT |
207 | def key(self): |
208 | if not self.data.key_id: | |
209 | return | |
9137135a | 210 | |
d629da45 | 211 | return self.pakfire.keys.get_by_id(self.data.key_id) |
9137135a | 212 | |
f6e6ff79 MT |
213 | @property |
214 | def arches(self): | |
9729c5b3 | 215 | return self.distro.arches + ["src"] |
f6e6ff79 | 216 | |
ce88a1bb MT |
217 | def set_mirrored(self, mirrored): |
218 | self._set_attribute("mirrored", mirrored) | |
219 | ||
220 | mirrored = property(lambda s: s.data.mirrored, set_mirrored) | |
9137135a | 221 | |
f6e6ff79 | 222 | def set_enabled_for_builds(self, state): |
d629da45 | 223 | self._set_attribute("enabled_for_builds", state) |
9137135a | 224 | |
d629da45 | 225 | enabled_for_builds = property(lambda s: s.data.enabled_for_builds, set_enabled_for_builds) |
9137135a MT |
226 | |
227 | @property | |
f6e6ff79 MT |
228 | def score_needed(self): |
229 | return self.data.score_needed | |
9137135a MT |
230 | |
231 | @property | |
f6e6ff79 MT |
232 | def time_min(self): |
233 | return self.data.time_min | |
9137135a MT |
234 | |
235 | @property | |
f6e6ff79 MT |
236 | def time_max(self): |
237 | return self.data.time_max | |
9137135a | 238 | |
f6e6ff79 MT |
239 | def _log_build(self, action, build, from_repo=None, to_repo=None, user=None): |
240 | user_id = None | |
241 | if user: | |
242 | user_id = user.id | |
9137135a | 243 | |
f6e6ff79 MT |
244 | from_repo_id = None |
245 | if from_repo: | |
246 | from_repo_id = from_repo.id | |
9137135a | 247 | |
f6e6ff79 MT |
248 | to_repo_id = None |
249 | if to_repo: | |
250 | to_repo_id = to_repo.id | |
9137135a | 251 | |
f6e6ff79 MT |
252 | self.db.execute("INSERT INTO repositories_history(action, build_id, from_repo_id, to_repo_id, user_id, time) \ |
253 | VALUES(%s, %s, %s, %s, %s, NOW())", action, build.id, from_repo_id, to_repo_id, user_id) | |
9137135a | 254 | |
f6e6ff79 MT |
255 | def add_build(self, build, user=None, log=True): |
256 | self.db.execute("INSERT INTO repositories_builds(repo_id, build_id, time_added)" | |
257 | " VALUES(%s, %s, NOW())", self.id, build.id) | |
9137135a | 258 | |
f6e6ff79 MT |
259 | # Update bug status. |
260 | build._update_bugs_helper(self) | |
9137135a | 261 | |
f6e6ff79 MT |
262 | if log: |
263 | self._log_build("added", build, to_repo=self, user=user) | |
9137135a | 264 | |
f6e6ff79 MT |
265 | def rem_build(self, build, user=None, log=True): |
266 | self.db.execute("DELETE FROM repositories_builds \ | |
267 | WHERE repo_id = %s AND build_id = %s", self.id, build.id) | |
9137135a | 268 | |
f6e6ff79 MT |
269 | if log: |
270 | self._log_build("removed", build, from_repo=self, user=user) | |
9137135a | 271 | |
f6e6ff79 MT |
272 | def move_build(self, build, to_repo, user=None, log=True): |
273 | self.db.execute("UPDATE repositories_builds SET repo_id = %s, time_added = NOW() \ | |
274 | WHERE repo_id = %s AND build_id = %s", to_repo.id, self.id, build.id) | |
9137135a | 275 | |
f6e6ff79 MT |
276 | # Update bug status. |
277 | build._update_bugs_helper(to_repo) | |
9137135a | 278 | |
f6e6ff79 MT |
279 | if log: |
280 | self._log_build("moved", build, from_repo=self, to_repo=to_repo, | |
281 | user=user) | |
9137135a | 282 | |
f6e6ff79 MT |
283 | def get_builds(self, limit=None, offset=None): |
284 | query = "SELECT build_id AS id FROM repositories_builds \ | |
285 | WHERE repo_id = %s ORDER BY time_added DESC" | |
286 | args = [self.id,] | |
9137135a | 287 | |
f6e6ff79 MT |
288 | if limit: |
289 | if offset: | |
290 | query += " LIMIT %s,%s" | |
291 | args += [offset, limit,] | |
292 | else: | |
293 | query += " LIMIT %s" | |
294 | args += [limit,] | |
295 | ||
296 | _builds = [] | |
297 | for build in self.db.query(query, *args): | |
2c909128 | 298 | build = self.pakfire.builds.get_by_id(build.id) |
f6e6ff79 MT |
299 | build._repo = self |
300 | ||
301 | _builds.append(build) | |
302 | ||
303 | return _builds | |
304 | ||
83be3106 | 305 | def _get_packages(self, arch): |
f6e6ff79 | 306 | if arch.name == "src": |
83be3106 | 307 | pkgs = self.db.query("SELECT packages.id AS id, packages.path AS path FROM packages \ |
f6e6ff79 MT |
308 | JOIN builds ON builds.pkg_id = packages.id \ |
309 | JOIN repositories_builds ON builds.id = repositories_builds.build_id \ | |
310 | WHERE packages.arch = %s AND repositories_builds.repo_id = %s", | |
22b715d7 | 311 | arch.name, self.id) |
9137135a | 312 | |
f6e6ff79 | 313 | else: |
83be3106 | 314 | pkgs = self.db.query("SELECT packages.id AS id, packages.path AS path FROM packages \ |
f6e6ff79 MT |
315 | JOIN jobs_packages ON jobs_packages.pkg_id = packages.id \ |
316 | JOIN jobs ON jobs_packages.job_id = jobs.id \ | |
317 | JOIN builds ON builds.id = jobs.build_id \ | |
318 | JOIN repositories_builds ON builds.id = repositories_builds.build_id \ | |
22b715d7 | 319 | WHERE (jobs.arch = %s OR jobs.arch = %s) AND \ |
f6e6ff79 | 320 | repositories_builds.repo_id = %s", |
d1a95124 | 321 | arch.name, "noarch", self.id) |
9137135a | 322 | |
83be3106 MT |
323 | return pkgs |
324 | ||
325 | def get_packages(self, arch): | |
2c909128 | 326 | pkgs = [self.pakfire.packages.get_by_id(p.id) for p in self._get_packages(arch)] |
83be3106 MT |
327 | pkgs.sort() |
328 | ||
329 | return pkgs | |
330 | ||
331 | def get_paths(self, arch): | |
332 | paths = [p.path for p in self._get_packages(arch)] | |
333 | paths.sort() | |
334 | ||
335 | return paths | |
f6e6ff79 MT |
336 | |
337 | @property | |
338 | def packages(self): | |
339 | return self.get_packages() | |
9137135a | 340 | |
9729c5b3 MT |
341 | @property |
342 | def unpushed_builds(self): | |
343 | return self.backend.builds._get_builds("SELECT builds.* FROM repositories \ | |
344 | LEFT JOIN repositories_builds ON repositories.id = repositories_builds.repo_id \ | |
345 | LEFT JOIN builds ON repositories_builds.build_id = builds.id \ | |
346 | WHERE repositories.id = %s \ | |
347 | AND repositories_builds.time_added >= repositories.last_update", self.id) | |
f6e6ff79 MT |
348 | |
349 | def get_obsolete_builds(self): | |
f6e6ff79 MT |
350 | return self.pakfire.builds.get_obsolete(self) |
351 | ||
9729c5b3 | 352 | @property |
f6e6ff79 | 353 | def needs_update(self): |
9729c5b3 | 354 | if self.unpushed_builds: |
9137135a MT |
355 | return True |
356 | ||
9137135a MT |
357 | return False |
358 | ||
f6e6ff79 MT |
359 | def updated(self): |
360 | self.db.execute("UPDATE repositories SET last_update = NOW() \ | |
361 | WHERE id = %s", self.id) | |
9137135a | 362 | |
9729c5b3 MT |
363 | def remaster(self): |
364 | log.info("Going to update repository %s..." % self.name) | |
365 | ||
366 | # Update the timestamp when we started at last. | |
367 | self.updated() | |
368 | ||
369 | for arch in self.arches: | |
370 | changed = False | |
371 | ||
372 | # Get all package paths that are to be included in this repository. | |
373 | paths = self.get_paths(arch) | |
374 | ||
375 | repo_path = os.path.join( | |
376 | REPOS_DIR, | |
377 | self.distro.identifier, | |
378 | self.identifier, | |
379 | arch | |
380 | ) | |
381 | ||
382 | if not os.path.exists(repo_path): | |
383 | os.makedirs(repo_path) | |
384 | ||
385 | source_files = [] | |
386 | remove_files = [] | |
387 | ||
388 | for filename in os.listdir(repo_path): | |
389 | path = os.path.join(repo_path, filename) | |
390 | ||
391 | if not os.path.isfile(path): | |
392 | continue | |
393 | ||
394 | remove_files.append(path) | |
395 | ||
396 | for path in paths: | |
397 | filename = os.path.basename(path) | |
398 | ||
399 | source_file = os.path.join(PACKAGES_DIR, path) | |
400 | target_file = os.path.join(repo_path, filename) | |
401 | ||
402 | # Do not add duplicate files twice. | |
403 | if source_file in source_files: | |
404 | continue | |
405 | ||
406 | source_files.append(source_file) | |
407 | ||
408 | try: | |
409 | remove_files.remove(target_file) | |
410 | except ValueError: | |
411 | changed = True | |
412 | ||
413 | if remove_files: | |
414 | changed = True | |
415 | ||
416 | # If nothing in the repository data has changed, there | |
417 | # is nothing to do. | |
418 | if changed: | |
419 | log.info("The repository has updates...") | |
420 | else: | |
421 | log.info("Nothing to update.") | |
422 | continue | |
423 | ||
424 | # Find the key to sign the package. | |
425 | key_id = None | |
426 | if repo.key: | |
427 | key_id = self.key.fingerprint | |
428 | ||
429 | # Create package index. | |
430 | p = pakfire.PakfireServer(arch=arch) | |
431 | ||
432 | p.repo_create(repo_path, source_files, | |
433 | name="%s - %s.%s" % (self.distro.name, self.name, arch), | |
434 | key_id=key_id) | |
435 | ||
436 | # Remove files afterwards. | |
437 | for file in remove_files: | |
438 | file = os.path.join(repo_path, file) | |
439 | ||
440 | try: | |
441 | os.remove(file) | |
442 | except OSError: | |
443 | log.warning("Could not remove %s." % file) | |
444 | ||
f6e6ff79 MT |
445 | def get_history(self, **kwargs): |
446 | kwargs.update({ | |
447 | "repo" : self, | |
448 | }) | |
9137135a | 449 | |
f6e6ff79 | 450 | return self.pakfire.repos.get_history(**kwargs) |
9137135a | 451 | |
f6e6ff79 | 452 | def get_build_times(self): |
f6e6ff79 | 453 | times = [] |
e459cbba MT |
454 | for arch in self.arches: |
455 | time = self.db.get("SELECT SUM(jobs.time_finished - jobs.time_started) AS time FROM jobs \ | |
f6e6ff79 MT |
456 | JOIN builds ON builds.id = jobs.build_id \ |
457 | JOIN repositories_builds ON builds.id = repositories_builds.build_id \ | |
e459cbba | 458 | WHERE (jobs.arch = %s OR jobs.arch = %s) AND \ |
a577f40c | 459 | jobs.type = 'build' AND \ |
e459cbba | 460 | repositories_builds.repo_id = %s", arch, "noarch", self.id) |
f6e6ff79 | 461 | |
e459cbba | 462 | times.append((arch, time.time.total_seconds())) |
f6e6ff79 MT |
463 | |
464 | return times | |
9137135a | 465 | |
9137135a | 466 | |
d629da45 MT |
467 | class RepositoryAux(base.DataObject): |
468 | table = "repositories_aux" | |
f6e6ff79 MT |
469 | |
470 | @property | |
471 | def name(self): | |
472 | return self.data.name | |
473 | ||
474 | @property | |
475 | def description(self): | |
476 | return self.data.description or "" | |
477 | ||
478 | @property | |
479 | def url(self): | |
480 | return self.data.url | |
481 | ||
482 | @property | |
483 | def identifier(self): | |
484 | return self.name.lower() | |
485 | ||
486 | @property | |
487 | def distro(self): | |
d629da45 | 488 | return self.pakfire.distros.get_by_id(self.data.distro_id) |
f6e6ff79 MT |
489 | |
490 | def get_conf(self): | |
491 | lines = [ | |
492 | "[repo:%s]" % self.identifier, | |
493 | "description = %s - %s" % (self.distro.name, self.name), | |
494 | "enabled = 1", | |
495 | "baseurl = %s" % self.url, | |
496 | "priority = 0", | |
497 | ] | |
498 | ||
499 | return "\n".join(lines) |