]>
git.ipfire.org Git - people/jschlag/pbs.git/blob - src/buildservice/repository.py
8 log
= logging
.getLogger("repositories")
14 from .constants
import *
15 from .decorators
import *
17 class Repositories(base
.Object
):
18 def _get_repository(self
, query
, *args
):
19 res
= self
.db
.get(query
, *args
)
22 return Repository(self
.backend
, res
.id, data
=res
)
24 def _get_repositories(self
, query
, *args
):
25 res
= self
.db
.query(query
, *args
)
28 yield Repository(self
.backend
, row
.id, data
=row
)
31 repositories
= self
._get
_repositories
("SELECT * FROM repositories \
32 WHERE deleted IS FALSE ORDER BY distro_id, name")
34 return iter(repositories
)
36 def create(self
, distro
, name
, description
):
37 return self
._get
_repository
("INSERT INTO repositories(distro_id, name, description) \
38 VALUES(%s, %s, %s) RETURNING *", distro
.id, name
, description
)
40 def get_by_id(self
, repo_id
):
41 return self
._get
_repository
("SELECT * FROM repositories \
42 WHERE id = %s", repo_id
)
44 def get_history(self
, limit
=None, offset
=None, build
=None, repo
=None, user
=None):
45 query
= "SELECT * FROM repositories_history"
48 query
+= " ORDER BY time DESC"
52 query
+= " LIMIT %s,%s"
53 args
+= [offset
, limit
,]
59 for entry
in self
.db
.query(query
, *args
):
60 entry
= logs
.RepositoryLogEntry(self
.pakfire
, entry
)
67 Remasters all repositories
70 # Skip all repositories that don't need an update
71 if not repo
.needs_update
:
72 log
.debug("Repository %s does not need an update" % repo
)
75 with self
.db
.transaction():
80 Cleans up all repositories
83 with self
.db
.transaction():
87 class Repository(base
.DataObject
):
88 table
= "repositories"
90 def __eq__(self
, other
):
91 if isinstance(other
, self
.__class
__):
92 return self
.id == other
.id
94 def __lt__(self
, other
):
95 if isinstance(other
, self
.__class
__):
96 return self
.parent_id
== other
.id
99 builds
= self
.backend
.builds
._get
_builds
("SELECT builds.* FROM repositories_builds \
100 LEFT JOIN builds ON repositories_builds.build_id = builds.id \
101 WHERE repositories_builds.repo_id = %s", self
.id)
106 res
= self
.db
.get("SELECT COUNT(*) AS len FROM repositories_builds \
107 WHERE repo_id = %s", self
.id)
111 def __nonzero__(self
):
116 return self
.backend
.repos
._get
_repository
("SELECT * FROM repositories \
117 WHERE parent_id = %s", self
.id)
121 if self
.data
.parent_id
:
122 return self
.backend
.repos
._get
_repository
("SELECT * FROM repositories \
123 WHERE id = %s", self
.data
.parent_id
)
127 return self
.backend
.distros
.get_by_id(self
.data
.distro_id
)
129 def set_priority(self
, priority
):
130 self
._set
_attribute
("priority", priority
)
132 priority
= property(lambda s
: s
.data
.priority
, set_priority
)
135 if self
.data
.user_id
:
136 return self
.backend
.users
.get_by_id(self
.data
.user_id
)
138 def set_user(self
, user
):
139 self
._set
_attribute
("user_id", user
.id)
141 user
= property(get_user
, set_user
)
147 "distro" : self
.distro
.info
,
149 "arches" : self
.arches
,
155 self
.distro
.identifier
,
161 return os
.path
.join(REPOS_DIR
, self
.basepath
)
166 self
.settings
.get("baseurl", "https://pakfire.ipfire.org"),
172 def mirrorlist(self
):
174 self
.settings
.get("baseurl", "https://pakfire.ipfire.org"),
175 "distro", self
.distro
.identifier
,
176 "repo", self
.identifier
,
177 "mirrorlist?arch=%{arch}"
180 def get_conf(self
, local
=False):
182 "[repo:%s]" % self
.identifier
,
183 "description = %s - %s" % (self
.distro
.name
, self
.summary
),
185 "baseurl = %s/%{arch}" % (self
.path
if local
else self
.url
),
188 if self
.mirrored
and not local
:
189 lines
.append("mirrors = %s" % self
.mirrorlist
)
192 lines
.append("priority = %s" % self
.priority
)
194 return "\n".join(lines
)
198 return self
.data
.name
201 def identifier(self
):
202 return self
.name
.lower()
206 return self
.data
.type
210 lines
= self
.description
.splitlines()
218 def description(self
):
219 return self
.data
.description
or ""
223 return self
.data
.parent_id
227 if not self
.data
.key_id
:
230 return self
.pakfire
.keys
.get_by_id(self
.data
.key_id
)
234 return self
.distro
.arches
+ ["src"]
236 def set_mirrored(self
, mirrored
):
237 self
._set
_attribute
("mirrored", mirrored
)
239 mirrored
= property(lambda s
: s
.data
.mirrored
, set_mirrored
)
241 def set_enabled_for_builds(self
, state
):
242 self
._set
_attribute
("enabled_for_builds", state
)
244 enabled_for_builds
= property(lambda s
: s
.data
.enabled_for_builds
, set_enabled_for_builds
)
247 def score_needed(self
):
248 return self
.data
.score_needed
252 return self
.data
.time_min
256 return self
.data
.time_max
258 def set_update_forced(self
, update_forced
):
259 self
._set
_attribute
("update_forced", update_forced
)
261 update_forced
= property(lambda s
: s
.data
.update_forced
, set_update_forced
)
263 def _log_build(self
, action
, build
, from_repo
=None, to_repo
=None, user
=None):
270 from_repo_id
= from_repo
.id
274 to_repo_id
= to_repo
.id
276 self
.db
.execute("INSERT INTO repositories_history(action, build_id, from_repo_id, to_repo_id, user_id, time) \
277 VALUES(%s, %s, %s, %s, %s, NOW())", action
, build
.id, from_repo_id
, to_repo_id
, user_id
)
279 def add_build(self
, build
, user
=None, log
=True):
280 self
.db
.execute("INSERT INTO repositories_builds(repo_id, build_id, time_added)"
281 " VALUES(%s, %s, NOW())", self
.id, build
.id)
284 build
._update
_bugs
_helper
(self
)
287 self
._log
_build
("added", build
, to_repo
=self
, user
=user
)
289 def rem_build(self
, build
, user
=None, log
=True):
290 self
.db
.execute("DELETE FROM repositories_builds \
291 WHERE repo_id = %s AND build_id = %s", self
.id, build
.id)
293 # Force regenerating the index
294 self
.update_forced
= True
297 self
._log
_build
("removed", build
, from_repo
=self
, user
=user
)
299 def move_build(self
, build
, to_repo
, user
=None, log
=True):
300 self
.db
.execute("UPDATE repositories_builds SET repo_id = %s, time_added = NOW() \
301 WHERE repo_id = %s AND build_id = %s", to_repo
.id, self
.id, build
.id)
303 # Force regenerating the index
304 self
.update_forced
= True
307 build
._update
_bugs
_helper
(to_repo
)
310 self
._log
_build
("moved", build
, from_repo
=self
, to_repo
=to_repo
,
313 def get_builds(self
, limit
=None, offset
=None):
314 query
= "SELECT build_id AS id FROM repositories_builds \
315 WHERE repo_id = %s ORDER BY time_added DESC"
320 query
+= " LIMIT %s,%s"
321 args
+= [offset
, limit
,]
327 for build
in self
.db
.query(query
, *args
):
328 build
= self
.pakfire
.builds
.get_by_id(build
.id)
331 _builds
.append(build
)
335 def get_packages(self
, arch
):
337 return self
.backend
.packages
._get
_packages
("SELECT packages.* FROM repositories_builds \
338 LEFT JOIN builds ON repositories_builds.build_id = builds.id \
339 LEFT JOIN packages ON builds.pkg_id = packages.id \
340 WHERE repositories_builds.repo_id = %s", self
.id)
342 return self
.backend
.packages
._get
_packages
("SELECT packages.* FROM repositories_builds \
343 LEFT JOIN builds ON repositories_builds.build_id = builds.id \
344 LEFT JOIN jobs ON builds.id = jobs.build_id \
345 LEFT JOIN jobs_packages ON jobs.id = jobs_packages.job_id \
346 LEFT JOIN packages ON jobs_packages.pkg_id = packages.id \
347 WHERE repositories_builds.repo_id = %s \
348 AND (jobs.arch = %s OR jobs.arch = %s) \
349 AND (packages.arch = %s OR packages.arch = %s)",
350 self
.id, arch
, "noarch", arch
, "noarch")
353 def unpushed_builds(self
):
354 return self
.backend
.builds
._get
_builds
("SELECT builds.* FROM repositories \
355 LEFT JOIN repositories_builds ON repositories.id = repositories_builds.repo_id \
356 LEFT JOIN builds ON repositories_builds.build_id = builds.id \
357 WHERE repositories.id = %s \
358 AND repositories_builds.time_added >= repositories.last_update", self
.id)
360 def get_obsolete_builds(self
):
361 return self
.pakfire
.builds
.get_obsolete(self
)
364 def needs_update(self
):
365 if self
.unpushed_builds
:
371 self
.db
.execute("UPDATE repositories SET last_update = NOW() \
372 WHERE id = %s", self
.id)
374 # Reset forced update flag
375 self
.update_forced
= False
378 log
.info("Going to update repository %s..." % self
.name
)
380 for arch
in self
.arches
:
383 repo_path
= os
.path
.join(self
.path
, arch
)
384 log
.debug(" Path: %s" % repo_path
)
386 if not os
.path
.exists(repo_path
):
387 os
.makedirs(repo_path
)
389 # Get all packages that are to be included in this repository
391 for p
in self
.get_packages(arch
):
392 path
= os
.path
.join(repo_path
, p
.filename
)
393 packages
.append(path
)
395 # Nothing to do if the package already exists
396 if os
.path
.exists(path
):
399 # Copy the package into the repository
400 log
.info("Adding %s..." % p
)
403 # XXX need to sign the new package here
405 # The repository has been changed
408 # No need to regenerate the index if the repository hasn't changed
409 if not changed
and not self
.update_forced
:
412 # Find the key to sign the package.
415 key_id
= self
.key
.fingerprint
417 # Create package index.
418 p
= pakfire
.PakfireServer(arch
=arch
)
419 p
.repo_create(repo_path
, packages
,
420 name
="%s - %s.%s" % (self
.distro
.name
, self
.name
, arch
),
423 # Update the timestamp when we started at last
427 log
.info("Cleaning up repository %s..." % self
.name
)
429 for arch
in self
.arches
:
430 repo_path
= os
.path
.join(self
.path
, arch
)
432 # Get a list of all files in the repository directory right now
433 filelist
= [e
for e
in os
.listdir(repo_path
)
434 if os
.path
.isfile(os
.path
.join(repo_path
, e
))]
436 # Get a list of all packages that should be in the repository
437 # and remove them from the filelist
438 for p
in self
.get_packages(arch
):
440 filelist
.remove(p
.filename
)
444 # For any files that do not belong into the repository
445 # any more, we will just delete them
446 for filename
in filelist
:
447 path
= os
.path
.join(repo_path
, filename
)
448 self
.backend
.delete_file(path
)
450 def get_history(self
, **kwargs
):
455 return self
.pakfire
.repos
.get_history(**kwargs
)
457 def get_build_times(self
):
459 for arch
in self
.arches
:
460 time
= self
.db
.get("SELECT SUM(jobs.time_finished - jobs.time_started) AS time FROM jobs \
461 JOIN builds ON builds.id = jobs.build_id \
462 JOIN repositories_builds ON builds.id = repositories_builds.build_id \
463 WHERE (jobs.arch = %s OR jobs.arch = %s) AND \
464 jobs.type = 'build' AND \
465 repositories_builds.repo_id = %s", arch
, "noarch", self
.id)
467 times
.append((arch
, time
.time
.total_seconds()))
472 class RepositoryAux(base
.DataObject
):
473 table
= "repositories_aux"
477 return self
.data
.name
480 def description(self
):
481 return self
.data
.description
or ""
488 def identifier(self
):
489 return self
.name
.lower()
493 return self
.pakfire
.distros
.get_by_id(self
.data
.distro_id
)
495 def get_conf(self
, local
=False):
497 "[repo:%s]" % self
.identifier
,
498 "description = %s - %s" % (self
.distro
.name
, self
.name
),
500 "baseurl = %s" % self
.url
,
504 return "\n".join(lines
)