src/buildservice/repository.py

   1 #!/usr/bin/python
   2
   3 import logging
   4 import os.path
   5
   6 log = logging.getLogger("repositories")
   7 log.propagate = 1
   8
   9 from . import base
  10 from . import logs
  11
  12 from .constants import *
  13 from .decorators import *
  14
  15 class Repositories(base.Object):
  16         def _get_repository(self, query, *args):
  17                 res = self.db.get(query, *args)
  18
  19                 if res:
  20                         return Repository(self.backend, res.id, data=res)
  21
  22         def _get_repositories(self, query, *args):
  23                 res = self.db.query(query, *args)
  24
  25                 for row in res:
  26                         yield Repository(self.backend, row.id, data=row)
  27
  28         def __iter__(self):
  29                 repositories = self._get_repositories("SELECT * FROM repositories \
  30                         WHERE deleted IS FALSE ORDER BY distro_id, name")
  31
  32                 return iter(repositories)
  33
  34         def create(self, distro, name, description):
  35                 return self._get_repository("INSERT INTO repositories(distro_id, name, description) \
  36                         VALUES(%s, %s, %s) RETURNING *", distro.id, name, description)
  37
  38         def get_by_id(self, repo_id):
  39                 return self._get_repository("SELECT * FROM repositories \
  40                         WHERE id = %s", repo_id)
  41
  42         def get_history(self, limit=None, offset=None, build=None, repo=None, user=None):
  43                 query = "SELECT * FROM repositories_history"
  44                 args  = []
  45
  46                 query += " ORDER BY time DESC"
  47
  48                 if limit:
  49                         if offset:
  50                                 query += " LIMIT %s,%s"
  51                                 args  += [offset, limit,]
  52                         else:
  53                                 query += " LIMIT %s"
  54                                 args  += [limit,]
  55
  56                 entries = []
  57                 for entry in self.db.query(query, *args):
  58                         entry = logs.RepositoryLogEntry(self.pakfire, entry)
  59                         entries.append(entry)
  60
  61                 return entries
  62
  63         def remaster(self):
  64                 """
  65                         Remasters all repositories
  66                 """
  67                 for repo in self:
  68                         # Skip all repositories that don't need an update
  69                         if not repo.needs_update:
  70                                 log.debug("Repository %s does not need an update" % repo)
  71                                 continue
  72
  73                         with self.db.transaction():
  74                                 repo.remaster()
  75
  76
  77 class Repository(base.DataObject):
  78         table = "repositories"
  79
  80         def __eq__(self, other):
  81                 if isinstance(other, self.__class__):
  82                         return self.id == other.id
  83
  84         def __lt__(self, other):
  85                 if isinstance(other, self.__class__):
  86                         return self.parent_id == other.id
  87
  88         def __iter__(self):
  89                 builds = self.backend.builds._get_builds("SELECT builds.* FROM repositories_builds \
  90                         LEFT JOIN builds ON repositories_builds.build_id = builds.id \
  91                         WHERE repositories_builds.repo_id = %s", self.id)
  92
  93                 return iter(builds)
  94
  95         def __len__(self):
  96                 res = self.db.get("SELECT COUNT(*) AS len FROM repositories_builds \
  97                         WHERE repo_id = %s", self.id)
  98
  99                 return res.len
 100
 101         @lazy_property
 102         def next(self):
 103                 return self.backend.repos._get_repository("SELECT * FROM repositories \
 104                         WHERE parent_id = %s", self.id)
 105
 106         @lazy_property
 107         def parent(self):
 108                 if self.data.parent_id:
 109                         return self.backend.repos._get_repository("SELECT * FROM repositories \
 110                                 WHERE id = %s", self.data.parent_id)
 111
 112         @lazy_property
 113         def distro(self):
 114                 return self.backend.distros.get_by_id(self.data.distro_id)
 115
 116         def set_priority(self, priority):
 117                 self._set_attribute("priority", priority)
 118
 119         priority = property(lambda s: s.data.priority, set_priority)
 120
 121         def get_user(self):
 122                 if self.data.user_id:
 123                         return self.backend.users.get_by_id(self.data.user_id)
 124
 125         def set_user(self, user):
 126                 self._set_attribute("user_id", user.id)
 127
 128         user = property(get_user, set_user)
 129
 130         @property
 131         def info(self):
 132                 return {
 133                         "id"     : self.id,
 134                         "distro" : self.distro.info,
 135                         "name"   : self.name,
 136                         "arches" : self.arches,
 137                 }
 138
 139         @property
 140         def url(self):
 141                 url = os.path.join(
 142                         self.settings.get("repository_baseurl", "http://pakfire.ipfire.org/repositories/"),
 143                         self.distro.identifier,
 144                         self.identifier,
 145                         "%{arch}"
 146                 )
 147
 148                 return url
 149
 150         @property
 151         def mirrorlist(self):
 152                 url = os.path.join(
 153                         self.settings.get("mirrorlist_baseurl", "https://pakfire.ipfire.org/"),
 154                         "distro", self.distro.identifier,
 155                         "repo", self.identifier,
 156                         "mirrorlist?arch=%{arch}"
 157                 )
 158
 159                 return url
 160
 161         def get_conf(self):
 162                 lines = [
 163                         "[repo:%s]" % self.identifier,
 164                         "description = %s - %s" % (self.distro.name, self.summary),
 165                         "enabled = 1",
 166                         "baseurl = %s" % self.url,
 167                 ]
 168
 169                 if self.mirrored:
 170                         lines.append("mirrors = %s" % self.mirrorlist)
 171
 172                 if self.priority:
 173                         lines.append("priority = %s" % self.priority)
 174
 175                 return "\n".join(lines)
 176
 177         @property
 178         def name(self):
 179                 return self.data.name
 180
 181         @property
 182         def identifier(self):
 183                 return self.name.lower()
 184
 185         @property
 186         def type(self):
 187                 return self.data.type
 188
 189         @property
 190         def summary(self):
 191                 lines = self.description.splitlines()
 192
 193                 if lines:
 194                         return lines[0]
 195
 196                 return "N/A"
 197
 198         @property
 199         def description(self):
 200                 return self.data.description or ""
 201
 202         @property
 203         def parent_id(self):
 204                 return self.data.parent_id
 205
 206         @lazy_property
 207         def key(self):
 208                 if not self.data.key_id:
 209                         return
 210
 211                 return self.pakfire.keys.get_by_id(self.data.key_id)
 212
 213         @property
 214         def arches(self):
 215                 return self.distro.arches + ["src"]
 216
 217         def set_mirrored(self, mirrored):
 218                 self._set_attribute("mirrored", mirrored)
 219
 220         mirrored = property(lambda s: s.data.mirrored, set_mirrored)
 221
 222         def set_enabled_for_builds(self, state):
 223                 self._set_attribute("enabled_for_builds", state)
 224
 225         enabled_for_builds = property(lambda s: s.data.enabled_for_builds, set_enabled_for_builds)
 226
 227         @property
 228         def score_needed(self):
 229                 return self.data.score_needed
 230
 231         @property
 232         def time_min(self):
 233                 return self.data.time_min
 234
 235         @property
 236         def time_max(self):
 237                 return self.data.time_max
 238
 239         def _log_build(self, action, build, from_repo=None, to_repo=None, user=None):
 240                 user_id = None
 241                 if user:
 242                         user_id = user.id
 243
 244                 from_repo_id = None
 245                 if from_repo:
 246                         from_repo_id = from_repo.id
 247
 248                 to_repo_id = None
 249                 if to_repo:
 250                         to_repo_id = to_repo.id
 251
 252                 self.db.execute("INSERT INTO repositories_history(action, build_id, from_repo_id, to_repo_id, user_id, time) \
 253                         VALUES(%s, %s, %s, %s, %s, NOW())", action, build.id, from_repo_id, to_repo_id, user_id)
 254
 255         def add_build(self, build, user=None, log=True):
 256                 self.db.execute("INSERT INTO repositories_builds(repo_id, build_id, time_added)"
 257                         " VALUES(%s, %s, NOW())", self.id, build.id)
 258
 259                 # Update bug status.
 260                 build._update_bugs_helper(self)
 261
 262                 if log:
 263                         self._log_build("added", build, to_repo=self, user=user)
 264
 265         def rem_build(self, build, user=None, log=True):
 266                 self.db.execute("DELETE FROM repositories_builds \
 267                         WHERE repo_id = %s AND build_id = %s", self.id, build.id)
 268
 269                 if log:
 270                         self._log_build("removed", build, from_repo=self, user=user)
 271
 272         def move_build(self, build, to_repo, user=None, log=True):
 273                 self.db.execute("UPDATE repositories_builds SET repo_id = %s, time_added = NOW() \
 274                         WHERE repo_id = %s AND build_id = %s", to_repo.id, self.id, build.id)
 275
 276                 # Update bug status.
 277                 build._update_bugs_helper(to_repo)
 278
 279                 if log:
 280                         self._log_build("moved", build, from_repo=self, to_repo=to_repo,
 281                                 user=user)
 282
 283         def get_builds(self, limit=None, offset=None):
 284                 query = "SELECT build_id AS id FROM repositories_builds \
 285                         WHERE repo_id = %s ORDER BY time_added DESC"
 286                 args  = [self.id,]
 287
 288                 if limit:
 289                         if offset:
 290                                 query += " LIMIT %s,%s"
 291                                 args  += [offset, limit,]
 292                         else:
 293                                 query += " LIMIT %s"
 294                                 args  += [limit,]
 295
 296                 _builds = []
 297                 for build in self.db.query(query, *args):
 298                         build = self.pakfire.builds.get_by_id(build.id)
 299                         build._repo = self
 300
 301                         _builds.append(build)
 302
 303                 return _builds
 304
 305         def _get_packages(self, arch):
 306                 if arch.name == "src":
 307                         pkgs = self.db.query("SELECT packages.id AS id, packages.path AS path FROM packages \
 308                                 JOIN builds ON builds.pkg_id = packages.id \
 309                                 JOIN repositories_builds ON builds.id = repositories_builds.build_id \
 310                                 WHERE packages.arch = %s AND repositories_builds.repo_id = %s",
 311                                 arch.name, self.id)
 312
 313                 else:
 314                         pkgs = self.db.query("SELECT packages.id AS id, packages.path AS path FROM packages \
 315                                 JOIN jobs_packages ON jobs_packages.pkg_id = packages.id \
 316                                 JOIN jobs ON jobs_packages.job_id = jobs.id \
 317                                 JOIN builds ON builds.id = jobs.build_id \
 318                                 JOIN repositories_builds ON builds.id = repositories_builds.build_id \
 319                                 WHERE (jobs.arch = %s OR jobs.arch = %s) AND \
 320                                 repositories_builds.repo_id = %s",
 321                                 arch.name, "noarch", self.id)
 322
 323                 return pkgs
 324
 325         def get_packages(self, arch):
 326                 pkgs =  [self.pakfire.packages.get_by_id(p.id) for p in self._get_packages(arch)]
 327                 pkgs.sort()
 328
 329                 return pkgs
 330
 331         def get_paths(self, arch):
 332                 paths = [p.path for p in self._get_packages(arch)]
 333                 paths.sort()
 334
 335                 return paths
 336
 337         @property
 338         def packages(self):
 339                 return self.get_packages()
 340
 341         @property
 342         def unpushed_builds(self):
 343                 return self.backend.builds._get_builds("SELECT builds.* FROM repositories \
 344                         LEFT JOIN repositories_builds ON repositories.id = repositories_builds.repo_id \
 345                         LEFT JOIN builds ON repositories_builds.build_id = builds.id \
 346                         WHERE repositories.id = %s \
 347                                 AND repositories_builds.time_added >= repositories.last_update", self.id)
 348
 349         def get_obsolete_builds(self):
 350                 return self.pakfire.builds.get_obsolete(self)
 351
 352         @property
 353         def needs_update(self):
 354                 if self.unpushed_builds:
 355                         return True
 356
 357                 return False
 358
 359         def updated(self):
 360                 self.db.execute("UPDATE repositories SET last_update = NOW() \
 361                         WHERE id = %s", self.id)
 362
 363         def remaster(self):
 364                 log.info("Going to update repository %s..." % self.name)
 365
 366                 # Update the timestamp when we started at last.
 367                 self.updated()
 368
 369                 for arch in self.arches:
 370                         changed = False
 371
 372                         # Get all package paths that are to be included in this repository.
 373                         paths = self.get_paths(arch)
 374
 375                         repo_path = os.path.join(
 376                                 REPOS_DIR,
 377                                 self.distro.identifier,
 378                                 self.identifier,
 379                                 arch
 380                         )
 381
 382                         if not os.path.exists(repo_path):
 383                                 os.makedirs(repo_path)
 384
 385                         source_files = []
 386                         remove_files = []
 387
 388                         for filename in os.listdir(repo_path):
 389                                 path = os.path.join(repo_path, filename)
 390
 391                                 if not os.path.isfile(path):
 392                                         continue
 393
 394                                 remove_files.append(path)
 395
 396                         for path in paths:
 397                                 filename = os.path.basename(path)
 398
 399                                 source_file = os.path.join(PACKAGES_DIR, path)
 400                                 target_file = os.path.join(repo_path, filename)
 401
 402                                 # Do not add duplicate files twice.
 403                                 if source_file in source_files:
 404                                         continue
 405
 406                                 source_files.append(source_file)
 407
 408                                 try:
 409                                         remove_files.remove(target_file)
 410                                 except ValueError:
 411                                         changed = True
 412
 413                         if remove_files:
 414                                 changed = True
 415
 416                         # If nothing in the repository data has changed, there
 417                         # is nothing to do.
 418                         if changed:
 419                                 log.info("The repository has updates...")
 420                         else:
 421                                 log.info("Nothing to update.")
 422                                 continue
 423
 424                         # Find the key to sign the package.
 425                         key_id = None
 426                         if repo.key:
 427                                 key_id = self.key.fingerprint
 428
 429                         # Create package index.
 430                         p = pakfire.PakfireServer(arch=arch)
 431
 432                         p.repo_create(repo_path, source_files,
 433                                 name="%s - %s.%s" % (self.distro.name, self.name, arch),
 434                                 key_id=key_id)
 435
 436                         # Remove files afterwards.
 437                         for file in remove_files:
 438                                 file = os.path.join(repo_path, file)
 439
 440                                 try:
 441                                         os.remove(file)
 442                                 except OSError:
 443                                         log.warning("Could not remove %s." % file)
 444
 445         def get_history(self, **kwargs):
 446                 kwargs.update({
 447                         "repo" : self,
 448                 })
 449
 450                 return self.pakfire.repos.get_history(**kwargs)
 451
 452         def get_build_times(self):
 453                 times = []
 454                 for arch in self.arches:
 455                         time = self.db.get("SELECT SUM(jobs.time_finished - jobs.time_started) AS time FROM jobs \
 456                                 JOIN builds ON builds.id = jobs.build_id \
 457                                 JOIN repositories_builds ON builds.id = repositories_builds.build_id \
 458                                 WHERE (jobs.arch = %s OR jobs.arch = %s) AND \
 459                                 jobs.type = 'build' AND \
 460                                 repositories_builds.repo_id = %s", arch, "noarch", self.id)
 461
 462                         times.append((arch, time.time.total_seconds()))
 463
 464                 return times
 465
 466
 467 class RepositoryAux(base.DataObject):
 468         table = "repositories_aux"
 469
 470         @property
 471         def name(self):
 472                 return self.data.name
 473
 474         @property
 475         def description(self):
 476                 return self.data.description or ""
 477
 478         @property
 479         def url(self):
 480                 return self.data.url
 481
 482         @property
 483         def identifier(self):
 484                 return self.name.lower()
 485
 486         @property
 487         def distro(self):
 488                 return self.pakfire.distros.get_by_id(self.data.distro_id)
 489
 490         def get_conf(self):
 491                 lines = [
 492                         "[repo:%s]" % self.identifier,
 493                         "description = %s - %s" % (self.distro.name, self.name),
 494                         "enabled = 1",
 495                         "baseurl = %s" % self.url,
 496                         "priority = 0",
 497                 ]
 498
 499                 return "\n".join(lines)