src/buildservice/repository.py

   1 #!/usr/bin/python
   2
   3 import logging
   4 import os.path
   5
   6 log = logging.getLogger("repositories")
   7 log.propagate = 1
   8
   9 from . import base
  10 from . import logs
  11
  12 from .constants import *
  13 from .decorators import *
  14
  15 class Repositories(base.Object):
  16         def _get_repository(self, query, *args):
  17                 res = self.db.get(query, *args)
  18
  19                 if res:
  20                         return Repository(self.backend, res.id, data=res)
  21
  22         def _get_repositories(self, query, *args):
  23                 res = self.db.query(query, *args)
  24
  25                 for row in res:
  26                         yield Repository(self.backend, row.id, data=row)
  27
  28         def __iter__(self):
  29                 repositories = self._get_repositories("SELECT * FROM repositories \
  30                         WHERE deleted IS FALSE ORDER BY distro_id, name")
  31
  32                 return iter(repositories)
  33
  34         def create(self, distro, name, description):
  35                 return self._get_repository("INSERT INTO repositories(distro_id, name, description) \
  36                         VALUES(%s, %s, %s) RETURNING *", distro.id, name, description)
  37
  38         def get_by_id(self, repo_id):
  39                 return self._get_repository("SELECT * FROM repositories \
  40                         WHERE id = %s", repo_id)
  41
  42         def get_history(self, limit=None, offset=None, build=None, repo=None, user=None):
  43                 query = "SELECT * FROM repositories_history"
  44                 args  = []
  45
  46                 query += " ORDER BY time DESC"
  47
  48                 if limit:
  49                         if offset:
  50                                 query += " LIMIT %s,%s"
  51                                 args  += [offset, limit,]
  52                         else:
  53                                 query += " LIMIT %s"
  54                                 args  += [limit,]
  55
  56                 entries = []
  57                 for entry in self.db.query(query, *args):
  58                         entry = logs.RepositoryLogEntry(self.pakfire, entry)
  59                         entries.append(entry)
  60
  61                 return entries
  62
  63         def remaster(self):
  64                 """
  65                         Remasters all repositories
  66                 """
  67                 for repo in self:
  68                         # Skip all repositories that don't need an update
  69                         if not repo.needs_update:
  70                                 log.debug("Repository %s does not need an update" % repo)
  71                                 continue
  72
  73                         with self.db.transaction():
  74                                 repo.remaster()
  75
  76
  77 class Repository(base.DataObject):
  78         table = "repositories"
  79
  80         def __eq__(self, other):
  81                 if isinstance(other, self.__class__):
  82                         return self.id == other.id
  83
  84         def __lt__(self, other):
  85                 if isinstance(other, self.__class__):
  86                         return self.parent_id == other.id
  87
  88         def __iter__(self):
  89                 builds = self.backend.builds._get_builds("SELECT builds.* FROM repositories_builds \
  90                         LEFT JOIN builds ON repositories_builds.build_id = builds.id \
  91                         WHERE repositories_builds.repo_id = %s", self.id)
  92
  93                 return iter(builds)
  94
  95         def __len__(self):
  96                 res = self.db.get("SELECT COUNT(*) AS len FROM repositories_builds \
  97                         WHERE repo_id = %s", self.id)
  98
  99                 return res.len
 100
 101         @lazy_property
 102         def next(self):
 103                 return self.backend.repos._get_repository("SELECT * FROM repositories \
 104                         WHERE parent_id = %s", self.id)
 105
 106         @lazy_property
 107         def parent(self):
 108                 if self.data.parent_id:
 109                         return self.backend.repos._get_repository("SELECT * FROM repositories \
 110                                 WHERE id = %s", self.data.parent_id)
 111
 112         @lazy_property
 113         def distro(self):
 114                 return self.backend.distros.get_by_id(self.data.distro_id)
 115
 116         def set_priority(self, priority):
 117                 self._set_attribute("priority", priority)
 118
 119         priority = property(lambda s: s.data.priority, set_priority)
 120
 121         def get_user(self):
 122                 if self.data.user_id:
 123                         return self.backend.users.get_by_id(self.data.user_id)
 124
 125         def set_user(self, user):
 126                 self._set_attribute("user_id", user.id)
 127
 128         user = property(get_user, set_user)
 129
 130         @property
 131         def info(self):
 132                 return {
 133                         "id"     : self.id,
 134                         "distro" : self.distro.info,
 135                         "name"   : self.name,
 136                         "arches" : self.arches,
 137                 }
 138
 139         @property
 140         def basepath(self):
 141                 return "/".join((
 142                         self.distro.identifier,
 143                         self.identifier,
 144                 ))
 145
 146         @property
 147         def path(self):
 148                 return os.path.join(REPOS_DIR, self.basepath, "%{arch}")
 149
 150         @property
 151         def url(self):
 152                 return os.path.join(
 153                         self.settings.get("baseurl", "https://pakfire.ipfire.org"),
 154                         "repositories",
 155                         self.basepath,
 156                         "%{arch}"
 157                 )
 158
 159         @property
 160         def mirrorlist(self):
 161                 return os.path.join(
 162                         self.settings.get("baseurl", "https://pakfire.ipfire.org"),
 163                         "distro", self.distro.identifier,
 164                         "repo", self.identifier,
 165                         "mirrorlist?arch=%{arch}"
 166                 )
 167
 168         def get_conf(self, local=False):
 169                 lines = [
 170                         "[repo:%s]" % self.identifier,
 171                         "description = %s - %s" % (self.distro.name, self.summary),
 172                         "enabled = 1",
 173                         "baseurl = %s" % (self.path if local else self.url),
 174                 ]
 175
 176                 if self.mirrored and not local:
 177                         lines.append("mirrors = %s" % self.mirrorlist)
 178
 179                 if self.priority:
 180                         lines.append("priority = %s" % self.priority)
 181
 182                 return "\n".join(lines)
 183
 184         @property
 185         def name(self):
 186                 return self.data.name
 187
 188         @property
 189         def identifier(self):
 190                 return self.name.lower()
 191
 192         @property
 193         def type(self):
 194                 return self.data.type
 195
 196         @property
 197         def summary(self):
 198                 lines = self.description.splitlines()
 199
 200                 if lines:
 201                         return lines[0]
 202
 203                 return "N/A"
 204
 205         @property
 206         def description(self):
 207                 return self.data.description or ""
 208
 209         @property
 210         def parent_id(self):
 211                 return self.data.parent_id
 212
 213         @lazy_property
 214         def key(self):
 215                 if not self.data.key_id:
 216                         return
 217
 218                 return self.pakfire.keys.get_by_id(self.data.key_id)
 219
 220         @property
 221         def arches(self):
 222                 return self.distro.arches + ["src"]
 223
 224         def set_mirrored(self, mirrored):
 225                 self._set_attribute("mirrored", mirrored)
 226
 227         mirrored = property(lambda s: s.data.mirrored, set_mirrored)
 228
 229         def set_enabled_for_builds(self, state):
 230                 self._set_attribute("enabled_for_builds", state)
 231
 232         enabled_for_builds = property(lambda s: s.data.enabled_for_builds, set_enabled_for_builds)
 233
 234         @property
 235         def score_needed(self):
 236                 return self.data.score_needed
 237
 238         @property
 239         def time_min(self):
 240                 return self.data.time_min
 241
 242         @property
 243         def time_max(self):
 244                 return self.data.time_max
 245
 246         def _log_build(self, action, build, from_repo=None, to_repo=None, user=None):
 247                 user_id = None
 248                 if user:
 249                         user_id = user.id
 250
 251                 from_repo_id = None
 252                 if from_repo:
 253                         from_repo_id = from_repo.id
 254
 255                 to_repo_id = None
 256                 if to_repo:
 257                         to_repo_id = to_repo.id
 258
 259                 self.db.execute("INSERT INTO repositories_history(action, build_id, from_repo_id, to_repo_id, user_id, time) \
 260                         VALUES(%s, %s, %s, %s, %s, NOW())", action, build.id, from_repo_id, to_repo_id, user_id)
 261
 262         def add_build(self, build, user=None, log=True):
 263                 self.db.execute("INSERT INTO repositories_builds(repo_id, build_id, time_added)"
 264                         " VALUES(%s, %s, NOW())", self.id, build.id)
 265
 266                 # Update bug status.
 267                 build._update_bugs_helper(self)
 268
 269                 if log:
 270                         self._log_build("added", build, to_repo=self, user=user)
 271
 272         def rem_build(self, build, user=None, log=True):
 273                 self.db.execute("DELETE FROM repositories_builds \
 274                         WHERE repo_id = %s AND build_id = %s", self.id, build.id)
 275
 276                 if log:
 277                         self._log_build("removed", build, from_repo=self, user=user)
 278
 279         def move_build(self, build, to_repo, user=None, log=True):
 280                 self.db.execute("UPDATE repositories_builds SET repo_id = %s, time_added = NOW() \
 281                         WHERE repo_id = %s AND build_id = %s", to_repo.id, self.id, build.id)
 282
 283                 # Update bug status.
 284                 build._update_bugs_helper(to_repo)
 285
 286                 if log:
 287                         self._log_build("moved", build, from_repo=self, to_repo=to_repo,
 288                                 user=user)
 289
 290         def get_builds(self, limit=None, offset=None):
 291                 query = "SELECT build_id AS id FROM repositories_builds \
 292                         WHERE repo_id = %s ORDER BY time_added DESC"
 293                 args  = [self.id,]
 294
 295                 if limit:
 296                         if offset:
 297                                 query += " LIMIT %s,%s"
 298                                 args  += [offset, limit,]
 299                         else:
 300                                 query += " LIMIT %s"
 301                                 args  += [limit,]
 302
 303                 _builds = []
 304                 for build in self.db.query(query, *args):
 305                         build = self.pakfire.builds.get_by_id(build.id)
 306                         build._repo = self
 307
 308                         _builds.append(build)
 309
 310                 return _builds
 311
 312         def _get_packages(self, arch):
 313                 if arch.name == "src":
 314                         pkgs = self.db.query("SELECT packages.id AS id, packages.path AS path FROM packages \
 315                                 JOIN builds ON builds.pkg_id = packages.id \
 316                                 JOIN repositories_builds ON builds.id = repositories_builds.build_id \
 317                                 WHERE packages.arch = %s AND repositories_builds.repo_id = %s",
 318                                 arch.name, self.id)
 319
 320                 else:
 321                         pkgs = self.db.query("SELECT packages.id AS id, packages.path AS path FROM packages \
 322                                 JOIN jobs_packages ON jobs_packages.pkg_id = packages.id \
 323                                 JOIN jobs ON jobs_packages.job_id = jobs.id \
 324                                 JOIN builds ON builds.id = jobs.build_id \
 325                                 JOIN repositories_builds ON builds.id = repositories_builds.build_id \
 326                                 WHERE (jobs.arch = %s OR jobs.arch = %s) AND \
 327                                 repositories_builds.repo_id = %s",
 328                                 arch.name, "noarch", self.id)
 329
 330                 return pkgs
 331
 332         def get_packages(self, arch):
 333                 pkgs =  [self.pakfire.packages.get_by_id(p.id) for p in self._get_packages(arch)]
 334                 pkgs.sort()
 335
 336                 return pkgs
 337
 338         def get_paths(self, arch):
 339                 paths = [p.path for p in self._get_packages(arch)]
 340                 paths.sort()
 341
 342                 return paths
 343
 344         @property
 345         def packages(self):
 346                 return self.get_packages()
 347
 348         @property
 349         def unpushed_builds(self):
 350                 return self.backend.builds._get_builds("SELECT builds.* FROM repositories \
 351                         LEFT JOIN repositories_builds ON repositories.id = repositories_builds.repo_id \
 352                         LEFT JOIN builds ON repositories_builds.build_id = builds.id \
 353                         WHERE repositories.id = %s \
 354                                 AND repositories_builds.time_added >= repositories.last_update", self.id)
 355
 356         def get_obsolete_builds(self):
 357                 return self.pakfire.builds.get_obsolete(self)
 358
 359         @property
 360         def needs_update(self):
 361                 if self.unpushed_builds:
 362                         return True
 363
 364                 return False
 365
 366         def updated(self):
 367                 self.db.execute("UPDATE repositories SET last_update = NOW() \
 368                         WHERE id = %s", self.id)
 369
 370         def remaster(self):
 371                 log.info("Going to update repository %s..." % self.name)
 372
 373                 # Update the timestamp when we started at last.
 374                 self.updated()
 375
 376                 for arch in self.arches:
 377                         changed = False
 378
 379                         # Get all package paths that are to be included in this repository.
 380                         paths = self.get_paths(arch)
 381
 382                         repo_path = os.path.join(
 383                                 REPOS_DIR,
 384                                 self.distro.identifier,
 385                                 self.identifier,
 386                                 arch
 387                         )
 388
 389                         if not os.path.exists(repo_path):
 390                                 os.makedirs(repo_path)
 391
 392                         source_files = []
 393                         remove_files = []
 394
 395                         for filename in os.listdir(repo_path):
 396                                 path = os.path.join(repo_path, filename)
 397
 398                                 if not os.path.isfile(path):
 399                                         continue
 400
 401                                 remove_files.append(path)
 402
 403                         for path in paths:
 404                                 filename = os.path.basename(path)
 405
 406                                 source_file = os.path.join(PACKAGES_DIR, path)
 407                                 target_file = os.path.join(repo_path, filename)
 408
 409                                 # Do not add duplicate files twice.
 410                                 if source_file in source_files:
 411                                         continue
 412
 413                                 source_files.append(source_file)
 414
 415                                 try:
 416                                         remove_files.remove(target_file)
 417                                 except ValueError:
 418                                         changed = True
 419
 420                         if remove_files:
 421                                 changed = True
 422
 423                         # If nothing in the repository data has changed, there
 424                         # is nothing to do.
 425                         if changed:
 426                                 log.info("The repository has updates...")
 427                         else:
 428                                 log.info("Nothing to update.")
 429                                 continue
 430
 431                         # Find the key to sign the package.
 432                         key_id = None
 433                         if repo.key:
 434                                 key_id = self.key.fingerprint
 435
 436                         # Create package index.
 437                         p = pakfire.PakfireServer(arch=arch)
 438
 439                         p.repo_create(repo_path, source_files,
 440                                 name="%s - %s.%s" % (self.distro.name, self.name, arch),
 441                                 key_id=key_id)
 442
 443                         # Remove files afterwards.
 444                         for file in remove_files:
 445                                 file = os.path.join(repo_path, file)
 446
 447                                 try:
 448                                         os.remove(file)
 449                                 except OSError:
 450                                         log.warning("Could not remove %s." % file)
 451
 452         def get_history(self, **kwargs):
 453                 kwargs.update({
 454                         "repo" : self,
 455                 })
 456
 457                 return self.pakfire.repos.get_history(**kwargs)
 458
 459         def get_build_times(self):
 460                 times = []
 461                 for arch in self.arches:
 462                         time = self.db.get("SELECT SUM(jobs.time_finished - jobs.time_started) AS time FROM jobs \
 463                                 JOIN builds ON builds.id = jobs.build_id \
 464                                 JOIN repositories_builds ON builds.id = repositories_builds.build_id \
 465                                 WHERE (jobs.arch = %s OR jobs.arch = %s) AND \
 466                                 jobs.type = 'build' AND \
 467                                 repositories_builds.repo_id = %s", arch, "noarch", self.id)
 468
 469                         times.append((arch, time.time.total_seconds()))
 470
 471                 return times
 472
 473
 474 class RepositoryAux(base.DataObject):
 475         table = "repositories_aux"
 476
 477         @property
 478         def name(self):
 479                 return self.data.name
 480
 481         @property
 482         def description(self):
 483                 return self.data.description or ""
 484
 485         @property
 486         def url(self):
 487                 return self.data.url
 488
 489         @property
 490         def identifier(self):
 491                 return self.name.lower()
 492
 493         @property
 494         def distro(self):
 495                 return self.pakfire.distros.get_by_id(self.data.distro_id)
 496
 497         def get_conf(self, local=False):
 498                 lines = [
 499                         "[repo:%s]" % self.identifier,
 500                         "description = %s - %s" % (self.distro.name, self.name),
 501                         "enabled = 1",
 502                         "baseurl = %s" % self.url,
 503                         "priority = 0",
 504                 ]
 505
 506                 return "\n".join(lines)