src/buildservice/repository.py

   1 #!/usr/bin/python
   2
   3 import logging
   4 import os.path
   5
   6 log = logging.getLogger("repositories")
   7 log.propagate = 1
   8
   9 from . import base
  10 from . import logs
  11
  12 from .constants import *
  13 from .decorators import *
  14
  15 class Repositories(base.Object):
  16         def _get_repository(self, query, *args):
  17                 res = self.db.get(query, *args)
  18
  19                 if res:
  20                         return Repository(self.backend, res.id, data=res)
  21
  22         def _get_repositories(self, query, *args):
  23                 res = self.db.query(query, *args)
  24
  25                 for row in res:
  26                         yield Repository(self.backend, row.id, data=row)
  27
  28         def __iter__(self):
  29                 repositories = self._get_repositories("SELECT * FROM repositories \
  30                         WHERE deleted IS FALSE ORDER BY distro_id, name")
  31
  32                 return iter(repositories)
  33
  34         def create(self, distro, name, description):
  35                 return self._get_repository("INSERT INTO repositories(distro_id, name, description) \
  36                         VALUES(%s, %s, %s) RETURNING *", distro.id, name, description)
  37
  38         def get_by_id(self, repo_id):
  39                 return self._get_repository("SELECT * FROM repositories \
  40                         WHERE id = %s", repo_id)
  41
  42         def get_history(self, limit=None, offset=None, build=None, repo=None, user=None):
  43                 query = "SELECT * FROM repositories_history"
  44                 args  = []
  45
  46                 query += " ORDER BY time DESC"
  47
  48                 if limit:
  49                         if offset:
  50                                 query += " LIMIT %s,%s"
  51                                 args  += [offset, limit,]
  52                         else:
  53                                 query += " LIMIT %s"
  54                                 args  += [limit,]
  55
  56                 entries = []
  57                 for entry in self.db.query(query, *args):
  58                         entry = logs.RepositoryLogEntry(self.pakfire, entry)
  59                         entries.append(entry)
  60
  61                 return entries
  62
  63         def remaster(self):
  64                 """
  65                         Remasters all repositories
  66                 """
  67                 for repo in self:
  68                         # Skip all repositories that don't need an update
  69                         if not repo.needs_update:
  70                                 log.debug("Repository %s does not need an update" % repo)
  71                                 continue
  72
  73                         with self.db.transaction():
  74                                 repo.remaster()
  75
  76
  77 class Repository(base.DataObject):
  78         table = "repositories"
  79
  80         def __eq__(self, other):
  81                 if isinstance(other, self.__class__):
  82                         return self.id == other.id
  83
  84         def __lt__(self, other):
  85                 if isinstance(other, self.__class__):
  86                         return self.parent_id == other.id
  87
  88         def __iter__(self):
  89                 builds = self.backend.builds._get_builds("SELECT builds.* FROM repositories_builds \
  90                         LEFT JOIN builds ON repositories_builds.build_id = builds.id \
  91                         WHERE repositories_builds.repo_id = %s", self.id)
  92
  93                 return iter(builds)
  94
  95         def __len__(self):
  96                 res = self.db.get("SELECT COUNT(*) AS len FROM repositories_builds \
  97                         WHERE repo_id = %s", self.id)
  98
  99                 return res.len
 100
 101         @lazy_property
 102         def next(self):
 103                 return self.backend.repos._get_repository("SELECT * FROM repositories \
 104                         WHERE parent_id = %s", self.id)
 105
 106         @lazy_property
 107         def parent(self):
 108                 if self.data.parent_id:
 109                         return self.backend.repos._get_repository("SELECT * FROM repositories \
 110                                 WHERE id = %s", self.data.parent_id)
 111
 112         @lazy_property
 113         def distro(self):
 114                 return self.backend.distros.get_by_id(self.data.distro_id)
 115
 116         @property
 117         def info(self):
 118                 return {
 119                         "id"     : self.id,
 120                         "distro" : self.distro.info,
 121                         "name"   : self.name,
 122                         "arches" : self.arches,
 123                 }
 124
 125         @property
 126         def url(self):
 127                 url = os.path.join(
 128                         self.settings.get("repository_baseurl", "http://pakfire.ipfire.org/repositories/"),
 129                         self.distro.identifier,
 130                         self.identifier,
 131                         "%{arch}"
 132                 )
 133
 134                 return url
 135
 136         @property
 137         def mirrorlist(self):
 138                 url = os.path.join(
 139                         self.settings.get("mirrorlist_baseurl", "https://pakfire.ipfire.org/"),
 140                         "distro", self.distro.identifier,
 141                         "repo", self.identifier,
 142                         "mirrorlist?arch=%{arch}"
 143                 )
 144
 145                 return url
 146
 147         def get_conf(self):
 148                 prioritymap = {
 149                         "stable"   : 500,
 150                         "unstable" : 200,
 151                         "testing"  : 100,
 152                 }
 153
 154                 try:
 155                         priority = prioritymap[self.type]
 156                 except KeyError:
 157                         priority = None
 158
 159                 lines = [
 160                         "[repo:%s]" % self.identifier,
 161                         "description = %s - %s" % (self.distro.name, self.summary),
 162                         "enabled = 1",
 163                         "baseurl = %s" % self.url,
 164                         "mirrors = %s" % self.mirrorlist,
 165                 ]
 166
 167                 if priority:
 168                         lines.append("priority = %s" % priority)
 169
 170                 return "\n".join(lines)
 171
 172         @property
 173         def name(self):
 174                 return self.data.name
 175
 176         @property
 177         def identifier(self):
 178                 return self.name.lower()
 179
 180         @property
 181         def type(self):
 182                 return self.data.type
 183
 184         @property
 185         def summary(self):
 186                 lines = self.description.splitlines()
 187
 188                 if lines:
 189                         return lines[0]
 190
 191                 return "N/A"
 192
 193         @property
 194         def description(self):
 195                 return self.data.description or ""
 196
 197         @property
 198         def parent_id(self):
 199                 return self.data.parent_id
 200
 201         @lazy_property
 202         def key(self):
 203                 if not self.data.key_id:
 204                         return
 205
 206                 return self.pakfire.keys.get_by_id(self.data.key_id)
 207
 208         @property
 209         def arches(self):
 210                 return self.distro.arches + ["src"]
 211
 212         @property
 213         def mirrored(self):
 214                 return self.data.mirrored
 215
 216         def set_enabled_for_builds(self, state):
 217                 self._set_attribute("enabled_for_builds", state)
 218
 219         enabled_for_builds = property(lambda s: s.data.enabled_for_builds, set_enabled_for_builds)
 220
 221         @property
 222         def score_needed(self):
 223                 return self.data.score_needed
 224
 225         @property
 226         def time_min(self):
 227                 return self.data.time_min
 228
 229         @property
 230         def time_max(self):
 231                 return self.data.time_max
 232
 233         def _log_build(self, action, build, from_repo=None, to_repo=None, user=None):
 234                 user_id = None
 235                 if user:
 236                         user_id = user.id
 237
 238                 from_repo_id = None
 239                 if from_repo:
 240                         from_repo_id = from_repo.id
 241
 242                 to_repo_id = None
 243                 if to_repo:
 244                         to_repo_id = to_repo.id
 245
 246                 self.db.execute("INSERT INTO repositories_history(action, build_id, from_repo_id, to_repo_id, user_id, time) \
 247                         VALUES(%s, %s, %s, %s, %s, NOW())", action, build.id, from_repo_id, to_repo_id, user_id)
 248
 249         def add_build(self, build, user=None, log=True):
 250                 self.db.execute("INSERT INTO repositories_builds(repo_id, build_id, time_added)"
 251                         " VALUES(%s, %s, NOW())", self.id, build.id)
 252
 253                 # Update bug status.
 254                 build._update_bugs_helper(self)
 255
 256                 if log:
 257                         self._log_build("added", build, to_repo=self, user=user)
 258
 259         def rem_build(self, build, user=None, log=True):
 260                 self.db.execute("DELETE FROM repositories_builds \
 261                         WHERE repo_id = %s AND build_id = %s", self.id, build.id)
 262
 263                 if log:
 264                         self._log_build("removed", build, from_repo=self, user=user)
 265
 266         def move_build(self, build, to_repo, user=None, log=True):
 267                 self.db.execute("UPDATE repositories_builds SET repo_id = %s, time_added = NOW() \
 268                         WHERE repo_id = %s AND build_id = %s", to_repo.id, self.id, build.id)
 269
 270                 # Update bug status.
 271                 build._update_bugs_helper(to_repo)
 272
 273                 if log:
 274                         self._log_build("moved", build, from_repo=self, to_repo=to_repo,
 275                                 user=user)
 276
 277         def get_builds(self, limit=None, offset=None):
 278                 query = "SELECT build_id AS id FROM repositories_builds \
 279                         WHERE repo_id = %s ORDER BY time_added DESC"
 280                 args  = [self.id,]
 281
 282                 if limit:
 283                         if offset:
 284                                 query += " LIMIT %s,%s"
 285                                 args  += [offset, limit,]
 286                         else:
 287                                 query += " LIMIT %s"
 288                                 args  += [limit,]
 289
 290                 _builds = []
 291                 for build in self.db.query(query, *args):
 292                         build = self.pakfire.builds.get_by_id(build.id)
 293                         build._repo = self
 294
 295                         _builds.append(build)
 296
 297                 return _builds
 298
 299         def _get_packages(self, arch):
 300                 if arch.name == "src":
 301                         pkgs = self.db.query("SELECT packages.id AS id, packages.path AS path FROM packages \
 302                                 JOIN builds ON builds.pkg_id = packages.id \
 303                                 JOIN repositories_builds ON builds.id = repositories_builds.build_id \
 304                                 WHERE packages.arch = %s AND repositories_builds.repo_id = %s",
 305                                 arch.name, self.id)
 306
 307                 else:
 308                         pkgs = self.db.query("SELECT packages.id AS id, packages.path AS path FROM packages \
 309                                 JOIN jobs_packages ON jobs_packages.pkg_id = packages.id \
 310                                 JOIN jobs ON jobs_packages.job_id = jobs.id \
 311                                 JOIN builds ON builds.id = jobs.build_id \
 312                                 JOIN repositories_builds ON builds.id = repositories_builds.build_id \
 313                                 WHERE (jobs.arch = %s OR jobs.arch = %s) AND \
 314                                 repositories_builds.repo_id = %s",
 315                                 arch.name, "noarch", self.id)
 316
 317                 return pkgs
 318
 319         def get_packages(self, arch):
 320                 pkgs =  [self.pakfire.packages.get_by_id(p.id) for p in self._get_packages(arch)]
 321                 pkgs.sort()
 322
 323                 return pkgs
 324
 325         def get_paths(self, arch):
 326                 paths = [p.path for p in self._get_packages(arch)]
 327                 paths.sort()
 328
 329                 return paths
 330
 331         @property
 332         def packages(self):
 333                 return self.get_packages()
 334
 335         @property
 336         def unpushed_builds(self):
 337                 return self.backend.builds._get_builds("SELECT builds.* FROM repositories \
 338                         LEFT JOIN repositories_builds ON repositories.id = repositories_builds.repo_id \
 339                         LEFT JOIN builds ON repositories_builds.build_id = builds.id \
 340                         WHERE repositories.id = %s \
 341                                 AND repositories_builds.time_added >= repositories.last_update", self.id)
 342
 343         def get_obsolete_builds(self):
 344                 return self.pakfire.builds.get_obsolete(self)
 345
 346         @property
 347         def needs_update(self):
 348                 if self.unpushed_builds:
 349                         return True
 350
 351                 return False
 352
 353         def updated(self):
 354                 self.db.execute("UPDATE repositories SET last_update = NOW() \
 355                         WHERE id = %s", self.id)
 356
 357         def remaster(self):
 358                 log.info("Going to update repository %s..." % self.name)
 359
 360                 # Update the timestamp when we started at last.
 361                 self.updated()
 362
 363                 for arch in self.arches:
 364                         changed = False
 365
 366                         # Get all package paths that are to be included in this repository.
 367                         paths = self.get_paths(arch)
 368
 369                         repo_path = os.path.join(
 370                                 REPOS_DIR,
 371                                 self.distro.identifier,
 372                                 self.identifier,
 373                                 arch
 374                         )
 375
 376                         if not os.path.exists(repo_path):
 377                                 os.makedirs(repo_path)
 378
 379                         source_files = []
 380                         remove_files = []
 381
 382                         for filename in os.listdir(repo_path):
 383                                 path = os.path.join(repo_path, filename)
 384
 385                                 if not os.path.isfile(path):
 386                                         continue
 387
 388                                 remove_files.append(path)
 389
 390                         for path in paths:
 391                                 filename = os.path.basename(path)
 392
 393                                 source_file = os.path.join(PACKAGES_DIR, path)
 394                                 target_file = os.path.join(repo_path, filename)
 395
 396                                 # Do not add duplicate files twice.
 397                                 if source_file in source_files:
 398                                         continue
 399
 400                                 source_files.append(source_file)
 401
 402                                 try:
 403                                         remove_files.remove(target_file)
 404                                 except ValueError:
 405                                         changed = True
 406
 407                         if remove_files:
 408                                 changed = True
 409
 410                         # If nothing in the repository data has changed, there
 411                         # is nothing to do.
 412                         if changed:
 413                                 log.info("The repository has updates...")
 414                         else:
 415                                 log.info("Nothing to update.")
 416                                 continue
 417
 418                         # Find the key to sign the package.
 419                         key_id = None
 420                         if repo.key:
 421                                 key_id = self.key.fingerprint
 422
 423                         # Create package index.
 424                         p = pakfire.PakfireServer(arch=arch)
 425
 426                         p.repo_create(repo_path, source_files,
 427                                 name="%s - %s.%s" % (self.distro.name, self.name, arch),
 428                                 key_id=key_id)
 429
 430                         # Remove files afterwards.
 431                         for file in remove_files:
 432                                 file = os.path.join(repo_path, file)
 433
 434                                 try:
 435                                         os.remove(file)
 436                                 except OSError:
 437                                         log.warning("Could not remove %s." % file)
 438
 439         def get_history(self, **kwargs):
 440                 kwargs.update({
 441                         "repo" : self,
 442                 })
 443
 444                 return self.pakfire.repos.get_history(**kwargs)
 445
 446         def get_build_times(self):
 447                 times = []
 448                 for arch in self.arches:
 449                         time = self.db.get("SELECT SUM(jobs.time_finished - jobs.time_started) AS time FROM jobs \
 450                                 JOIN builds ON builds.id = jobs.build_id \
 451                                 JOIN repositories_builds ON builds.id = repositories_builds.build_id \
 452                                 WHERE (jobs.arch = %s OR jobs.arch = %s) AND \
 453                                 jobs.type = 'build' AND \
 454                                 repositories_builds.repo_id = %s", arch, "noarch", self.id)
 455
 456                         times.append((arch, time.time.total_seconds()))
 457
 458                 return times
 459
 460
 461 class RepositoryAux(base.DataObject):
 462         table = "repositories_aux"
 463
 464         @property
 465         def name(self):
 466                 return self.data.name
 467
 468         @property
 469         def description(self):
 470                 return self.data.description or ""
 471
 472         @property
 473         def url(self):
 474                 return self.data.url
 475
 476         @property
 477         def identifier(self):
 478                 return self.name.lower()
 479
 480         @property
 481         def distro(self):
 482                 return self.pakfire.distros.get_by_id(self.data.distro_id)
 483
 484         def get_conf(self):
 485                 lines = [
 486                         "[repo:%s]" % self.identifier,
 487                         "description = %s - %s" % (self.distro.name, self.name),
 488                         "enabled = 1",
 489                         "baseurl = %s" % self.url,
 490                         "priority = 0",
 491                 ]
 492
 493                 return "\n".join(lines)