src/buildservice/repository.py

   1 #!/usr/bin/python
   2
   3 import logging
   4 import os.path
   5
   6 log = logging.getLogger("repositories")
   7 log.propagate = 1
   8
   9 from . import base
  10 from . import logs
  11
  12 from .constants import *
  13 from .decorators import *
  14
  15 class Repositories(base.Object):
  16         def _get_repository(self, query, *args):
  17                 res = self.db.get(query, *args)
  18
  19                 if res:
  20                         return Repository(self.backend, res.id, data=res)
  21
  22         def _get_repositories(self, query, *args):
  23                 res = self.db.query(query, *args)
  24
  25                 for row in res:
  26                         yield Repository(self.backend, row.id, data=row)
  27
  28         def __iter__(self):
  29                 repositories = self._get_repositories("SELECT * FROM repositories \
  30                         WHERE deleted IS FALSE ORDER BY distro_id, name")
  31
  32                 return iter(repositories)
  33
  34         def create(self, distro, name, description):
  35                 return self._get_repository("INSERT INTO repositories(distro_id, name, description) \
  36                         VALUES(%s, %s, %s) RETURNING *", distro.id, name, description)
  37
  38         def get_by_id(self, repo_id):
  39                 return self._get_repository("SELECT * FROM repositories \
  40                         WHERE id = %s", repo_id)
  41
  42         def get_history(self, limit=None, offset=None, build=None, repo=None, user=None):
  43                 query = "SELECT * FROM repositories_history"
  44                 args  = []
  45
  46                 query += " ORDER BY time DESC"
  47
  48                 if limit:
  49                         if offset:
  50                                 query += " LIMIT %s,%s"
  51                                 args  += [offset, limit,]
  52                         else:
  53                                 query += " LIMIT %s"
  54                                 args  += [limit,]
  55
  56                 entries = []
  57                 for entry in self.db.query(query, *args):
  58                         entry = logs.RepositoryLogEntry(self.pakfire, entry)
  59                         entries.append(entry)
  60
  61                 return entries
  62
  63         def remaster(self):
  64                 """
  65                         Remasters all repositories
  66                 """
  67                 for repo in self:
  68                         # Skip all repositories that don't need an update
  69                         if not repo.needs_update:
  70                                 log.debug("Repository %s does not need an update" % repo)
  71                                 continue
  72
  73                         with self.db.transaction():
  74                                 repo.remaster()
  75
  76
  77 class Repository(base.DataObject):
  78         table = "repositories"
  79
  80         def __eq__(self, other):
  81                 if isinstance(other, self.__class__):
  82                         return self.id == other.id
  83
  84         def __lt__(self, other):
  85                 if isinstance(other, self.__class__):
  86                         return self.parent_id == other.id
  87
  88         def __iter__(self):
  89                 builds = self.backend.builds._get_builds("SELECT builds.* FROM repositories_builds \
  90                         LEFT JOIN builds ON repositories_builds.build_id = builds.id \
  91                         WHERE repositories_builds.repo_id = %s", self.id)
  92
  93                 return iter(builds)
  94
  95         def __len__(self):
  96                 res = self.db.get("SELECT COUNT(*) AS len FROM repositories_builds \
  97                         WHERE repo_id = %s", self.id)
  98
  99                 return res.len
 100
 101         @lazy_property
 102         def next(self):
 103                 return self.backend.repos._get_repository("SELECT * FROM repositories \
 104                         WHERE parent_id = %s", self.id)
 105
 106         @lazy_property
 107         def parent(self):
 108                 if self.data.parent_id:
 109                         return self.backend.repos._get_repository("SELECT * FROM repositories \
 110                                 WHERE id = %s", self.data.parent_id)
 111
 112         @lazy_property
 113         def distro(self):
 114                 return self.backend.distros.get_by_id(self.data.distro_id)
 115
 116         def set_priority(self, priority):
 117                 self._set_attribute("priority", priority)
 118
 119         priority = property(lambda s: s.data.priority, set_priority)
 120
 121         @property
 122         def info(self):
 123                 return {
 124                         "id"     : self.id,
 125                         "distro" : self.distro.info,
 126                         "name"   : self.name,
 127                         "arches" : self.arches,
 128                 }
 129
 130         @property
 131         def url(self):
 132                 url = os.path.join(
 133                         self.settings.get("repository_baseurl", "http://pakfire.ipfire.org/repositories/"),
 134                         self.distro.identifier,
 135                         self.identifier,
 136                         "%{arch}"
 137                 )
 138
 139                 return url
 140
 141         @property
 142         def mirrorlist(self):
 143                 url = os.path.join(
 144                         self.settings.get("mirrorlist_baseurl", "https://pakfire.ipfire.org/"),
 145                         "distro", self.distro.identifier,
 146                         "repo", self.identifier,
 147                         "mirrorlist?arch=%{arch}"
 148                 )
 149
 150                 return url
 151
 152         def get_conf(self):
 153                 lines = [
 154                         "[repo:%s]" % self.identifier,
 155                         "description = %s - %s" % (self.distro.name, self.summary),
 156                         "enabled = 1",
 157                         "baseurl = %s" % self.url,
 158                         "mirrors = %s" % self.mirrorlist,
 159                 ]
 160
 161                 if self.priority:
 162                         lines.append("priority = %s" % self.priority)
 163
 164                 return "\n".join(lines)
 165
 166         @property
 167         def name(self):
 168                 return self.data.name
 169
 170         @property
 171         def identifier(self):
 172                 return self.name.lower()
 173
 174         @property
 175         def type(self):
 176                 return self.data.type
 177
 178         @property
 179         def summary(self):
 180                 lines = self.description.splitlines()
 181
 182                 if lines:
 183                         return lines[0]
 184
 185                 return "N/A"
 186
 187         @property
 188         def description(self):
 189                 return self.data.description or ""
 190
 191         @property
 192         def parent_id(self):
 193                 return self.data.parent_id
 194
 195         @lazy_property
 196         def key(self):
 197                 if not self.data.key_id:
 198                         return
 199
 200                 return self.pakfire.keys.get_by_id(self.data.key_id)
 201
 202         @property
 203         def arches(self):
 204                 return self.distro.arches + ["src"]
 205
 206         @property
 207         def mirrored(self):
 208                 return self.data.mirrored
 209
 210         def set_enabled_for_builds(self, state):
 211                 self._set_attribute("enabled_for_builds", state)
 212
 213         enabled_for_builds = property(lambda s: s.data.enabled_for_builds, set_enabled_for_builds)
 214
 215         @property
 216         def score_needed(self):
 217                 return self.data.score_needed
 218
 219         @property
 220         def time_min(self):
 221                 return self.data.time_min
 222
 223         @property
 224         def time_max(self):
 225                 return self.data.time_max
 226
 227         def _log_build(self, action, build, from_repo=None, to_repo=None, user=None):
 228                 user_id = None
 229                 if user:
 230                         user_id = user.id
 231
 232                 from_repo_id = None
 233                 if from_repo:
 234                         from_repo_id = from_repo.id
 235
 236                 to_repo_id = None
 237                 if to_repo:
 238                         to_repo_id = to_repo.id
 239
 240                 self.db.execute("INSERT INTO repositories_history(action, build_id, from_repo_id, to_repo_id, user_id, time) \
 241                         VALUES(%s, %s, %s, %s, %s, NOW())", action, build.id, from_repo_id, to_repo_id, user_id)
 242
 243         def add_build(self, build, user=None, log=True):
 244                 self.db.execute("INSERT INTO repositories_builds(repo_id, build_id, time_added)"
 245                         " VALUES(%s, %s, NOW())", self.id, build.id)
 246
 247                 # Update bug status.
 248                 build._update_bugs_helper(self)
 249
 250                 if log:
 251                         self._log_build("added", build, to_repo=self, user=user)
 252
 253         def rem_build(self, build, user=None, log=True):
 254                 self.db.execute("DELETE FROM repositories_builds \
 255                         WHERE repo_id = %s AND build_id = %s", self.id, build.id)
 256
 257                 if log:
 258                         self._log_build("removed", build, from_repo=self, user=user)
 259
 260         def move_build(self, build, to_repo, user=None, log=True):
 261                 self.db.execute("UPDATE repositories_builds SET repo_id = %s, time_added = NOW() \
 262                         WHERE repo_id = %s AND build_id = %s", to_repo.id, self.id, build.id)
 263
 264                 # Update bug status.
 265                 build._update_bugs_helper(to_repo)
 266
 267                 if log:
 268                         self._log_build("moved", build, from_repo=self, to_repo=to_repo,
 269                                 user=user)
 270
 271         def get_builds(self, limit=None, offset=None):
 272                 query = "SELECT build_id AS id FROM repositories_builds \
 273                         WHERE repo_id = %s ORDER BY time_added DESC"
 274                 args  = [self.id,]
 275
 276                 if limit:
 277                         if offset:
 278                                 query += " LIMIT %s,%s"
 279                                 args  += [offset, limit,]
 280                         else:
 281                                 query += " LIMIT %s"
 282                                 args  += [limit,]
 283
 284                 _builds = []
 285                 for build in self.db.query(query, *args):
 286                         build = self.pakfire.builds.get_by_id(build.id)
 287                         build._repo = self
 288
 289                         _builds.append(build)
 290
 291                 return _builds
 292
 293         def _get_packages(self, arch):
 294                 if arch.name == "src":
 295                         pkgs = self.db.query("SELECT packages.id AS id, packages.path AS path FROM packages \
 296                                 JOIN builds ON builds.pkg_id = packages.id \
 297                                 JOIN repositories_builds ON builds.id = repositories_builds.build_id \
 298                                 WHERE packages.arch = %s AND repositories_builds.repo_id = %s",
 299                                 arch.name, self.id)
 300
 301                 else:
 302                         pkgs = self.db.query("SELECT packages.id AS id, packages.path AS path FROM packages \
 303                                 JOIN jobs_packages ON jobs_packages.pkg_id = packages.id \
 304                                 JOIN jobs ON jobs_packages.job_id = jobs.id \
 305                                 JOIN builds ON builds.id = jobs.build_id \
 306                                 JOIN repositories_builds ON builds.id = repositories_builds.build_id \
 307                                 WHERE (jobs.arch = %s OR jobs.arch = %s) AND \
 308                                 repositories_builds.repo_id = %s",
 309                                 arch.name, "noarch", self.id)
 310
 311                 return pkgs
 312
 313         def get_packages(self, arch):
 314                 pkgs =  [self.pakfire.packages.get_by_id(p.id) for p in self._get_packages(arch)]
 315                 pkgs.sort()
 316
 317                 return pkgs
 318
 319         def get_paths(self, arch):
 320                 paths = [p.path for p in self._get_packages(arch)]
 321                 paths.sort()
 322
 323                 return paths
 324
 325         @property
 326         def packages(self):
 327                 return self.get_packages()
 328
 329         @property
 330         def unpushed_builds(self):
 331                 return self.backend.builds._get_builds("SELECT builds.* FROM repositories \
 332                         LEFT JOIN repositories_builds ON repositories.id = repositories_builds.repo_id \
 333                         LEFT JOIN builds ON repositories_builds.build_id = builds.id \
 334                         WHERE repositories.id = %s \
 335                                 AND repositories_builds.time_added >= repositories.last_update", self.id)
 336
 337         def get_obsolete_builds(self):
 338                 return self.pakfire.builds.get_obsolete(self)
 339
 340         @property
 341         def needs_update(self):
 342                 if self.unpushed_builds:
 343                         return True
 344
 345                 return False
 346
 347         def updated(self):
 348                 self.db.execute("UPDATE repositories SET last_update = NOW() \
 349                         WHERE id = %s", self.id)
 350
 351         def remaster(self):
 352                 log.info("Going to update repository %s..." % self.name)
 353
 354                 # Update the timestamp when we started at last.
 355                 self.updated()
 356
 357                 for arch in self.arches:
 358                         changed = False
 359
 360                         # Get all package paths that are to be included in this repository.
 361                         paths = self.get_paths(arch)
 362
 363                         repo_path = os.path.join(
 364                                 REPOS_DIR,
 365                                 self.distro.identifier,
 366                                 self.identifier,
 367                                 arch
 368                         )
 369
 370                         if not os.path.exists(repo_path):
 371                                 os.makedirs(repo_path)
 372
 373                         source_files = []
 374                         remove_files = []
 375
 376                         for filename in os.listdir(repo_path):
 377                                 path = os.path.join(repo_path, filename)
 378
 379                                 if not os.path.isfile(path):
 380                                         continue
 381
 382                                 remove_files.append(path)
 383
 384                         for path in paths:
 385                                 filename = os.path.basename(path)
 386
 387                                 source_file = os.path.join(PACKAGES_DIR, path)
 388                                 target_file = os.path.join(repo_path, filename)
 389
 390                                 # Do not add duplicate files twice.
 391                                 if source_file in source_files:
 392                                         continue
 393
 394                                 source_files.append(source_file)
 395
 396                                 try:
 397                                         remove_files.remove(target_file)
 398                                 except ValueError:
 399                                         changed = True
 400
 401                         if remove_files:
 402                                 changed = True
 403
 404                         # If nothing in the repository data has changed, there
 405                         # is nothing to do.
 406                         if changed:
 407                                 log.info("The repository has updates...")
 408                         else:
 409                                 log.info("Nothing to update.")
 410                                 continue
 411
 412                         # Find the key to sign the package.
 413                         key_id = None
 414                         if repo.key:
 415                                 key_id = self.key.fingerprint
 416
 417                         # Create package index.
 418                         p = pakfire.PakfireServer(arch=arch)
 419
 420                         p.repo_create(repo_path, source_files,
 421                                 name="%s - %s.%s" % (self.distro.name, self.name, arch),
 422                                 key_id=key_id)
 423
 424                         # Remove files afterwards.
 425                         for file in remove_files:
 426                                 file = os.path.join(repo_path, file)
 427
 428                                 try:
 429                                         os.remove(file)
 430                                 except OSError:
 431                                         log.warning("Could not remove %s." % file)
 432
 433         def get_history(self, **kwargs):
 434                 kwargs.update({
 435                         "repo" : self,
 436                 })
 437
 438                 return self.pakfire.repos.get_history(**kwargs)
 439
 440         def get_build_times(self):
 441                 times = []
 442                 for arch in self.arches:
 443                         time = self.db.get("SELECT SUM(jobs.time_finished - jobs.time_started) AS time FROM jobs \
 444                                 JOIN builds ON builds.id = jobs.build_id \
 445                                 JOIN repositories_builds ON builds.id = repositories_builds.build_id \
 446                                 WHERE (jobs.arch = %s OR jobs.arch = %s) AND \
 447                                 jobs.type = 'build' AND \
 448                                 repositories_builds.repo_id = %s", arch, "noarch", self.id)
 449
 450                         times.append((arch, time.time.total_seconds()))
 451
 452                 return times
 453
 454
 455 class RepositoryAux(base.DataObject):
 456         table = "repositories_aux"
 457
 458         @property
 459         def name(self):
 460                 return self.data.name
 461
 462         @property
 463         def description(self):
 464                 return self.data.description or ""
 465
 466         @property
 467         def url(self):
 468                 return self.data.url
 469
 470         @property
 471         def identifier(self):
 472                 return self.name.lower()
 473
 474         @property
 475         def distro(self):
 476                 return self.pakfire.distros.get_by_id(self.data.distro_id)
 477
 478         def get_conf(self):
 479                 lines = [
 480                         "[repo:%s]" % self.identifier,
 481                         "description = %s - %s" % (self.distro.name, self.name),
 482                         "enabled = 1",
 483                         "baseurl = %s" % self.url,
 484                         "priority = 0",
 485                 ]
 486
 487                 return "\n".join(lines)