src/buildservice/sources.py

   1 #!/usr/bin/python
   2
   3 import datetime
   4 import logging
   5 import os
   6 import pakfire
   7 import pakfire.config
   8 import shutil
   9 import subprocess
  10 import tempfile
  11
  12 from . import base
  13 from . import database
  14 from . import git
  15
  16 from .constants import *
  17 from .decorators import *
  18
  19 class Sources(base.Object):
  20         def _get_source(self, query, *args):
  21                 res = self.db.get(query, *args)
  22
  23                 if res:
  24                         return Source(self.backend, res.id, data=res)
  25
  26         def _get_sources(self, query, *args):
  27                 res = self.db.query(query, *args)
  28
  29                 for row in res:
  30                         yield Source(self.backend, row.id, data=row)
  31
  32         def _get_commit(self, query, *args):
  33                 res = self.db.get(query, *args)
  34
  35                 if res:
  36                         return Commit(self.backend, res.id, data=res)
  37
  38         def _get_commits(self, query, *args):
  39                 res = self.db.query(query, *args)
  40
  41                 for row in res:
  42                         yield Commit(self.backend, row.id, data=row)
  43
  44         def __iter__(self):
  45                 return self._get_sources("SELECT * FROM sources")
  46
  47         def get_by_id(self, id):
  48                 return self._get_source("SELECT * FROM sources \
  49                         WHERE id = %s", id)
  50
  51         def get_by_distro(self, distro):
  52                 return self._get_sources("SELECT * FROM sources \
  53                         WHERE distro_id = %s", distro.id)
  54
  55         def update_revision(self, source_id, revision):
  56                 query = "UPDATE sources SET revision = %s WHERE id = %s"
  57
  58                 return self.db.execute(query, revision, source_id)
  59
  60         def get_commit_by_id(self, commit_id):
  61                 commit = self.db.get("SELECT id FROM sources_commits WHERE id = %s", commit_id)
  62
  63                 if commit:
  64                         return Commit(self.pakfire, commit.id)
  65
  66         def pull(self):
  67                 for source in self:
  68                         with git.Repo(self.backend, source, mode="mirror") as repo:
  69                                 # Fetch the latest updates
  70                                 repo.fetch()
  71
  72                                 # Import all new revisions
  73                                 repo.import_revisions()
  74
  75         def dist(self):
  76                 # Walk through all source repositories
  77                 for source in self:
  78                         # Get access to the git repo
  79                         with git.Repo(self.pakfire, source) as repo:
  80                                 # Walk through all pending commits
  81                                 for commit in source.pending_commits:
  82                                         commit.state = "running"
  83
  84                                         logging.debug("Processing commit %s: %s" % (commit.revision, commit.subject))
  85
  86                                         # Navigate to the right revision.
  87                                         repo.checkout(commit.revision)
  88
  89                                         # Get all changed makefiles.
  90                                         deleted_files = []
  91                                         updated_files = []
  92
  93                                         for file in repo.changed_files(commit.revision):
  94                                                 # Don't care about files that are not a makefile.
  95                                                 if not file.endswith(".%s" % MAKEFILE_EXTENSION):
  96                                                         continue
  97
  98                                                 if os.path.exists(file):
  99                                                         updated_files.append(file)
 100                                                 else:
 101                                                         deleted_files.append(file)
 102
 103                                                 if updated_files:
 104                                                         # Create a temporary directory where to put all the files
 105                                                         # that are generated here.
 106                                                         pkg_dir = tempfile.mkdtemp()
 107
 108                                                         try:
 109                                                                 config = pakfire.config.Config(["general.conf",])
 110                                                                 config.parse(source.distro.get_config())
 111
 112                                                                 p = pakfire.PakfireServer(config=config)
 113
 114                                                                 pkgs = []
 115                                                                 for file in updated_files:
 116                                                                         try:
 117                                                                                 pkg_file = p.dist(file, pkg_dir)
 118                                                                                 pkgs.append(pkg_file)
 119                                                                         except:
 120                                                                                 raise
 121
 122                                                                 # Import all packages in one swoop.
 123                                                                 for pkg in pkgs:
 124                                                                         # Import the package file and create a build out of it.
 125                                                                         from . import builds
 126                                                                         builds.import_from_package(_pakfire, pkg,
 127                                                                                 distro=source.distro, commit=commit, type="release")
 128
 129                                                         except:
 130                                                                 if commit:
 131                                                                         commit.state = "failed"
 132
 133                                                                 raise
 134
 135                                                         finally:
 136                                                                 if os.path.exists(pkg_dir):
 137                                                                         shutil.rmtree(pkg_dir)
 138
 139                                                 for file in deleted_files:
 140                                                         # Determine the name of the package.
 141                                                         name = os.path.basename(file)
 142                                                         name = name[:len(MAKEFILE_EXTENSION) + 1]
 143
 144                                                         source.distro.delete_package(name)
 145
 146                                                 if commit:
 147                                                         commit.state = "finished"
 148
 149
 150 class Commit(base.DataObject):
 151         table = "sources_commits"
 152
 153         @property
 154         def revision(self):
 155                 return self.data.revision
 156
 157         @lazy_property
 158         def source(self):
 159                 return self.backend.sources.get_by_id(self.data.source_id)
 160
 161         @property
 162         def distro(self):
 163                 """
 164                         A shortcut to the distribution this commit
 165                         belongs to.
 166                 """
 167                 return self.source.distro
 168
 169         def set_state(self, state):
 170                 self._set_attribute("state", state)
 171
 172         state = property(lambda s: s.data.state, set_state)
 173
 174         @property
 175         def author(self):
 176                 return self.data.author
 177
 178         @property
 179         def committer(self):
 180                 return self.data.committer
 181
 182         @property
 183         def subject(self):
 184                 return self.data.subject.strip()
 185
 186         @property
 187         def message(self):
 188                 return self.data.body.strip()
 189
 190         @property
 191         def message_full(self):
 192                 msg = [self.subject, ""] + self.message.splitlines()
 193
 194                 return "\n".join(msg)
 195
 196         @property
 197         def date(self):
 198                 return self.data.date
 199
 200         @lazy_property
 201         def packages(self):
 202                 return self.backend.packages._get_packages("SELECT * FROM packages \
 203                         WHERE commit_id = %s", self.id)
 204
 205         def reset(self):
 206                 """
 207                         Removes all packages that have been created by this commit and
 208                         resets the state so it will be processed again.
 209                 """
 210                 # Remove all packages and corresponding builds.
 211                 for pkg in self.packages:
 212                         # Check if there is a build associated with the package.
 213                         # If so, the whole build will be deleted.
 214                         if pkg.build:
 215                                 pkg.build.delete()
 216
 217                         else:
 218                                 # Delete the package.
 219                                 pkg.delete()
 220
 221                 # Clear the cache.
 222                 del self.packages
 223
 224                 # Reset the state to 'pending'.
 225                 self.state = "pending"
 226
 227
 228 class Source(base.DataObject):
 229         table = "sources"
 230
 231         def __eq__(self, other):
 232                 return self.id == other.id
 233
 234         def __len__(self):
 235                 ret = self.db.get("SELECT COUNT(*) AS len FROM sources_commits \
 236                         WHERE source_id = %s", self.id)
 237
 238                 return ret.len
 239
 240         def create_commit(self, revision, author, committer, subject, body, date):
 241                 commit = self.backend.sources._get_commit("INSERT INTO sources_commits(source_id, \
 242                         revision, author, committer, subject, body, date) VALUES(%s, %s, %s, %s, %s, %s, %s) \
 243                         RETURNING *", self.id, revision, author, committer, subject, body, date)
 244
 245                 # Commit
 246                 commit.source = self
 247
 248                 return commit
 249
 250         @property
 251         def info(self):
 252                 return {
 253                         "id"         : self.id,
 254                         "name"       : self.name,
 255                         "url"        : self.url,
 256                         "path"       : self.path,
 257                         "targetpath" : self.targetpath,
 258                         "revision"   : self.revision,
 259                         "branch"     : self.branch,
 260                 }
 261
 262         @property
 263         def name(self):
 264                 return self.data.name
 265
 266         @property
 267         def identifier(self):
 268                 return self.data.identifier
 269
 270         @property
 271         def url(self):
 272                 return self.data.url
 273
 274         @property
 275         def gitweb(self):
 276                 return self.data.gitweb
 277
 278         @property
 279         def revision(self):
 280                 return self.data.revision
 281
 282         @property
 283         def branch(self):
 284                 return self.data.branch
 285
 286         @property
 287         def builds(self):
 288                 return self.pakfire.builds.get_by_source(self.id)
 289
 290         @lazy_property
 291         def distro(self):
 292                 return self.pakfire.distros.get_by_id(self.data.distro_id)
 293
 294         @property
 295         def start_revision(self):
 296                 return self.data.revision
 297
 298         @lazy_property
 299         def head_revision(self):
 300                 return self.backend.sources._get_commit("SELECT * FROM sources_commits \
 301                         WHERE source_id = %s ORDER BY id DESC LIMIT 1", self.id)
 302
 303         def get_commits(self, limit=None, offset=None):
 304                 return self.backend.sources._get_commits("SELECT * FROM sources_commits \
 305                         WHERE source_id = %s ORDER BY id DESC LIMIT %s OFFSET %s", limit, offset)
 306
 307         def get_commit(self, revision):
 308                 commit = self.backend.sources._get_commit("SELECT * FROM sources_commits \
 309                         WHERE source_id = %s AND revision = %s", self.id, revision)
 310
 311                 if commit:
 312                         commit.source = self
 313                         return commit
 314
 315         @property
 316         def pending_commits(self):
 317                 return self.backend.sources._get_commits("SELECT * FROM sources_commits \
 318                         WHERE state = %s ORDER BY imported_at", "pending")