src/buildservice/sources.py

   1 #!/usr/bin/python
   2
   3 import datetime
   4 import logging
   5 import os
   6 import pakfire
   7 import pakfire.config
   8 import shutil
   9 import subprocess
  10 import tempfile
  11
  12 from . import base
  13 from . import git
  14
  15 from .constants import *
  16 from .decorators import *
  17
  18 class Sources(base.Object):
  19         def _get_source(self, query, *args):
  20                 res = self.db.get(query, *args)
  21
  22                 if res:
  23                         return Source(self.backend, res.id, data=res)
  24
  25         def _get_sources(self, query, *args):
  26                 res = self.db.query(query, *args)
  27
  28                 for row in res:
  29                         yield Source(self.backend, row.id, data=row)
  30
  31         def _get_commit(self, query, *args):
  32                 res = self.db.get(query, *args)
  33
  34                 if res:
  35                         return Commit(self.backend, res.id, data=res)
  36
  37         def _get_commits(self, query, *args):
  38                 res = self.db.query(query, *args)
  39
  40                 for row in res:
  41                         yield Commit(self.backend, row.id, data=row)
  42
  43         def __iter__(self):
  44                 return self._get_sources("SELECT * FROM sources")
  45
  46         def get_by_id(self, id):
  47                 return self._get_source("SELECT * FROM sources \
  48                         WHERE id = %s", id)
  49
  50         def get_by_distro(self, distro):
  51                 return self._get_sources("SELECT * FROM sources \
  52                         WHERE distro_id = %s", distro.id)
  53
  54         def update_revision(self, source_id, revision):
  55                 query = "UPDATE sources SET revision = %s WHERE id = %s"
  56
  57                 return self.db.execute(query, revision, source_id)
  58
  59         def get_commit_by_id(self, commit_id):
  60                 commit = self.db.get("SELECT id FROM sources_commits WHERE id = %s", commit_id)
  61
  62                 if commit:
  63                         return Commit(self.pakfire, commit.id)
  64
  65         def pull(self):
  66                 for source in self:
  67                         with git.Repo(self.backend, source, mode="mirror") as repo:
  68                                 # Fetch the latest updates
  69                                 repo.fetch()
  70
  71                                 # Import all new revisions
  72                                 repo.import_revisions()
  73
  74         def dist(self):
  75                 # Walk through all source repositories
  76                 for source in self:
  77                         # Get access to the git repo
  78                         with git.Repo(self.pakfire, source) as repo:
  79                                 # Walk through all pending commits
  80                                 for commit in source.pending_commits:
  81                                         commit.state = "running"
  82
  83                                         logging.debug("Processing commit %s: %s" % (commit.revision, commit.subject))
  84
  85                                         # Navigate to the right revision.
  86                                         repo.checkout(commit.revision)
  87
  88                                         # Get all changed makefiles.
  89                                         deleted_files = []
  90                                         updated_files = []
  91
  92                                         for file in repo.changed_files(commit.revision):
  93                                                 # Don't care about files that are not a makefile.
  94                                                 if not file.endswith(".%s" % MAKEFILE_EXTENSION):
  95                                                         continue
  96
  97                                                 if os.path.exists(file):
  98                                                         updated_files.append(file)
  99                                                 else:
 100                                                         deleted_files.append(file)
 101
 102                                                 if updated_files:
 103                                                         # Create a temporary directory where to put all the files
 104                                                         # that are generated here.
 105                                                         pkg_dir = tempfile.mkdtemp()
 106
 107                                                         try:
 108                                                                 config = pakfire.config.Config(["general.conf",])
 109                                                                 config.parse(source.distro.get_config())
 110
 111                                                                 p = pakfire.PakfireServer(config=config)
 112
 113                                                                 pkgs = []
 114                                                                 for file in updated_files:
 115                                                                         try:
 116                                                                                 pkg_file = p.dist(file, pkg_dir)
 117                                                                                 pkgs.append(pkg_file)
 118                                                                         except:
 119                                                                                 raise
 120
 121                                                                 # Import all packages in one swoop.
 122                                                                 for pkg in pkgs:
 123                                                                         with self.db.transaction():
 124                                                                                 self.backend.builds.create_from_source_package(pkg,
 125                                                                                         source.distro, commit=commit, type="release")
 126
 127                                                         except:
 128                                                                 if commit:
 129                                                                         commit.state = "failed"
 130
 131                                                                 raise
 132
 133                                                         finally:
 134                                                                 if os.path.exists(pkg_dir):
 135                                                                         shutil.rmtree(pkg_dir)
 136
 137                                                 for file in deleted_files:
 138                                                         # Determine the name of the package.
 139                                                         name = os.path.basename(file)
 140                                                         name = name[:len(MAKEFILE_EXTENSION) + 1]
 141
 142                                                         source.distro.delete_package(name)
 143
 144                                                 if commit:
 145                                                         commit.state = "finished"
 146
 147
 148 class Commit(base.DataObject):
 149         table = "sources_commits"
 150
 151         @property
 152         def revision(self):
 153                 return self.data.revision
 154
 155         @lazy_property
 156         def source(self):
 157                 return self.backend.sources.get_by_id(self.data.source_id)
 158
 159         @property
 160         def distro(self):
 161                 """
 162                         A shortcut to the distribution this commit
 163                         belongs to.
 164                 """
 165                 return self.source.distro
 166
 167         def set_state(self, state):
 168                 self._set_attribute("state", state)
 169
 170         state = property(lambda s: s.data.state, set_state)
 171
 172         @property
 173         def author(self):
 174                 return self.data.author
 175
 176         @property
 177         def committer(self):
 178                 return self.data.committer
 179
 180         @property
 181         def subject(self):
 182                 return self.data.subject.strip()
 183
 184         @property
 185         def message(self):
 186                 return self.data.body.strip()
 187
 188         @property
 189         def message_full(self):
 190                 msg = [self.subject, ""] + self.message.splitlines()
 191
 192                 return "\n".join(msg)
 193
 194         @property
 195         def date(self):
 196                 return self.data.date
 197
 198         @lazy_property
 199         def packages(self):
 200                 return self.backend.packages._get_packages("SELECT * FROM packages \
 201                         WHERE commit_id = %s", self.id)
 202
 203         def reset(self):
 204                 """
 205                         Removes all packages that have been created by this commit and
 206                         resets the state so it will be processed again.
 207                 """
 208                 # Remove all packages and corresponding builds.
 209                 for pkg in self.packages:
 210                         # Check if there is a build associated with the package.
 211                         # If so, the whole build will be deleted.
 212                         if pkg.build:
 213                                 pkg.build.delete()
 214
 215                         else:
 216                                 # Delete the package.
 217                                 pkg.delete()
 218
 219                 # Clear the cache.
 220                 del self.packages
 221
 222                 # Reset the state to 'pending'.
 223                 self.state = "pending"
 224
 225
 226 class Source(base.DataObject):
 227         table = "sources"
 228
 229         def __eq__(self, other):
 230                 return self.id == other.id
 231
 232         def __len__(self):
 233                 ret = self.db.get("SELECT COUNT(*) AS len FROM sources_commits \
 234                         WHERE source_id = %s", self.id)
 235
 236                 return ret.len
 237
 238         def create_commit(self, revision, author, committer, subject, body, date):
 239                 commit = self.backend.sources._get_commit("INSERT INTO sources_commits(source_id, \
 240                         revision, author, committer, subject, body, date) VALUES(%s, %s, %s, %s, %s, %s, %s) \
 241                         RETURNING *", self.id, revision, author, committer, subject, body, date)
 242
 243                 # Commit
 244                 commit.source = self
 245
 246                 return commit
 247
 248         @property
 249         def info(self):
 250                 return {
 251                         "id"         : self.id,
 252                         "name"       : self.name,
 253                         "url"        : self.url,
 254                         "path"       : self.path,
 255                         "targetpath" : self.targetpath,
 256                         "revision"   : self.revision,
 257                         "branch"     : self.branch,
 258                 }
 259
 260         @property
 261         def name(self):
 262                 return self.data.name
 263
 264         @property
 265         def identifier(self):
 266                 return self.data.identifier
 267
 268         @property
 269         def url(self):
 270                 return self.data.url
 271
 272         @property
 273         def gitweb(self):
 274                 return self.data.gitweb
 275
 276         @property
 277         def revision(self):
 278                 return self.data.revision
 279
 280         @property
 281         def branch(self):
 282                 return self.data.branch
 283
 284         @property
 285         def builds(self):
 286                 return self.pakfire.builds.get_by_source(self.id)
 287
 288         @lazy_property
 289         def distro(self):
 290                 return self.pakfire.distros.get_by_id(self.data.distro_id)
 291
 292         @property
 293         def start_revision(self):
 294                 return self.data.revision
 295
 296         @lazy_property
 297         def head_revision(self):
 298                 return self.backend.sources._get_commit("SELECT * FROM sources_commits \
 299                         WHERE source_id = %s ORDER BY id DESC LIMIT 1", self.id)
 300
 301         def get_commits(self, limit=None, offset=None):
 302                 return self.backend.sources._get_commits("SELECT * FROM sources_commits \
 303                         WHERE source_id = %s ORDER BY id DESC LIMIT %s OFFSET %s", limit, offset)
 304
 305         def get_commit(self, revision):
 306                 commit = self.backend.sources._get_commit("SELECT * FROM sources_commits \
 307                         WHERE source_id = %s AND revision = %s", self.id, revision)
 308
 309                 if commit:
 310                         commit.source = self
 311                         return commit
 312
 313         @property
 314         def pending_commits(self):
 315                 return self.backend.sources._get_commits("SELECT * FROM sources_commits \
 316                         WHERE state = %s ORDER BY imported_at", "pending")