src/buildservice/sources.py

   1 #!/usr/bin/python
   2
   3 import datetime
   4 import logging
   5 import os
   6 import pakfire
   7 import pakfire.config
   8 import re
   9 import shutil
  10 import subprocess
  11 import tempfile
  12
  13 from . import base
  14 from . import git
  15
  16 from .constants import *
  17 from .decorators import *
  18
  19 VALID_TAGS = (
  20         "Acked-by",
  21         "Cc",
  22         "Fixes",
  23         "Reported-by",
  24         "Reviewed-by",
  25         "Signed-off-by",
  26         "Suggested-by",
  27         "Tested-by",
  28 )
  29
  30 class Sources(base.Object):
  31         def _get_source(self, query, *args):
  32                 res = self.db.get(query, *args)
  33
  34                 if res:
  35                         return Source(self.backend, res.id, data=res)
  36
  37         def _get_sources(self, query, *args):
  38                 res = self.db.query(query, *args)
  39
  40                 for row in res:
  41                         yield Source(self.backend, row.id, data=row)
  42
  43         def _get_commit(self, query, *args):
  44                 res = self.db.get(query, *args)
  45
  46                 if res:
  47                         return Commit(self.backend, res.id, data=res)
  48
  49         def _get_commits(self, query, *args):
  50                 res = self.db.query(query, *args)
  51
  52                 for row in res:
  53                         yield Commit(self.backend, row.id, data=row)
  54
  55         def __iter__(self):
  56                 return self._get_sources("SELECT * FROM sources")
  57
  58         def get_by_id(self, id):
  59                 return self._get_source("SELECT * FROM sources \
  60                         WHERE id = %s", id)
  61
  62         def get_by_distro(self, distro):
  63                 return self._get_sources("SELECT * FROM sources \
  64                         WHERE distro_id = %s", distro.id)
  65
  66         def update_revision(self, source_id, revision):
  67                 query = "UPDATE sources SET revision = %s WHERE id = %s"
  68
  69                 return self.db.execute(query, revision, source_id)
  70
  71         def get_commit_by_id(self, commit_id):
  72                 commit = self.db.get("SELECT id FROM sources_commits WHERE id = %s", commit_id)
  73
  74                 if commit:
  75                         return Commit(self.pakfire, commit.id)
  76
  77         def pull(self):
  78                 for source in self:
  79                         with git.Repo(self.backend, source, mode="mirror") as repo:
  80                                 # Fetch the latest updates
  81                                 repo.fetch()
  82
  83                                 # Import all new revisions
  84                                 repo.import_revisions()
  85
  86         def dist(self):
  87                 # Walk through all source repositories
  88                 for source in self:
  89                         # Get access to the git repo
  90                         with git.Repo(self.pakfire, source) as repo:
  91                                 # Walk through all pending commits
  92                                 for commit in source.pending_commits:
  93                                         commit.state = "running"
  94
  95                                         logging.debug("Processing commit %s: %s" % (commit.revision, commit.subject))
  96
  97                                         # Navigate to the right revision.
  98                                         repo.checkout(commit.revision)
  99
 100                                         # Get all changed makefiles.
 101                                         deleted_files = []
 102                                         updated_files = []
 103
 104                                         for file in repo.changed_files(commit.revision):
 105                                                 # Don't care about files that are not a makefile.
 106                                                 if not file.endswith(".%s" % MAKEFILE_EXTENSION):
 107                                                         continue
 108
 109                                                 if os.path.exists(file):
 110                                                         updated_files.append(file)
 111                                                 else:
 112                                                         deleted_files.append(file)
 113
 114                                                 if updated_files:
 115                                                         # Create a temporary directory where to put all the files
 116                                                         # that are generated here.
 117                                                         pkg_dir = tempfile.mkdtemp()
 118
 119                                                         try:
 120                                                                 config = pakfire.config.Config(["general.conf",])
 121                                                                 config.parse(source.distro.get_config())
 122
 123                                                                 p = pakfire.PakfireServer(config=config)
 124
 125                                                                 pkgs = []
 126                                                                 for file in updated_files:
 127                                                                         try:
 128                                                                                 pkg_file = p.dist(file, pkg_dir)
 129                                                                                 pkgs.append(pkg_file)
 130                                                                         except:
 131                                                                                 raise
 132
 133                                                                 # Import all packages in one swoop.
 134                                                                 for pkg in pkgs:
 135                                                                         with self.db.transaction():
 136                                                                                 self.backend.builds.create_from_source_package(pkg,
 137                                                                                         source.distro, commit=commit, type="release")
 138
 139                                                         except:
 140                                                                 if commit:
 141                                                                         commit.state = "failed"
 142
 143                                                                 raise
 144
 145                                                         finally:
 146                                                                 if os.path.exists(pkg_dir):
 147                                                                         shutil.rmtree(pkg_dir)
 148
 149                                                 for file in deleted_files:
 150                                                         # Determine the name of the package.
 151                                                         name = os.path.basename(file)
 152                                                         name = name[:len(MAKEFILE_EXTENSION) + 1]
 153
 154                                                         source.distro.delete_package(name)
 155
 156                                                 if commit:
 157                                                         commit.state = "finished"
 158
 159
 160 class Commit(base.DataObject):
 161         table = "sources_commits"
 162
 163         @property
 164         def revision(self):
 165                 return self.data.revision
 166
 167         @lazy_property
 168         def source(self):
 169                 return self.backend.sources.get_by_id(self.data.source_id)
 170
 171         @property
 172         def distro(self):
 173                 """
 174                         A shortcut to the distribution this commit
 175                         belongs to.
 176                 """
 177                 return self.source.distro
 178
 179         def set_state(self, state):
 180                 self._set_attribute("state", state)
 181
 182         state = property(lambda s: s.data.state, set_state)
 183
 184         @lazy_property
 185         def author(self):
 186                 return self.backend.users.find_maintainer(self.data.author) or self.data.author
 187
 188         @lazy_property
 189         def committer(self):
 190                 return self.backend.users.find_maintainer(self.data.committer) or self.data.committer
 191
 192         @property
 193         def subject(self):
 194                 return self.data.subject.strip()
 195
 196         @property
 197         def body(self):
 198                 return self.data.body.strip()
 199
 200         @lazy_property
 201         def message(self):
 202                 """
 203                         Returns the message without any Git tags
 204                 """
 205                 # Compile regex
 206                 r = re.compile("^(%s):?" % "|".join(VALID_TAGS), re.IGNORECASE)
 207
 208                 message = []
 209                 for line in self.body.splitlines():
 210                         # Find lines that start with a known Git tag
 211                         if r.match(line):
 212                                 continue
 213
 214                         message.append(line)
 215
 216                 # If all lines are empty lines, we send back an empty message
 217                 if all((l == "" for l in message)):
 218                         return
 219
 220                 # We will now break the message into paragraphs
 221                 paragraphs = re.split("\n\n+", "\n".join(message))
 222                 print paragraphs
 223
 224                 message = []
 225                 for paragraph in paragraphs:
 226                         # Remove all line breaks that are not following a colon
 227                         # and where the next line does not start with a star.
 228                         paragraph = re.sub("(?<=\:)\n(?=[\*\s])", " ", paragraph)
 229
 230                         message.append(paragraph)
 231
 232                 return "\n\n".join(message)
 233
 234         @property
 235         def message_full(self):
 236                 message = self.subject
 237
 238                 if self.message:
 239                         message += "\n\n%s" % self.message
 240
 241                 return message
 242
 243         def get_tag(self, tag):
 244                 """
 245                         Returns a list of the values of this Git tag
 246                 """
 247                 if not tag in VALID_TAGS:
 248                         raise ValueError("Unknown tag: %s" % tag)
 249
 250                 # Compile regex
 251                 r = re.compile("^%s:? (.*)$" % tag, re.IGNORECASE)
 252
 253                 values = []
 254                 for line in self.body.splitlines():
 255                         # Skip all empty lines
 256                         if not line:
 257                                 continue
 258
 259                         # Check if line matches the regex
 260                         m = r.match(line)
 261                         if m:
 262                                 values.append(m.group(1))
 263
 264                 return values
 265
 266         @lazy_property
 267         def contributors(self):
 268                 contributors = [
 269                         self.data.author,
 270                         self.data.committer,
 271                 ]
 272
 273                 for tag in ("Acked-by", "Cc", "Reported-by", "Reviewed-by", "Signed-off-by", "Suggested-by", "Tested-by"):
 274                         contributors += self.get_tag(tag)
 275
 276                 # Get all user accounts that we know
 277                 users = self.backend.users.find_maintainers(contributors)
 278
 279                 # Add all email addresses where a user could not be found
 280                 for contributor in contributors[:]:
 281                         for user in users:
 282                                 if user.has_email_address(contributor):
 283                                         try:
 284                                                 contributors.remove(contributor)
 285                                         except:
 286                                                 pass
 287
 288                 return sorted(contributors + users)
 289
 290         @lazy_property
 291         def testers(self):
 292                 users = []
 293
 294                 for tag in ("Acked-by", "Reviewed-by", "Signed-off-by", "Tested-by"):
 295                         users += self.get_tag(tag)
 296
 297                 return self.backend.users.find_maintainers(users)
 298
 299         @property
 300         def fixed_bugs(self):
 301                 """
 302                         Returns a list of all fixed bugs
 303                 """
 304                 return self.get_tag("Fixes")
 305
 306         @property
 307         def date(self):
 308                 return self.data.date
 309
 310         @lazy_property
 311         def packages(self):
 312                 return self.backend.packages._get_packages("SELECT * FROM packages \
 313                         WHERE commit_id = %s", self.id)
 314
 315         def reset(self):
 316                 """
 317                         Removes all packages that have been created by this commit and
 318                         resets the state so it will be processed again.
 319                 """
 320                 # Remove all packages and corresponding builds.
 321                 for pkg in self.packages:
 322                         # Check if there is a build associated with the package.
 323                         # If so, the whole build will be deleted.
 324                         if pkg.build:
 325                                 pkg.build.delete()
 326
 327                         else:
 328                                 # Delete the package.
 329                                 pkg.delete()
 330
 331                 # Clear the cache.
 332                 del self.packages
 333
 334                 # Reset the state to 'pending'.
 335                 self.state = "pending"
 336
 337
 338 class Source(base.DataObject):
 339         table = "sources"
 340
 341         def __eq__(self, other):
 342                 return self.id == other.id
 343
 344         def __len__(self):
 345                 ret = self.db.get("SELECT COUNT(*) AS len FROM sources_commits \
 346                         WHERE source_id = %s", self.id)
 347
 348                 return ret.len
 349
 350         def create_commit(self, revision, author, committer, subject, body, date):
 351                 commit = self.backend.sources._get_commit("INSERT INTO sources_commits(source_id, \
 352                         revision, author, committer, subject, body, date) VALUES(%s, %s, %s, %s, %s, %s, %s) \
 353                         RETURNING *", self.id, revision, author, committer, subject, body, date)
 354
 355                 # Commit
 356                 commit.source = self
 357
 358                 return commit
 359
 360         @property
 361         def info(self):
 362                 return {
 363                         "id"         : self.id,
 364                         "name"       : self.name,
 365                         "url"        : self.url,
 366                         "path"       : self.path,
 367                         "targetpath" : self.targetpath,
 368                         "revision"   : self.revision,
 369                         "branch"     : self.branch,
 370                 }
 371
 372         @property
 373         def name(self):
 374                 return self.data.name
 375
 376         @property
 377         def identifier(self):
 378                 return self.data.identifier
 379
 380         @property
 381         def url(self):
 382                 return self.data.url
 383
 384         @property
 385         def gitweb(self):
 386                 return self.data.gitweb
 387
 388         @property
 389         def revision(self):
 390                 return self.data.revision
 391
 392         @property
 393         def branch(self):
 394                 return self.data.branch
 395
 396         @property
 397         def builds(self):
 398                 return self.pakfire.builds.get_by_source(self.id)
 399
 400         @lazy_property
 401         def distro(self):
 402                 return self.pakfire.distros.get_by_id(self.data.distro_id)
 403
 404         @property
 405         def start_revision(self):
 406                 return self.data.revision
 407
 408         @lazy_property
 409         def head_revision(self):
 410                 return self.backend.sources._get_commit("SELECT * FROM sources_commits \
 411                         WHERE source_id = %s ORDER BY id DESC LIMIT 1", self.id)
 412
 413         def get_commits(self, limit=None, offset=None):
 414                 return self.backend.sources._get_commits("SELECT * FROM sources_commits \
 415                         WHERE source_id = %s ORDER BY id DESC LIMIT %s OFFSET %s", self.id, limit, offset)
 416
 417         def get_commit(self, revision):
 418                 commit = self.backend.sources._get_commit("SELECT * FROM sources_commits \
 419                         WHERE source_id = %s AND revision = %s", self.id, revision)
 420
 421                 if commit:
 422                         commit.source = self
 423                         return commit
 424
 425         @property
 426         def pending_commits(self):
 427                 return self.backend.sources._get_commits("SELECT * FROM sources_commits \
 428                         WHERE state = %s ORDER BY imported_at", "pending")