pakfire/server/master.py

   1 #!/usr/bin/python
   2
   3 import logging
   4 import os
   5 import random
   6 import shutil
   7 import subprocess
   8 import xmlrpclib
   9
  10 import pakfire
  11 import pakfire.base
  12
  13 import pakfire.packages as packages
  14 import pakfire.repository as repository
  15 import pakfire.util as util
  16 from pakfire.constants import *
  17
  18 class Source(object):
  19         def __init__(self, master, id, name, path, targetpath, revision, branch):
  20                 self.master = master
  21                 self.id = id
  22                 self.name = name
  23                 self.path = path
  24                 self.targetpath = targetpath
  25                 self.revision = revision
  26                 self.branch = branch
  27
  28         @property
  29         def pakfire(self):
  30                 return self.master.pakfire
  31
  32         def _git(self, cmd):
  33                 cmd = "cd %s; git %s" % (self.path, cmd)
  34
  35                 logging.debug("Running command: %s" % cmd)
  36
  37                 return subprocess.check_output(["/bin/sh", "-c", cmd])
  38
  39         def _git_rev_list(self, revision=None):
  40                 if not revision:
  41                         revision = self.revision
  42
  43                 command = "rev-list %s..origin/%s" % (revision, self.branch)
  44
  45                 # Get all normal commits.
  46                 commits = self._git("%s --no-merges" % command)
  47                 commits = commits.splitlines()
  48
  49                 revisions = []
  50                 for commit in self._git(command).splitlines():
  51                         # Check if commit is a normal commit or merge commit.
  52                         merge = not commit in commits
  53
  54                         revisions.append((commit, merge))
  55
  56                 return reversed(revisions)
  57
  58         def _git_changed_files(self, revision1, revision2=""):
  59                 files = self._git("diff --name-only %s %s" % (revision1, revision2))
  60
  61                 return [os.path.join(self.path, f) for f in files.splitlines()]
  62
  63         def _git_checkout_revision(self, revision):
  64                 self._git("checkout %s" % revision)
  65
  66         def update_revision(self, (revision, merge)):
  67                 if not merge:
  68                         self._git_checkout_revision(revision)
  69
  70                         # Get list of all changes files between the current revision and
  71                         # the previous one.
  72                         files = self._git_changed_files("HEAD^", "HEAD")
  73
  74                         self.update_files([f for f in files if f.endswith(".%s" % MAKEFILE_EXTENSION)])
  75
  76                 # Send update to the server.
  77                 self.master.update_revision(self, revision)
  78
  79         def update_files(self, files):
  80                 rnd = random.randint(0, 1024**2)
  81                 tmpdir = "/tmp/pakfire-source-%s" % rnd
  82
  83                 pkgs = []
  84                 for file in files:
  85                         if os.path.exists(file):
  86                                 pkgs.append(file)
  87                         else:
  88                                 pkg_name = os.path.basename(os.path.dirname(file))
  89
  90                                 # Send deleted package to server.
  91                                 self.master.package_remove(self, pkg_name)
  92
  93                 if not pkgs:
  94                         return
  95
  96                 # XXX This totally ignores the local configuration.
  97                 for pkg in pkgs:
  98                         pakfire.dist(pkg, resultdirs=[tmpdir,])
  99
 100                 # Create a kind of dummy repository to link the packages against it.
 101                 repo = repository.LocalSourceRepository(self.pakfire,
 102                         "source-%s" % rnd, "Source packages", tmpdir, idx="directory")
 103                 repo.update(force=True)
 104
 105                 for pkg in repo.get_all():
 106                         logging.debug("Processing package: %s" % pkg)
 107
 108                         pkg_path = "%(name)s/%(epoch)s-%(version)s-%(release)s/%(arch)s" % pkg.info
 109
 110                         file = os.path.join(self.targetpath, pkg_path, os.path.basename(pkg.filename))
 111                         dir  = os.path.dirname(file)
 112
 113                         print file
 114
 115                         if os.path.exists(file):
 116                                 logging.warning("Package does already exist: %s" % file)
 117
 118                         else:
 119                                 if not os.path.exists(dir):
 120                                         os.makedirs(dir)
 121
 122                                 # Copy the source file to the designated data pool.
 123                                 shutil.copy2(pkg.filename, file)
 124
 125                         # Register package in database and get an ID.
 126                         pkg_id = self.master.package_add(self, pkg)
 127
 128                         # Re-read the package metadata (mainly update filenames).
 129                         pkg = packages.SourcePackage(self.pakfire, repo, file)
 130
 131                         self.master.package_file_add(self, pkg_id, pkg)
 132
 133                 util.rm(tmpdir)
 134
 135         def update(self):
 136                 # Update files from server.
 137                 self._git("fetch")
 138
 139                 # If there has been no data, yet we need to import all packages
 140                 # that are currently checked out.
 141                 if not self.revision:
 142                         self.update_all()
 143
 144                 for rev in self._git_rev_list():
 145                         self.update_revision(rev)
 146
 147         def update_all(self):
 148                 _files = []
 149                 for dir, subdirs, files in os.walk(self.path):
 150                         for f in files:
 151                                 if not f.endswith(".%s" % MAKEFILE_EXTENSION):
 152                                         continue
 153
 154                                 _files.append(os.path.join(dir, f))
 155
 156                 self.update_files(_files)
 157
 158
 159 class Master(object):
 160         def __init__(self, **pakfire_args):
 161                 self.pakfire = pakfire.base.Pakfire(**pakfire_args)
 162
 163                 server = self.pakfire.config._master.get("server")
 164
 165                 logging.info("Establishing RPC connection to: %s" % server)
 166
 167                 self.conn = xmlrpclib.Server(server)
 168
 169         def update_sources(self):
 170                 sources = self.conn.sources_get_all()
 171
 172                 for source in sources:
 173                         source = Source(self, **source)
 174
 175                         source.update()
 176
 177         def update_revision(self, source, revision):
 178                 self.conn.sources_update_revision(source.id, revision)
 179
 180         def package_add(self, source, pkg):
 181                 logging.info("Adding package: %s" % pkg.friendly_name)
 182
 183                 # Collect data that is sent to the database...
 184                 info = {
 185                         "name"             : pkg.name,
 186                         "epoch"            : pkg.epoch,
 187                         "version"          : pkg.version,
 188                         "release"          : pkg.release,
 189                         "groups"           : " ".join(pkg.groups),
 190                         "maintainer"       : pkg.maintainer,
 191                         "license"          : pkg.license,
 192                         "url"              : pkg.url,
 193                         "summary"          : pkg.summary,
 194                         "description"      : pkg.description,
 195                         "supported_arches" : pkg.supported_arches,
 196                         "source_id"        : source.id,
 197                 }
 198
 199                 return self.conn.package_add(info)
 200
 201         def package_file_add(self, source, pkg_id, pkg):
 202                 logging.info("Adding package file: %s" % pkg.filename)
 203
 204                 info = {
 205                         "path"        : pkg.filename[len(source.path) + 1:],
 206                         "source_id"   : source.id,
 207                         "type"        : pkg.type,
 208                         "arch"        : pkg.arch,
 209                         "summary"     : pkg.summary,
 210                         "description" : pkg.description,
 211                         "requires"    : " ".join(pkg.requires),
 212                         "provides"    : "",
 213                         "obsoletes"   : "",
 214                         "conflicts"   : "",
 215                         "url"         : pkg.url,
 216                         "license"     : pkg.license,
 217                         "maintainer"  : pkg.maintainer,
 218                         "size"        : pkg.size,
 219                         "hash1"       : pkg.hash1,
 220                         "build_host"  : pkg.build_host,
 221                         "build_id"    : pkg.build_id,
 222                         "build_time"  : pkg.build_time,
 223                         "uuid"        : pkg.uuid,
 224                 }
 225
 226                 if isinstance(pkg, packages.BinaryPackage):
 227                         info.update({
 228                                 "provides"    : " ".join(pkg.provides),
 229                                 "obsoletes"   : " ".join(pkg.obsoletes),
 230                                 "conflicts"   : " ".join(pkg.conflicts),
 231                         })
 232
 233                 return self.conn.package_file_add(pkg_id, info)
 234
 235         def package_remove(self, source, pkg):
 236                 logging.info("Package '%s' has been removed." % pkg)
 237