src/buildservice/__init__.py

   1 #!/usr/bin/python
   2
   3 import asyncio
   4 import configparser
   5 import logging
   6 import os
   7 import pakfire
   8 import tempfile
   9 import urllib.parse
  10
  11 from . import aws
  12 from . import bugtracker
  13 from . import builders
  14 from . import builds
  15 from . import cache
  16 from . import database
  17 from . import distribution
  18 from . import events
  19 from . import jobqueue
  20 from . import jobs
  21 from . import keys
  22 from . import logs
  23 from . import messages
  24 from . import mirrors
  25 from . import packages
  26 from . import repository
  27 from . import settings
  28 from . import sessions
  29 from . import sources
  30 from . import updates
  31 from . import uploads
  32 from . import users
  33
  34 log = logging.getLogger("backend")
  35 log.propagate = 1
  36
  37 # Import version
  38 from .__version__ import VERSION as __version__
  39
  40 from .decorators import *
  41 from .constants import *
  42
  43 class Backend(object):
  44         version = __version__
  45
  46         def __init__(self, config_file=None):
  47                 # Read configuration file.
  48                 self.config = self.read_config(config_file)
  49
  50                 # Global pakfire settings (from database).
  51                 self.settings = settings.Settings(self)
  52
  53                 self.aws         = aws.AWS(self)
  54                 self.builds      = builds.Builds(self)
  55                 self.cache       = cache.Cache(self)
  56                 self.jobs        = jobs.Jobs(self)
  57                 self.builders    = builders.Builders(self)
  58                 self.distros     = distribution.Distributions(self)
  59                 self.events      = events.Events(self)
  60                 self.jobqueue    = jobqueue.JobQueue(self)
  61                 self.keys        = keys.Keys(self)
  62                 self.messages    = messages.Messages(self)
  63                 self.mirrors     = mirrors.Mirrors(self)
  64                 self.packages    = packages.Packages(self)
  65                 self.repos       = repository.Repositories(self)
  66                 self.sessions    = sessions.Sessions(self)
  67                 self.sources     = sources.Sources(self)
  68                 self.updates     = updates.Updates(self)
  69                 self.uploads     = uploads.Uploads(self)
  70                 self.users       = users.Users(self)
  71
  72                 # Open a connection to bugzilla.
  73                 self.bugzilla    = bugtracker.Bugzilla(self)
  74
  75         @lazy_property
  76         def _environment_configuration(self):
  77                 env = {}
  78
  79                 # Get database configuration
  80                 env["database"] = {
  81                         "name"     : os.environ.get("PBS_DATABASE_NAME"),
  82                         "hostname" : os.environ.get("PBS_DATABASE_HOSTNAME"),
  83                         "user"     : os.environ.get("PBS_DATABASE_USER"),
  84                         "password" : os.environ.get("PBS_DATABASE_PASSWORD"),
  85                 }
  86
  87                 return env
  88
  89         def read_config(self, path):
  90                 c = configparser.SafeConfigParser()
  91
  92                 # Import configuration from environment
  93                 for section in self._environment_configuration:
  94                         c.add_section(section)
  95
  96                         for k in self._environment_configuration[section]:
  97                                 c.set(section, k, self._environment_configuration[section][k] or "")
  98
  99                 # Load default configuration file first
 100                 paths = [
 101                         os.path.join(CONFIGSDIR, "pbs.conf"),
 102                 ]
 103
 104                 if path:
 105                         paths.append(path)
 106
 107                 # Load all configuration files
 108                 for path in paths:
 109                         if os.path.exists(path):
 110                                 log.debug("Loading configuration from %s" % path)
 111                                 c.read(path)
 112                         else:
 113                                 log.error("No such file %s" % path)
 114
 115                 return c
 116
 117         @lazy_property
 118         def db(self):
 119                 try:
 120                         name     = self.config.get("database", "name")
 121                         hostname = self.config.get("database", "hostname")
 122                         user     = self.config.get("database", "user")
 123                         password = self.config.get("database", "password")
 124                 except configparser.Error as e:
 125                         log.error("Error parsing the config: %s" % e.message)
 126
 127                 log.debug("Connecting to database %s @ %s" % (name, hostname))
 128
 129                 return database.Connection(hostname, name, user=user, password=password)
 130
 131         def path_to_url(self, path):
 132                 """
 133                         Takes a path to a file on the file system and converts it into a URL
 134                 """
 135                 # The base URL
 136                 baseurl = self.settings.get("baseurl")
 137
 138                 # Path to package
 139                 path = os.path.join(
 140                         "files", os.path.relpath(path, PAKFIRE_DIR),
 141                 )
 142
 143                 # Join it all together
 144                 return urllib.parse.urljoin(baseurl, path)
 145
 146         def pakfire(self, config, offline=True, **kwargs):
 147                 """
 148                         Launches a new Pakfire instance with the given configuration
 149                 """
 150                 log.debug("Launching pakfire with configuration:\n%s" % config)
 151
 152                 # Write configuration to file
 153                 t = self._write_tempfile(config)
 154
 155                 # Launch a new Pakfire instance
 156                 try:
 157                         return pakfire.Pakfire(conf=t, logger=log.log, offline=offline, **kwargs)
 158
 159                 finally:
 160                         # Delete the configuration file
 161                         os.unlink(t)
 162
 163         # Commands
 164
 165         async def command(self, *args, krb5_auth=False, **kwargs):
 166                 """
 167                         Runs this shell command
 168                 """
 169                 # Authenticate using Kerberos
 170                 if krb5_auth:
 171                         await self.krb5_auth()
 172
 173                 log.debug("Running command: %s" % " ".join(args))
 174
 175                 # Fork child process
 176                 process = await asyncio.create_subprocess_exec(
 177                         *args,
 178                         stdin=asyncio.subprocess.DEVNULL,
 179                         stdout=asyncio.subprocess.PIPE,
 180                         stderr=asyncio.subprocess.STDOUT,
 181                         **kwargs,
 182                 )
 183
 184                 # Fetch output of command and send it to the logger
 185                 while True:
 186                         line = await process.stdout.readline()
 187                         if not line:
 188                                 break
 189
 190                         # Decode line
 191                         line = line.decode()
 192
 193                         # Strip newline
 194                         line = line.rstrip()
 195
 196                         log.info(line)
 197
 198                 # Wait until the process has finished
 199                 await process.wait()
 200
 201         async def krb5_auth(self):
 202                 log.debug("Performing Kerberos authentication...")
 203
 204                 # Fetch path to keytab
 205                 keytab = self.settings.get("krb5-keytab")
 206                 if not keytab:
 207                         log.warning("No keytab configured")
 208                         return
 209
 210                 # Fetch Kerberos principal
 211                 principal = self.settings.get("krb5-principal")
 212                 if not principal:
 213                         log.warning("No Kerberos principal configured")
 214                         return
 215
 216                 # Fetch a Kerberos ticket
 217                 await self.command(
 218                         "kinit", "-k", "-t", keytab, principal,
 219                 )
 220
 221         async def copy(self, src, dst):
 222                 """
 223                         Copies a file from src to dst
 224                 """
 225                 log.debug("Copying %s to %s" % (src, dst))
 226
 227                 path = os.path.dirname(dst)
 228
 229                 # Create destination path (if it does not exist)
 230                 try:
 231                         await asyncio.to_thread(os.makedirs, path)
 232                 except FileExistsError:
 233                         pass
 234
 235                 # Copy data without any metadata
 236                 await asyncio.to_thread(shutil.copyfile, src, dst)
 237
 238         def _write_tempfile(self, content):
 239                 """
 240                         Writes the content to a temporary file and returns its path
 241                 """
 242                 t = tempfile.NamedTemporaryFile(delete=False)
 243
 244                 # Write the content
 245                 t.write(content.encode())
 246                 t.close()
 247
 248                 return t.name
 249
 250         async def open(self, path):
 251                 """
 252                         Opens a package and returns the archive
 253                 """
 254                 return await asyncio.to_thread(self._open, path)
 255
 256         def _open(self, path):
 257                 # Create a dummy Pakfire instance
 258                 p = pakfire.Pakfire(offline=True)
 259
 260                 # Open the archive
 261                 return p.open(path)
 262
 263         def delete_file(self, path, not_before=None):
 264                 self.db.execute("INSERT INTO queue_delete(path, not_before) \
 265                         VALUES(%s, %s)", path, not_before)
 266
 267         def cleanup_files(self):
 268                 query = self.db.query("SELECT * FROM queue_delete \
 269                         WHERE (not_before IS NULL OR not_before <= NOW())")
 270
 271                 for row in query:
 272                         if not row.path:
 273                                 continue
 274
 275                         path = row.path
 276
 277                         if not path or not path.startswith("%s/" % PAKFIRE_DIR):
 278                                 log.warning("Cannot delete file outside of the tree")
 279                                 continue
 280
 281                         try:
 282                                 logging.debug("Removing %s..." % path)
 283                                 os.unlink(path)
 284                         except OSError as e:
 285                                 logging.error("Could not remove %s: %s" % (path, e))
 286
 287                         while True:
 288                                 path = os.path.dirname(path)
 289
 290                                 # Stop if we are running outside of the tree.
 291                                 if not path.startswith(PAKFIRE_DIR):
 292                                         break
 293
 294                                 # If the directory is not empty, we cannot remove it.
 295                                 if os.path.exists(path) and os.listdir(path):
 296                                         break
 297
 298                                 try:
 299                                         logging.debug("Removing %s..." % path)
 300                                         os.rmdir(path)
 301                                 except OSError as e:
 302                                         logging.error("Could not remove %s: %s" % (path, e))
 303                                         break
 304
 305                         self.db.execute("DELETE FROM queue_delete WHERE id = %s", row.id)