src/buildservice/__init__.py

   1 #!/usr/bin/python
   2
   3 import asyncio
   4 import configparser
   5 import logging
   6 import os
   7 import pakfire
   8 import tempfile
   9
  10 from . import aws
  11 from . import bugtracker
  12 from . import builders
  13 from . import builds
  14 from . import cache
  15 from . import database
  16 from . import distribution
  17 from . import jobqueue
  18 from . import jobs
  19 from . import keys
  20 from . import logs
  21 from . import messages
  22 from . import mirrors
  23 from . import packages
  24 from . import repository
  25 from . import settings
  26 from . import sessions
  27 from . import sources
  28 from . import updates
  29 from . import uploads
  30 from . import users
  31
  32 log = logging.getLogger("backend")
  33 log.propagate = 1
  34
  35 # Import version
  36 from .__version__ import VERSION as __version__
  37
  38 from .decorators import *
  39 from .constants import *
  40
  41 class Backend(object):
  42         version = __version__
  43
  44         def __init__(self, config_file=None):
  45                 # Read configuration file.
  46                 self.config = self.read_config(config_file)
  47
  48                 # Global pakfire settings (from database).
  49                 self.settings = settings.Settings(self)
  50
  51                 self.aws         = aws.AWS(self)
  52                 self.builds      = builds.Builds(self)
  53                 self.cache       = cache.Cache(self)
  54                 self.jobs        = jobs.Jobs(self)
  55                 self.builders    = builders.Builders(self)
  56                 self.distros     = distribution.Distributions(self)
  57                 self.jobqueue    = jobqueue.JobQueue(self)
  58                 self.keys        = keys.Keys(self)
  59                 self.messages    = messages.Messages(self)
  60                 self.mirrors     = mirrors.Mirrors(self)
  61                 self.packages    = packages.Packages(self)
  62                 self.repos       = repository.Repositories(self)
  63                 self.sessions    = sessions.Sessions(self)
  64                 self.sources     = sources.Sources(self)
  65                 self.updates     = updates.Updates(self)
  66                 self.uploads     = uploads.Uploads(self)
  67                 self.users       = users.Users(self)
  68
  69                 # Open a connection to bugzilla.
  70                 self.bugzilla    = bugtracker.Bugzilla(self)
  71
  72         @lazy_property
  73         def _environment_configuration(self):
  74                 env = {}
  75
  76                 # Get database configuration
  77                 env["database"] = {
  78                         "name"     : os.environ.get("PBS_DATABASE_NAME"),
  79                         "hostname" : os.environ.get("PBS_DATABASE_HOSTNAME"),
  80                         "user"     : os.environ.get("PBS_DATABASE_USER"),
  81                         "password" : os.environ.get("PBS_DATABASE_PASSWORD"),
  82                 }
  83
  84                 return env
  85
  86         def read_config(self, path):
  87                 c = configparser.SafeConfigParser()
  88
  89                 # Import configuration from environment
  90                 for section in self._environment_configuration:
  91                         c.add_section(section)
  92
  93                         for k in self._environment_configuration[section]:
  94                                 c.set(section, k, self._environment_configuration[section][k] or "")
  95
  96                 # Load default configuration file first
  97                 paths = [
  98                         os.path.join(CONFIGSDIR, "pbs.conf"),
  99                 ]
 100
 101                 if path:
 102                         paths.append(path)
 103
 104                 # Load all configuration files
 105                 for path in paths:
 106                         if os.path.exists(path):
 107                                 log.debug("Loading configuration from %s" % path)
 108                                 c.read(path)
 109                         else:
 110                                 log.error("No such file %s" % path)
 111
 112                 return c
 113
 114         @lazy_property
 115         def db(self):
 116                 try:
 117                         name     = self.config.get("database", "name")
 118                         hostname = self.config.get("database", "hostname")
 119                         user     = self.config.get("database", "user")
 120                         password = self.config.get("database", "password")
 121                 except configparser.Error as e:
 122                         log.error("Error parsing the config: %s" % e.message)
 123
 124                 log.debug("Connecting to database %s @ %s" % (name, hostname))
 125
 126                 return database.Connection(hostname, name, user=user, password=password)
 127
 128         def pakfire(self, config, offline=True, **kwargs):
 129                 """
 130                         Launches a new Pakfire instance with the given configuration
 131                 """
 132                 log.debug("Launching pakfire with configuration:\n%s" % config)
 133
 134                 # Write configuration to file
 135                 t = self._write_tempfile(config)
 136
 137                 # Launch a new Pakfire instance
 138                 try:
 139                         return pakfire.Pakfire(conf=t, logger=log.log, offline=offline, **kwargs)
 140
 141                 finally:
 142                         # Delete the configuration file
 143                         os.unlink(t)
 144
 145         def _write_tempfile(self, content):
 146                 """
 147                         Writes the content to a temporary file and returns its path
 148                 """
 149                 t = tempfile.NamedTemporaryFile(delete=False)
 150
 151                 # Write the content
 152                 t.write(content.encode())
 153                 t.close()
 154
 155                 return t.name
 156
 157         async def open(self, path):
 158                 """
 159                         Opens a package and returns the archive
 160                 """
 161                 return await asyncio.to_thread(self._open, path)
 162
 163         def _open(self, path):
 164                 # Create a dummy Pakfire instance
 165                 p = pakfire.Pakfire(offline=True)
 166
 167                 # Open the archive
 168                 return p.open(path)
 169
 170         def delete_file(self, path, not_before=None):
 171                 self.db.execute("INSERT INTO queue_delete(path, not_before) \
 172                         VALUES(%s, %s)", path, not_before)
 173
 174         def cleanup_files(self):
 175                 query = self.db.query("SELECT * FROM queue_delete \
 176                         WHERE (not_before IS NULL OR not_before <= NOW())")
 177
 178                 for row in query:
 179                         if not row.path:
 180                                 continue
 181
 182                         path = row.path
 183
 184                         if not path or not path.startswith("%s/" % PAKFIRE_DIR):
 185                                 log.warning("Cannot delete file outside of the tree")
 186                                 continue
 187
 188                         try:
 189                                 logging.debug("Removing %s..." % path)
 190                                 os.unlink(path)
 191                         except OSError as e:
 192                                 logging.error("Could not remove %s: %s" % (path, e))
 193
 194                         while True:
 195                                 path = os.path.dirname(path)
 196
 197                                 # Stop if we are running outside of the tree.
 198                                 if not path.startswith(PAKFIRE_DIR):
 199                                         break
 200
 201                                 # If the directory is not empty, we cannot remove it.
 202                                 if os.path.exists(path) and os.listdir(path):
 203                                         break
 204
 205                                 try:
 206                                         logging.debug("Removing %s..." % path)
 207                                         os.rmdir(path)
 208                                 except OSError as e:
 209                                         logging.error("Could not remove %s: %s" % (path, e))
 210                                         break
 211
 212                         self.db.execute("DELETE FROM queue_delete WHERE id = %s", row.id)