python/pakfire/packages/make.py

   1 #!/usr/bin/python
   2 ###############################################################################
   3 #                                                                             #
   4 # Pakfire - The IPFire package management system                              #
   5 # Copyright (C) 2011 Pakfire development team                                 #
   6 #                                                                             #
   7 # This program is free software: you can redistribute it and/or modify        #
   8 # it under the terms of the GNU General Public License as published by        #
   9 # the Free Software Foundation, either version 3 of the License, or           #
  10 # (at your option) any later version.                                         #
  11 #                                                                             #
  12 # This program is distributed in the hope that it will be useful,             #
  13 # but WITHOUT ANY WARRANTY; without even the implied warranty of              #
  14 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the               #
  15 # GNU General Public License for more details.                                #
  16 #                                                                             #
  17 # You should have received a copy of the GNU General Public License           #
  18 # along with this program.  If not, see <http://www.gnu.org/licenses/>.       #
  19 #                                                                             #
  20 ###############################################################################
  21
  22 import os
  23 import re
  24 import shutil
  25 import socket
  26 import tarfile
  27 import uuid
  28
  29 from urlgrabber.grabber import URLGrabber, URLGrabError
  30 from urlgrabber.progress import TextMeter
  31
  32 import lexer
  33 import packager
  34
  35 import logging
  36 log = logging.getLogger("pakfire")
  37
  38 import pakfire.chroot as chroot
  39 import pakfire.downloader as downloader
  40 import pakfire.util as util
  41
  42 from base import Package
  43 from file import SourcePackage
  44
  45 from pakfire.constants import *
  46 from pakfire.i18n import _
  47
  48 class MakefileBase(Package):
  49         def __init__(self, pakfire, filename):
  50                 Package.__init__(self, pakfire)
  51
  52                 # Save the filename of the makefile.
  53                 self.filename = os.path.abspath(filename)
  54
  55                 # Update environment.
  56                 environ = self.pakfire.environ
  57                 environ.update({
  58                         "BASEDIR"          : os.path.dirname(self.filename),
  59                         "PARALLELISMFLAGS" : "-j%s" % util.calc_parallelism(),
  60                 })
  61
  62                 # Open and parse the makefile.
  63                 self.lexer = lexer.RootLexer.open(self.filename, environ=environ)
  64
  65         @property
  66         def package_filename(self):
  67                 return PACKAGE_FILENAME_FMT % {
  68                         "arch"    : self.arch,
  69                         "ext"     : PACKAGE_EXTENSION,
  70                         "name"    : self.name,
  71                         "release" : self.release,
  72                         "version" : self.version,
  73                 }
  74
  75         def lint(self):
  76                 errors = []
  77
  78                 if not self.name:
  79                         errors.append(_("Package name is undefined."))
  80
  81                 if not self.version:
  82                         errors.append(_("Package version is undefined."))
  83
  84                 # XXX to do...
  85
  86                 return errors
  87
  88         @property
  89         def name(self):
  90                 return self.lexer.get_var("name")
  91
  92         @property
  93         def epoch(self):
  94                 epoch = self.lexer.get_var("epoch")
  95                 if not epoch:
  96                         return 0
  97
  98                 return int(epoch)
  99
 100         @property
 101         def version(self):
 102                 return self.lexer.get_var("version")
 103
 104         @property
 105         def release(self):
 106                 release = self.lexer.get_var("release")
 107                 assert release
 108
 109                 tag = self.lexer.get_var("DISTRO_DISTTAG")
 110                 assert tag
 111
 112                 return ".".join((release, tag))
 113
 114         @property
 115         def summary(self):
 116                 return self.lexer.get_var("summary")
 117
 118         @property
 119         def description(self):
 120                 description = self.lexer.get_var("description")
 121
 122                 # Replace all backslashes at the end of a line.
 123                 return description.replace("\\\n", "\n")
 124
 125         @property
 126         def groups(self):
 127                 groups = self.lexer.get_var("groups").split()
 128
 129                 return sorted(groups)
 130
 131         @property
 132         def url(self):
 133                 return self.lexer.get_var("url")
 134
 135         @property
 136         def license(self):
 137                 return self.lexer.get_var("license")
 138
 139         @property
 140         def maintainer(self):
 141                 maintainer = self.lexer.get_var("maintainer")
 142
 143                 if not maintainer:
 144                         maintainer = self.lexer.get_var("DISTRO_MAINTAINER")
 145
 146                 return maintainer
 147
 148         @property
 149         def vendor(self):
 150                 return self.lexer.get_var("DISTRO_VENDOR")
 151
 152         @property
 153         def buildroot(self):
 154                 return self.lexer.get_var("BUILDROOT")
 155
 156         @property
 157         def sourcedir(self):
 158                 return self.lexer.get_var("DIR_SRC")
 159
 160         @property
 161         def build_host(self):
 162                 return socket.gethostname()
 163
 164         # XXX build_id and build_time are used to create a source package
 165
 166         @property
 167         def build_id(self):
 168                 # XXX todo
 169                 # Not existant for Makefiles
 170                 return None
 171
 172         @property
 173         def build_time(self):
 174                 # XXX todo
 175                 # Not existant for Makefiles
 176                 return None
 177
 178         @property
 179         def supported_arches(self):
 180                 """
 181                         These are the supported arches. Which means, packages of these
 182                         architectures can be built out of this source package.
 183                 """
 184                 # If the package architecture is "noarch", the package
 185                 # needs only to be built for that.
 186                 if self.arch == "noarch":
 187                         return "noarch"
 188
 189                 return self.lexer.get_var("sup_arches", "all")
 190
 191
 192 class Makefile(MakefileBase):
 193         @property
 194         def uuid(self):
 195                 hash1 = util.calc_hash1(self.filename)
 196
 197                 # Return UUID version 5 (SHA1 hash)
 198                 return "%8s-%4s-5%3s-%4s-%11s" % \
 199                         (hash1[0:8], hash1[9:13], hash1[14:17], hash1[18:22], hash1[23:34])
 200
 201         @property
 202         def path(self):
 203                 return os.path.dirname(self.filename)
 204
 205         @property
 206         def arch(self):
 207                 return "src"
 208
 209         @property
 210         def packages(self):
 211                 pkgs = []
 212
 213                 for lexer in self.lexer.packages:
 214                         name = lexer.get_var("_name")
 215
 216                         pkg = MakefilePackage(self.pakfire, name, lexer)
 217                         pkgs.append(pkg)
 218
 219                 return pkgs
 220
 221         @property
 222         def source_dl(self):
 223                 dls = []
 224
 225                 if self.pakfire.distro.source_dl:
 226                         dls.append(self.pakfire.distro.source_dl)
 227
 228                 dl = self.lexer.get_var("source_dl")
 229                 if dl:
 230                         dls.append(dl)
 231
 232                 return dls
 233
 234         def download(self):
 235                 """
 236                         Download all external sources and return a list with the local
 237                         copies.
 238                 """
 239                 # Download source files.
 240                 grabber = downloader.SourceDownloader(self.pakfire,
 241                         mirrors=self.source_dl)
 242
 243                 return grabber.download(self.sources)
 244
 245         def dist(self, resultdirs):
 246                 """
 247                         Create a source package.
 248                 """
 249                 # Download all files we need for this package.
 250                 self.download()
 251
 252                 p = packager.SourcePackager(self.pakfire, self)
 253                 p.run(resultdirs)
 254
 255         def dump(self, *args, **kwargs):
 256                 dump = MakefileBase.dump(self, *args, **kwargs)
 257                 dump = dump.splitlines()
 258
 259                 #dump += ["", _("Containing the following binary packages:"),]
 260                 #
 261                 #for pkg in self.packages:
 262                 #       _dump = pkg.dump(*args, **kwargs)
 263                 #
 264                 #       for line in _dump.splitlines():
 265                 #               dump.append("  %s" % line)
 266                 #       dump.append("")
 267
 268                 return "\n".join(dump)
 269
 270         def get_buildscript(self, stage):
 271                 return self.lexer.build.get_var("_%s" % stage)
 272
 273         @property
 274         def prerequires(self):
 275                 return []
 276
 277         @property
 278         def requires(self):
 279                 reqs = []
 280
 281                 for line in self.lexer.build.get_var("requires", "").splitlines():
 282                         reqs += [r.strip() for r in line.split(",")]
 283
 284                 return self.filter_deps(reqs)
 285
 286         @property
 287         def provides(self):
 288                 return []
 289
 290         @property
 291         def obsoletes(self):
 292                 return []
 293
 294         @property
 295         def conflicts(self):
 296                 return []
 297
 298         @property
 299         def files(self):
 300                 files = []
 301                 basedir = os.path.dirname(self.filename)
 302
 303                 for dirs, subdirs, _files in os.walk(basedir):
 304                         for f in _files:
 305                                 files.append(os.path.join(dirs, f))
 306
 307                 return files
 308
 309         @property
 310         def sources(self):
 311                 return self.lexer.get_var("sources").split()
 312
 313         @property
 314         def exports(self):
 315                 exports = {}
 316
 317                 # Include quality agent exports.
 318                 exports.update(self.lexer.quality_agent.exports)
 319
 320                 for export in self.lexer.build.exports:
 321                         exports[export] = self.lexer.build.get_var(export)
 322
 323                 return exports
 324
 325         def extract(self, message=None, prefix=None):
 326                 # XXX neeed to make this waaaaaaaaaay better.
 327
 328                 files = self.files
 329
 330                 # Load progressbar.
 331                 pb = None
 332                 if message:
 333                         message = "%-10s : %s" % (message, self.friendly_name)
 334                         pb = util.make_progress(message, len(files), eta=False)
 335
 336                 dir_len = len(os.path.dirname(self.filename))
 337
 338                 # Copy all files that belong to the package
 339                 i = 0
 340                 for f in files:
 341                         if pb:
 342                                 i += 1
 343                                 pb.update(i)
 344
 345                         _f = f[dir_len:]
 346                         log.debug("%s/%s" % (prefix, _f))
 347
 348                         path = "%s/%s" % (prefix, _f)
 349
 350                         path_dir = os.path.dirname(path)
 351                         if not os.path.exists(path_dir):
 352                                 os.makedirs(path_dir)
 353
 354                         shutil.copy2(f, path)
 355
 356                 if pb:
 357                         pb.finish()
 358
 359                 # Download source files.
 360                 for _filename in self.download():
 361                         filename = "%s/files/%s" % (prefix, os.path.basename(_filename))
 362                         dirname = os.path.dirname(filename)
 363
 364                         if not os.path.exists(dirname):
 365                                 os.makedirs(dirname)
 366
 367                         shutil.copy2(_filename, filename)
 368
 369
 370 class MakefilePackage(MakefileBase):
 371         def __init__(self, pakfire, name, lexer):
 372                 Package.__init__(self, pakfire)
 373
 374                 self._name = name
 375                 self.lexer = lexer
 376
 377                 # Store additional dependencies in here.
 378                 self._dependencies = {}
 379
 380                 # Generate a random identifier.
 381                 self._uuid = "%s" % uuid.uuid4()
 382
 383         @property
 384         def name(self):
 385                 return self._name
 386
 387         @property
 388         def arch(self):
 389                 return self.lexer.get_var("arch", "%{DISTRO_ARCH}")
 390
 391         @property
 392         def configfiles(self):
 393                 return self.lexer.get_var("configfiles").split()
 394
 395         @property
 396         def files(self):
 397                 return self.lexer.get_var("files").split()
 398
 399         @property
 400         def uuid(self):
 401                 return self._uuid
 402
 403         def track_dependencies(self, builder, path):
 404                 # Build filelist with all files that have been installed.
 405                 filelist = builder.mktemp()
 406
 407                 try:
 408                         f = open(filelist, "w")
 409                         for dir, subdirs, files in os.walk(path):
 410                                 f.write("%s\n" % dir)
 411
 412                                 for file in files:
 413                                         f.write("%s\n" % os.path.join(dir, file))
 414                         f.close()
 415
 416                         log.info(_("Searching for automatic dependencies for %s...") % self.friendly_name)
 417
 418                         # Search for provides.
 419                         res = builder.do("/usr/lib/pakfire/find-provides %s %s" \
 420                                 % (path, filelist), returnOutput=True)
 421                         provides = set(res.splitlines())
 422
 423                         # Search for requires.
 424                         res = builder.do("/usr/lib/pakfire/find-requires %s %s" \
 425                                 % (path, filelist), returnOutput=True)
 426                         requires = set(res.splitlines()) - provides
 427
 428                 finally:
 429                         if os.path.exists(filelist):
 430                                 os.unlink(filelist)
 431
 432                 self._dependencies["provides"] = provides
 433                 self._dependencies["requires"] = requires
 434
 435                 # Filter dependencies.
 436                 for key in ("prerequires", "requires", "provides", "conflicts", "obsoletes"):
 437                         # Make sure this is a list.
 438                         try:
 439                                 self._dependencies[key] = list(self._dependencies[key])
 440                         except KeyError:
 441                                 self._dependencies[key] = []
 442
 443                         # Filter out unwanted elements.
 444                         self._dependencies[key] = self.filter_deps(
 445                                 self._dependencies[key], self.lexer.get_var("filter_%s" % key)
 446                         )
 447
 448         @staticmethod
 449         def filter_deps(deps, filters):
 450                 if not filters:
 451                         return deps
 452
 453                 _filters = []
 454                 filtered_deps = []
 455
 456                 # Compile all filters.
 457                 for filter in filters.splitlines():
 458                         # Convert to raw string to make escaping characters
 459                         # easy to the user.
 460                         try:
 461                                 _filter = re.compile("%r" % filter)
 462                         except re.error:
 463                                 log.warning(_("Regular experession is invalid and has been skipped: %s") % filter)
 464                                 continue
 465
 466                         _filters.append(_filter)
 467
 468                 filters = _filters
 469
 470                 for dep in deps:
 471                         filtered = False
 472                         for filter in filters:
 473                                 # Search for a match anywhere in the line.
 474                                 m = re.search(filter, dep)
 475                                 if not m:
 476                                         continue
 477
 478                                 # Let the user know what has been done.
 479                                 log.info(_("Filter %s filtered %s.") % (filter, dep))
 480
 481                                 # Yes, we found a match.
 482                                 filtered = True
 483                                 break
 484
 485                         if not filtered:
 486                                 filtered_deps.append(dep)
 487
 488                 return filtered_deps
 489
 490         def get_deps(self, key):
 491                 # Collect all dependencies that were set in the makefile by the user.
 492                 deps = self.lexer.get_var(key).splitlines()
 493
 494                 # Collect all dependencies that were discovered by the tracker.
 495                 deps += self._dependencies.get(key, [])
 496
 497                 # Add the UUID.
 498                 if key == "provides":
 499                         deps.append("uuid(%s)" % self.uuid)
 500
 501                 # Remove duplicates.
 502                 deps = set(deps)
 503                 deps = list(deps)
 504
 505                 return sorted(deps)
 506
 507         @property
 508         def prerequires(self):
 509                 return self.get_deps("prerequires")
 510
 511         @property
 512         def requires(self):
 513                 # Make sure that no self-provides are in the list
 514                 # of requirements.
 515                 provides = self.provides
 516
 517                 return [r for r in self.get_deps("requires") if not r in provides]
 518
 519         @property
 520         def provides(self):
 521                 return self.get_deps("provides")
 522
 523         @property
 524         def obsoletes(self):
 525                 return self.get_deps("obsoletes")
 526
 527         @property
 528         def conflicts(self):
 529                 return self.get_deps("conflicts")
 530
 531         def get_scriptlet(self, type):
 532                 return self.lexer.get_scriptlet(type)
 533
 534         @property
 535         def inst_size(self):
 536                 # The size of this is unknown.
 537                 return 0