+++ /dev/null
-#!/usr/bin/python
-###############################################################################
-# #
-# Pakfire - The IPFire package management system #
-# Copyright (C) 2011 Pakfire development team #
-# #
-# This program is free software: you can redistribute it and/or modify #
-# it under the terms of the GNU General Public License as published by #
-# the Free Software Foundation, either version 3 of the License, or #
-# (at your option) any later version. #
-# #
-# This program is distributed in the hope that it will be useful, #
-# but WITHOUT ANY WARRANTY; without even the implied warranty of #
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the #
-# GNU General Public License for more details. #
-# #
-# You should have received a copy of the GNU General Public License #
-# along with this program. If not, see <http://www.gnu.org/licenses/>. #
-# #
-###############################################################################
-
-import collections
-import fnmatch
-import glob
-import hashlib
-import os
-import re
-import shutil
-import sys
-import tarfile
-import tempfile
-import time
-import uuid
-import zlib
-
-import logging
-log = logging.getLogger("pakfire")
-
-import pakfire.util as util
-
-from pakfire.constants import *
-from pakfire.i18n import _
-
-from . import tar
-
-class Packager(object):
- payload_compression = None
-
- def __init__(self, pakfire, pkg):
- self.pakfire = pakfire
- self.pkg = pkg
-
- self.files = []
- self.tmpfiles = []
-
- def __del__(self):
- for file in self.tmpfiles:
- if not os.path.exists(file):
- continue
-
- log.debug("Removing tmpfile: %s" % file)
-
- if os.path.isdir(file):
- util.rm(file)
- else:
- os.remove(file)
-
- def mktemp(self, directory=False):
- if directory:
- filename = os.path.join("/", LOCAL_TMP_PATH, util.random_string())
- os.makedirs(filename)
- else:
- f = tempfile.NamedTemporaryFile(mode="w", delete=False)
- f.close()
-
- filename = f.name
-
- self.tmpfiles.append(filename)
-
- return filename
-
- def save(self, filename):
- # Create a new tar archive.
- tar = tarfile.TarFile(filename, mode="w", format=tarfile.PAX_FORMAT)
-
- # Add package formation information.
- # Must always be the first file in the archive.
- formatfile = self.create_package_format()
- tar.add(formatfile, arcname="pakfire-format")
-
- # XXX make sure all files belong to the root user
-
- # Create checksum file.
- chksumsfile = self.mktemp()
- chksums = open(chksumsfile, "w")
-
- # Add all files to tar file.
- for arcname, filename in self.files:
- tar.add(filename, arcname=arcname)
-
- # Calculating the hash sum of the added file
- # and store it in the chksums file.
- f = open(filename)
- h = hashlib.new("sha512")
- while True:
- buf = f.read(BUFFER_SIZE)
- if not buf:
- break
-
- h.update(buf)
- f.close()
-
- chksums.write("%-10s %s\n" % (arcname, h.hexdigest()))
-
- # Close checksum file and attach it to the end.
- chksums.close()
- tar.add(chksumsfile, "chksums")
-
- # Close the tar file.
- tar.close()
-
- def add(self, filename, arcname=None):
- if not arcname:
- arcname = os.path.basename(filename)
-
- log.debug("Adding %s (as %s) to tarball." % (filename, arcname))
- self.files.append((arcname, filename))
-
- def create_package_format(self):
- filename = self.mktemp()
-
- f = open(filename, "w")
- f.write("%s\n" % PACKAGE_FORMAT)
- f.close()
-
- return filename
-
- def create_filelist(self, datafile):
- filelist = self.mktemp()
-
- f = open(filelist, "w")
-
- if self.payload_compression == "xz":
- datafile = tar.InnerTarFileXz.open(datafile)
- else:
- datafile = tar.InnerTarFile.open(datafile)
-
- while True:
- m = next(datafile)
- if not m:
- break
-
- log.debug(" %s %-8s %-8s %s %6s %s" % \
- (tarfile.filemode(m.mode), m.uname, m.gname,
- "%d-%02d-%02d %02d:%02d:%02d" % time.localtime(m.mtime)[:6],
- util.format_size(m.size), m.name))
-
- f.write("%(type)1s %(size)-10d %(uname)-10s %(gname)-10s %(mode)-6d %(mtime)-12d" \
- % m.get_info(tarfile.ENCODING, "strict"))
-
- # Calculate SHA512 hash of regular files.
- if m.isreg():
- mobj = datafile.extractfile(m)
- h = hashlib.new("sha512")
-
- while True:
- buf = mobj.read(BUFFER_SIZE)
- if not buf:
- break
- h.update(buf)
-
- mobj.close()
- f.write(" %s" % h.hexdigest())
- else:
- f.write(" -")
-
- caps = m.pax_headers.get("PAKFIRE.capabilities", None)
- if caps:
- f.write(" %s" % caps)
- else:
- f.write(" -")
-
- # The file name must be the last argument to contain spaces.
- f.write(" %s" % m.name)
-
- f.write("\n")
-
- log.info("")
-
- datafile.close()
- f.close()
-
- return filelist
-
- def run(self):
- raise NotImplementedError
-
- def getsize(self, datafile):
- size = 0
-
- if self.payload_compression == "xz":
- t = tar.InnerTarFileXz.open(datafile)
- else:
- t = tar.InnerTarFile.open(datafile)
-
- while True:
- m = next(t)
- if not m:
- break
-
- size += m.size
-
- t.close()
- return size
-
-
-class BinaryPackager(Packager):
- payload_compression = "xz"
-
- def __init__(self, pakfire, pkg, builder, buildroot):
- Packager.__init__(self, pakfire, pkg)
-
- self.builder = builder
- self.buildroot = buildroot
-
- def create_metafile(self, datafile):
- info = collections.defaultdict(lambda: "")
-
- # Extract datafile in temporary directory and scan for dependencies.
- tmpdir = self.mktemp(directory=True)
-
- if self.payload_compression == "xz":
- tarfile = tar.InnerTarFileXz.open(datafile)
- else:
- tarfile = tar.InnerTarFile.open(datafile)
-
- tarfile.extractall(path=tmpdir)
- tarfile.close()
-
- # Run the dependency tracker.
- self.pkg.track_dependencies(self.builder, tmpdir)
-
- # Generic package information including Pakfire information.
- info.update({
- "pakfire_version" : PAKFIRE_VERSION,
- "uuid" : self.pkg.uuid,
- "type" : "binary",
- })
-
- # Include distribution information.
- info.update(self.pakfire.distro.info)
- info.update(self.pkg.info)
-
- # Update package information for string formatting.
- info.update({
- "groups" : " ".join(self.pkg.groups),
- "prerequires" : "\n".join([PACKAGE_INFO_DEPENDENCY_LINE % d \
- for d in self.pkg.prerequires]),
- "requires" : "\n".join([PACKAGE_INFO_DEPENDENCY_LINE % d \
- for d in self.pkg.requires]),
- "provides" : "\n".join([PACKAGE_INFO_DEPENDENCY_LINE % d \
- for d in self.pkg.provides]),
- "conflicts" : "\n".join([PACKAGE_INFO_DEPENDENCY_LINE % d \
- for d in self.pkg.conflicts]),
- "obsoletes" : "\n".join([PACKAGE_INFO_DEPENDENCY_LINE % d \
- for d in self.pkg.obsoletes]),
- "recommends" : "\n".join([PACKAGE_INFO_DEPENDENCY_LINE % d \
- for d in self.pkg.recommends]),
- "suggests" : "\n".join([PACKAGE_INFO_DEPENDENCY_LINE % d \
- for d in self.pkg.suggests]),
- })
-
- # Format description.
- description = [PACKAGE_INFO_DESCRIPTION_LINE % l \
- for l in util.text_wrap(self.pkg.description, length=80)]
- info["description"] = "\n".join(description)
-
- # Build information.
- info.update({
- # Package it built right now.
- "build_time" : int(time.time()),
- "build_id" : uuid.uuid4(),
- })
-
- # Installed size (equals size of the uncompressed tarball).
- info.update({
- "inst_size" : self.getsize(datafile),
- })
-
- metafile = self.mktemp()
-
- f = open(metafile, "w")
- f.write(PACKAGE_INFO % info)
- f.close()
-
- return metafile
-
- def create_datafile(self):
- includes = []
- excludes = []
-
- # List of all patterns, which grows.
- patterns = self.pkg.files
-
- # ...
- orphan_directories = []
- for d in ORPHAN_DIRECTORIES:
- if d.startswith("usr/"):
- b = os.path.basename(d)
- b = os.path.join(self.buildroot, b)
-
- if os.path.islink(b):
- continue
-
- d = os.path.join(self.buildroot, d)
- if not os.path.islink(d):
- orphan_directories.append(d)
-
- for pattern in patterns:
- # Check if we are running in include or exclude mode.
- if pattern.startswith("!"):
- files = excludes
-
- # Strip the ! character.
- pattern = pattern[1:]
- else:
- files = includes
-
- # Expand file to point to chroot.
- if pattern.startswith("/"):
- pattern = pattern[1:]
- pattern = os.path.join(self.buildroot, pattern)
-
- # Recognize the type of the pattern. Patterns could be a glob
- # pattern that is expanded here or just a directory which will
- # be included recursively.
- if "*" in pattern or "?" in pattern or ("[" in pattern and "]" in pattern):
- _patterns = glob.glob(pattern)
- else:
- _patterns = [pattern,]
-
- for pattern in _patterns:
- # Try to stat the pattern. If that is not successful, we cannot go on.
- try:
- os.lstat(pattern)
- except OSError:
- continue
-
- # Add directories recursively but skip those symlinks
- # that point to a directory.
- if os.path.isdir(pattern) and not os.path.islink(pattern):
- # Add directory itself.
- files.append(pattern)
-
- for dir, subdirs, _files in os.walk(pattern):
- for subdir in subdirs:
- if subdir in orphan_directories:
- continue
-
- subdir = os.path.join(dir, subdir)
- files.append(subdir)
-
- for file in _files:
- file = os.path.join(dir, file)
- files.append(file)
-
- # All other files are just added.
- else:
- files.append(pattern)
-
- files = []
- for file in includes:
- # Skip if file is already in the file set or
- # marked to be excluded from this archive.
- if file in excludes or file in files:
- continue
-
- # Skip orphan directories.
- if file in orphan_directories and not os.listdir(file):
- log.debug("Found an orphaned directory: %s" % file)
- continue
-
- files.append(file)
-
- while True:
- file = os.path.dirname(file)
-
- if file == self.buildroot:
- break
-
- if not file in files:
- files.append(file)
-
- files.sort()
-
- # Load progressbar.
- message = "%-10s : %s" % (_("Packaging"), self.pkg.friendly_name)
- pb = util.make_progress(message, len(files), eta=False)
-
- datafile = self.mktemp()
- if self.payload_compression == "xz":
- t = tar.InnerTarFileXz.open(datafile, mode="w")
- else:
- t = tar.InnerTarFile.open(datafile, mode="w")
-
- # All files in the tarball are relative to this directory.
- basedir = self.buildroot
-
- i = 0
- for file in files:
- if pb:
- i += 1
- pb.update(i)
-
- # Never package /.
- if os.path.normpath(file) == os.path.normpath(basedir):
- continue
-
- # Name of the file in the archive.
- arcname = "/%s" % os.path.relpath(file, basedir)
-
- # Add file to tarball.
- t.add(file, arcname=arcname, recursive=False)
-
- # Remove all packaged files.
- for file in reversed(files):
- # It's okay if we cannot remove directories,
- # when they are not empty.
- if os.path.isdir(file):
- try:
- os.rmdir(file)
- except OSError:
- continue
- else:
- try:
- os.unlink(file)
- except OSError:
- pass
-
- while True:
- file = os.path.dirname(file)
-
- if not file.startswith(basedir):
- break
-
- try:
- os.rmdir(file)
- except OSError:
- break
-
- # Close the tarfile.
- t.close()
-
- # Finish progressbar.
- if pb:
- pb.finish()
-
- return datafile
-
- def create_scriptlets(self):
- scriptlets = []
-
- # Collect all prerequires for the scriptlets.
- prerequires = []
-
- for scriptlet_name in SCRIPTS:
- scriptlet = self.pkg.get_scriptlet(scriptlet_name)
-
- if not scriptlet:
- continue
-
- # Write script to a file.
- scriptlet_file = self.mktemp()
-
- lang = scriptlet["lang"]
-
- if lang == "bin":
- path = lang["path"]
- try:
- f = open(path, "b")
- except OSError:
- raise Exception("Cannot open script file: %s" % lang["path"])
-
- s = open(scriptlet_file, "wb")
-
- while True:
- buf = f.read(BUFFER_SIZE)
- if not buf:
- break
-
- s.write(buf)
-
- f.close()
- s.close()
-
- elif lang == "shell":
- s = open(scriptlet_file, "w")
-
- # Write shell script to file.
- s.write("#!/bin/sh -e\n\n")
- s.write(scriptlet["scriptlet"])
- s.write("\n\nexit 0\n")
- s.close()
-
- if scriptlet_name in SCRIPTS_PREREQUIRES:
- # Shell scripts require a shell to be executed.
- prerequires.append("/bin/sh")
-
- prerequires += self.builder.find_prerequires(scriptlet_file)
-
- elif lang == "python":
- # Write the code to the scriptlet file.
- s = open(scriptlet_file, "w")
- s.write(scriptlet["scriptlet"])
- s.close()
-
- else:
- raise Exception("Unknown scriptlet language: %s" % scriptlet["lang"])
-
- scriptlets.append((scriptlet_name, scriptlet_file))
-
- # Cleanup prerequires.
- self.pkg.update_prerequires(prerequires)
-
- return scriptlets
-
- def find_files(self, datafile, patterns):
- if self.payload_compression == "xz":
- datafile = tar.InnerTarFileXz.open(datafile)
- else:
- datafile = tar.InnerTarFile.open(datafile)
-
- members = datafile.getmembers()
-
- files = []
- dirs = []
-
- # Find all directories in the file list.
- for file in patterns:
- if file.startswith("/"):
- file = file[1:]
-
- for member in members:
- if member.name == file and member.isdir():
- dirs.append(file)
-
- for d in dirs:
- for member in members:
- if not member.isdir() and member.name.startswith(d):
- files.append(member.name)
-
- for pattern in patterns:
- if pattern.startswith("/"):
- pattern = pattern[1:]
-
- for member in members:
- if not fnmatch.fnmatch(member.name, pattern):
- continue
-
- if member.name in files:
- continue
-
- files.append(member.name)
-
- # Sort list alphabetically.
- files.sort()
-
- return files
-
- def create_configfiles(self, datafile):
- files = self.find_files(datafile, self.pkg.configfiles)
-
- configsfile = self.mktemp()
-
- f = open(configsfile, "w")
- for file in files:
- f.write("%s\n" % file)
- f.close()
-
- return configsfile
-
- def create_datafiles(self, datafile):
- files = self.find_files(datafile, self.pkg.datafiles)
-
- datafile = self.mktemp()
-
- f = open(datafile, "w")
- for file in files:
- f.write("%s\n" % file)
- f.close()
-
- return datafile
-
- def run(self, resultdir):
- # Add all files to this package.
- datafile = self.create_datafile()
-
- # Get filelist from datafile.
- filelist = self.create_filelist(datafile)
- configfiles = self.create_configfiles(datafile)
- datafiles = self.create_datafiles(datafile)
-
- # Create script files.
- scriptlets = self.create_scriptlets()
-
- metafile = self.create_metafile(datafile)
-
- # Add files to the tar archive in correct order.
- self.add(metafile, "info")
- self.add(filelist, "filelist")
- self.add(configfiles, "configfiles")
- self.add(datafiles, "datafiles")
- self.add(datafile, "data.img")
-
- for scriptlet_name, scriptlet_file in scriptlets:
- self.add(scriptlet_file, "scriptlets/%s" % scriptlet_name)
-
- # Build the final package.
- tempfile = self.mktemp()
- self.save(tempfile)
-
- # Add architecture information to path.
- resultdir = "%s/%s" % (resultdir, self.pkg.arch)
-
- if not os.path.exists(resultdir):
- os.makedirs(resultdir)
-
- resultfile = os.path.join(resultdir, self.pkg.package_filename)
- log.info("Saving package to %s" % resultfile)
- try:
- os.link(tempfile, resultfile)
- except OSError:
- shutil.copy2(tempfile, resultfile)
-
-
-class SourcePackager(Packager):
- payload_compression = None
-
- def create_metafile(self, datafile):
- info = collections.defaultdict(lambda: "")
-
- # Generic package information including Pakfire information.
- info.update({
- "pakfire_version" : PAKFIRE_VERSION,
- "type" : "source",
- })
-
- # Include distribution information.
- info.update(self.pakfire.distro.info)
- info.update(self.pkg.info)
-
- # Size is the size of the (uncompressed) datafile.
- info["inst_size"] = self.getsize(datafile)
-
- # Update package information for string formatting.
- requires = [PACKAGE_INFO_DEPENDENCY_LINE % r for r in self.pkg.requires]
- info.update({
- "groups" : " ".join(self.pkg.groups),
- "requires" : "\n".join(requires),
- })
-
- # Format description.
- description = [PACKAGE_INFO_DESCRIPTION_LINE % l \
- for l in util.text_wrap(self.pkg.description, length=80)]
- info["description"] = "\n".join(description)
-
- # Build information.
- info.update({
- # Package it built right now.
- "build_time" : int(time.time()),
- "build_id" : uuid.uuid4(),
- })
-
- # Arches equals supported arches.
- info["arch"] = self.pkg.supported_arches
-
- # Set UUID
- # XXX replace this by the payload hash
- info.update({
- "uuid" : uuid.uuid4(),
- })
-
- metafile = self.mktemp()
-
- f = open(metafile, "w")
- f.write(PACKAGE_INFO % info)
- f.close()
-
- return metafile
-
- def create_datafile(self):
- # Create a list of all files that have to be put into the
- # package.
- files = []
-
- # Download all files that go into the package.
- for file in self.pkg.download():
- assert os.path.getsize(file), "Don't package empty files"
- files.append(("files/%s" % os.path.basename(file), file))
-
- # Add all files in the package directory.
- for file in self.pkg.files:
- files.append((os.path.relpath(file, self.pkg.path), file))
-
- # Add files in alphabetical order.
- files.sort()
-
- # Load progressbar.
- message = "%-10s : %s" % (_("Packaging"), self.pkg.friendly_name)
- pb = util.make_progress(message, len(files), eta=False)
-
- filename = self.mktemp()
- if self.payload_compression == "xz":
- datafile = tar.InnerTarFileXz.open(filename, mode="w")
- else:
- datafile = tar.InnerTarFile.open(filename, mode="w")
-
- i = 0
- for arcname, file in files:
- if pb:
- i += 1
- pb.update(i)
-
- datafile.add(file, arcname)
- datafile.close()
-
- if pb:
- pb.finish()
-
- return filename
-
- def run(self, resultdir):
- # Create resultdir if it does not exist yet.
- if not os.path.exists(resultdir):
- os.makedirs(resultdir)
-
- log.info(_("Building source package %s:") % self.pkg.package_filename)
-
- # The filename where this source package is saved at.
- target_filename = os.path.join(resultdir, self.pkg.package_filename)
-
- # Add datafile to package.
- datafile = self.create_datafile()
-
- # Create filelist out of data.
- filelist = self.create_filelist(datafile)
-
- # Create metadata.
- metafile = self.create_metafile(datafile)
-
- # Add files to the tar archive in correct order.
- self.add(metafile, "info")
- self.add(filelist, "filelist")
- self.add(datafile, "data.img")
-
- # Build the final tarball.
- try:
- self.save(target_filename)
- except:
- # Remove the target file when anything went wrong.
- os.unlink(target_filename)
- raise
-
- return target_filename
projects / people / stevee / pakfire.git / commitdiff
