From c2808056ee3a8f4aca8d6936092ae29fbec425ce Mon Sep 17 00:00:00 2001 From: Michael Tremer Date: Fri, 14 Oct 2011 20:55:21 +0000 Subject: [PATCH] Add proper handling of configuration files. --- po/pakfire.pot | 76 +++++++++------ python/pakfire/constants.py | 7 +- python/pakfire/filelist.py | 46 ++++++++- python/pakfire/packages/base.py | 25 ++++- python/pakfire/packages/file.py | 134 ++++++++++++++++++++++---- python/pakfire/repository/database.py | 26 +++-- 6 files changed, 256 insertions(+), 58 deletions(-) diff --git a/po/pakfire.pot b/po/pakfire.pot index f733d031..8726c366 100644 --- a/po/pakfire.pot +++ b/po/pakfire.pot @@ -8,7 +8,7 @@ msgid "" msgstr "" "Project-Id-Version: PACKAGE VERSION\n" "Report-Msgid-Bugs-To: \n" -"POT-Creation-Date: 2011-10-11 21:54+0200\n" +"POT-Creation-Date: 2011-10-14 20:51+0000\n" "PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" "Last-Translator: FULL NAME \n" "Language-Team: LANGUAGE \n" @@ -386,112 +386,132 @@ msgstr "" msgid "%(commas)s and %(last)s" msgstr "" -#: ../python/pakfire/packages/base.py:94 +#: ../python/pakfire/packages/base.py:97 msgid "Name" msgstr "" -#: ../python/pakfire/packages/base.py:102 ../python/pakfire/transaction.py:315 +#: ../python/pakfire/packages/base.py:105 ../python/pakfire/transaction.py:315 msgid "Arch" msgstr "" -#: ../python/pakfire/packages/base.py:105 ../python/pakfire/transaction.py:315 +#: ../python/pakfire/packages/base.py:108 ../python/pakfire/transaction.py:315 msgid "Version" msgstr "" -#: ../python/pakfire/packages/base.py:106 +#: ../python/pakfire/packages/base.py:109 msgid "Release" msgstr "" -#: ../python/pakfire/packages/base.py:110 ../python/pakfire/transaction.py:316 +#: ../python/pakfire/packages/base.py:113 ../python/pakfire/transaction.py:316 msgid "Size" msgstr "" -#: ../python/pakfire/packages/base.py:114 +#: ../python/pakfire/packages/base.py:117 msgid "Repo" msgstr "" -#: ../python/pakfire/packages/base.py:117 +#: ../python/pakfire/packages/base.py:120 msgid "Summary" msgstr "" -#: ../python/pakfire/packages/base.py:118 +#: ../python/pakfire/packages/base.py:121 msgid "Groups" msgstr "" -#: ../python/pakfire/packages/base.py:119 +#: ../python/pakfire/packages/base.py:122 msgid "URL" msgstr "" -#: ../python/pakfire/packages/base.py:120 +#: ../python/pakfire/packages/base.py:123 msgid "License" msgstr "" -#: ../python/pakfire/packages/base.py:123 +#: ../python/pakfire/packages/base.py:126 msgid "Description" msgstr "" -#: ../python/pakfire/packages/base.py:130 +#: ../python/pakfire/packages/base.py:133 msgid "Maintainer" msgstr "" -#: ../python/pakfire/packages/base.py:132 +#: ../python/pakfire/packages/base.py:135 msgid "Vendor" msgstr "" -#: ../python/pakfire/packages/base.py:134 +#: ../python/pakfire/packages/base.py:137 msgid "UUID" msgstr "" -#: ../python/pakfire/packages/base.py:135 +#: ../python/pakfire/packages/base.py:138 msgid "Build ID" msgstr "" -#: ../python/pakfire/packages/base.py:136 +#: ../python/pakfire/packages/base.py:139 msgid "Build date" msgstr "" -#: ../python/pakfire/packages/base.py:137 +#: ../python/pakfire/packages/base.py:140 msgid "Build host" msgstr "" -#: ../python/pakfire/packages/base.py:139 +#: ../python/pakfire/packages/base.py:142 msgid "Provides" msgstr "" -#: ../python/pakfire/packages/base.py:144 +#: ../python/pakfire/packages/base.py:147 msgid "Pre-requires" msgstr "" -#: ../python/pakfire/packages/base.py:149 +#: ../python/pakfire/packages/base.py:152 msgid "Requires" msgstr "" -#: ../python/pakfire/packages/base.py:154 +#: ../python/pakfire/packages/base.py:157 msgid "Conflicts" msgstr "" -#: ../python/pakfire/packages/base.py:159 +#: ../python/pakfire/packages/base.py:162 msgid "Obsoletes" msgstr "" -#: ../python/pakfire/packages/base.py:167 +#: ../python/pakfire/packages/base.py:170 msgid "File" msgstr "" -#: ../python/pakfire/packages/base.py:353 +#: ../python/pakfire/packages/base.py:356 msgid "Not set" msgstr "" -#: ../python/pakfire/packages/file.py:109 +#: ../python/pakfire/packages/base.py:495 +#, python-format +msgid "Config file saved as %s." +msgstr "" + +#: ../python/pakfire/packages/file.py:111 #, python-format msgid "Could not extract file: /%(src)s - %(dst)s" msgstr "" -#: ../python/pakfire/packages/file.py:162 +#: ../python/pakfire/packages/file.py:164 #, python-format msgid "Filename: %s" msgstr "" +#: ../python/pakfire/packages/file.py:278 +#, python-format +msgid "File in archive is missing in file metadata: /%s. Skipping." +msgstr "" + +#: ../python/pakfire/packages/file.py:330 +#, python-format +msgid "Config file created as %s" +msgstr "" + +#: ../python/pakfire/packages/file.py:344 +#, python-format +msgid "Could not remove file: /%s" +msgstr "" + #: ../python/pakfire/packages/make.py:75 msgid "Package name is undefined." msgstr "" @@ -514,7 +534,7 @@ msgstr "" msgid "The format of the database is not supported by this version of pakfire." msgstr "" -#: ../python/pakfire/repository/database.py:212 +#: ../python/pakfire/repository/database.py:217 #, python-format msgid "Migrating database from format %s to %s." msgstr "" diff --git a/python/pakfire/constants.py b/python/pakfire/constants.py index eb4a5529..a78a2a10 100644 --- a/python/pakfire/constants.py +++ b/python/pakfire/constants.py @@ -63,8 +63,8 @@ PACKAGE_FORMATS_SUPPORTED = [0, 1, 2] PACKAGE_EXTENSION = "pfm" MAKEFILE_EXTENSION = "nm" -DATABASE_FORMAT = 1 -DATABASE_FORMATS_SUPPORTED = [0, 1] +DATABASE_FORMAT = 2 +DATABASE_FORMATS_SUPPORTED = [0, 1, 2] PACKAGE_FILENAME_FMT = "%(name)s-%(version)s-%(release)s.%(arch)s.%(ext)s" @@ -190,3 +190,6 @@ SCRIPTS = ( ) LDCONFIG = "/sbin/ldconfig" + +CONFIG_FILE_SUFFIX_NEW = ".paknew" +CONFIG_FILE_SUFFIX_SAVE = ".paksave" diff --git a/python/pakfire/filelist.py b/python/pakfire/filelist.py index 14e7af92..05a06e0d 100644 --- a/python/pakfire/filelist.py +++ b/python/pakfire/filelist.py @@ -19,6 +19,18 @@ # # ############################################################################### +import tarfile + +TYPE_REG = tarfile.REGTYPE # regular file +TYPE_AREG = tarfile.AREGTYPE # regular file +TYPE_LNK = tarfile.LNKTYPE # link (inside tarfile) +TYPE_SYM = tarfile.SYMTYPE # symbolic link +TYPE_CHR = tarfile.CHRTYPE # character special device +TYPE_BLK = tarfile.BLKTYPE # block special device +TYPE_DIR = tarfile.DIRTYPE # directory +TYPE_FIFO = tarfile.FIFOTYPE # fifo special device +TYPE_CONT = tarfile.CONTTYPE # contiguous file + class _File(object): def __init__(self, pakfire): self.pakfire = pakfire @@ -46,9 +58,18 @@ class File(_File): _File.__init__(self, pakfire) self.name = "" + self.config = False self.pkg = None self.size = -1 - self.hash1 = "" + self.hash1 = None + self.type = TYPE_REG + self.mode = 0 + self.user = 0 + self.group = 0 + self.mtime = 0 + + def is_config(self): + return self.config class FileDatabase(_File): @@ -80,6 +101,9 @@ class FileDatabase(_File): return self.__row + def is_config(self): + return self.row["config"] == 1 + @property def pkg(self): return self.db.get_package_by_id(self.row["pkg"]) @@ -95,3 +119,23 @@ class FileDatabase(_File): @property def hash1(self): return self.row["hash1"] + + @property + def type(self): + return self.row["type"] + + @property + def mode(self): + return self.row["mode"] + + @property + def user(self): + return self.row["user"] + + @property + def group(self): + return self.row["group"] + + @property + def mtime(self): + return self.row["mtime"] diff --git a/python/pakfire/packages/base.py b/python/pakfire/packages/base.py index c9ce4e95..b2316e94 100644 --- a/python/pakfire/packages/base.py +++ b/python/pakfire/packages/base.py @@ -22,9 +22,12 @@ import datetime import logging import os +import shutil import xml.sax.saxutils import pakfire.util as util + +from pakfire.constants import * from pakfire.i18n import _ class Package(object): @@ -456,6 +459,9 @@ class Package(object): # a directory first and then check, if there are any files left. files.sort(cmp=lambda x,y: cmp(len(x.name), len(y.name)), reverse=True) + # Messages to the user. + messages = [] + i = 0 for _file in files: # Update progress. @@ -475,6 +481,20 @@ class Package(object): if not os.path.exists(file): continue + # Rename configuration files. + if _file.is_config(): + file_save = "%s%s" % (file, CONFIG_FILE_SUFFIX_SAVE) + + try: + shutil.move(file, file_save) + except shutil.Error, e: + print e + + if prefix: + file_save = os.path.relpath(file_save, prefix) + messages.append(_("Config file saved as %s.") % file_save) + continue + # Handle regular files and symlinks. if os.path.isfile(file) or os.path.islink(file): try: @@ -494,9 +514,10 @@ class Package(object): # Log all unhandled types. else: - logging.warning("Cannot remove file: %s. Filetype is unhandled." % _file) + logging.warning("Cannot remove file: %s. Filetype is unhandled." % file) if pb: pb.finish() - # XXX Rename config files + for msg in messages: + logging.warning(msg) diff --git a/python/pakfire/packages/file.py b/python/pakfire/packages/file.py index 632c903e..a249677d 100644 --- a/python/pakfire/packages/file.py +++ b/python/pakfire/packages/file.py @@ -19,9 +19,11 @@ # # ############################################################################### +import hashlib import logging import os import re +import shutil import tarfile import tempfile import xattr @@ -242,18 +244,19 @@ class FilePackage(Package): # Open the tarball in the package. payload_archive = InnerTarFile.open(fileobj=payload) - members = payload_archive.getmembers() - # Load progressbar. pb = None if message: message = "%-10s : %s" % (message, self.friendly_name) - pb = util.make_progress(message, len(members), eta=False) + pb = util.make_progress(message, len(self.filelist), eta=False) # Collect messages with errors and warnings, that are passed to # the user. messages = [] + # Get a list of files in the archive. + members = payload_archive.getmembers() + i = 0 for member in members: # Update progress. @@ -261,11 +264,73 @@ class FilePackage(Package): i += 1 pb.update(i) + file = None + for f in self.filelist: + m_name = "/%s" % member.name + if f.is_dir(): + m_name = "%s/" % m_name + + if m_name == f.name: + file = f + break + + if not file: + logging.warning(_("File in archive is missing in file metadata: /%s. Skipping.") % member.name) + continue + target = os.path.join(prefix, member.name) + # Check if a configuration file is already present. We don't want to + # overwrite that. + if file.is_config(): + config_save = "%s%s" % (target, CONFIG_FILE_SUFFIX_SAVE) + config_new = "%s%s" % (target, CONFIG_FILE_SUFFIX_NEW) + + if os.path.exists(config_save) and not os.path.exists(target): + # Extract new configuration file, save it as CONFIG_FILE_SUFFIX_NEW, + # and reuse _SAVE. + payload_archive.extract(member, path=prefix) + + shutil.move(target, config_new) + shutil.move(config_save, target) + continue + + elif os.path.exists(target): + # If the files are identical, we skip the extraction of a + # new configuration file. We also do that when the new configuration file + # is a dummy file. + if file.size == 0: + continue + + # Calc hash of the current configuration file. + config_hash1 = hashlib.sha512() + f = open(target) + while True: + buf = f.read(BUFFER_SIZE) + if not buf: + break + config_hash1.update(buf) + f.close() + + if file.hash1 == config_hash1.hexdigest(): + continue + + # Backup old configuration file and extract new one. + shutil.move(target, config_save) + payload_archive.extract(member, path=prefix) + + # Save new configuration file as CONFIG_FILE_SUFFIX_NEW and + # restore old configuration file. + shutil.move(target, config_new) + shutil.move(config_save, target) + + if prefix: + config_new = os.path.relpath(config_new, prefix) + messages.append(_("Config file created as %s") % config_new) + continue + # If the member is a directory and if it already exists, we # don't need to create it again. - if os.path.exists(target): if member.isdir(): continue @@ -357,32 +422,71 @@ class FilePackage(Package): ret = [] a = self.open_archive() - f = a.extractfile("filelist") + # Cache configfiles. + configfiles = [] + f = a.extractfile("configs") + for line in f.readlines(): + line = line.rstrip() + if not line.startswith("/"): + line = "/%s" % line + configfiles.append(line) + f.close() + + f = a.extractfile("filelist") for line in f.readlines(): line = line.strip() file = pakfire.filelist.File(self.pakfire) if self.format >= 1: + line = line.rstrip() line = line.split() name = line[0] + if not name.startswith("/"): + name = "/%s" % name + + # Check if configfiles. + if name in configfiles: + file.config = True + + # Parse file type. + try: + file.type = int(line[1]) + except ValueError: + file.type = 0 + # Parse the size information. try: file.size = int(line[2]) except ValueError: - print "PARSE ERROR", line[2] file.size = 0 - # XXX need to parse the rest of the information from the - # file + # Parse user and group. + file.user, file.group = line[3], line[4] + + # Parse mode. + try: + file.mode = int(line[5]) + except ValueError: + file.mode = 0 + + # Parse time. + try: + file.mtime = line[6] + except ValueError: + file.mtime = 0 + + # Parse hash1 (sha512). + if not line[7] == "-": + file.hash1 = line[7] else: name = line - if not name.startswith("/"): - name = "/%s" % name + if not name.startswith("/"): + name = "/%s" % name file.name = name file.pkg = self @@ -403,15 +507,7 @@ class FilePackage(Package): @property def configfiles(self): - a = self.open_archive() - - f = a.extractfile("configs") - for line in f.readlines(): - if not line.startswith("/"): - line = "/%s" % line - yield line - - a.close() + return [f for f in self.filelist if f.is_config()] @property def payload_compression(self): diff --git a/python/pakfire/repository/database.py b/python/pakfire/repository/database.py index 48adbe05..24d81541 100644 --- a/python/pakfire/repository/database.py +++ b/python/pakfire/repository/database.py @@ -149,16 +149,21 @@ class DatabaseLocal(Database): INSERT INTO settings(key, val) VALUES('version', '%s'); CREATE TABLE files( - id INTEGER PRIMARY KEY, + id INTEGER PRIMARY KEY, name TEXT, - pkg INTEGER, + pkg INTEGER, size INTEGER, type INTEGER, - hash1 TEXT + config INTEGER, + mode INTEGER, + user TEXT, + `group` TEXT, + hash1 TEXT, + mtime INTEGER ); CREATE TABLE packages( - id INTEGER PRIMARY KEY, + id INTEGER PRIMARY KEY, name TEXT, epoch INTEGER, version TEXT, @@ -218,10 +223,18 @@ class DatabaseLocal(Database): if self.format < 1: c.execute("ALTER TABLE packages ADD COLUMN vendor TEXT AFTER uuid") + if self.format < 2: + c.execute("ALTER TABLE files ADD COLUMN `config` INTEGER") + c.execute("ALTER TABLE files ADD COLUMN `mode` INTEGER") + c.execute("ALTER TABLE files ADD COLUMN `user` TEXT") + c.execute("ALTER TABLE files ADD COLUMN `group` TEXT") + c.execute("ALTER TABLE files ADD COLUMN `mtime` INTEGER") + # In the end, we can easily update the version of the database. c.execute("UPDATE settings SET val = ? WHERE key = 'version'", (DATABASE_FORMAT,)) self.__format = DATABASE_FORMAT + self.commit() c.close() def add_package(self, pkg, reason=None): @@ -289,8 +302,9 @@ class DatabaseLocal(Database): pkg_id = c.lastrowid - c.executemany("INSERT INTO files(name, pkg, size, hash1) VALUES(?, ?, ?, ?)", - ((f.name, pkg_id, f.size, f.hash1) for f in pkg.filelist)) + c.executemany("INSERT INTO files(`name`, `pkg`, `size`, `config`, `type`, `hash1`, `mode`, `user`, `group`, `mtime`)" + " VALUES(?, ?, ?, ?, ?, ?, ?, ?, ?, ?)", + ((f.name, pkg_id, f.size, f.is_config(), f.type, f.hash1, f.mode, f.user, f.group, f.mtime) for f in pkg.filelist)) except: raise -- 2.39.2