From: Trevor Gamblin Date: Tue, 24 Sep 2024 11:54:59 +0000 (-0400) Subject: patchtest: mbox.py: new data implementation X-Git-Tag: yocto-5.2~1741 X-Git-Url: http://git.ipfire.org/gitweb.cgi?a=commitdiff_plain;h=d4fbdb1d15f281b236137d63710c73bca8911a36;p=thirdparty%2Fopenembedded%2Fopenembedded-core-contrib.git patchtest: mbox.py: new data implementation Consolidate and improve some objects: - absorb utils.py functionality - repo.py: use mbox.py - repo.py: remove some cruft - utils.py: replace with logs.py - utils.py: delete - patch.py: delete - scripts/patchtest: use logging directly - general cleanup Signed-off-by: Trevor Gamblin Signed-off-by: Richard Purdie --- diff --git a/meta/lib/patchtest/mbox.py b/meta/lib/patchtest/mbox.py new file mode 100644 index 00000000000..1d95819b7ae --- /dev/null +++ b/meta/lib/patchtest/mbox.py @@ -0,0 +1,108 @@ +#! /usr/bin/env python3 + +# series.py +# +# Read a series' mbox file and get information about the patches +# contained +# +# Copyright (C) 2024 BayLibre SAS +# +# SPDX-License-Identifier: GPL-2.0-only +# + +import email +import re + +# From: https://stackoverflow.com/questions/59681461/read-a-big-mbox-file-with-python +class MboxReader: + def __init__(self, filepath): + self.handle = open(filepath, 'rb') + assert self.handle.readline().startswith(b'From ') + + def __enter__(self): + return self + + def __exit__(self, exc_type, exc_value, exc_traceback): + self.handle.close() + + def __iter__(self): + return iter(self.__next__()) + + def __next__(self): + lines = [] + while True: + line = self.handle.readline() + if line == b'' or line.startswith(b'From '): + yield email.message_from_bytes(b''.join(lines)) + if line == b'': + break + lines = [] + continue + lines.append(line) + +class Patch: + def __init__(self, data): + self.author = data['From'] + self.to = data['To'] + self.cc = data['Cc'] + self.subject = data['Subject'] + self.split_body = re.split('---', data.get_payload(), maxsplit=1) + self.commit_message = self.split_body[0] + self.diff = self.split_body[1] + +class PatchSeries: + def __init__(self, filepath): + with MboxReader(filepath) as mbox: + self.patches = [Patch(message) for message in mbox] + + assert self.patches + self.patch_count = len(self.patches) + self.path = filepath + + @property + def path(self): + return self.path + + self.branch = self.get_branch() + + def get_branch(self): + fullprefix = "" + pattern = re.compile(r"(\[.*\])", re.DOTALL) + + # There should be at least one patch in the series and it should + # include the branch name in the subject, so parse that + match = pattern.search(self.patches[0].subject) + if match: + fullprefix = match.group(1) + + branch, branches, valid_branches = None, [], [] + + if fullprefix: + prefix = fullprefix.strip('[]') + branches = [ b.strip() for b in prefix.split(',')] + valid_branches = [b for b in branches if PatchSeries.valid_branch(b)] + + if len(valid_branches): + branch = valid_branches[0] + + # Get the branch name excluding any brackets. If nothing was + # found, then assume there was no branch tag in the subject line + # and that the patch targets master + if branch is not None: + return branch.split(']')[0] + else: + return "master" + + @staticmethod + def valid_branch(branch): + """ Check if branch is valid name """ + lbranch = branch.lower() + + invalid = lbranch.startswith('patch') or \ + lbranch.startswith('rfc') or \ + lbranch.startswith('resend') or \ + re.search(r'^v\d+', lbranch) or \ + re.search(r'^\d+/\d+', lbranch) + + return not invalid + diff --git a/meta/lib/patchtest/patch.py b/meta/lib/patchtest/patch.py deleted file mode 100644 index 90faf3eeb4f..00000000000 --- a/meta/lib/patchtest/patch.py +++ /dev/null @@ -1,43 +0,0 @@ -# ex:ts=4:sw=4:sts=4:et -# -*- tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- -# -# patchtestpatch: PatchTestPatch class which abstracts a patch file -# -# Copyright (C) 2016 Intel Corporation -# -# SPDX-License-Identifier: GPL-2.0-only -# - -import logging -import utils - -logger = logging.getLogger('patchtest') - -class PatchTestPatch(object): - def __init__(self, path, forcereload=False): - self._path = path - self._forcereload = forcereload - - self._contents = None - self._branch = None - - @property - def contents(self): - if self._forcereload or (not self._contents): - logger.debug('Reading %s contents' % self._path) - try: - with open(self._path, newline='') as _f: - self._contents = _f.read() - except IOError: - logger.warn("Reading the mbox %s failed" % self.resource) - return self._contents - - @property - def path(self): - return self._path - - @property - def branch(self): - if not self._branch: - self._branch = utils.get_branch(self._path) - return self._branch diff --git a/meta/lib/patchtest/patterns.py b/meta/lib/patchtest/patterns.py index ba97a4ffe9f..b703b0c8b99 100644 --- a/meta/lib/patchtest/patterns.py +++ b/meta/lib/patchtest/patterns.py @@ -10,11 +10,8 @@ import pyparsing colon = pyparsing.Literal(":") line_start = pyparsing.LineStart() line_end = pyparsing.LineEnd() -at = pyparsing.Literal("@") lessthan = pyparsing.Literal("<") greaterthan = pyparsing.Literal(">") -opensquare = pyparsing.Literal("[") -closesquare = pyparsing.Literal("]") inappropriate = pyparsing.CaselessLiteral("Inappropriate") submitted = pyparsing.CaselessLiteral("Submitted") diff --git a/meta/lib/patchtest/repo.py b/meta/lib/patchtest/repo.py index 5f361ac5005..8ec8f68a0bc 100644 --- a/meta/lib/patchtest/repo.py +++ b/meta/lib/patchtest/repo.py @@ -8,40 +8,27 @@ # SPDX-License-Identifier: GPL-2.0-only # -import os -import utils -import logging import git -from patch import PatchTestPatch - -logger = logging.getLogger('patchtest') -info=logger.info +import os +import mbox class PatchTestRepo(object): # prefixes used for temporal branches/stashes prefix = 'patchtest' - def __init__(self, patch, repodir, commit=None, branch=None): - self._repodir = repodir - self._repo = git.Repo.init(repodir) - self._patch = PatchTestPatch(patch) - self._current_branch = self._repo.active_branch.name + self.repodir = repodir + self.repo = git.Repo.init(repodir) + self.patch = mbox.PatchSeries(patch) + self.current_branch = self.repo.active_branch.name # targeted branch defined on the patch may be invalid, so make sure there # is a corresponding remote branch valid_patch_branch = None - if self._patch.branch in self._repo.branches: - valid_patch_branch = self._patch.branch + if self.patch.branch in self.repo.branches: + valid_patch_branch = self.patch.branch - # Target Branch - # Priority (top has highest priority): - # 1. branch given at cmd line - # 2. branch given at the patch - # 3. current branch - self._branch = branch or valid_patch_branch or self._current_branch - # Target Commit # Priority (top has highest priority): # 1. commit given at cmd line @@ -57,7 +44,7 @@ class PatchTestRepo(object): # create working branch. Use the '-B' flag so that we just # check out the existing one if it's there - self._repo.git.execute(['git', 'checkout', '-B', self._workingbranch, self._commit]) + self.repo.git.execute(['git', 'checkout', '-B', self._workingbranch, self._commit]) self._patchmerged = False @@ -65,35 +52,13 @@ class PatchTestRepo(object): self._patchcanbemerged = True try: # Make sure to get the absolute path of the file - self._repo.git.execute(['git', 'apply', '--check', os.path.abspath(self._patch.path)], with_exceptions=True) + self.repo.git.execute(['git', 'apply', '--check', os.path.abspath(self.patch.path)], with_exceptions=True) except git.exc.GitCommandError as ce: self._patchcanbemerged = False - # for debugging purposes, print all repo parameters - logger.debug("Parameters") - logger.debug("\tRepository : %s" % self._repodir) - logger.debug("\tTarget Commit : %s" % self._commit) - logger.debug("\tTarget Branch : %s" % self._branch) - logger.debug("\tWorking branch : %s" % self._workingbranch) - logger.debug("\tPatch : %s" % self._patch) - - @property - def patch(self): - return self._patch.path - - @property - def branch(self): - return self._branch - - @property - def commit(self): - return self._commit - - @property def ismerged(self): return self._patchmerged - @property def canbemerged(self): return self._patchcanbemerged @@ -103,7 +68,7 @@ class PatchTestRepo(object): return None try: - return self._repo.rev_parse(commit).hexsha + return self.repo.rev_parse(commit).hexsha except Exception as e: print(f"Couldn't find commit {commit} in repo") @@ -111,10 +76,10 @@ class PatchTestRepo(object): def merge(self): if self._patchcanbemerged: - self._repo.git.execute(['git', 'am', '--keep-cr', os.path.abspath(self._patch.path)]) + self.repo.git.execute(['git', 'am', '--keep-cr', os.path.abspath(self.patch.path)]) self._patchmerged = True def clean(self): - self._repo.git.execute(['git', 'checkout', self._current_branch]) - self._repo.git.execute(['git', 'branch', '-D', self._workingbranch]) + self.repo.git.execute(['git', 'checkout', self.current_branch]) + self.repo.git.execute(['git', 'branch', '-D', self._workingbranch]) self._patchmerged = False diff --git a/meta/lib/patchtest/tests/base.py b/meta/lib/patchtest/tests/base.py index 424e61b5be2..911addb199a 100644 --- a/meta/lib/patchtest/tests/base.py +++ b/meta/lib/patchtest/tests/base.py @@ -37,7 +37,6 @@ class Base(unittest.TestCase): endcommit_messages_regex = re.compile(r'\(From \w+-\w+ rev:|(?