[people/ms/u-boot.git] / tools / patman / patchstream.py

# Copyright (c) 2011 The Chromium OS Authors.
#
# SPDX-License-Identifier:	GPL-2.0+
#

import math
import os
import re
import shutil
import tempfile

import command
import commit
import gitutil
from series import Series

# Tags that we detect and remove
re_remove = re.compile('^BUG=|^TEST=|^BRANCH=|^Change-Id:|^Review URL:'
    '|Reviewed-on:|Commit-\w*:')

# Lines which are allowed after a TEST= line
re_allowed_after_test = re.compile('^Signed-off-by:')

# Signoffs
re_signoff = re.compile('^Signed-off-by: *(.*)')

# The start of the cover letter
re_cover = re.compile('^Cover-letter:')

# A cover letter Cc
re_cover_cc = re.compile('^Cover-letter-cc: *(.*)')

# Patch series tag
re_series_tag = re.compile('^Series-([a-z-]*): *(.*)')

# Commit series tag
re_commit_tag = re.compile('^Commit-([a-z-]*): *(.*)')

# Commit tags that we want to collect and keep
re_tag = re.compile('^(Tested-by|Acked-by|Reviewed-by|Patch-cc): (.*)')

# The start of a new commit in the git log
re_commit = re.compile('^commit ([0-9a-f]*)$')

# We detect these since checkpatch doesn't always do it
re_space_before_tab = re.compile('^[+].* \t')

# States we can be in - can we use range() and still have comments?
STATE_MSG_HEADER = 0        # Still in the message header
STATE_PATCH_SUBJECT = 1     # In patch subject (first line of log for a commit)
STATE_PATCH_HEADER = 2      # In patch header (after the subject)
STATE_DIFFS = 3             # In the diff part (past --- line)

class PatchStream:
    """Class for detecting/injecting tags in a patch or series of patches

    We support processing the output of 'git log' to read out the tags we
    are interested in. We can also process a patch file in order to remove
    unwanted tags or inject additional ones. These correspond to the two
    phases of processing.
    """
    def __init__(self, series, name=None, is_log=False):
        self.skip_blank = False          # True to skip a single blank line
        self.found_test = False          # Found a TEST= line
        self.lines_after_test = 0        # MNumber of lines found after TEST=
        self.warn = []                   # List of warnings we have collected
        self.linenum = 1                 # Output line number we are up to
        self.in_section = None           # Name of start...END section we are in
        self.notes = []                  # Series notes
        self.section = []                # The current section...END section
        self.series = series             # Info about the patch series
        self.is_log = is_log             # True if indent like git log
        self.in_change = 0               # Non-zero if we are in a change list
        self.blank_count = 0             # Number of blank lines stored up
        self.state = STATE_MSG_HEADER    # What state are we in?
        self.signoff = []                # Contents of signoff line
        self.commit = None               # Current commit

    def AddToSeries(self, line, name, value):
        """Add a new Series-xxx tag.

        When a Series-xxx tag is detected, we come here to record it, if we
        are scanning a 'git log'.

        Args:
            line: Source line containing tag (useful for debug/error messages)
            name: Tag name (part after 'Series-')
            value: Tag value (part after 'Series-xxx: ')
        """
        if name == 'notes':
            self.in_section = name
            self.skip_blank = False
        if self.is_log:
            self.series.AddTag(self.commit, line, name, value)

    def AddToCommit(self, line, name, value):
        """Add a new Commit-xxx tag.

        When a Commit-xxx tag is detected, we come here to record it.

        Args:
            line: Source line containing tag (useful for debug/error messages)
            name: Tag name (part after 'Commit-')
            value: Tag value (part after 'Commit-xxx: ')
        """
        if name == 'notes':
            self.in_section = 'commit-' + name
            self.skip_blank = False

    def CloseCommit(self):
        """Save the current commit into our commit list, and reset our state"""
        if self.commit and self.is_log:
            self.series.AddCommit(self.commit)
            self.commit = None

    def ProcessLine(self, line):
        """Process a single line of a patch file or commit log

        This process a line and returns a list of lines to output. The list
        may be empty or may contain multiple output lines.

        This is where all the complicated logic is located. The class's
        state is used to move between different states and detect things
        properly.

        We can be in one of two modes:
            self.is_log == True: This is 'git log' mode, where most output is
                indented by 4 characters and we are scanning for tags

            self.is_log == False: This is 'patch' mode, where we already have
                all the tags, and are processing patches to remove junk we
                don't want, and add things we think are required.

        Args:
            line: text line to process

        Returns:
            list of output lines, or [] if nothing should be output
        """
        # Initially we have no output. Prepare the input line string
        out = []
        line = line.rstrip('\n')

        commit_match = re_commit.match(line) if self.is_log else None

        if self.is_log:
            if line[:4] == '    ':
                line = line[4:]

        # Handle state transition and skipping blank lines
        series_tag_match = re_series_tag.match(line)
        commit_tag_match = re_commit_tag.match(line)
        cover_cc_match = re_cover_cc.match(line)
        signoff_match = re_signoff.match(line)
        tag_match = None
        if self.state == STATE_PATCH_HEADER:
            tag_match = re_tag.match(line)
        is_blank = not line.strip()
        if is_blank:
            if (self.state == STATE_MSG_HEADER
                    or self.state == STATE_PATCH_SUBJECT):
                self.state += 1

            # We don't have a subject in the text stream of patch files
            # It has its own line with a Subject: tag
            if not self.is_log and self.state == STATE_PATCH_SUBJECT:
                self.state += 1
        elif commit_match:
            self.state = STATE_MSG_HEADER

        # If we are in a section, keep collecting lines until we see END
        if self.in_section:
            if line == 'END':
                if self.in_section == 'cover':
                    self.series.cover = self.section
                elif self.in_section == 'notes':
                    if self.is_log:
                        self.series.notes += self.section
                elif self.in_section == 'commit-notes':
                    if self.is_log:
                        self.commit.notes += self.section
                else:
                    self.warn.append("Unknown section '%s'" % self.in_section)
                self.in_section = None
                self.skip_blank = True
                self.section = []
            else:
                self.section.append(line)

        # Detect the commit subject
        elif not is_blank and self.state == STATE_PATCH_SUBJECT:
            self.commit.subject = line

        # Detect the tags we want to remove, and skip blank lines
        elif re_remove.match(line) and not commit_tag_match:
            self.skip_blank = True

            # TEST= should be the last thing in the commit, so remove
            # everything after it
            if line.startswith('TEST='):
                self.found_test = True
        elif self.skip_blank and is_blank:
            self.skip_blank = False

        # Detect the start of a cover letter section
        elif re_cover.match(line):
            self.in_section = 'cover'
            self.skip_blank = False

        elif cover_cc_match:
            value = cover_cc_match.group(1)
            self.AddToSeries(line, 'cover-cc', value)

        # If we are in a change list, key collected lines until a blank one
        elif self.in_change:
            if is_blank:
                # Blank line ends this change list
                self.in_change = 0
            elif line == '---':
                self.in_change = 0
                out = self.ProcessLine(line)
            else:
                if self.is_log:
                    self.series.AddChange(self.in_change, self.commit, line)
            self.skip_blank = False

        # Detect Series-xxx tags
        elif series_tag_match:
            name = series_tag_match.group(1)
            value = series_tag_match.group(2)
            if name == 'changes':
                # value is the version number: e.g. 1, or 2
                try:
                    value = int(value)
                except ValueError as str:
                    raise ValueError("%s: Cannot decode version info '%s'" %
                        (self.commit.hash, line))
                self.in_change = int(value)
            else:
                self.AddToSeries(line, name, value)
                self.skip_blank = True

        # Detect Commit-xxx tags
        elif commit_tag_match:
            name = commit_tag_match.group(1)
            value = commit_tag_match.group(2)
            if name == 'notes':
                self.AddToCommit(line, name, value)
                self.skip_blank = True

        # Detect the start of a new commit
        elif commit_match:
            self.CloseCommit()
            self.commit = commit.Commit(commit_match.group(1))

        # Detect tags in the commit message
        elif tag_match:
            # Remove Tested-by self, since few will take much notice
            if (tag_match.group(1) == 'Tested-by' and
                    tag_match.group(2).find(os.getenv('USER') + '@') != -1):
                self.warn.append("Ignoring %s" % line)
            elif tag_match.group(1) == 'Patch-cc':
                self.commit.AddCc(tag_match.group(2).split(','))
            else:
                out = [line]

        # Suppress duplicate signoffs
        elif signoff_match:
            if (self.is_log or not self.commit or
                self.commit.CheckDuplicateSignoff(signoff_match.group(1))):
                out = [line]

        # Well that means this is an ordinary line
        else:
            pos = 1
            # Look for ugly ASCII characters
            for ch in line:
                # TODO: Would be nicer to report source filename and line
                if ord(ch) > 0x80:
                    self.warn.append("Line %d/%d ('%s') has funny ascii char" %
                        (self.linenum, pos, line))
                pos += 1

            # Look for space before tab
            m = re_space_before_tab.match(line)
            if m:
                self.warn.append('Line %d/%d has space before tab' %
                    (self.linenum, m.start()))

            # OK, we have a valid non-blank line
            out = [line]
            self.linenum += 1
            self.skip_blank = False
            if self.state == STATE_DIFFS:
                pass

            # If this is the start of the diffs section, emit our tags and
            # change log
            elif line == '---':
                self.state = STATE_DIFFS

                # Output the tags (signeoff first), then change list
                out = []
                log = self.series.MakeChangeLog(self.commit)
                out += [line]
                if self.commit:
                    out += self.commit.notes
                out += [''] + log
            elif self.found_test:
                if not re_allowed_after_test.match(line):
                    self.lines_after_test += 1

        return out

    def Finalize(self):
        """Close out processing of this patch stream"""
        self.CloseCommit()
        if self.lines_after_test:
            self.warn.append('Found %d lines after TEST=' %
                    self.lines_after_test)

    def ProcessStream(self, infd, outfd):
        """Copy a stream from infd to outfd, filtering out unwanting things.

        This is used to process patch files one at a time.

        Args:
            infd: Input stream file object
            outfd: Output stream file object
        """
        # Extract the filename from each diff, for nice warnings
        fname = None
        last_fname = None
        re_fname = re.compile('diff --git a/(.*) b/.*')
        while True:
            line = infd.readline()
            if not line:
                break
            out = self.ProcessLine(line)

            # Try to detect blank lines at EOF
            for line in out:
                match = re_fname.match(line)
                if match:
                    last_fname = fname
                    fname = match.group(1)
                if line == '+':
                    self.blank_count += 1
                else:
                    if self.blank_count and (line == '-- ' or match):
                        self.warn.append("Found possible blank line(s) at "
                                "end of file '%s'" % last_fname)
                    outfd.write('+\n' * self.blank_count)
                    outfd.write(line + '\n')
                    self.blank_count = 0
        self.Finalize()


def GetMetaDataForList(commit_range, git_dir=None, count=None,
                       series = None, allow_overwrite=False):
    """Reads out patch series metadata from the commits

    This does a 'git log' on the relevant commits and pulls out the tags we
    are interested in.

    Args:
        commit_range: Range of commits to count (e.g. 'HEAD..base')
        git_dir: Path to git repositiory (None to use default)
        count: Number of commits to list, or None for no limit
        series: Series object to add information into. By default a new series
            is started.
        allow_overwrite: Allow tags to overwrite an existing tag
    Returns:
        A Series object containing information about the commits.
    """
    if not series:
        series = Series()
    series.allow_overwrite = allow_overwrite
    params = gitutil.LogCmd(commit_range,reverse=True, count=count,
                            git_dir=git_dir)
    stdout = command.RunPipe([params], capture=True).stdout
    ps = PatchStream(series, is_log=True)
    for line in stdout.splitlines():
        ps.ProcessLine(line)
    ps.Finalize()
    return series

def GetMetaData(start, count):
    """Reads out patch series metadata from the commits

    This does a 'git log' on the relevant commits and pulls out the tags we
    are interested in.

    Args:
        start: Commit to start from: 0=HEAD, 1=next one, etc.
        count: Number of commits to list
    """
    return GetMetaDataForList('HEAD~%d' % start, None, count)

def FixPatch(backup_dir, fname, series, commit):
    """Fix up a patch file, by adding/removing as required.

    We remove our tags from the patch file, insert changes lists, etc.
    The patch file is processed in place, and overwritten.

    A backup file is put into backup_dir (if not None).

    Args:
        fname: Filename to patch file to process
        series: Series information about this patch set
        commit: Commit object for this patch file
    Return:
        A list of errors, or [] if all ok.
    """
    handle, tmpname = tempfile.mkstemp()
    outfd = os.fdopen(handle, 'w')
    infd = open(fname, 'r')
    ps = PatchStream(series)
    ps.commit = commit
    ps.ProcessStream(infd, outfd)
    infd.close()
    outfd.close()

    # Create a backup file if required
    if backup_dir:
        shutil.copy(fname, os.path.join(backup_dir, os.path.basename(fname)))
    shutil.move(tmpname, fname)
    return ps.warn

def FixPatches(series, fnames):
    """Fix up a list of patches identified by filenames

    The patch files are processed in place, and overwritten.

    Args:
        series: The series object
        fnames: List of patch files to process
    """
    # Current workflow creates patches, so we shouldn't need a backup
    backup_dir = None  #tempfile.mkdtemp('clean-patch')
    count = 0
    for fname in fnames:
        commit = series.commits[count]
        commit.patch = fname
        result = FixPatch(backup_dir, fname, series, commit)
        if result:
            print '%d warnings for %s:' % (len(result), fname)
            for warn in result:
                print '\t', warn
            print
        count += 1
    print 'Cleaned %d patches' % count
    return series

def InsertCoverLetter(fname, series, count):
    """Inserts a cover letter with the required info into patch 0

    Args:
        fname: Input / output filename of the cover letter file
        series: Series object
        count: Number of patches in the series
    """
    fd = open(fname, 'r')
    lines = fd.readlines()
    fd.close()

    fd = open(fname, 'w')
    text = series.cover
    prefix = series.GetPatchPrefix()
    for line in lines:
        if line.startswith('Subject:'):
            # if more than 10 or 100 patches, it should say 00/xx, 000/xxx, etc
            zero_repeat = int(math.log10(count)) + 1
            zero = '0' * zero_repeat
            line = 'Subject: [%s %s/%d] %s\n' % (prefix, zero, count, text[0])

        # Insert our cover letter
        elif line.startswith('*** BLURB HERE ***'):
            # First the blurb test
            line = '\n'.join(text[1:]) + '\n'
            if series.get('notes'):
                line += '\n'.join(series.notes) + '\n'

            # Now the change list
            out = series.MakeChangeLog(None)
            line += '\n' + '\n'.join(out)
        fd.write(line)
    fd.close()
Commit	Line	Data
0d24de9d SG	1	# Copyright (c) 2011 The Chromium OS Authors.
0d24de9d SG	2	#
1a459660	3	# SPDX-License-Identifier: GPL-2.0+
0d24de9d SG	4	#
0d24de9d SG	5
35ce2dc4	6	import math
0d24de9d SG	7	import os
	8	import re
	9	import shutil
	10	import tempfile
	11
	12	import command
	13	import commit
	14	import gitutil
	15	from series import Series
	16
	17	# Tags that we detect and remove
619dd5de	18	re_remove = re.compile('^BUG=\|^TEST=\|^BRANCH=\|^Change-Id:\|^Review URL:'
3fefd5ef	19	'\|Reviewed-on:\|Commit-\w*:')
0d24de9d SG	20
	21	# Lines which are allowed after a TEST= line
	22	re_allowed_after_test = re.compile('^Signed-off-by:')
	23
05e5b735	24	# Signoffs
102061bd	25	re_signoff = re.compile('^Signed-off-by: (.)')
05e5b735	26
0d24de9d SG	27	# The start of the cover letter
	28	re_cover = re.compile('^Cover-letter:')
	29
fe2f8d9e SG	30	# A cover letter Cc
	31	re_cover_cc = re.compile('^Cover-letter-cc: (.)')
	32
0d24de9d	33	# Patch series tag
5c8fdd91 AA	34	re_series_tag = re.compile('^Series-([a-z-]): (.*)')
	35
	36	# Commit series tag
	37	re_commit_tag = re.compile('^Commit-([a-z-]): (.*)')
0d24de9d SG	38
0d24de9d SG	39	# Commit tags that we want to collect and keep
659c89da	40	re_tag = re.compile('^(Tested-by\|Acked-by\|Reviewed-by\|Patch-cc): (.*)')
0d24de9d SG	41
0d24de9d SG	42	# The start of a new commit in the git log
68618281	43	re_commit = re.compile('^commit ([0-9a-f]*)$')
0d24de9d SG	44
	45	# We detect these since checkpatch doesn't always do it
	46	re_space_before_tab = re.compile('^[+].* \t')
	47
	48	# States we can be in - can we use range() and still have comments?
	49	STATE_MSG_HEADER = 0 # Still in the message header
	50	STATE_PATCH_SUBJECT = 1 # In patch subject (first line of log for a commit)
	51	STATE_PATCH_HEADER = 2 # In patch header (after the subject)
	52	STATE_DIFFS = 3 # In the diff part (past --- line)
	53
	54	class PatchStream:
	55	"""Class for detecting/injecting tags in a patch or series of patches
	56
	57	We support processing the output of 'git log' to read out the tags we
	58	are interested in. We can also process a patch file in order to remove
	59	unwanted tags or inject additional ones. These correspond to the two
	60	phases of processing.
	61	"""
	62	def __init__(self, series, name=None, is_log=False):
	63	self.skip_blank = False # True to skip a single blank line
	64	self.found_test = False # Found a TEST= line
	65	self.lines_after_test = 0 # MNumber of lines found after TEST=
	66	self.warn = [] # List of warnings we have collected
	67	self.linenum = 1 # Output line number we are up to
	68	self.in_section = None # Name of start...END section we are in
	69	self.notes = [] # Series notes
	70	self.section = [] # The current section...END section
	71	self.series = series # Info about the patch series
	72	self.is_log = is_log # True if indent like git log
	73	self.in_change = 0 # Non-zero if we are in a change list
	74	self.blank_count = 0 # Number of blank lines stored up
	75	self.state = STATE_MSG_HEADER # What state are we in?
0d24de9d SG	76	self.signoff = [] # Contents of signoff line
	77	self.commit = None # Current commit
	78
	79	def AddToSeries(self, line, name, value):
	80	"""Add a new Series-xxx tag.
	81
	82	When a Series-xxx tag is detected, we come here to record it, if we
	83	are scanning a 'git log'.
	84
	85	Args:
	86	line: Source line containing tag (useful for debug/error messages)
	87	name: Tag name (part after 'Series-')
	88	value: Tag value (part after 'Series-xxx: ')
	89	"""
	90	if name == 'notes':
	91	self.in_section = name
	92	self.skip_blank = False
	93	if self.is_log:
	94	self.series.AddTag(self.commit, line, name, value)
	95
5c8fdd91 AA	96	def AddToCommit(self, line, name, value):
	97	"""Add a new Commit-xxx tag.
	98
	99	When a Commit-xxx tag is detected, we come here to record it.
	100
	101	Args:
	102	line: Source line containing tag (useful for debug/error messages)
	103	name: Tag name (part after 'Commit-')
	104	value: Tag value (part after 'Commit-xxx: ')
	105	"""
	106	if name == 'notes':
	107	self.in_section = 'commit-' + name
	108	self.skip_blank = False
	109
0d24de9d SG	110	def CloseCommit(self):
	111	"""Save the current commit into our commit list, and reset our state"""
	112	if self.commit and self.is_log:
	113	self.series.AddCommit(self.commit)
	114	self.commit = None
	115
0d24de9d SG	116	def ProcessLine(self, line):
	117	"""Process a single line of a patch file or commit log
	118
	119	This process a line and returns a list of lines to output. The list
	120	may be empty or may contain multiple output lines.
	121
	122	This is where all the complicated logic is located. The class's
	123	state is used to move between different states and detect things
	124	properly.
	125
	126	We can be in one of two modes:
	127	self.is_log == True: This is 'git log' mode, where most output is
	128	indented by 4 characters and we are scanning for tags
	129
	130	self.is_log == False: This is 'patch' mode, where we already have
	131	all the tags, and are processing patches to remove junk we
	132	don't want, and add things we think are required.
	133
	134	Args:
	135	line: text line to process
	136
	137	Returns:
	138	list of output lines, or [] if nothing should be output
	139	"""
	140	# Initially we have no output. Prepare the input line string
	141	out = []
	142	line = line.rstrip('\n')
4b89b813 SW	143
	144	commit_match = re_commit.match(line) if self.is_log else None
	145
0d24de9d SG	146	if self.is_log:
	147	if line[:4] == ' ':
	148	line = line[4:]
	149
	150	# Handle state transition and skipping blank lines
5c8fdd91 AA	151	series_tag_match = re_series_tag.match(line)
5c8fdd91 AA	152	commit_tag_match = re_commit_tag.match(line)
fe2f8d9e	153	cover_cc_match = re_cover_cc.match(line)
102061bd	154	signoff_match = re_signoff.match(line)
0d24de9d SG	155	tag_match = None
	156	if self.state == STATE_PATCH_HEADER:
	157	tag_match = re_tag.match(line)
	158	is_blank = not line.strip()
	159	if is_blank:
	160	if (self.state == STATE_MSG_HEADER
	161	or self.state == STATE_PATCH_SUBJECT):
	162	self.state += 1
	163
	164	# We don't have a subject in the text stream of patch files
	165	# It has its own line with a Subject: tag
	166	if not self.is_log and self.state == STATE_PATCH_SUBJECT:
	167	self.state += 1
	168	elif commit_match:
	169	self.state = STATE_MSG_HEADER
	170
	171	# If we are in a section, keep collecting lines until we see END
	172	if self.in_section:
	173	if line == 'END':
	174	if self.in_section == 'cover':
	175	self.series.cover = self.section
	176	elif self.in_section == 'notes':
	177	if self.is_log:
	178	self.series.notes += self.section
5c8fdd91 AA	179	elif self.in_section == 'commit-notes':
	180	if self.is_log:
	181	self.commit.notes += self.section
0d24de9d SG	182	else:
	183	self.warn.append("Unknown section '%s'" % self.in_section)
	184	self.in_section = None
	185	self.skip_blank = True
	186	self.section = []
	187	else:
	188	self.section.append(line)
	189
	190	# Detect the commit subject
	191	elif not is_blank and self.state == STATE_PATCH_SUBJECT:
	192	self.commit.subject = line
	193
	194	# Detect the tags we want to remove, and skip blank lines
5c8fdd91	195	elif re_remove.match(line) and not commit_tag_match:
0d24de9d SG	196	self.skip_blank = True
	197
	198	# TEST= should be the last thing in the commit, so remove
	199	# everything after it
	200	if line.startswith('TEST='):
	201	self.found_test = True
	202	elif self.skip_blank and is_blank:
	203	self.skip_blank = False
	204
	205	# Detect the start of a cover letter section
	206	elif re_cover.match(line):
	207	self.in_section = 'cover'
	208	self.skip_blank = False
	209
fe2f8d9e SG	210	elif cover_cc_match:
	211	value = cover_cc_match.group(1)
	212	self.AddToSeries(line, 'cover-cc', value)
	213
0d24de9d SG	214	# If we are in a change list, key collected lines until a blank one
	215	elif self.in_change:
	216	if is_blank:
	217	# Blank line ends this change list
	218	self.in_change = 0
102061bd	219	elif line == '---':
05e5b735 IY	220	self.in_change = 0
05e5b735 IY	221	out = self.ProcessLine(line)
0d24de9d	222	else:
a8840cb2 IY	223	if self.is_log:
a8840cb2 IY	224	self.series.AddChange(self.in_change, self.commit, line)
0d24de9d SG	225	self.skip_blank = False
	226
	227	# Detect Series-xxx tags
5c8fdd91 AA	228	elif series_tag_match:
	229	name = series_tag_match.group(1)
	230	value = series_tag_match.group(2)
0d24de9d SG	231	if name == 'changes':
	232	# value is the version number: e.g. 1, or 2
	233	try:
	234	value = int(value)
	235	except ValueError as str:
	236	raise ValueError("%s: Cannot decode version info '%s'" %
	237	(self.commit.hash, line))
	238	self.in_change = int(value)
	239	else:
	240	self.AddToSeries(line, name, value)
	241	self.skip_blank = True
	242
5c8fdd91 AA	243	# Detect Commit-xxx tags
	244	elif commit_tag_match:
	245	name = commit_tag_match.group(1)
	246	value = commit_tag_match.group(2)
	247	if name == 'notes':
	248	self.AddToCommit(line, name, value)
	249	self.skip_blank = True
	250
0d24de9d SG	251	# Detect the start of a new commit
	252	elif commit_match:
	253	self.CloseCommit()
0b5b409a	254	self.commit = commit.Commit(commit_match.group(1))
0d24de9d SG	255
	256	# Detect tags in the commit message
	257	elif tag_match:
0d24de9d	258	# Remove Tested-by self, since few will take much notice
c7379149	259	if (tag_match.group(1) == 'Tested-by' and
0d24de9d SG	260	tag_match.group(2).find(os.getenv('USER') + '@') != -1):
0d24de9d SG	261	self.warn.append("Ignoring %s" % line)
659c89da	262	elif tag_match.group(1) == 'Patch-cc':
0d24de9d SG	263	self.commit.AddCc(tag_match.group(2).split(','))
0d24de9d SG	264	else:
d0c5719d	265	out = [line]
0d24de9d	266
102061bd SG	267	# Suppress duplicate signoffs
102061bd SG	268	elif signoff_match:
e752edcb	269	if (self.is_log or not self.commit or
6be6b6bc	270	self.commit.CheckDuplicateSignoff(signoff_match.group(1))):
102061bd SG	271	out = [line]
102061bd SG	272
0d24de9d SG	273	# Well that means this is an ordinary line
	274	else:
	275	pos = 1
	276	# Look for ugly ASCII characters
	277	for ch in line:
	278	# TODO: Would be nicer to report source filename and line
	279	if ord(ch) > 0x80:
	280	self.warn.append("Line %d/%d ('%s') has funny ascii char" %
	281	(self.linenum, pos, line))
	282	pos += 1
	283
	284	# Look for space before tab
	285	m = re_space_before_tab.match(line)
	286	if m:
	287	self.warn.append('Line %d/%d has space before tab' %
	288	(self.linenum, m.start()))
	289
	290	# OK, we have a valid non-blank line
	291	out = [line]
	292	self.linenum += 1
	293	self.skip_blank = False
	294	if self.state == STATE_DIFFS:
	295	pass
	296
	297	# If this is the start of the diffs section, emit our tags and
	298	# change log
	299	elif line == '---':
	300	self.state = STATE_DIFFS
	301
	302	# Output the tags (signeoff first), then change list
	303	out = []
0d24de9d	304	log = self.series.MakeChangeLog(self.commit)
e752edcb SG	305	out += [line]
	306	if self.commit:
	307	out += self.commit.notes
	308	out += [''] + log
0d24de9d SG	309	elif self.found_test:
	310	if not re_allowed_after_test.match(line):
	311	self.lines_after_test += 1
	312
	313	return out
	314
	315	def Finalize(self):
	316	"""Close out processing of this patch stream"""
	317	self.CloseCommit()
	318	if self.lines_after_test:
	319	self.warn.append('Found %d lines after TEST=' %
	320	self.lines_after_test)
	321
	322	def ProcessStream(self, infd, outfd):
	323	"""Copy a stream from infd to outfd, filtering out unwanting things.
	324
	325	This is used to process patch files one at a time.
	326
	327	Args:
	328	infd: Input stream file object
	329	outfd: Output stream file object
	330	"""
	331	# Extract the filename from each diff, for nice warnings
	332	fname = None
	333	last_fname = None
	334	re_fname = re.compile('diff --git a/(.) b/.')
	335	while True:
	336	line = infd.readline()
	337	if not line:
	338	break
	339	out = self.ProcessLine(line)
	340
	341	# Try to detect blank lines at EOF
	342	for line in out:
	343	match = re_fname.match(line)
	344	if match:
	345	last_fname = fname
	346	fname = match.group(1)
	347	if line == '+':
	348	self.blank_count += 1
	349	else:
	350	if self.blank_count and (line == '-- ' or match):
	351	self.warn.append("Found possible blank line(s) at "
	352	"end of file '%s'" % last_fname)
	353	outfd.write('+\n' * self.blank_count)
	354	outfd.write(line + '\n')
	355	self.blank_count = 0
	356	self.Finalize()
	357
	358
e62f905e	359	def GetMetaDataForList(commit_range, git_dir=None, count=None,
950a2313	360	series = None, allow_overwrite=False):
0d24de9d SG	361	"""Reads out patch series metadata from the commits
	362
	363	This does a 'git log' on the relevant commits and pulls out the tags we
	364	are interested in.
	365
	366	Args:
e62f905e SG	367	commit_range: Range of commits to count (e.g. 'HEAD..base')
	368	git_dir: Path to git repositiory (None to use default)
	369	count: Number of commits to list, or None for no limit
	370	series: Series object to add information into. By default a new series
	371	is started.
950a2313	372	allow_overwrite: Allow tags to overwrite an existing tag
e62f905e SG	373	Returns:
e62f905e SG	374	A Series object containing information about the commits.
0d24de9d	375	"""
891b7a07 SG	376	if not series:
891b7a07 SG	377	series = Series()
950a2313	378	series.allow_overwrite = allow_overwrite
cda2a611 SG	379	params = gitutil.LogCmd(commit_range,reverse=True, count=count,
	380	git_dir=git_dir)
	381	stdout = command.RunPipe([params], capture=True).stdout
0d24de9d SG	382	ps = PatchStream(series, is_log=True)
	383	for line in stdout.splitlines():
	384	ps.ProcessLine(line)
	385	ps.Finalize()
	386	return series
	387
e62f905e SG	388	def GetMetaData(start, count):
	389	"""Reads out patch series metadata from the commits
	390
	391	This does a 'git log' on the relevant commits and pulls out the tags we
	392	are interested in.
	393
	394	Args:
	395	start: Commit to start from: 0=HEAD, 1=next one, etc.
	396	count: Number of commits to list
	397	"""
	398	return GetMetaDataForList('HEAD~%d' % start, None, count)
	399
0d24de9d SG	400	def FixPatch(backup_dir, fname, series, commit):
	401	"""Fix up a patch file, by adding/removing as required.
	402
	403	We remove our tags from the patch file, insert changes lists, etc.
	404	The patch file is processed in place, and overwritten.
	405
	406	A backup file is put into backup_dir (if not None).
	407
	408	Args:
	409	fname: Filename to patch file to process
	410	series: Series information about this patch set
	411	commit: Commit object for this patch file
	412	Return:
	413	A list of errors, or [] if all ok.
	414	"""
	415	handle, tmpname = tempfile.mkstemp()
	416	outfd = os.fdopen(handle, 'w')
	417	infd = open(fname, 'r')
	418	ps = PatchStream(series)
	419	ps.commit = commit
	420	ps.ProcessStream(infd, outfd)
	421	infd.close()
	422	outfd.close()
	423
	424	# Create a backup file if required
	425	if backup_dir:
	426	shutil.copy(fname, os.path.join(backup_dir, os.path.basename(fname)))
	427	shutil.move(tmpname, fname)
	428	return ps.warn
	429
	430	def FixPatches(series, fnames):
	431	"""Fix up a list of patches identified by filenames
	432
	433	The patch files are processed in place, and overwritten.
	434
	435	Args:
	436	series: The series object
	437	fnames: List of patch files to process
	438	"""
	439	# Current workflow creates patches, so we shouldn't need a backup
	440	backup_dir = None #tempfile.mkdtemp('clean-patch')
	441	count = 0
	442	for fname in fnames:
	443	commit = series.commits[count]
	444	commit.patch = fname
	445	result = FixPatch(backup_dir, fname, series, commit)
	446	if result:
	447	print '%d warnings for %s:' % (len(result), fname)
	448	for warn in result:
	449	print '\t', warn
	450	print
	451	count += 1
	452	print 'Cleaned %d patches' % count
	453	return series
	454
	455	def InsertCoverLetter(fname, series, count):
	456	"""Inserts a cover letter with the required info into patch 0
	457
	458	Args:
	459	fname: Input / output filename of the cover letter file
	460	series: Series object
	461	count: Number of patches in the series
	462	"""
	463	fd = open(fname, 'r')
464	lines = fd.readlines()
465	fd.close()
466
467	fd = open(fname, 'w')
468	text = series.cover
469	prefix = series.GetPatchPrefix()
470	for line in lines:
471	if line.startswith('Subject:'):
35ce2dc4 WJ	472	# if more than 10 or 100 patches, it should say 00/xx, 000/xxx, etc
	473	zero_repeat = int(math.log10(count)) + 1
	474	zero = '0' * zero_repeat
	475	line = 'Subject: [%s %s/%d] %s\n' % (prefix, zero, count, text[0])
0d24de9d SG	476
	477	# Insert our cover letter
	478	elif line.startswith('* BLURB HERE *'):
	479	# First the blurb test
	480	line = '\n'.join(text[1:]) + '\n'
	481	if series.get('notes'):
	482	line += '\n'.join(series.notes) + '\n'
	483
	484	# Now the change list
	485	out = series.MakeChangeLog(None)
	486	line += '\n' + '\n'.join(out)
	487	fd.write(line)
	488	fd.close()