tools/patman/patchstream.py

   1 # Copyright (c) 2011 The Chromium OS Authors.
   2 #
   3 # SPDX-License-Identifier:      GPL-2.0+
   4 #
   5
   6 import os
   7 import re
   8 import shutil
   9 import tempfile
  10
  11 import command
  12 import commit
  13 import gitutil
  14 from series import Series
  15
  16 # Tags that we detect and remove
  17 re_remove = re.compile('^BUG=|^TEST=|^BRANCH=|^Change-Id:|^Review URL:'
  18     '|Reviewed-on:|Commit-\w*:')
  19
  20 # Lines which are allowed after a TEST= line
  21 re_allowed_after_test = re.compile('^Signed-off-by:')
  22
  23 # Signoffs
  24 re_signoff = re.compile('^Signed-off-by:')
  25
  26 # The start of the cover letter
  27 re_cover = re.compile('^Cover-letter:')
  28
  29 # A cover letter Cc
  30 re_cover_cc = re.compile('^Cover-letter-cc: *(.*)')
  31
  32 # Patch series tag
  33 re_series = re.compile('^Series-([a-z-]*): *(.*)')
  34
  35 # Commit tags that we want to collect and keep
  36 re_tag = re.compile('^(Tested-by|Acked-by|Reviewed-by|Cc): (.*)')
  37
  38 # The start of a new commit in the git log
  39 re_commit = re.compile('^commit ([0-9a-f]*)$')
  40
  41 # We detect these since checkpatch doesn't always do it
  42 re_space_before_tab = re.compile('^[+].* \t')
  43
  44 # States we can be in - can we use range() and still have comments?
  45 STATE_MSG_HEADER = 0        # Still in the message header
  46 STATE_PATCH_SUBJECT = 1     # In patch subject (first line of log for a commit)
  47 STATE_PATCH_HEADER = 2      # In patch header (after the subject)
  48 STATE_DIFFS = 3             # In the diff part (past --- line)
  49
  50 class PatchStream:
  51     """Class for detecting/injecting tags in a patch or series of patches
  52
  53     We support processing the output of 'git log' to read out the tags we
  54     are interested in. We can also process a patch file in order to remove
  55     unwanted tags or inject additional ones. These correspond to the two
  56     phases of processing.
  57     """
  58     def __init__(self, series, name=None, is_log=False):
  59         self.skip_blank = False          # True to skip a single blank line
  60         self.found_test = False          # Found a TEST= line
  61         self.lines_after_test = 0        # MNumber of lines found after TEST=
  62         self.warn = []                   # List of warnings we have collected
  63         self.linenum = 1                 # Output line number we are up to
  64         self.in_section = None           # Name of start...END section we are in
  65         self.notes = []                  # Series notes
  66         self.section = []                # The current section...END section
  67         self.series = series             # Info about the patch series
  68         self.is_log = is_log             # True if indent like git log
  69         self.in_change = 0               # Non-zero if we are in a change list
  70         self.blank_count = 0             # Number of blank lines stored up
  71         self.state = STATE_MSG_HEADER    # What state are we in?
  72         self.tags = []                   # Tags collected, like Tested-by...
  73         self.signoff = []                # Contents of signoff line
  74         self.commit = None               # Current commit
  75
  76     def AddToSeries(self, line, name, value):
  77         """Add a new Series-xxx tag.
  78
  79         When a Series-xxx tag is detected, we come here to record it, if we
  80         are scanning a 'git log'.
  81
  82         Args:
  83             line: Source line containing tag (useful for debug/error messages)
  84             name: Tag name (part after 'Series-')
  85             value: Tag value (part after 'Series-xxx: ')
  86         """
  87         if name == 'notes':
  88             self.in_section = name
  89             self.skip_blank = False
  90         if self.is_log:
  91             self.series.AddTag(self.commit, line, name, value)
  92
  93     def CloseCommit(self):
  94         """Save the current commit into our commit list, and reset our state"""
  95         if self.commit and self.is_log:
  96             self.series.AddCommit(self.commit)
  97             self.commit = None
  98
  99     def FormatTags(self, tags):
 100         out_list = []
 101         for tag in sorted(tags):
 102             if tag.startswith('Cc:'):
 103                 tag_list = tag[4:].split(',')
 104                 out_list += gitutil.BuildEmailList(tag_list, 'Cc:')
 105             else:
 106                 out_list.append(tag)
 107         return out_list
 108
 109     def ProcessLine(self, line):
 110         """Process a single line of a patch file or commit log
 111
 112         This process a line and returns a list of lines to output. The list
 113         may be empty or may contain multiple output lines.
 114
 115         This is where all the complicated logic is located. The class's
 116         state is used to move between different states and detect things
 117         properly.
 118
 119         We can be in one of two modes:
 120             self.is_log == True: This is 'git log' mode, where most output is
 121                 indented by 4 characters and we are scanning for tags
 122
 123             self.is_log == False: This is 'patch' mode, where we already have
 124                 all the tags, and are processing patches to remove junk we
 125                 don't want, and add things we think are required.
 126
 127         Args:
 128             line: text line to process
 129
 130         Returns:
 131             list of output lines, or [] if nothing should be output
 132         """
 133         # Initially we have no output. Prepare the input line string
 134         out = []
 135         line = line.rstrip('\n')
 136         if self.is_log:
 137             if line[:4] == '    ':
 138                 line = line[4:]
 139
 140         # Handle state transition and skipping blank lines
 141         series_match = re_series.match(line)
 142         commit_match = re_commit.match(line) if self.is_log else None
 143         cover_cc_match = re_cover_cc.match(line)
 144         tag_match = None
 145         if self.state == STATE_PATCH_HEADER:
 146             tag_match = re_tag.match(line)
 147         is_blank = not line.strip()
 148         if is_blank:
 149             if (self.state == STATE_MSG_HEADER
 150                     or self.state == STATE_PATCH_SUBJECT):
 151                 self.state += 1
 152
 153             # We don't have a subject in the text stream of patch files
 154             # It has its own line with a Subject: tag
 155             if not self.is_log and self.state == STATE_PATCH_SUBJECT:
 156                 self.state += 1
 157         elif commit_match:
 158             self.state = STATE_MSG_HEADER
 159
 160         # If we are in a section, keep collecting lines until we see END
 161         if self.in_section:
 162             if line == 'END':
 163                 if self.in_section == 'cover':
 164                     self.series.cover = self.section
 165                 elif self.in_section == 'notes':
 166                     if self.is_log:
 167                         self.series.notes += self.section
 168                 else:
 169                     self.warn.append("Unknown section '%s'" % self.in_section)
 170                 self.in_section = None
 171                 self.skip_blank = True
 172                 self.section = []
 173             else:
 174                 self.section.append(line)
 175
 176         # Detect the commit subject
 177         elif not is_blank and self.state == STATE_PATCH_SUBJECT:
 178             self.commit.subject = line
 179
 180         # Detect the tags we want to remove, and skip blank lines
 181         elif re_remove.match(line):
 182             self.skip_blank = True
 183
 184             # TEST= should be the last thing in the commit, so remove
 185             # everything after it
 186             if line.startswith('TEST='):
 187                 self.found_test = True
 188         elif self.skip_blank and is_blank:
 189             self.skip_blank = False
 190
 191         # Detect the start of a cover letter section
 192         elif re_cover.match(line):
 193             self.in_section = 'cover'
 194             self.skip_blank = False
 195
 196         elif cover_cc_match:
 197             value = cover_cc_match.group(1)
 198             self.AddToSeries(line, 'cover-cc', value)
 199
 200         # If we are in a change list, key collected lines until a blank one
 201         elif self.in_change:
 202             if is_blank:
 203                 # Blank line ends this change list
 204                 self.in_change = 0
 205             elif line == '---' or re_signoff.match(line):
 206                 self.in_change = 0
 207                 out = self.ProcessLine(line)
 208             else:
 209                 if self.is_log:
 210                     self.series.AddChange(self.in_change, self.commit, line)
 211             self.skip_blank = False
 212
 213         # Detect Series-xxx tags
 214         elif series_match:
 215             name = series_match.group(1)
 216             value = series_match.group(2)
 217             if name == 'changes':
 218                 # value is the version number: e.g. 1, or 2
 219                 try:
 220                     value = int(value)
 221                 except ValueError as str:
 222                     raise ValueError("%s: Cannot decode version info '%s'" %
 223                         (self.commit.hash, line))
 224                 self.in_change = int(value)
 225             else:
 226                 self.AddToSeries(line, name, value)
 227                 self.skip_blank = True
 228
 229         # Detect the start of a new commit
 230         elif commit_match:
 231             self.CloseCommit()
 232             # TODO: We should store the whole hash, and just display a subset
 233             self.commit = commit.Commit(commit_match.group(1)[:8])
 234
 235         # Detect tags in the commit message
 236         elif tag_match:
 237             # Remove Tested-by self, since few will take much notice
 238             if (tag_match.group(1) == 'Tested-by' and
 239                     tag_match.group(2).find(os.getenv('USER') + '@') != -1):
 240                 self.warn.append("Ignoring %s" % line)
 241             elif tag_match.group(1) == 'Cc':
 242                 self.commit.AddCc(tag_match.group(2).split(','))
 243             else:
 244                 self.tags.append(line);
 245
 246         # Well that means this is an ordinary line
 247         else:
 248             pos = 1
 249             # Look for ugly ASCII characters
 250             for ch in line:
 251                 # TODO: Would be nicer to report source filename and line
 252                 if ord(ch) > 0x80:
 253                     self.warn.append("Line %d/%d ('%s') has funny ascii char" %
 254                         (self.linenum, pos, line))
 255                 pos += 1
 256
 257             # Look for space before tab
 258             m = re_space_before_tab.match(line)
 259             if m:
 260                 self.warn.append('Line %d/%d has space before tab' %
 261                     (self.linenum, m.start()))
 262
 263             # OK, we have a valid non-blank line
 264             out = [line]
 265             self.linenum += 1
 266             self.skip_blank = False
 267             if self.state == STATE_DIFFS:
 268                 pass
 269
 270             # If this is the start of the diffs section, emit our tags and
 271             # change log
 272             elif line == '---':
 273                 self.state = STATE_DIFFS
 274
 275                 # Output the tags (signeoff first), then change list
 276                 out = []
 277                 log = self.series.MakeChangeLog(self.commit)
 278                 out += self.FormatTags(self.tags)
 279                 out += [line] + log
 280             elif self.found_test:
 281                 if not re_allowed_after_test.match(line):
 282                     self.lines_after_test += 1
 283
 284         return out
 285
 286     def Finalize(self):
 287         """Close out processing of this patch stream"""
 288         self.CloseCommit()
 289         if self.lines_after_test:
 290             self.warn.append('Found %d lines after TEST=' %
 291                     self.lines_after_test)
 292
 293     def ProcessStream(self, infd, outfd):
 294         """Copy a stream from infd to outfd, filtering out unwanting things.
 295
 296         This is used to process patch files one at a time.
 297
 298         Args:
 299             infd: Input stream file object
 300             outfd: Output stream file object
 301         """
 302         # Extract the filename from each diff, for nice warnings
 303         fname = None
 304         last_fname = None
 305         re_fname = re.compile('diff --git a/(.*) b/.*')
 306         while True:
 307             line = infd.readline()
 308             if not line:
 309                 break
 310             out = self.ProcessLine(line)
 311
 312             # Try to detect blank lines at EOF
 313             for line in out:
 314                 match = re_fname.match(line)
 315                 if match:
 316                     last_fname = fname
 317                     fname = match.group(1)
 318                 if line == '+':
 319                     self.blank_count += 1
 320                 else:
 321                     if self.blank_count and (line == '-- ' or match):
 322                         self.warn.append("Found possible blank line(s) at "
 323                                 "end of file '%s'" % last_fname)
 324                     outfd.write('+\n' * self.blank_count)
 325                     outfd.write(line + '\n')
 326                     self.blank_count = 0
 327         self.Finalize()
 328
 329
 330 def GetMetaDataForList(commit_range, git_dir=None, count=None,
 331                        series = Series()):
 332     """Reads out patch series metadata from the commits
 333
 334     This does a 'git log' on the relevant commits and pulls out the tags we
 335     are interested in.
 336
 337     Args:
 338         commit_range: Range of commits to count (e.g. 'HEAD..base')
 339         git_dir: Path to git repositiory (None to use default)
 340         count: Number of commits to list, or None for no limit
 341         series: Series object to add information into. By default a new series
 342             is started.
 343     Returns:
 344         A Series object containing information about the commits.
 345     """
 346     params = ['git', 'log', '--no-color', '--reverse', '--no-decorate',
 347                     commit_range]
 348     if count is not None:
 349         params[2:2] = ['-n%d' % count]
 350     if git_dir:
 351         params[1:1] = ['--git-dir', git_dir]
 352     pipe = [params]
 353     stdout = command.RunPipe(pipe, capture=True).stdout
 354     ps = PatchStream(series, is_log=True)
 355     for line in stdout.splitlines():
 356         ps.ProcessLine(line)
 357     ps.Finalize()
 358     return series
 359
 360 def GetMetaData(start, count):
 361     """Reads out patch series metadata from the commits
 362
 363     This does a 'git log' on the relevant commits and pulls out the tags we
 364     are interested in.
 365
 366     Args:
 367         start: Commit to start from: 0=HEAD, 1=next one, etc.
 368         count: Number of commits to list
 369     """
 370     return GetMetaDataForList('HEAD~%d' % start, None, count)
 371
 372 def FixPatch(backup_dir, fname, series, commit):
 373     """Fix up a patch file, by adding/removing as required.
 374
 375     We remove our tags from the patch file, insert changes lists, etc.
 376     The patch file is processed in place, and overwritten.
 377
 378     A backup file is put into backup_dir (if not None).
 379
 380     Args:
 381         fname: Filename to patch file to process
 382         series: Series information about this patch set
 383         commit: Commit object for this patch file
 384     Return:
 385         A list of errors, or [] if all ok.
 386     """
 387     handle, tmpname = tempfile.mkstemp()
 388     outfd = os.fdopen(handle, 'w')
 389     infd = open(fname, 'r')
 390     ps = PatchStream(series)
 391     ps.commit = commit
 392     ps.ProcessStream(infd, outfd)
 393     infd.close()
 394     outfd.close()
 395
 396     # Create a backup file if required
 397     if backup_dir:
 398         shutil.copy(fname, os.path.join(backup_dir, os.path.basename(fname)))
 399     shutil.move(tmpname, fname)
 400     return ps.warn
 401
 402 def FixPatches(series, fnames):
 403     """Fix up a list of patches identified by filenames
 404
 405     The patch files are processed in place, and overwritten.
 406
 407     Args:
 408         series: The series object
 409         fnames: List of patch files to process
 410     """
 411     # Current workflow creates patches, so we shouldn't need a backup
 412     backup_dir = None  #tempfile.mkdtemp('clean-patch')
 413     count = 0
 414     for fname in fnames:
 415         commit = series.commits[count]
 416         commit.patch = fname
 417         result = FixPatch(backup_dir, fname, series, commit)
 418         if result:
 419             print '%d warnings for %s:' % (len(result), fname)
 420             for warn in result:
 421                 print '\t', warn
 422             print
 423         count += 1
 424     print 'Cleaned %d patches' % count
 425     return series
 426
 427 def InsertCoverLetter(fname, series, count):
 428     """Inserts a cover letter with the required info into patch 0
 429
 430     Args:
 431         fname: Input / output filename of the cover letter file
 432         series: Series object
 433         count: Number of patches in the series
 434     """
 435     fd = open(fname, 'r')
 436     lines = fd.readlines()
 437     fd.close()
 438
 439     fd = open(fname, 'w')
 440     text = series.cover
 441     prefix = series.GetPatchPrefix()
 442     for line in lines:
 443         if line.startswith('Subject:'):
 444             # TODO: if more than 10 patches this should save 00/xx, not 0/xx
 445             line = 'Subject: [%s 0/%d] %s\n' % (prefix, count, text[0])
 446
 447         # Insert our cover letter
 448         elif line.startswith('*** BLURB HERE ***'):
 449             # First the blurb test
 450             line = '\n'.join(text[1:]) + '\n'
 451             if series.get('notes'):
 452                 line += '\n'.join(series.notes) + '\n'
 453
 454             # Now the change list
 455             out = series.MakeChangeLog(None)
 456             line += '\n' + '\n'.join(out)
 457         fd.write(line)
 458     fd.close()