tools/patman/patchstream.py

   1 # Copyright (c) 2011 The Chromium OS Authors.
   2 #
   3 # SPDX-License-Identifier:      GPL-2.0+
   4 #
   5
   6 import os
   7 import re
   8 import shutil
   9 import tempfile
  10
  11 import command
  12 import commit
  13 import gitutil
  14 from series import Series
  15
  16 # Tags that we detect and remove
  17 re_remove = re.compile('^BUG=|^TEST=|^BRANCH=|^Change-Id:|^Review URL:'
  18     '|Reviewed-on:|Commit-\w*:')
  19
  20 # Lines which are allowed after a TEST= line
  21 re_allowed_after_test = re.compile('^Signed-off-by:')
  22
  23 # Signoffs
  24 re_signoff = re.compile('^Signed-off-by: *(.*)')
  25
  26 # The start of the cover letter
  27 re_cover = re.compile('^Cover-letter:')
  28
  29 # A cover letter Cc
  30 re_cover_cc = re.compile('^Cover-letter-cc: *(.*)')
  31
  32 # Patch series tag
  33 re_series_tag = re.compile('^Series-([a-z-]*): *(.*)')
  34
  35 # Commit series tag
  36 re_commit_tag = re.compile('^Commit-([a-z-]*): *(.*)')
  37
  38 # Commit tags that we want to collect and keep
  39 re_tag = re.compile('^(Tested-by|Acked-by|Reviewed-by|Patch-cc): (.*)')
  40
  41 # The start of a new commit in the git log
  42 re_commit = re.compile('^commit ([0-9a-f]*)$')
  43
  44 # We detect these since checkpatch doesn't always do it
  45 re_space_before_tab = re.compile('^[+].* \t')
  46
  47 # States we can be in - can we use range() and still have comments?
  48 STATE_MSG_HEADER = 0        # Still in the message header
  49 STATE_PATCH_SUBJECT = 1     # In patch subject (first line of log for a commit)
  50 STATE_PATCH_HEADER = 2      # In patch header (after the subject)
  51 STATE_DIFFS = 3             # In the diff part (past --- line)
  52
  53 class PatchStream:
  54     """Class for detecting/injecting tags in a patch or series of patches
  55
  56     We support processing the output of 'git log' to read out the tags we
  57     are interested in. We can also process a patch file in order to remove
  58     unwanted tags or inject additional ones. These correspond to the two
  59     phases of processing.
  60     """
  61     def __init__(self, series, name=None, is_log=False):
  62         self.skip_blank = False          # True to skip a single blank line
  63         self.found_test = False          # Found a TEST= line
  64         self.lines_after_test = 0        # MNumber of lines found after TEST=
  65         self.warn = []                   # List of warnings we have collected
  66         self.linenum = 1                 # Output line number we are up to
  67         self.in_section = None           # Name of start...END section we are in
  68         self.notes = []                  # Series notes
  69         self.section = []                # The current section...END section
  70         self.series = series             # Info about the patch series
  71         self.is_log = is_log             # True if indent like git log
  72         self.in_change = 0               # Non-zero if we are in a change list
  73         self.blank_count = 0             # Number of blank lines stored up
  74         self.state = STATE_MSG_HEADER    # What state are we in?
  75         self.tags = []                   # Tags collected, like Tested-by...
  76         self.signoff = []                # Contents of signoff line
  77         self.commit = None               # Current commit
  78
  79     def AddToSeries(self, line, name, value):
  80         """Add a new Series-xxx tag.
  81
  82         When a Series-xxx tag is detected, we come here to record it, if we
  83         are scanning a 'git log'.
  84
  85         Args:
  86             line: Source line containing tag (useful for debug/error messages)
  87             name: Tag name (part after 'Series-')
  88             value: Tag value (part after 'Series-xxx: ')
  89         """
  90         if name == 'notes':
  91             self.in_section = name
  92             self.skip_blank = False
  93         if self.is_log:
  94             self.series.AddTag(self.commit, line, name, value)
  95
  96     def AddToCommit(self, line, name, value):
  97         """Add a new Commit-xxx tag.
  98
  99         When a Commit-xxx tag is detected, we come here to record it.
 100
 101         Args:
 102             line: Source line containing tag (useful for debug/error messages)
 103             name: Tag name (part after 'Commit-')
 104             value: Tag value (part after 'Commit-xxx: ')
 105         """
 106         if name == 'notes':
 107             self.in_section = 'commit-' + name
 108             self.skip_blank = False
 109
 110     def CloseCommit(self):
 111         """Save the current commit into our commit list, and reset our state"""
 112         if self.commit and self.is_log:
 113             self.series.AddCommit(self.commit)
 114             self.commit = None
 115
 116     def FormatTags(self, tags):
 117         out_list = []
 118         for tag in sorted(tags):
 119             if tag.startswith('Cc:'):
 120                 tag_list = tag[4:].split(',')
 121                 out_list += gitutil.BuildEmailList(tag_list, 'Cc:')
 122             else:
 123                 out_list.append(tag)
 124         return out_list
 125
 126     def ProcessLine(self, line):
 127         """Process a single line of a patch file or commit log
 128
 129         This process a line and returns a list of lines to output. The list
 130         may be empty or may contain multiple output lines.
 131
 132         This is where all the complicated logic is located. The class's
 133         state is used to move between different states and detect things
 134         properly.
 135
 136         We can be in one of two modes:
 137             self.is_log == True: This is 'git log' mode, where most output is
 138                 indented by 4 characters and we are scanning for tags
 139
 140             self.is_log == False: This is 'patch' mode, where we already have
 141                 all the tags, and are processing patches to remove junk we
 142                 don't want, and add things we think are required.
 143
 144         Args:
 145             line: text line to process
 146
 147         Returns:
 148             list of output lines, or [] if nothing should be output
 149         """
 150         # Initially we have no output. Prepare the input line string
 151         out = []
 152         line = line.rstrip('\n')
 153         if self.is_log:
 154             if line[:4] == '    ':
 155                 line = line[4:]
 156
 157         # Handle state transition and skipping blank lines
 158         series_tag_match = re_series_tag.match(line)
 159         commit_tag_match = re_commit_tag.match(line)
 160         commit_match = re_commit.match(line) if self.is_log else None
 161         cover_cc_match = re_cover_cc.match(line)
 162         signoff_match = re_signoff.match(line)
 163         tag_match = None
 164         if self.state == STATE_PATCH_HEADER:
 165             tag_match = re_tag.match(line)
 166         is_blank = not line.strip()
 167         if is_blank:
 168             if (self.state == STATE_MSG_HEADER
 169                     or self.state == STATE_PATCH_SUBJECT):
 170                 self.state += 1
 171
 172             # We don't have a subject in the text stream of patch files
 173             # It has its own line with a Subject: tag
 174             if not self.is_log and self.state == STATE_PATCH_SUBJECT:
 175                 self.state += 1
 176         elif commit_match:
 177             self.state = STATE_MSG_HEADER
 178
 179         # If we are in a section, keep collecting lines until we see END
 180         if self.in_section:
 181             if line == 'END':
 182                 if self.in_section == 'cover':
 183                     self.series.cover = self.section
 184                 elif self.in_section == 'notes':
 185                     if self.is_log:
 186                         self.series.notes += self.section
 187                 elif self.in_section == 'commit-notes':
 188                     if self.is_log:
 189                         self.commit.notes += self.section
 190                 else:
 191                     self.warn.append("Unknown section '%s'" % self.in_section)
 192                 self.in_section = None
 193                 self.skip_blank = True
 194                 self.section = []
 195             else:
 196                 self.section.append(line)
 197
 198         # Detect the commit subject
 199         elif not is_blank and self.state == STATE_PATCH_SUBJECT:
 200             self.commit.subject = line
 201
 202         # Detect the tags we want to remove, and skip blank lines
 203         elif re_remove.match(line) and not commit_tag_match:
 204             self.skip_blank = True
 205
 206             # TEST= should be the last thing in the commit, so remove
 207             # everything after it
 208             if line.startswith('TEST='):
 209                 self.found_test = True
 210         elif self.skip_blank and is_blank:
 211             self.skip_blank = False
 212
 213         # Detect the start of a cover letter section
 214         elif re_cover.match(line):
 215             self.in_section = 'cover'
 216             self.skip_blank = False
 217
 218         elif cover_cc_match:
 219             value = cover_cc_match.group(1)
 220             self.AddToSeries(line, 'cover-cc', value)
 221
 222         # If we are in a change list, key collected lines until a blank one
 223         elif self.in_change:
 224             if is_blank:
 225                 # Blank line ends this change list
 226                 self.in_change = 0
 227             elif line == '---':
 228                 self.in_change = 0
 229                 out = self.ProcessLine(line)
 230             else:
 231                 if self.is_log:
 232                     self.series.AddChange(self.in_change, self.commit, line)
 233             self.skip_blank = False
 234
 235         # Detect Series-xxx tags
 236         elif series_tag_match:
 237             name = series_tag_match.group(1)
 238             value = series_tag_match.group(2)
 239             if name == 'changes':
 240                 # value is the version number: e.g. 1, or 2
 241                 try:
 242                     value = int(value)
 243                 except ValueError as str:
 244                     raise ValueError("%s: Cannot decode version info '%s'" %
 245                         (self.commit.hash, line))
 246                 self.in_change = int(value)
 247             else:
 248                 self.AddToSeries(line, name, value)
 249                 self.skip_blank = True
 250
 251         # Detect Commit-xxx tags
 252         elif commit_tag_match:
 253             name = commit_tag_match.group(1)
 254             value = commit_tag_match.group(2)
 255             if name == 'notes':
 256                 self.AddToCommit(line, name, value)
 257                 self.skip_blank = True
 258
 259         # Detect the start of a new commit
 260         elif commit_match:
 261             self.CloseCommit()
 262             # TODO: We should store the whole hash, and just display a subset
 263             self.commit = commit.Commit(commit_match.group(1)[:8])
 264
 265         # Detect tags in the commit message
 266         elif tag_match:
 267             # Remove Tested-by self, since few will take much notice
 268             if (tag_match.group(1) == 'Tested-by' and
 269                     tag_match.group(2).find(os.getenv('USER') + '@') != -1):
 270                 self.warn.append("Ignoring %s" % line)
 271             elif tag_match.group(1) == 'Patch-cc':
 272                 self.commit.AddCc(tag_match.group(2).split(','))
 273             else:
 274                 self.tags.append(line);
 275
 276         # Suppress duplicate signoffs
 277         elif signoff_match:
 278             if self.commit.CheckDuplicateSignoff(signoff_match.group(1)):
 279                 out = [line]
 280
 281         # Well that means this is an ordinary line
 282         else:
 283             pos = 1
 284             # Look for ugly ASCII characters
 285             for ch in line:
 286                 # TODO: Would be nicer to report source filename and line
 287                 if ord(ch) > 0x80:
 288                     self.warn.append("Line %d/%d ('%s') has funny ascii char" %
 289                         (self.linenum, pos, line))
 290                 pos += 1
 291
 292             # Look for space before tab
 293             m = re_space_before_tab.match(line)
 294             if m:
 295                 self.warn.append('Line %d/%d has space before tab' %
 296                     (self.linenum, m.start()))
 297
 298             # OK, we have a valid non-blank line
 299             out = [line]
 300             self.linenum += 1
 301             self.skip_blank = False
 302             if self.state == STATE_DIFFS:
 303                 pass
 304
 305             # If this is the start of the diffs section, emit our tags and
 306             # change log
 307             elif line == '---':
 308                 self.state = STATE_DIFFS
 309
 310                 # Output the tags (signeoff first), then change list
 311                 out = []
 312                 log = self.series.MakeChangeLog(self.commit)
 313                 out += self.FormatTags(self.tags)
 314                 out += [line] + self.commit.notes + [''] + log
 315             elif self.found_test:
 316                 if not re_allowed_after_test.match(line):
 317                     self.lines_after_test += 1
 318
 319         return out
 320
 321     def Finalize(self):
 322         """Close out processing of this patch stream"""
 323         self.CloseCommit()
 324         if self.lines_after_test:
 325             self.warn.append('Found %d lines after TEST=' %
 326                     self.lines_after_test)
 327
 328     def ProcessStream(self, infd, outfd):
 329         """Copy a stream from infd to outfd, filtering out unwanting things.
 330
 331         This is used to process patch files one at a time.
 332
 333         Args:
 334             infd: Input stream file object
 335             outfd: Output stream file object
 336         """
 337         # Extract the filename from each diff, for nice warnings
 338         fname = None
 339         last_fname = None
 340         re_fname = re.compile('diff --git a/(.*) b/.*')
 341         while True:
 342             line = infd.readline()
 343             if not line:
 344                 break
 345             out = self.ProcessLine(line)
 346
 347             # Try to detect blank lines at EOF
 348             for line in out:
 349                 match = re_fname.match(line)
 350                 if match:
 351                     last_fname = fname
 352                     fname = match.group(1)
 353                 if line == '+':
 354                     self.blank_count += 1
 355                 else:
 356                     if self.blank_count and (line == '-- ' or match):
 357                         self.warn.append("Found possible blank line(s) at "
 358                                 "end of file '%s'" % last_fname)
 359                     outfd.write('+\n' * self.blank_count)
 360                     outfd.write(line + '\n')
 361                     self.blank_count = 0
 362         self.Finalize()
 363
 364
 365 def GetMetaDataForList(commit_range, git_dir=None, count=None,
 366                        series = Series()):
 367     """Reads out patch series metadata from the commits
 368
 369     This does a 'git log' on the relevant commits and pulls out the tags we
 370     are interested in.
 371
 372     Args:
 373         commit_range: Range of commits to count (e.g. 'HEAD..base')
 374         git_dir: Path to git repositiory (None to use default)
 375         count: Number of commits to list, or None for no limit
 376         series: Series object to add information into. By default a new series
 377             is started.
 378     Returns:
 379         A Series object containing information about the commits.
 380     """
 381     params = ['git', 'log', '--no-color', '--reverse', '--no-decorate',
 382                     commit_range]
 383     if count is not None:
 384         params[2:2] = ['-n%d' % count]
 385     if git_dir:
 386         params[1:1] = ['--git-dir', git_dir]
 387     pipe = [params]
 388     stdout = command.RunPipe(pipe, capture=True).stdout
 389     ps = PatchStream(series, is_log=True)
 390     for line in stdout.splitlines():
 391         ps.ProcessLine(line)
 392     ps.Finalize()
 393     return series
 394
 395 def GetMetaData(start, count):
 396     """Reads out patch series metadata from the commits
 397
 398     This does a 'git log' on the relevant commits and pulls out the tags we
 399     are interested in.
 400
 401     Args:
 402         start: Commit to start from: 0=HEAD, 1=next one, etc.
 403         count: Number of commits to list
 404     """
 405     return GetMetaDataForList('HEAD~%d' % start, None, count)
 406
 407 def FixPatch(backup_dir, fname, series, commit):
 408     """Fix up a patch file, by adding/removing as required.
 409
 410     We remove our tags from the patch file, insert changes lists, etc.
 411     The patch file is processed in place, and overwritten.
 412
 413     A backup file is put into backup_dir (if not None).
 414
 415     Args:
 416         fname: Filename to patch file to process
 417         series: Series information about this patch set
 418         commit: Commit object for this patch file
 419     Return:
 420         A list of errors, or [] if all ok.
 421     """
 422     handle, tmpname = tempfile.mkstemp()
 423     outfd = os.fdopen(handle, 'w')
 424     infd = open(fname, 'r')
 425     ps = PatchStream(series)
 426     ps.commit = commit
 427     ps.ProcessStream(infd, outfd)
 428     infd.close()
 429     outfd.close()
 430
 431     # Create a backup file if required
 432     if backup_dir:
 433         shutil.copy(fname, os.path.join(backup_dir, os.path.basename(fname)))
 434     shutil.move(tmpname, fname)
 435     return ps.warn
 436
 437 def FixPatches(series, fnames):
 438     """Fix up a list of patches identified by filenames
 439
 440     The patch files are processed in place, and overwritten.
 441
 442     Args:
 443         series: The series object
 444         fnames: List of patch files to process
 445     """
 446     # Current workflow creates patches, so we shouldn't need a backup
 447     backup_dir = None  #tempfile.mkdtemp('clean-patch')
 448     count = 0
 449     for fname in fnames:
 450         commit = series.commits[count]
 451         commit.patch = fname
 452         result = FixPatch(backup_dir, fname, series, commit)
 453         if result:
 454             print '%d warnings for %s:' % (len(result), fname)
 455             for warn in result:
 456                 print '\t', warn
 457             print
 458         count += 1
 459     print 'Cleaned %d patches' % count
 460     return series
 461
 462 def InsertCoverLetter(fname, series, count):
 463     """Inserts a cover letter with the required info into patch 0
 464
 465     Args:
 466         fname: Input / output filename of the cover letter file
 467         series: Series object
 468         count: Number of patches in the series
 469     """
 470     fd = open(fname, 'r')
 471     lines = fd.readlines()
 472     fd.close()
 473
 474     fd = open(fname, 'w')
 475     text = series.cover
 476     prefix = series.GetPatchPrefix()
 477     for line in lines:
 478         if line.startswith('Subject:'):
 479             # TODO: if more than 10 patches this should save 00/xx, not 0/xx
 480             line = 'Subject: [%s 0/%d] %s\n' % (prefix, count, text[0])
 481
 482         # Insert our cover letter
 483         elif line.startswith('*** BLURB HERE ***'):
 484             # First the blurb test
 485             line = '\n'.join(text[1:]) + '\n'
 486             if series.get('notes'):
 487                 line += '\n'.join(series.notes) + '\n'
 488
 489             # Now the change list
 490             out = series.MakeChangeLog(None)
 491             line += '\n' + '\n'.join(out)
 492         fd.write(line)
 493     fd.close()