tools/patman/patchstream.py

   1 # Copyright (c) 2011 The Chromium OS Authors.
   2 #
   3 # SPDX-License-Identifier:      GPL-2.0+
   4 #
   5
   6 import math
   7 import os
   8 import re
   9 import shutil
  10 import tempfile
  11
  12 import command
  13 import commit
  14 import gitutil
  15 from series import Series
  16
  17 # Tags that we detect and remove
  18 re_remove = re.compile('^BUG=|^TEST=|^BRANCH=|^Change-Id:|^Review URL:'
  19     '|Reviewed-on:|Commit-\w*:')
  20
  21 # Lines which are allowed after a TEST= line
  22 re_allowed_after_test = re.compile('^Signed-off-by:')
  23
  24 # Signoffs
  25 re_signoff = re.compile('^Signed-off-by: *(.*)')
  26
  27 # The start of the cover letter
  28 re_cover = re.compile('^Cover-letter:')
  29
  30 # A cover letter Cc
  31 re_cover_cc = re.compile('^Cover-letter-cc: *(.*)')
  32
  33 # Patch series tag
  34 re_series_tag = re.compile('^Series-([a-z-]*): *(.*)')
  35
  36 # Commit series tag
  37 re_commit_tag = re.compile('^Commit-([a-z-]*): *(.*)')
  38
  39 # Commit tags that we want to collect and keep
  40 re_tag = re.compile('^(Tested-by|Acked-by|Reviewed-by|Patch-cc): (.*)')
  41
  42 # The start of a new commit in the git log
  43 re_commit = re.compile('^commit ([0-9a-f]*)$')
  44
  45 # We detect these since checkpatch doesn't always do it
  46 re_space_before_tab = re.compile('^[+].* \t')
  47
  48 # States we can be in - can we use range() and still have comments?
  49 STATE_MSG_HEADER = 0        # Still in the message header
  50 STATE_PATCH_SUBJECT = 1     # In patch subject (first line of log for a commit)
  51 STATE_PATCH_HEADER = 2      # In patch header (after the subject)
  52 STATE_DIFFS = 3             # In the diff part (past --- line)
  53
  54 class PatchStream:
  55     """Class for detecting/injecting tags in a patch or series of patches
  56
  57     We support processing the output of 'git log' to read out the tags we
  58     are interested in. We can also process a patch file in order to remove
  59     unwanted tags or inject additional ones. These correspond to the two
  60     phases of processing.
  61     """
  62     def __init__(self, series, name=None, is_log=False):
  63         self.skip_blank = False          # True to skip a single blank line
  64         self.found_test = False          # Found a TEST= line
  65         self.lines_after_test = 0        # MNumber of lines found after TEST=
  66         self.warn = []                   # List of warnings we have collected
  67         self.linenum = 1                 # Output line number we are up to
  68         self.in_section = None           # Name of start...END section we are in
  69         self.notes = []                  # Series notes
  70         self.section = []                # The current section...END section
  71         self.series = series             # Info about the patch series
  72         self.is_log = is_log             # True if indent like git log
  73         self.in_change = 0               # Non-zero if we are in a change list
  74         self.blank_count = 0             # Number of blank lines stored up
  75         self.state = STATE_MSG_HEADER    # What state are we in?
  76         self.signoff = []                # Contents of signoff line
  77         self.commit = None               # Current commit
  78
  79     def AddToSeries(self, line, name, value):
  80         """Add a new Series-xxx tag.
  81
  82         When a Series-xxx tag is detected, we come here to record it, if we
  83         are scanning a 'git log'.
  84
  85         Args:
  86             line: Source line containing tag (useful for debug/error messages)
  87             name: Tag name (part after 'Series-')
  88             value: Tag value (part after 'Series-xxx: ')
  89         """
  90         if name == 'notes':
  91             self.in_section = name
  92             self.skip_blank = False
  93         if self.is_log:
  94             self.series.AddTag(self.commit, line, name, value)
  95
  96     def AddToCommit(self, line, name, value):
  97         """Add a new Commit-xxx tag.
  98
  99         When a Commit-xxx tag is detected, we come here to record it.
 100
 101         Args:
 102             line: Source line containing tag (useful for debug/error messages)
 103             name: Tag name (part after 'Commit-')
 104             value: Tag value (part after 'Commit-xxx: ')
 105         """
 106         if name == 'notes':
 107             self.in_section = 'commit-' + name
 108             self.skip_blank = False
 109
 110     def CloseCommit(self):
 111         """Save the current commit into our commit list, and reset our state"""
 112         if self.commit and self.is_log:
 113             self.series.AddCommit(self.commit)
 114             self.commit = None
 115         # If 'END' is missing in a 'Cover-letter' section, and that section
 116         # happens to show up at the very end of the commit message, this is
 117         # the chance for us to fix it up.
 118         if self.in_section == 'cover' and self.is_log:
 119             self.series.cover = self.section
 120             self.in_section = None
 121             self.skip_blank = True
 122             self.section = []
 123
 124     def ProcessLine(self, line):
 125         """Process a single line of a patch file or commit log
 126
 127         This process a line and returns a list of lines to output. The list
 128         may be empty or may contain multiple output lines.
 129
 130         This is where all the complicated logic is located. The class's
 131         state is used to move between different states and detect things
 132         properly.
 133
 134         We can be in one of two modes:
 135             self.is_log == True: This is 'git log' mode, where most output is
 136                 indented by 4 characters and we are scanning for tags
 137
 138             self.is_log == False: This is 'patch' mode, where we already have
 139                 all the tags, and are processing patches to remove junk we
 140                 don't want, and add things we think are required.
 141
 142         Args:
 143             line: text line to process
 144
 145         Returns:
 146             list of output lines, or [] if nothing should be output
 147         """
 148         # Initially we have no output. Prepare the input line string
 149         out = []
 150         line = line.rstrip('\n')
 151
 152         commit_match = re_commit.match(line) if self.is_log else None
 153
 154         if self.is_log:
 155             if line[:4] == '    ':
 156                 line = line[4:]
 157
 158         # Handle state transition and skipping blank lines
 159         series_tag_match = re_series_tag.match(line)
 160         commit_tag_match = re_commit_tag.match(line)
 161         cover_match = re_cover.match(line)
 162         cover_cc_match = re_cover_cc.match(line)
 163         signoff_match = re_signoff.match(line)
 164         tag_match = None
 165         if self.state == STATE_PATCH_HEADER:
 166             tag_match = re_tag.match(line)
 167         is_blank = not line.strip()
 168         if is_blank:
 169             if (self.state == STATE_MSG_HEADER
 170                     or self.state == STATE_PATCH_SUBJECT):
 171                 self.state += 1
 172
 173             # We don't have a subject in the text stream of patch files
 174             # It has its own line with a Subject: tag
 175             if not self.is_log and self.state == STATE_PATCH_SUBJECT:
 176                 self.state += 1
 177         elif commit_match:
 178             self.state = STATE_MSG_HEADER
 179
 180         # If a tag is detected
 181         if series_tag_match or commit_tag_match or \
 182            cover_match or cover_cc_match or signoff_match:
 183             # but we are already in a section, this means 'END' is missing
 184             # for that section, fix it up.
 185             if self.in_section:
 186                 self.warn.append("Missing 'END' in section '%s'" % self.in_section)
 187                 if self.in_section == 'cover':
 188                     self.series.cover = self.section
 189                 elif self.in_section == 'notes':
 190                     if self.is_log:
 191                         self.series.notes += self.section
 192                 elif self.in_section == 'commit-notes':
 193                     if self.is_log:
 194                         self.commit.notes += self.section
 195                 else:
 196                     self.warn.append("Unknown section '%s'" % self.in_section)
 197                 self.in_section = None
 198                 self.skip_blank = True
 199                 self.section = []
 200             # but we are already in a change list, that means a blank line
 201             # is missing, fix it up.
 202             if self.in_change:
 203                 self.warn.append("Missing 'blank line' in section 'Series-changes'")
 204                 self.in_change = 0
 205
 206         # If we are in a section, keep collecting lines until we see END
 207         if self.in_section:
 208             if line == 'END':
 209                 if self.in_section == 'cover':
 210                     self.series.cover = self.section
 211                 elif self.in_section == 'notes':
 212                     if self.is_log:
 213                         self.series.notes += self.section
 214                 elif self.in_section == 'commit-notes':
 215                     if self.is_log:
 216                         self.commit.notes += self.section
 217                 else:
 218                     self.warn.append("Unknown section '%s'" % self.in_section)
 219                 self.in_section = None
 220                 self.skip_blank = True
 221                 self.section = []
 222             else:
 223                 self.section.append(line)
 224
 225         # Detect the commit subject
 226         elif not is_blank and self.state == STATE_PATCH_SUBJECT:
 227             self.commit.subject = line
 228
 229         # Detect the tags we want to remove, and skip blank lines
 230         elif re_remove.match(line) and not commit_tag_match:
 231             self.skip_blank = True
 232
 233             # TEST= should be the last thing in the commit, so remove
 234             # everything after it
 235             if line.startswith('TEST='):
 236                 self.found_test = True
 237         elif self.skip_blank and is_blank:
 238             self.skip_blank = False
 239
 240         # Detect the start of a cover letter section
 241         elif cover_match:
 242             self.in_section = 'cover'
 243             self.skip_blank = False
 244
 245         elif cover_cc_match:
 246             value = cover_cc_match.group(1)
 247             self.AddToSeries(line, 'cover-cc', value)
 248
 249         # If we are in a change list, key collected lines until a blank one
 250         elif self.in_change:
 251             if is_blank:
 252                 # Blank line ends this change list
 253                 self.in_change = 0
 254             elif line == '---':
 255                 self.in_change = 0
 256                 out = self.ProcessLine(line)
 257             else:
 258                 if self.is_log:
 259                     self.series.AddChange(self.in_change, self.commit, line)
 260             self.skip_blank = False
 261
 262         # Detect Series-xxx tags
 263         elif series_tag_match:
 264             name = series_tag_match.group(1)
 265             value = series_tag_match.group(2)
 266             if name == 'changes':
 267                 # value is the version number: e.g. 1, or 2
 268                 try:
 269                     value = int(value)
 270                 except ValueError as str:
 271                     raise ValueError("%s: Cannot decode version info '%s'" %
 272                         (self.commit.hash, line))
 273                 self.in_change = int(value)
 274             else:
 275                 self.AddToSeries(line, name, value)
 276                 self.skip_blank = True
 277
 278         # Detect Commit-xxx tags
 279         elif commit_tag_match:
 280             name = commit_tag_match.group(1)
 281             value = commit_tag_match.group(2)
 282             if name == 'notes':
 283                 self.AddToCommit(line, name, value)
 284                 self.skip_blank = True
 285
 286         # Detect the start of a new commit
 287         elif commit_match:
 288             self.CloseCommit()
 289             self.commit = commit.Commit(commit_match.group(1))
 290
 291         # Detect tags in the commit message
 292         elif tag_match:
 293             # Remove Tested-by self, since few will take much notice
 294             if (tag_match.group(1) == 'Tested-by' and
 295                     tag_match.group(2).find(os.getenv('USER') + '@') != -1):
 296                 self.warn.append("Ignoring %s" % line)
 297             elif tag_match.group(1) == 'Patch-cc':
 298                 self.commit.AddCc(tag_match.group(2).split(','))
 299             else:
 300                 out = [line]
 301
 302         # Suppress duplicate signoffs
 303         elif signoff_match:
 304             if (self.is_log or not self.commit or
 305                 self.commit.CheckDuplicateSignoff(signoff_match.group(1))):
 306                 out = [line]
 307
 308         # Well that means this is an ordinary line
 309         else:
 310             pos = 1
 311             # Look for ugly ASCII characters
 312             for ch in line:
 313                 # TODO: Would be nicer to report source filename and line
 314                 if ord(ch) > 0x80:
 315                     self.warn.append("Line %d/%d ('%s') has funny ascii char" %
 316                         (self.linenum, pos, line))
 317                 pos += 1
 318
 319             # Look for space before tab
 320             m = re_space_before_tab.match(line)
 321             if m:
 322                 self.warn.append('Line %d/%d has space before tab' %
 323                     (self.linenum, m.start()))
 324
 325             # OK, we have a valid non-blank line
 326             out = [line]
 327             self.linenum += 1
 328             self.skip_blank = False
 329             if self.state == STATE_DIFFS:
 330                 pass
 331
 332             # If this is the start of the diffs section, emit our tags and
 333             # change log
 334             elif line == '---':
 335                 self.state = STATE_DIFFS
 336
 337                 # Output the tags (signeoff first), then change list
 338                 out = []
 339                 log = self.series.MakeChangeLog(self.commit)
 340                 out += [line]
 341                 if self.commit:
 342                     out += self.commit.notes
 343                 out += [''] + log
 344             elif self.found_test:
 345                 if not re_allowed_after_test.match(line):
 346                     self.lines_after_test += 1
 347
 348         return out
 349
 350     def Finalize(self):
 351         """Close out processing of this patch stream"""
 352         self.CloseCommit()
 353         if self.lines_after_test:
 354             self.warn.append('Found %d lines after TEST=' %
 355                     self.lines_after_test)
 356
 357     def ProcessStream(self, infd, outfd):
 358         """Copy a stream from infd to outfd, filtering out unwanting things.
 359
 360         This is used to process patch files one at a time.
 361
 362         Args:
 363             infd: Input stream file object
 364             outfd: Output stream file object
 365         """
 366         # Extract the filename from each diff, for nice warnings
 367         fname = None
 368         last_fname = None
 369         re_fname = re.compile('diff --git a/(.*) b/.*')
 370         while True:
 371             line = infd.readline()
 372             if not line:
 373                 break
 374             out = self.ProcessLine(line)
 375
 376             # Try to detect blank lines at EOF
 377             for line in out:
 378                 match = re_fname.match(line)
 379                 if match:
 380                     last_fname = fname
 381                     fname = match.group(1)
 382                 if line == '+':
 383                     self.blank_count += 1
 384                 else:
 385                     if self.blank_count and (line == '-- ' or match):
 386                         self.warn.append("Found possible blank line(s) at "
 387                                 "end of file '%s'" % last_fname)
 388                     outfd.write('+\n' * self.blank_count)
 389                     outfd.write(line + '\n')
 390                     self.blank_count = 0
 391         self.Finalize()
 392
 393
 394 def GetMetaDataForList(commit_range, git_dir=None, count=None,
 395                        series = None, allow_overwrite=False):
 396     """Reads out patch series metadata from the commits
 397
 398     This does a 'git log' on the relevant commits and pulls out the tags we
 399     are interested in.
 400
 401     Args:
 402         commit_range: Range of commits to count (e.g. 'HEAD..base')
 403         git_dir: Path to git repositiory (None to use default)
 404         count: Number of commits to list, or None for no limit
 405         series: Series object to add information into. By default a new series
 406             is started.
 407         allow_overwrite: Allow tags to overwrite an existing tag
 408     Returns:
 409         A Series object containing information about the commits.
 410     """
 411     if not series:
 412         series = Series()
 413     series.allow_overwrite = allow_overwrite
 414     params = gitutil.LogCmd(commit_range, reverse=True, count=count,
 415                             git_dir=git_dir)
 416     stdout = command.RunPipe([params], capture=True).stdout
 417     ps = PatchStream(series, is_log=True)
 418     for line in stdout.splitlines():
 419         ps.ProcessLine(line)
 420     ps.Finalize()
 421     return series
 422
 423 def GetMetaData(start, count):
 424     """Reads out patch series metadata from the commits
 425
 426     This does a 'git log' on the relevant commits and pulls out the tags we
 427     are interested in.
 428
 429     Args:
 430         start: Commit to start from: 0=HEAD, 1=next one, etc.
 431         count: Number of commits to list
 432     """
 433     return GetMetaDataForList('HEAD~%d' % start, None, count)
 434
 435 def FixPatch(backup_dir, fname, series, commit):
 436     """Fix up a patch file, by adding/removing as required.
 437
 438     We remove our tags from the patch file, insert changes lists, etc.
 439     The patch file is processed in place, and overwritten.
 440
 441     A backup file is put into backup_dir (if not None).
 442
 443     Args:
 444         fname: Filename to patch file to process
 445         series: Series information about this patch set
 446         commit: Commit object for this patch file
 447     Return:
 448         A list of errors, or [] if all ok.
 449     """
 450     handle, tmpname = tempfile.mkstemp()
 451     outfd = os.fdopen(handle, 'w')
 452     infd = open(fname, 'r')
 453     ps = PatchStream(series)
 454     ps.commit = commit
 455     ps.ProcessStream(infd, outfd)
 456     infd.close()
 457     outfd.close()
 458
 459     # Create a backup file if required
 460     if backup_dir:
 461         shutil.copy(fname, os.path.join(backup_dir, os.path.basename(fname)))
 462     shutil.move(tmpname, fname)
 463     return ps.warn
 464
 465 def FixPatches(series, fnames):
 466     """Fix up a list of patches identified by filenames
 467
 468     The patch files are processed in place, and overwritten.
 469
 470     Args:
 471         series: The series object
 472         fnames: List of patch files to process
 473     """
 474     # Current workflow creates patches, so we shouldn't need a backup
 475     backup_dir = None  #tempfile.mkdtemp('clean-patch')
 476     count = 0
 477     for fname in fnames:
 478         commit = series.commits[count]
 479         commit.patch = fname
 480         result = FixPatch(backup_dir, fname, series, commit)
 481         if result:
 482             print '%d warnings for %s:' % (len(result), fname)
 483             for warn in result:
 484                 print '\t', warn
 485             print
 486         count += 1
 487     print 'Cleaned %d patches' % count
 488     return series
 489
 490 def InsertCoverLetter(fname, series, count):
 491     """Inserts a cover letter with the required info into patch 0
 492
 493     Args:
 494         fname: Input / output filename of the cover letter file
 495         series: Series object
 496         count: Number of patches in the series
 497     """
 498     fd = open(fname, 'r')
 499     lines = fd.readlines()
 500     fd.close()
 501
 502     fd = open(fname, 'w')
 503     text = series.cover
 504     prefix = series.GetPatchPrefix()
 505     for line in lines:
 506         if line.startswith('Subject:'):
 507             # if more than 10 or 100 patches, it should say 00/xx, 000/xxx, etc
 508             zero_repeat = int(math.log10(count)) + 1
 509             zero = '0' * zero_repeat
 510             line = 'Subject: [%s %s/%d] %s\n' % (prefix, zero, count, text[0])
 511
 512         # Insert our cover letter
 513         elif line.startswith('*** BLURB HERE ***'):
 514             # First the blurb test
 515             line = '\n'.join(text[1:]) + '\n'
 516             if series.get('notes'):
 517                 line += '\n'.join(series.notes) + '\n'
 518
 519             # Now the change list
 520             out = series.MakeChangeLog(None)
 521             line += '\n' + '\n'.join(out)
 522         fd.write(line)
 523     fd.close()