contrib/gcc-changelog/git_commit.py

   1 #!/usr/bin/env python3
   2 #
   3 # This file is part of GCC.
   4 #
   5 # GCC is free software; you can redistribute it and/or modify it under
   6 # the terms of the GNU General Public License as published by the Free
   7 # Software Foundation; either version 3, or (at your option) any later
   8 # version.
   9 #
  10 # GCC is distributed in the hope that it will be useful, but WITHOUT ANY
  11 # WARRANTY; without even the implied warranty of MERCHANTABILITY or
  12 # FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
  13 # for more details.
  14 #
  15 # You should have received a copy of the GNU General Public License
  16 # along with GCC; see the file COPYING3.  If not see
  17 # <http://www.gnu.org/licenses/>.  */
  18
  19 import os
  20 import re
  21
  22 changelog_locations = set([
  23     'config',
  24     'contrib',
  25     'contrib/header-tools',
  26     'contrib/reghunt',
  27     'contrib/regression',
  28     'fixincludes',
  29     'gcc/ada',
  30     'gcc/analyzer',
  31     'gcc/brig',
  32     'gcc/c',
  33     'gcc/c-family',
  34     'gcc',
  35     'gcc/cp',
  36     'gcc/d',
  37     'gcc/fortran',
  38     'gcc/go',
  39     'gcc/jit',
  40     'gcc/lto',
  41     'gcc/objc',
  42     'gcc/objcp',
  43     'gcc/po',
  44     'gcc/testsuite',
  45     'gnattools',
  46     'gotools',
  47     'include',
  48     'intl',
  49     'libada',
  50     'libatomic',
  51     'libbacktrace',
  52     'libcc1',
  53     'libcpp',
  54     'libcpp/po',
  55     'libdecnumber',
  56     'libffi',
  57     'libgcc',
  58     'libgcc/config/avr/libf7',
  59     'libgcc/config/libbid',
  60     'libgfortran',
  61     'libgomp',
  62     'libhsail-rt',
  63     'libiberty',
  64     'libitm',
  65     'libobjc',
  66     'liboffloadmic',
  67     'libphobos',
  68     'libquadmath',
  69     'libsanitizer',
  70     'libssp',
  71     'libstdc++-v3',
  72     'libvtv',
  73     'lto-plugin',
  74     'maintainer-scripts',
  75     'zlib'])
  76
  77 bug_components = set([
  78     'ada',
  79     'analyzer',
  80     'boehm-gc',
  81     'bootstrap',
  82     'c',
  83     'c++',
  84     'd',
  85     'debug',
  86     'demangler',
  87     'driver',
  88     'fastjar',
  89     'fortran',
  90     'gcov-profile',
  91     'go',
  92     'hsa',
  93     'inline-asm',
  94     'ipa',
  95     'java',
  96     'jit',
  97     'libbacktrace',
  98     'libf2c',
  99     'libffi',
 100     'libfortran',
 101     'libgcc',
 102     'libgcj',
 103     'libgomp',
 104     'libitm',
 105     'libobjc',
 106     'libquadmath',
 107     'libstdc++',
 108     'lto',
 109     'middle-end',
 110     'modula2',
 111     'objc',
 112     'objc++',
 113     'other',
 114     'pch',
 115     'pending',
 116     'plugins',
 117     'preprocessor',
 118     'regression',
 119     'rtl-optimization',
 120     'sanitizer',
 121     'spam',
 122     'target',
 123     'testsuite',
 124     'translation',
 125     'tree-optimization',
 126     'web'])
 127
 128 ignored_prefixes = [
 129     'gcc/d/dmd/',
 130     'gcc/go/gofrontend/',
 131     'gcc/testsuite/go.test/test/',
 132     'libgo/',
 133     'libphobos/libdruntime/',
 134     'libphobos/src/',
 135     'libsanitizer/',
 136     ]
 137
 138 misc_files = [
 139     'gcc/DATESTAMP',
 140     'gcc/BASE-VER',
 141     'gcc/DEV-PHASE'
 142     ]
 143
 144 author_line_regex = \
 145         re.compile(r'^(?P<datetime>\d{4}-\d{2}-\d{2})\ {2}(?P<name>.*  <.*>)')
 146 additional_author_regex = re.compile(r'^\t(?P<spaces>\ *)?(?P<name>.*  <.*>)')
 147 changelog_regex = re.compile(r'^([a-z0-9+-/]*)/ChangeLog:?')
 148 pr_regex = re.compile(r'\tPR (?P<component>[a-z+-]+\/)?([0-9]+)$')
 149 dr_regex = re.compile(r'\tDR ([0-9]+)$')
 150 star_prefix_regex = re.compile(r'\t\*(?P<spaces>\ *)(?P<content>.*)')
 151
 152 LINE_LIMIT = 100
 153 TAB_WIDTH = 8
 154 CO_AUTHORED_BY_PREFIX = 'co-authored-by: '
 155 CHERRY_PICK_PREFIX = '(cherry picked from commit '
 156 REVIEWED_BY_PREFIX = 'reviewed-by: '
 157 REVIEWED_ON_PREFIX = 'reviewed-on: '
 158 SIGNED_OFF_BY_PREFIX = 'signed-off-by: '
 159
 160 REVIEW_PREFIXES = (REVIEWED_BY_PREFIX, REVIEWED_ON_PREFIX,
 161                    SIGNED_OFF_BY_PREFIX)
 162
 163
 164 class Error:
 165     def __init__(self, message, line=None):
 166         self.message = message
 167         self.line = line
 168
 169     def __repr__(self):
 170         s = self.message
 171         if self.line:
 172             s += ':"%s"' % self.line
 173         return s
 174
 175
 176 class ChangeLogEntry:
 177     def __init__(self, folder, authors, prs):
 178         self.folder = folder
 179         # Python2 has not 'copy' function
 180         self.author_lines = list(authors)
 181         self.initial_prs = list(prs)
 182         self.prs = list(prs)
 183         self.lines = []
 184
 185     @property
 186     def files(self):
 187         files = []
 188         for line in self.lines:
 189             m = star_prefix_regex.match(line)
 190             if m:
 191                 line = m.group('content')
 192                 if '(' in line:
 193                     line = line[:line.index('(')]
 194                 if ':' in line:
 195                     line = line[:line.index(':')]
 196                 for file in line.split(','):
 197                     file = file.strip()
 198                     if file:
 199                         files.append(file)
 200         return files
 201
 202     @property
 203     def datetime(self):
 204         for author in self.author_lines:
 205             if author[1]:
 206                 return author[1]
 207         return None
 208
 209     @property
 210     def authors(self):
 211         return [author_line[0] for author_line in self.author_lines]
 212
 213     @property
 214     def is_empty(self):
 215         return not self.lines and self.prs == self.initial_prs
 216
 217
 218 class GitCommit:
 219     def __init__(self, hexsha, date, author, body, modified_files,
 220                  strict=True):
 221         self.hexsha = hexsha
 222         self.lines = body
 223         self.modified_files = modified_files
 224         self.message = None
 225         self.changes = None
 226         self.changelog_entries = []
 227         self.errors = []
 228         self.date = date
 229         self.author = author
 230         self.top_level_authors = []
 231         self.co_authors = []
 232         self.top_level_prs = []
 233
 234         project_files = [f for f in self.modified_files
 235                          if self.is_changelog_filename(f[0])
 236                          or f[0] in misc_files]
 237         ignored_files = [f for f in self.modified_files
 238                          if self.in_ignored_location(f[0])]
 239         if len(project_files) == len(self.modified_files):
 240             # All modified files are only MISC files
 241             return
 242         elif project_files and strict:
 243             self.errors.append(Error('ChangeLog, DATESTAMP, BASE-VER and '
 244                                      'DEV-PHASE updates should be done '
 245                                      'separately from normal commits'))
 246             return
 247
 248         all_are_ignored = (len(project_files) + len(ignored_files)
 249                            == len(self.modified_files))
 250         self.parse_lines(all_are_ignored)
 251         if self.changes:
 252             self.parse_changelog()
 253             self.deduce_changelog_locations()
 254             if not self.errors:
 255                 self.check_mentioned_files()
 256                 self.check_for_correct_changelog()
 257
 258     @property
 259     def success(self):
 260         return not self.errors
 261
 262     @property
 263     def new_files(self):
 264         return [x[0] for x in self.modified_files if x[1] == 'A']
 265
 266     @classmethod
 267     def is_changelog_filename(cls, path):
 268         return path.endswith('/ChangeLog') or path == 'ChangeLog'
 269
 270     @classmethod
 271     def find_changelog_location(cls, name):
 272         if name.startswith('\t'):
 273             name = name[1:]
 274         if name.endswith(':'):
 275             name = name[:-1]
 276         if name.endswith('/'):
 277             name = name[:-1]
 278         return name if name in changelog_locations else None
 279
 280     @classmethod
 281     def format_git_author(cls, author):
 282         assert '<' in author
 283         return author.replace('<', ' <')
 284
 285     @classmethod
 286     def parse_git_name_status(cls, string):
 287         modified_files = []
 288         for entry in string.split('\n'):
 289             parts = entry.split('\t')
 290             t = parts[0]
 291             if t == 'A' or t == 'D' or t == 'M':
 292                 modified_files.append((parts[1], t))
 293             elif t == 'R':
 294                 modified_files.append((parts[1], 'D'))
 295                 modified_files.append((parts[2], 'A'))
 296         return modified_files
 297
 298     def parse_lines(self, all_are_ignored):
 299         body = self.lines
 300
 301         for i, b in enumerate(body):
 302             if not b:
 303                 continue
 304             if (changelog_regex.match(b) or self.find_changelog_location(b)
 305                     or star_prefix_regex.match(b) or pr_regex.match(b)
 306                     or dr_regex.match(b) or author_line_regex.match(b)):
 307                 self.changes = body[i:]
 308                 return
 309         if not all_are_ignored:
 310             self.errors.append(Error('cannot find a ChangeLog location in '
 311                                      'message'))
 312
 313     def parse_changelog(self):
 314         last_entry = None
 315         will_deduce = False
 316         for line in self.changes:
 317             if not line:
 318                 if last_entry and will_deduce:
 319                     last_entry = None
 320                 continue
 321             if line != line.rstrip():
 322                 self.errors.append(Error('trailing whitespace', line))
 323             if len(line.replace('\t', ' ' * TAB_WIDTH)) > LINE_LIMIT:
 324                 self.errors.append(Error('line limit exceeds %d characters'
 325                                          % LINE_LIMIT, line))
 326             m = changelog_regex.match(line)
 327             if m:
 328                 last_entry = ChangeLogEntry(m.group(1), self.top_level_authors,
 329                                             self.top_level_prs)
 330                 self.changelog_entries.append(last_entry)
 331             elif self.find_changelog_location(line):
 332                 last_entry = ChangeLogEntry(self.find_changelog_location(line),
 333                                             self.top_level_authors,
 334                                             self.top_level_prs)
 335                 self.changelog_entries.append(last_entry)
 336             else:
 337                 author_tuple = None
 338                 pr_line = None
 339                 if author_line_regex.match(line):
 340                     m = author_line_regex.match(line)
 341                     author_tuple = (m.group('name'), m.group('datetime'))
 342                 elif additional_author_regex.match(line):
 343                     m = additional_author_regex.match(line)
 344                     if len(m.group('spaces')) != 4:
 345                         msg = 'additional author must prepend with tab ' \
 346                               'and 4 spaces'
 347                         self.errors.append(Error(msg, line))
 348                     else:
 349                         author_tuple = (m.group('name'), None)
 350                 elif pr_regex.match(line):
 351                     component = pr_regex.match(line).group('component')
 352                     if not component:
 353                         self.errors.append(Error('missing PR component', line))
 354                         continue
 355                     elif not component[:-1] in bug_components:
 356                         self.errors.append(Error('invalid PR component', line))
 357                         continue
 358                     else:
 359                         pr_line = line.lstrip()
 360                 elif dr_regex.match(line):
 361                     pr_line = line.lstrip()
 362
 363                 lowered_line = line.lower()
 364                 if lowered_line.startswith(CO_AUTHORED_BY_PREFIX):
 365                     name = line[len(CO_AUTHORED_BY_PREFIX):]
 366                     author = self.format_git_author(name)
 367                     self.co_authors.append(author)
 368                     continue
 369                 elif lowered_line.startswith(REVIEW_PREFIXES):
 370                     continue
 371                 elif line.startswith(CHERRY_PICK_PREFIX):
 372                     continue
 373
 374                 # ChangeLog name will be deduced later
 375                 if not last_entry:
 376                     if author_tuple:
 377                         self.top_level_authors.append(author_tuple)
 378                         continue
 379                     elif pr_line:
 380                         # append to top_level_prs only when we haven't met
 381                         # a ChangeLog entry
 382                         if (pr_line not in self.top_level_prs
 383                                 and not self.changelog_entries):
 384                             self.top_level_prs.append(pr_line)
 385                         continue
 386                     else:
 387                         last_entry = ChangeLogEntry(None,
 388                                                     self.top_level_authors,
 389                                                     self.top_level_prs)
 390                         self.changelog_entries.append(last_entry)
 391                         will_deduce = True
 392                 elif author_tuple:
 393                     if author_tuple not in last_entry.author_lines:
 394                         last_entry.author_lines.append(author_tuple)
 395                     continue
 396
 397                 if not line.startswith('\t'):
 398                     err = Error('line should start with a tab', line)
 399                     self.errors.append(err)
 400                 elif pr_line:
 401                     last_entry.prs.append(pr_line)
 402                 else:
 403                     m = star_prefix_regex.match(line)
 404                     if m:
 405                         if len(m.group('spaces')) != 1:
 406                             err = Error('one space should follow asterisk',
 407                                         line)
 408                             self.errors.append(err)
 409                         else:
 410                             last_entry.lines.append(line)
 411                     else:
 412                         if last_entry.is_empty:
 413                             msg = 'first line should start with a tab, ' \
 414                                   'asterisk and space'
 415                             self.errors.append(Error(msg, line))
 416                         else:
 417                             last_entry.lines.append(line)
 418
 419     def get_file_changelog_location(self, changelog_file):
 420         for file in self.modified_files:
 421             if file[0] == changelog_file:
 422                 # root ChangeLog file
 423                 return ''
 424             index = file[0].find('/' + changelog_file)
 425             if index != -1:
 426                 return file[0][:index]
 427         return None
 428
 429     def deduce_changelog_locations(self):
 430         for entry in self.changelog_entries:
 431             if not entry.folder:
 432                 changelog = None
 433                 for file in entry.files:
 434                     location = self.get_file_changelog_location(file)
 435                     if (location == ''
 436                        or (location and location in changelog_locations)):
 437                         if changelog and changelog != location:
 438                             msg = 'could not deduce ChangeLog file, ' \
 439                                   'not unique location'
 440                             self.errors.append(Error(msg))
 441                             return
 442                         changelog = location
 443                 if changelog is not None:
 444                     entry.folder = changelog
 445                 else:
 446                     msg = 'could not deduce ChangeLog file'
 447                     self.errors.append(Error(msg))
 448
 449     @classmethod
 450     def in_ignored_location(cls, path):
 451         for ignored in ignored_prefixes:
 452             if path.startswith(ignored):
 453                 return True
 454         return False
 455
 456     @classmethod
 457     def get_changelog_by_path(cls, path):
 458         components = path.split('/')
 459         while components:
 460             if '/'.join(components) in changelog_locations:
 461                 break
 462             components = components[:-1]
 463         return '/'.join(components)
 464
 465     def check_mentioned_files(self):
 466         folder_count = len([x.folder for x in self.changelog_entries])
 467         assert folder_count == len(self.changelog_entries)
 468
 469         mentioned_files = set()
 470         for entry in self.changelog_entries:
 471             if not entry.files:
 472                 msg = 'ChangeLog must contain a file entry'
 473                 self.errors.append(Error(msg, entry.folder))
 474             assert not entry.folder.endswith('/')
 475             for file in entry.files:
 476                 if not self.is_changelog_filename(file):
 477                     mentioned_files.add(os.path.join(entry.folder, file))
 478
 479         cand = [x[0] for x in self.modified_files
 480                 if not self.is_changelog_filename(x[0])]
 481         changed_files = set(cand)
 482         for file in sorted(mentioned_files - changed_files):
 483             self.errors.append(Error('file not changed in a patch', file))
 484         for file in sorted(changed_files - mentioned_files):
 485             if not self.in_ignored_location(file):
 486                 if file in self.new_files:
 487                     changelog_location = self.get_changelog_by_path(file)
 488                     # Python2: we cannot use next(filter(...))
 489                     entries = filter(lambda x: x.folder == changelog_location,
 490                                      self.changelog_entries)
 491                     entries = list(entries)
 492                     entry = entries[0] if entries else None
 493                     if not entry:
 494                         prs = self.top_level_prs
 495                         if not prs:
 496                             # if all ChangeLog entries have identical PRs
 497                             # then use them
 498                             prs = self.changelog_entries[0].prs
 499                             for entry in self.changelog_entries:
 500                                 if entry.prs != prs:
 501                                     prs = []
 502                                     break
 503                         entry = ChangeLogEntry(changelog_location,
 504                                                self.top_level_authors,
 505                                                prs)
 506                         self.changelog_entries.append(entry)
 507                     # strip prefix of the file
 508                     assert file.startswith(entry.folder)
 509                     file = file[len(entry.folder):].lstrip('/')
 510                     entry.lines.append('\t* %s: New file.' % file)
 511                 else:
 512                     msg = 'changed file not mentioned in a ChangeLog'
 513                     self.errors.append(Error(msg, file))
 514
 515     def check_for_correct_changelog(self):
 516         for entry in self.changelog_entries:
 517             for file in entry.files:
 518                 full_path = os.path.join(entry.folder, file)
 519                 changelog_location = self.get_changelog_by_path(full_path)
 520                 if changelog_location != entry.folder:
 521                     msg = 'wrong ChangeLog location "%s", should be "%s"'
 522                     err = Error(msg % (entry.folder, changelog_location), file)
 523                     self.errors.append(err)
 524
 525     def to_changelog_entries(self, use_commit_ts=False):
 526         for entry in self.changelog_entries:
 527             output = ''
 528             timestamp = entry.datetime
 529             if not timestamp or use_commit_ts:
 530                 timestamp = self.date.strftime('%Y-%m-%d')
 531             authors = entry.authors if entry.authors else [self.author]
 532             # add Co-Authored-By authors to all ChangeLog entries
 533             for author in self.co_authors:
 534                 if author not in authors:
 535                     authors.append(author)
 536
 537             for i, author in enumerate(authors):
 538                 if i == 0:
 539                     output += '%s  %s\n' % (timestamp, author)
 540                 else:
 541                     output += '\t    %s\n' % author
 542             output += '\n'
 543             for pr in entry.prs:
 544                 output += '\t%s\n' % pr
 545             for line in entry.lines:
 546                 output += line + '\n'
 547             yield (entry.folder, output.rstrip())
 548
 549     def print_output(self):
 550         for entry, output in self.to_changelog_entries():
 551             print('------ %s/ChangeLog ------ ' % entry)
 552             print(output)
 553
 554     def print_errors(self):
 555         print('Errors:')
 556         for error in self.errors:
 557             print(error)