contrib/gcc-changelog/git_commit.py

   1 #!/usr/bin/env python3
   2 #
   3 # This file is part of GCC.
   4 #
   5 # GCC is free software; you can redistribute it and/or modify it under
   6 # the terms of the GNU General Public License as published by the Free
   7 # Software Foundation; either version 3, or (at your option) any later
   8 # version.
   9 #
  10 # GCC is distributed in the hope that it will be useful, but WITHOUT ANY
  11 # WARRANTY; without even the implied warranty of MERCHANTABILITY or
  12 # FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
  13 # for more details.
  14 #
  15 # You should have received a copy of the GNU General Public License
  16 # along with GCC; see the file COPYING3.  If not see
  17 # <http://www.gnu.org/licenses/>.  */
  18
  19 import os
  20 import re
  21
  22 changelog_locations = set([
  23     'config',
  24     'contrib',
  25     'contrib/header-tools',
  26     'contrib/reghunt',
  27     'contrib/regression',
  28     'fixincludes',
  29     'gcc/ada',
  30     'gcc/analyzer',
  31     'gcc/brig',
  32     'gcc/c',
  33     'gcc/c-family',
  34     'gcc',
  35     'gcc/cp',
  36     'gcc/d',
  37     'gcc/fortran',
  38     'gcc/go',
  39     'gcc/jit',
  40     'gcc/lto',
  41     'gcc/objc',
  42     'gcc/objcp',
  43     'gcc/po',
  44     'gcc/testsuite',
  45     'gnattools',
  46     'gotools',
  47     'include',
  48     'intl',
  49     'libada',
  50     'libatomic',
  51     'libbacktrace',
  52     'libcc1',
  53     'libcpp',
  54     'libcpp/po',
  55     'libdecnumber',
  56     'libffi',
  57     'libgcc',
  58     'libgcc/config/avr/libf7',
  59     'libgcc/config/libbid',
  60     'libgfortran',
  61     'libgomp',
  62     'libhsail-rt',
  63     'libiberty',
  64     'libitm',
  65     'libobjc',
  66     'liboffloadmic',
  67     'libphobos',
  68     'libquadmath',
  69     'libsanitizer',
  70     'libssp',
  71     'libstdc++-v3',
  72     'libvtv',
  73     'lto-plugin',
  74     'maintainer-scripts',
  75     'zlib'])
  76
  77 bug_components = set([
  78     'ada',
  79     'analyzer',
  80     'boehm-gc',
  81     'bootstrap',
  82     'c',
  83     'c++',
  84     'd',
  85     'debug',
  86     'demangler',
  87     'driver',
  88     'fastjar',
  89     'fortran',
  90     'gcov-profile',
  91     'go',
  92     'hsa',
  93     'inline-asm',
  94     'ipa',
  95     'java',
  96     'jit',
  97     'libbacktrace',
  98     'libf2c',
  99     'libffi',
 100     'libfortran',
 101     'libgcc',
 102     'libgcj',
 103     'libgomp',
 104     'libitm',
 105     'libobjc',
 106     'libquadmath',
 107     'libstdc++',
 108     'lto',
 109     'middle-end',
 110     'modula2',
 111     'objc',
 112     'objc++',
 113     'other',
 114     'pch',
 115     'pending',
 116     'plugins',
 117     'preprocessor',
 118     'regression',
 119     'rtl-optimization',
 120     'sanitizer',
 121     'spam',
 122     'target',
 123     'testsuite',
 124     'translation',
 125     'tree-optimization',
 126     'web'])
 127
 128 ignored_prefixes = [
 129     'gcc/d/dmd/',
 130     'gcc/go/gofrontend/',
 131     'gcc/testsuite/gdc.test/',
 132     'gcc/testsuite/go.test/test/',
 133     'libgo/',
 134     'libphobos/libdruntime/',
 135     'libphobos/src/',
 136     'libsanitizer/',
 137     ]
 138
 139 wildcard_prefixes = [
 140     'gcc/testsuite/',
 141     'libstdc++-v3/doc/html/'
 142     ]
 143
 144 misc_files = [
 145     'gcc/DATESTAMP',
 146     'gcc/BASE-VER',
 147     'gcc/DEV-PHASE'
 148     ]
 149
 150 author_line_regex = \
 151         re.compile(r'^(?P<datetime>\d{4}-\d{2}-\d{2})\ {2}(?P<name>.*  <.*>)')
 152 additional_author_regex = re.compile(r'^\t(?P<spaces>\ *)?(?P<name>.*  <.*>)')
 153 changelog_regex = re.compile(r'^(?:[fF]or +)?([a-z0-9+-/]*)ChangeLog:?')
 154 pr_regex = re.compile(r'\tPR (?P<component>[a-z+-]+\/)?([0-9]+)$')
 155 dr_regex = re.compile(r'\tDR ([0-9]+)$')
 156 star_prefix_regex = re.compile(r'\t\*(?P<spaces>\ *)(?P<content>.*)')
 157
 158 LINE_LIMIT = 100
 159 TAB_WIDTH = 8
 160 CO_AUTHORED_BY_PREFIX = 'co-authored-by: '
 161 CHERRY_PICK_PREFIX = '(cherry picked from commit '
 162
 163 REVIEW_PREFIXES = ('reviewed-by: ', 'reviewed-on: ', 'signed-off-by: ',
 164                    'acked-by: ', 'tested-by: ', 'reported-by: ',
 165                    'suggested-by: ')
 166 DATE_FORMAT = '%Y-%m-%d'
 167
 168
 169 class Error:
 170     def __init__(self, message, line=None):
 171         self.message = message
 172         self.line = line
 173
 174     def __repr__(self):
 175         s = self.message
 176         if self.line:
 177             s += ':"%s"' % self.line
 178         return s
 179
 180
 181 class ChangeLogEntry:
 182     def __init__(self, folder, authors, prs):
 183         self.folder = folder
 184         # The 'list.copy()' function is not available before Python 3.3
 185         self.author_lines = list(authors)
 186         self.initial_prs = list(prs)
 187         self.prs = list(prs)
 188         self.lines = []
 189         self.files = []
 190         self.file_patterns = []
 191
 192     def parse_file_names(self):
 193         # Whether the content currently processed is between a star prefix the
 194         # end of the file list: a colon or an open paren.
 195         in_location = False
 196
 197         for line in self.lines:
 198             # If this line matches the star prefix, start the location
 199             # processing on the information that follows the star.
 200             m = star_prefix_regex.match(line)
 201             if m:
 202                 in_location = True
 203                 line = m.group('content')
 204
 205             if in_location:
 206                 # Strip everything that is not a filename in "line": entities
 207                 # "(NAME)", entry text (the colon, if present, and anything
 208                 # that follows it).
 209                 if '(' in line:
 210                     line = line[:line.index('(')]
 211                     in_location = False
 212                 if ':' in line:
 213                     line = line[:line.index(':')]
 214                     in_location = False
 215
 216                 # At this point, all that's left is a list of filenames
 217                 # separated by commas and whitespaces.
 218                 for file in line.split(','):
 219                     file = file.strip()
 220                     if file:
 221                         if file.endswith('*'):
 222                             self.file_patterns.append(file[:-1])
 223                         else:
 224                             self.files.append(file)
 225
 226     @property
 227     def datetime(self):
 228         for author in self.author_lines:
 229             if author[1]:
 230                 return author[1]
 231         return None
 232
 233     @property
 234     def authors(self):
 235         return [author_line[0] for author_line in self.author_lines]
 236
 237     @property
 238     def is_empty(self):
 239         return not self.lines and self.prs == self.initial_prs
 240
 241     def contains_author(self, author):
 242         for author_lines in self.author_lines:
 243             if author_lines[0] == author:
 244                 return True
 245         return False
 246
 247
 248 class GitCommit:
 249     def __init__(self, hexsha, date, author, body, modified_files,
 250                  strict=True, commit_to_date_hook=None):
 251         self.hexsha = hexsha
 252         self.lines = body
 253         self.modified_files = modified_files
 254         self.message = None
 255         self.changes = None
 256         self.changelog_entries = []
 257         self.errors = []
 258         self.date = date
 259         self.author = author
 260         self.top_level_authors = []
 261         self.co_authors = []
 262         self.top_level_prs = []
 263         self.cherry_pick_commit = None
 264         self.commit_to_date_hook = commit_to_date_hook
 265
 266         project_files = [f for f in self.modified_files
 267                          if self.is_changelog_filename(f[0])
 268                          or f[0] in misc_files]
 269         ignored_files = [f for f in self.modified_files
 270                          if self.in_ignored_location(f[0])]
 271         if len(project_files) == len(self.modified_files):
 272             # All modified files are only MISC files
 273             return
 274         elif project_files and strict:
 275             self.errors.append(Error('ChangeLog, DATESTAMP, BASE-VER and '
 276                                      'DEV-PHASE updates should be done '
 277                                      'separately from normal commits'))
 278             return
 279
 280         all_are_ignored = (len(project_files) + len(ignored_files)
 281                            == len(self.modified_files))
 282         self.parse_lines(all_are_ignored)
 283         if self.changes:
 284             self.parse_changelog()
 285             self.parse_file_names()
 286             self.check_for_empty_description()
 287             self.deduce_changelog_locations()
 288             self.check_file_patterns()
 289             if not self.errors:
 290                 self.check_mentioned_files()
 291                 self.check_for_correct_changelog()
 292
 293     @property
 294     def success(self):
 295         return not self.errors
 296
 297     @property
 298     def new_files(self):
 299         return [x[0] for x in self.modified_files if x[1] == 'A']
 300
 301     @classmethod
 302     def is_changelog_filename(cls, path):
 303         return path.endswith('/ChangeLog') or path == 'ChangeLog'
 304
 305     @classmethod
 306     def find_changelog_location(cls, name):
 307         if name.startswith('\t'):
 308             name = name[1:]
 309         if name.endswith(':'):
 310             name = name[:-1]
 311         if name.endswith('/'):
 312             name = name[:-1]
 313         return name if name in changelog_locations else None
 314
 315     @classmethod
 316     def format_git_author(cls, author):
 317         assert '<' in author
 318         return author.replace('<', ' <')
 319
 320     @classmethod
 321     def parse_git_name_status(cls, string):
 322         modified_files = []
 323         for entry in string.split('\n'):
 324             parts = entry.split('\t')
 325             t = parts[0]
 326             if t == 'A' or t == 'D' or t == 'M':
 327                 modified_files.append((parts[1], t))
 328             elif t.startswith('R'):
 329                 modified_files.append((parts[1], 'D'))
 330                 modified_files.append((parts[2], 'A'))
 331         return modified_files
 332
 333     def parse_lines(self, all_are_ignored):
 334         body = self.lines
 335
 336         for i, b in enumerate(body):
 337             if not b:
 338                 continue
 339             if (changelog_regex.match(b) or self.find_changelog_location(b)
 340                     or star_prefix_regex.match(b) or pr_regex.match(b)
 341                     or dr_regex.match(b) or author_line_regex.match(b)):
 342                 self.changes = body[i:]
 343                 return
 344         if not all_are_ignored:
 345             self.errors.append(Error('cannot find a ChangeLog location in '
 346                                      'message'))
 347
 348     def parse_changelog(self):
 349         last_entry = None
 350         will_deduce = False
 351         for line in self.changes:
 352             if not line:
 353                 if last_entry and will_deduce:
 354                     last_entry = None
 355                 continue
 356             if line != line.rstrip():
 357                 self.errors.append(Error('trailing whitespace', line))
 358             if len(line.replace('\t', ' ' * TAB_WIDTH)) > LINE_LIMIT:
 359                 self.errors.append(Error('line exceeds %d character limit'
 360                                          % LINE_LIMIT, line))
 361             m = changelog_regex.match(line)
 362             if m:
 363                 last_entry = ChangeLogEntry(m.group(1).rstrip('/'),
 364                                             self.top_level_authors,
 365                                             self.top_level_prs)
 366                 self.changelog_entries.append(last_entry)
 367             elif self.find_changelog_location(line):
 368                 last_entry = ChangeLogEntry(self.find_changelog_location(line),
 369                                             self.top_level_authors,
 370                                             self.top_level_prs)
 371                 self.changelog_entries.append(last_entry)
 372             else:
 373                 author_tuple = None
 374                 pr_line = None
 375                 if author_line_regex.match(line):
 376                     m = author_line_regex.match(line)
 377                     author_tuple = (m.group('name'), m.group('datetime'))
 378                 elif additional_author_regex.match(line):
 379                     m = additional_author_regex.match(line)
 380                     if len(m.group('spaces')) != 4:
 381                         msg = 'additional author must be indented with '\
 382                               'one tab and four spaces'
 383                         self.errors.append(Error(msg, line))
 384                     else:
 385                         author_tuple = (m.group('name'), None)
 386                 elif pr_regex.match(line):
 387                     component = pr_regex.match(line).group('component')
 388                     if not component:
 389                         self.errors.append(Error('missing PR component', line))
 390                         continue
 391                     elif not component[:-1] in bug_components:
 392                         self.errors.append(Error('invalid PR component', line))
 393                         continue
 394                     else:
 395                         pr_line = line.lstrip()
 396                 elif dr_regex.match(line):
 397                     pr_line = line.lstrip()
 398
 399                 lowered_line = line.lower()
 400                 if lowered_line.startswith(CO_AUTHORED_BY_PREFIX):
 401                     name = line[len(CO_AUTHORED_BY_PREFIX):]
 402                     author = self.format_git_author(name)
 403                     self.co_authors.append(author)
 404                     continue
 405                 elif lowered_line.startswith(REVIEW_PREFIXES):
 406                     continue
 407                 elif line.startswith(CHERRY_PICK_PREFIX):
 408                     commit = line[len(CHERRY_PICK_PREFIX):].rstrip(')')
 409                     self.cherry_pick_commit = commit
 410                     continue
 411
 412                 # ChangeLog name will be deduced later
 413                 if not last_entry:
 414                     if author_tuple:
 415                         self.top_level_authors.append(author_tuple)
 416                         continue
 417                     elif pr_line:
 418                         # append to top_level_prs only when we haven't met
 419                         # a ChangeLog entry
 420                         if (pr_line not in self.top_level_prs
 421                                 and not self.changelog_entries):
 422                             self.top_level_prs.append(pr_line)
 423                         continue
 424                     else:
 425                         last_entry = ChangeLogEntry(None,
 426                                                     self.top_level_authors,
 427                                                     self.top_level_prs)
 428                         self.changelog_entries.append(last_entry)
 429                         will_deduce = True
 430                 elif author_tuple:
 431                     if not last_entry.contains_author(author_tuple[0]):
 432                         last_entry.author_lines.append(author_tuple)
 433                     continue
 434
 435                 if not line.startswith('\t'):
 436                     err = Error('line should start with a tab', line)
 437                     self.errors.append(err)
 438                 elif pr_line:
 439                     last_entry.prs.append(pr_line)
 440                 else:
 441                     m = star_prefix_regex.match(line)
 442                     if m:
 443                         if len(m.group('spaces')) != 1:
 444                             msg = 'one space should follow asterisk'
 445                             self.errors.append(Error(msg, line))
 446                         else:
 447                             last_entry.lines.append(line)
 448                     else:
 449                         if last_entry.is_empty:
 450                             msg = 'first line should start with a tab, ' \
 451                                   'an asterisk and a space'
 452                             self.errors.append(Error(msg, line))
 453                         else:
 454                             last_entry.lines.append(line)
 455
 456     def parse_file_names(self):
 457         for entry in self.changelog_entries:
 458             entry.parse_file_names()
 459
 460     def check_file_patterns(self):
 461         for entry in self.changelog_entries:
 462             for pattern in entry.file_patterns:
 463                 name = os.path.join(entry.folder, pattern)
 464                 if name not in wildcard_prefixes:
 465                     msg = 'unsupported wildcard prefix'
 466                     self.errors.append(Error(msg, name))
 467
 468     def check_for_empty_description(self):
 469         for entry in self.changelog_entries:
 470             for i, line in enumerate(entry.lines):
 471                 if (star_prefix_regex.match(line) and line.endswith(':') and
 472                     (i == len(entry.lines) - 1
 473                      or star_prefix_regex.match(entry.lines[i + 1]))):
 474                     msg = 'missing description of a change'
 475                     self.errors.append(Error(msg, line))
 476
 477     def get_file_changelog_location(self, changelog_file):
 478         for file in self.modified_files:
 479             if file[0] == changelog_file:
 480                 # root ChangeLog file
 481                 return ''
 482             index = file[0].find('/' + changelog_file)
 483             if index != -1:
 484                 return file[0][:index]
 485         return None
 486
 487     def deduce_changelog_locations(self):
 488         for entry in self.changelog_entries:
 489             if not entry.folder:
 490                 changelog = None
 491                 for file in entry.files:
 492                     location = self.get_file_changelog_location(file)
 493                     if (location == ''
 494                        or (location and location in changelog_locations)):
 495                         if changelog and changelog != location:
 496                             msg = 'could not deduce ChangeLog file, ' \
 497                                   'not unique location'
 498                             self.errors.append(Error(msg))
 499                             return
 500                         changelog = location
 501                 if changelog is not None:
 502                     entry.folder = changelog
 503                 else:
 504                     msg = 'could not deduce ChangeLog file'
 505                     self.errors.append(Error(msg))
 506
 507     @classmethod
 508     def in_ignored_location(cls, path):
 509         for ignored in ignored_prefixes:
 510             if path.startswith(ignored):
 511                 return True
 512         return False
 513
 514     @classmethod
 515     def get_changelog_by_path(cls, path):
 516         components = path.split('/')
 517         while components:
 518             if '/'.join(components) in changelog_locations:
 519                 break
 520             components = components[:-1]
 521         return '/'.join(components)
 522
 523     def check_mentioned_files(self):
 524         folder_count = len([x.folder for x in self.changelog_entries])
 525         assert folder_count == len(self.changelog_entries)
 526
 527         mentioned_files = set()
 528         mentioned_patterns = []
 529         used_patterns = set()
 530         for entry in self.changelog_entries:
 531             if not entry.files:
 532                 msg = 'no files mentioned for ChangeLog in directory'
 533                 self.errors.append(Error(msg, entry.folder))
 534             assert not entry.folder.endswith('/')
 535             for file in entry.files:
 536                 if not self.is_changelog_filename(file):
 537                     mentioned_files.add(os.path.join(entry.folder, file))
 538             for pattern in entry.file_patterns:
 539                 mentioned_patterns.append(os.path.join(entry.folder, pattern))
 540
 541         cand = [x[0] for x in self.modified_files
 542                 if not self.is_changelog_filename(x[0])]
 543         changed_files = set(cand)
 544         for file in sorted(mentioned_files - changed_files):
 545             msg = 'unchanged file mentioned in a ChangeLog'
 546             self.errors.append(Error(msg, file))
 547         for file in sorted(changed_files - mentioned_files):
 548             if not self.in_ignored_location(file):
 549                 if file in self.new_files:
 550                     changelog_location = self.get_changelog_by_path(file)
 551                     # Python2: we cannot use next(filter(...))
 552                     entries = filter(lambda x: x.folder == changelog_location,
 553                                      self.changelog_entries)
 554                     entries = list(entries)
 555                     entry = entries[0] if entries else None
 556                     if not entry:
 557                         prs = self.top_level_prs
 558                         if not prs:
 559                             # if all ChangeLog entries have identical PRs
 560                             # then use them
 561                             prs = self.changelog_entries[0].prs
 562                             for entry in self.changelog_entries:
 563                                 if entry.prs != prs:
 564                                     prs = []
 565                                     break
 566                         entry = ChangeLogEntry(changelog_location,
 567                                                self.top_level_authors,
 568                                                prs)
 569                         self.changelog_entries.append(entry)
 570                     # strip prefix of the file
 571                     assert file.startswith(entry.folder)
 572                     file = file[len(entry.folder):].lstrip('/')
 573                     entry.lines.append('\t* %s: New file.' % file)
 574                     entry.files.append(file)
 575                 else:
 576                     used_pattern = [p for p in mentioned_patterns
 577                                     if file.startswith(p)]
 578                     used_pattern = used_pattern[0] if used_pattern else None
 579                     if used_pattern:
 580                         used_patterns.add(used_pattern)
 581                     else:
 582                         msg = 'changed file not mentioned in a ChangeLog'
 583                         self.errors.append(Error(msg, file))
 584
 585         for pattern in mentioned_patterns:
 586             if pattern not in used_patterns:
 587                 error = 'pattern doesn''t match any changed files'
 588                 self.errors.append(Error(error, pattern))
 589
 590     def check_for_correct_changelog(self):
 591         for entry in self.changelog_entries:
 592             for file in entry.files:
 593                 full_path = os.path.join(entry.folder, file)
 594                 changelog_location = self.get_changelog_by_path(full_path)
 595                 if changelog_location != entry.folder:
 596                     msg = 'wrong ChangeLog location "%s", should be "%s"'
 597                     err = Error(msg % (entry.folder, changelog_location), file)
 598                     self.errors.append(err)
 599
 600     @classmethod
 601     def format_authors_in_changelog(cls, authors, timestamp, prefix=''):
 602         output = ''
 603         for i, author in enumerate(authors):
 604             if i == 0:
 605                 output += '%s%s  %s\n' % (prefix, timestamp, author)
 606             else:
 607                 output += '%s\t    %s\n' % (prefix, author)
 608         output += '\n'
 609         return output
 610
 611     def to_changelog_entries(self, use_commit_ts=False):
 612         current_timestamp = self.date.strftime(DATE_FORMAT)
 613         for entry in self.changelog_entries:
 614             output = ''
 615             timestamp = entry.datetime
 616             if self.cherry_pick_commit:
 617                 timestamp = self.commit_to_date_hook(self.cherry_pick_commit)
 618                 if timestamp:
 619                     timestamp = timestamp.strftime(DATE_FORMAT)
 620             if not timestamp or use_commit_ts:
 621                 timestamp = current_timestamp
 622             authors = entry.authors if entry.authors else [self.author]
 623             # add Co-Authored-By authors to all ChangeLog entries
 624             for author in self.co_authors:
 625                 if author not in authors:
 626                     authors.append(author)
 627
 628             if self.cherry_pick_commit:
 629                 output += self.format_authors_in_changelog([self.author],
 630                                                            current_timestamp)
 631                 output += '\tBackported from master:\n'
 632                 output += self.format_authors_in_changelog(authors,
 633                                                            timestamp, '\t')
 634             else:
 635                 output += self.format_authors_in_changelog(authors, timestamp)
 636             for pr in entry.prs:
 637                 output += '\t%s\n' % pr
 638             for line in entry.lines:
 639                 output += line + '\n'
 640             yield (entry.folder, output.rstrip())
 641
 642     def print_output(self):
 643         for entry, output in self.to_changelog_entries():
 644             print('------ %s/ChangeLog ------ ' % entry)
 645             print(output)
 646
 647     def print_errors(self):
 648         print('Errors:')
 649         for error in self.errors:
 650             print(error)