contrib/gcc-changelog/git_commit.py

   1 #!/usr/bin/env python3
   2 #
   3 # This file is part of GCC.
   4 #
   5 # GCC is free software; you can redistribute it and/or modify it under
   6 # the terms of the GNU General Public License as published by the Free
   7 # Software Foundation; either version 3, or (at your option) any later
   8 # version.
   9 #
  10 # GCC is distributed in the hope that it will be useful, but WITHOUT ANY
  11 # WARRANTY; without even the implied warranty of MERCHANTABILITY or
  12 # FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
  13 # for more details.
  14 #
  15 # You should have received a copy of the GNU General Public License
  16 # along with GCC; see the file COPYING3.  If not see
  17 # <http://www.gnu.org/licenses/>.  */
  18
  19 import os
  20 import re
  21
  22 changelog_locations = set([
  23     'config',
  24     'contrib',
  25     'contrib/header-tools',
  26     'contrib/reghunt',
  27     'contrib/regression',
  28     'fixincludes',
  29     'gcc/ada',
  30     'gcc/analyzer',
  31     'gcc/brig',
  32     'gcc/c',
  33     'gcc/c-family',
  34     'gcc',
  35     'gcc/cp',
  36     'gcc/d',
  37     'gcc/fortran',
  38     'gcc/go',
  39     'gcc/jit',
  40     'gcc/lto',
  41     'gcc/objc',
  42     'gcc/objcp',
  43     'gcc/po',
  44     'gcc/testsuite',
  45     'gnattools',
  46     'gotools',
  47     'include',
  48     'intl',
  49     'libada',
  50     'libatomic',
  51     'libbacktrace',
  52     'libcc1',
  53     'libcpp',
  54     'libcpp/po',
  55     'libdecnumber',
  56     'libffi',
  57     'libgcc',
  58     'libgcc/config/avr/libf7',
  59     'libgcc/config/libbid',
  60     'libgfortran',
  61     'libgomp',
  62     'libhsail-rt',
  63     'libiberty',
  64     'libitm',
  65     'libobjc',
  66     'liboffloadmic',
  67     'libphobos',
  68     'libquadmath',
  69     'libsanitizer',
  70     'libssp',
  71     'libstdc++-v3',
  72     'libvtv',
  73     'lto-plugin',
  74     'maintainer-scripts',
  75     'zlib'])
  76
  77 bug_components = set([
  78     'ada',
  79     'analyzer',
  80     'boehm-gc',
  81     'bootstrap',
  82     'c',
  83     'c++',
  84     'd',
  85     'debug',
  86     'demangler',
  87     'driver',
  88     'fastjar',
  89     'fortran',
  90     'gcov-profile',
  91     'go',
  92     'hsa',
  93     'inline-asm',
  94     'ipa',
  95     'java',
  96     'jit',
  97     'libbacktrace',
  98     'libf2c',
  99     'libffi',
 100     'libfortran',
 101     'libgcc',
 102     'libgcj',
 103     'libgomp',
 104     'libitm',
 105     'libobjc',
 106     'libquadmath',
 107     'libstdc++',
 108     'lto',
 109     'middle-end',
 110     'modula2',
 111     'objc',
 112     'objc++',
 113     'other',
 114     'pch',
 115     'pending',
 116     'plugins',
 117     'preprocessor',
 118     'regression',
 119     'rtl-optimization',
 120     'sanitizer',
 121     'spam',
 122     'target',
 123     'testsuite',
 124     'translation',
 125     'tree-optimization',
 126     'web'])
 127
 128 ignored_prefixes = [
 129     'gcc/d/dmd/',
 130     'gcc/go/frontend/',
 131     'libgo/',
 132     'libphobos/libdruntime',
 133     'libphobos/src/',
 134     'libsanitizer/',
 135     ]
 136
 137 misc_files = [
 138     'gcc/DATESTAMP',
 139     'gcc/BASE-VER',
 140     'gcc/DEV-PHASE'
 141     ]
 142
 143 author_line_regex = \
 144         re.compile(r'^(?P<datetime>\d{4}-\d{2}-\d{2})\ {2}(?P<name>.*  <.*>)')
 145 additional_author_regex = re.compile(r'^\t(?P<spaces>\ *)?(?P<name>.*  <.*>)')
 146 changelog_regex = re.compile(r'^([a-z0-9+-/]*)/ChangeLog:?')
 147 pr_regex = re.compile(r'\tPR (?P<component>[a-z+-]+\/)?([0-9]+)$')
 148 star_prefix_regex = re.compile(r'\t\*(?P<spaces>\ *)(?P<content>.*)')
 149
 150 LINE_LIMIT = 100
 151 TAB_WIDTH = 8
 152 CO_AUTHORED_BY_PREFIX = 'co-authored-by: '
 153 CHERRY_PICK_PREFIX = '(cherry picked from commit '
 154
 155
 156 class Error:
 157     def __init__(self, message, line=None):
 158         self.message = message
 159         self.line = line
 160
 161     def __repr__(self):
 162         s = self.message
 163         if self.line:
 164             s += ':"%s"' % self.line
 165         return s
 166
 167
 168 class ChangeLogEntry:
 169     def __init__(self, folder, authors, prs):
 170         self.folder = folder
 171         # Python2 has not 'copy' function
 172         self.author_lines = list(authors)
 173         self.initial_prs = list(prs)
 174         self.prs = list(prs)
 175         self.lines = []
 176
 177     @property
 178     def files(self):
 179         files = []
 180         for line in self.lines:
 181             m = star_prefix_regex.match(line)
 182             if m:
 183                 line = m.group('content')
 184                 if '(' in line:
 185                     line = line[:line.index('(')]
 186                 if ':' in line:
 187                     line = line[:line.index(':')]
 188                 for file in line.split(','):
 189                     file = file.strip()
 190                     if file:
 191                         files.append(file)
 192         return files
 193
 194     @property
 195     def datetime(self):
 196         for author in self.author_lines:
 197             if author[1]:
 198                 return author[1]
 199         return None
 200
 201     @property
 202     def authors(self):
 203         return [author_line[0] for author_line in self.author_lines]
 204
 205     @property
 206     def is_empty(self):
 207         return not self.lines and self.prs == self.initial_prs
 208
 209
 210 class GitCommit:
 211     def __init__(self, hexsha, date, author, body, modified_files,
 212                  strict=True):
 213         self.hexsha = hexsha
 214         self.lines = body
 215         self.modified_files = modified_files
 216         self.message = None
 217         self.changes = None
 218         self.changelog_entries = []
 219         self.errors = []
 220         self.date = date
 221         self.author = author
 222         self.top_level_authors = []
 223         self.co_authors = []
 224         self.top_level_prs = []
 225
 226         project_files = [f for f in self.modified_files
 227                          if self.is_changelog_filename(f[0])
 228                          or f[0] in misc_files]
 229         if len(project_files) == len(self.modified_files):
 230             # All modified files are only MISC files
 231             return
 232         elif project_files and strict:
 233             self.errors.append(Error('ChangeLog, DATESTAMP, BASE-VER and '
 234                                      'DEV-PHASE updates should be done '
 235                                      'separately from normal commits'))
 236             return
 237
 238         self.parse_lines()
 239         if self.changes:
 240             self.parse_changelog()
 241             self.deduce_changelog_locations()
 242             if not self.errors:
 243                 self.check_mentioned_files()
 244                 self.check_for_correct_changelog()
 245
 246     @property
 247     def success(self):
 248         return not self.errors
 249
 250     @property
 251     def new_files(self):
 252         return [x[0] for x in self.modified_files if x[1] == 'A']
 253
 254     @classmethod
 255     def is_changelog_filename(cls, path):
 256         return path.endswith('/ChangeLog') or path == 'ChangeLog'
 257
 258     @classmethod
 259     def find_changelog_location(cls, name):
 260         if name.startswith('\t'):
 261             name = name[1:]
 262         if name.endswith(':'):
 263             name = name[:-1]
 264         if name.endswith('/'):
 265             name = name[:-1]
 266         return name if name in changelog_locations else None
 267
 268     @classmethod
 269     def format_git_author(cls, author):
 270         assert '<' in author
 271         return author.replace('<', ' <')
 272
 273     @classmethod
 274     def parse_git_name_status(cls, string):
 275         modified_files = []
 276         for entry in string.split('\n'):
 277             parts = entry.split('\t')
 278             t = parts[0]
 279             if t == 'A' or t == 'D' or t == 'M':
 280                 modified_files.append((parts[1], t))
 281             elif t == 'R':
 282                 modified_files.append((parts[1], 'D'))
 283                 modified_files.append((parts[2], 'A'))
 284         return modified_files
 285
 286     def parse_lines(self):
 287         body = self.lines
 288
 289         for i, b in enumerate(body):
 290             if not b:
 291                 continue
 292             if (changelog_regex.match(b) or self.find_changelog_location(b)
 293                     or star_prefix_regex.match(b) or pr_regex.match(b)
 294                     or author_line_regex.match(b)):
 295                 self.changes = body[i:]
 296                 return
 297         self.errors.append(Error('cannot find a ChangeLog location in '
 298                                  'message'))
 299
 300     def parse_changelog(self):
 301         last_entry = None
 302         will_deduce = False
 303         for line in self.changes:
 304             if not line:
 305                 if last_entry and will_deduce:
 306                     last_entry = None
 307                 continue
 308             if line != line.rstrip():
 309                 self.errors.append(Error('trailing whitespace', line))
 310             if len(line.replace('\t', ' ' * TAB_WIDTH)) > LINE_LIMIT:
 311                 self.errors.append(Error('line limit exceeds %d characters'
 312                                          % LINE_LIMIT, line))
 313             m = changelog_regex.match(line)
 314             if m:
 315                 last_entry = ChangeLogEntry(m.group(1), self.top_level_authors,
 316                                             self.top_level_prs)
 317                 self.changelog_entries.append(last_entry)
 318             elif self.find_changelog_location(line):
 319                 last_entry = ChangeLogEntry(self.find_changelog_location(line),
 320                                             self.top_level_authors,
 321                                             self.top_level_prs)
 322                 self.changelog_entries.append(last_entry)
 323             else:
 324                 author_tuple = None
 325                 pr_line = None
 326                 if author_line_regex.match(line):
 327                     m = author_line_regex.match(line)
 328                     author_tuple = (m.group('name'), m.group('datetime'))
 329                 elif additional_author_regex.match(line):
 330                     m = additional_author_regex.match(line)
 331                     if len(m.group('spaces')) != 4:
 332                         msg = 'additional author must prepend with tab ' \
 333                               'and 4 spaces'
 334                         self.errors.append(Error(msg, line))
 335                     else:
 336                         author_tuple = (m.group('name'), None)
 337                 elif pr_regex.match(line):
 338                     component = pr_regex.match(line).group('component')
 339                     if not component:
 340                         self.errors.append(Error('missing PR component', line))
 341                         continue
 342                     elif not component[:-1] in bug_components:
 343                         self.errors.append(Error('invalid PR component', line))
 344                         continue
 345                     else:
 346                         pr_line = line.lstrip()
 347
 348                 if line.lower().startswith(CO_AUTHORED_BY_PREFIX):
 349                     name = line[len(CO_AUTHORED_BY_PREFIX):]
 350                     author = self.format_git_author(name)
 351                     self.co_authors.append(author)
 352                     continue
 353                 elif line.startswith(CHERRY_PICK_PREFIX):
 354                     continue
 355
 356                 # ChangeLog name will be deduced later
 357                 if not last_entry:
 358                     if author_tuple:
 359                         self.top_level_authors.append(author_tuple)
 360                         continue
 361                     elif pr_line:
 362                         # append to top_level_prs only when we haven't met
 363                         # a ChangeLog entry
 364                         if (pr_line not in self.top_level_prs
 365                                 and not self.changelog_entries):
 366                             self.top_level_prs.append(pr_line)
 367                         continue
 368                     else:
 369                         last_entry = ChangeLogEntry(None,
 370                                                     self.top_level_authors,
 371                                                     self.top_level_prs)
 372                         self.changelog_entries.append(last_entry)
 373                         will_deduce = True
 374                 elif author_tuple:
 375                     last_entry.author_lines.append(author_tuple)
 376                     continue
 377
 378                 if not line.startswith('\t'):
 379                     err = Error('line should start with a tab', line)
 380                     self.errors.append(err)
 381                 elif pr_line:
 382                     last_entry.prs.append(pr_line)
 383                 else:
 384                     m = star_prefix_regex.match(line)
 385                     if m:
 386                         if len(m.group('spaces')) != 1:
 387                             err = Error('one space should follow asterisk',
 388                                         line)
 389                             self.errors.append(err)
 390                         else:
 391                             last_entry.lines.append(line)
 392                     else:
 393                         if last_entry.is_empty:
 394                             msg = 'first line should start with a tab, ' \
 395                                   'asterisk and space'
 396                             self.errors.append(Error(msg, line))
 397                         else:
 398                             last_entry.lines.append(line)
 399
 400     def get_file_changelog_location(self, changelog_file):
 401         for file in self.modified_files:
 402             if file[0] == changelog_file:
 403                 # root ChangeLog file
 404                 return ''
 405             index = file[0].find('/' + changelog_file)
 406             if index != -1:
 407                 return file[0][:index]
 408         return None
 409
 410     def deduce_changelog_locations(self):
 411         for entry in self.changelog_entries:
 412             if not entry.folder:
 413                 changelog = None
 414                 for file in entry.files:
 415                     location = self.get_file_changelog_location(file)
 416                     if (location == ''
 417                        or (location and location in changelog_locations)):
 418                         if changelog and changelog != location:
 419                             msg = 'could not deduce ChangeLog file, ' \
 420                                   'not unique location'
 421                             self.errors.append(Error(msg))
 422                             return
 423                         changelog = location
 424                 if changelog is not None:
 425                     entry.folder = changelog
 426                 else:
 427                     msg = 'could not deduce ChangeLog file'
 428                     self.errors.append(Error(msg))
 429
 430     @classmethod
 431     def in_ignored_location(cls, path):
 432         for ignored in ignored_prefixes:
 433             if path.startswith(ignored):
 434                 return True
 435         return False
 436
 437     @classmethod
 438     def get_changelog_by_path(cls, path):
 439         components = path.split('/')
 440         while components:
 441             if '/'.join(components) in changelog_locations:
 442                 break
 443             components = components[:-1]
 444         return '/'.join(components)
 445
 446     def check_mentioned_files(self):
 447         folder_count = len([x.folder for x in self.changelog_entries])
 448         assert folder_count == len(self.changelog_entries)
 449
 450         mentioned_files = set()
 451         for entry in self.changelog_entries:
 452             if not entry.files:
 453                 msg = 'ChangeLog must contain a file entry'
 454                 self.errors.append(Error(msg, entry.folder))
 455             assert not entry.folder.endswith('/')
 456             for file in entry.files:
 457                 if not self.is_changelog_filename(file):
 458                     mentioned_files.add(os.path.join(entry.folder, file))
 459
 460         cand = [x[0] for x in self.modified_files
 461                 if not self.is_changelog_filename(x[0])]
 462         changed_files = set(cand)
 463         for file in sorted(mentioned_files - changed_files):
 464             self.errors.append(Error('file not changed in a patch', file))
 465         for file in sorted(changed_files - mentioned_files):
 466             if not self.in_ignored_location(file):
 467                 if file in self.new_files:
 468                     changelog_location = self.get_changelog_by_path(file)
 469                     # Python2: we cannot use next(filter(...))
 470                     entries = filter(lambda x: x.folder == changelog_location,
 471                                      self.changelog_entries)
 472                     entries = list(entries)
 473                     entry = entries[0] if entries else None
 474                     if not entry:
 475                         prs = self.top_level_prs
 476                         if not prs:
 477                             # if all ChangeLog entries have identical PRs
 478                             # then use them
 479                             prs = self.changelog_entries[0].prs
 480                             for entry in self.changelog_entries:
 481                                 if entry.prs != prs:
 482                                     prs = []
 483                                     break
 484                         entry = ChangeLogEntry(changelog_location,
 485                                                self.top_level_authors,
 486                                                prs)
 487                         self.changelog_entries.append(entry)
 488                     # strip prefix of the file
 489                     assert file.startswith(entry.folder)
 490                     file = file[len(entry.folder):].lstrip('/')
 491                     entry.lines.append('\t* %s: New file.' % file)
 492                 else:
 493                     msg = 'changed file not mentioned in a ChangeLog'
 494                     self.errors.append(Error(msg, file))
 495
 496     def check_for_correct_changelog(self):
 497         for entry in self.changelog_entries:
 498             for file in entry.files:
 499                 full_path = os.path.join(entry.folder, file)
 500                 changelog_location = self.get_changelog_by_path(full_path)
 501                 if changelog_location != entry.folder:
 502                     msg = 'wrong ChangeLog location "%s", should be "%s"'
 503                     err = Error(msg % (entry.folder, changelog_location), file)
 504                     self.errors.append(err)
 505
 506     def to_changelog_entries(self, use_commit_ts=False):
 507         for entry in self.changelog_entries:
 508             output = ''
 509             timestamp = entry.datetime
 510             if not timestamp or use_commit_ts:
 511                 timestamp = self.date.strftime('%Y-%m-%d')
 512             authors = entry.authors if entry.authors else [self.author]
 513             # add Co-Authored-By authors to all ChangeLog entries
 514             for author in self.co_authors:
 515                 if author not in authors:
 516                     authors.append(author)
 517
 518             for i, author in enumerate(authors):
 519                 if i == 0:
 520                     output += '%s  %s\n' % (timestamp, author)
 521                 else:
 522                     output += '\t    %s\n' % author
 523             output += '\n'
 524             for pr in entry.prs:
 525                 output += '\t%s\n' % pr
 526             for line in entry.lines:
 527                 output += line + '\n'
 528             yield (entry.folder, output.rstrip())
 529
 530     def print_output(self):
 531         for entry, output in self.to_changelog_entries():
 532             print('------ %s/ChangeLog ------ ' % entry)
 533             print(output)
 534
 535     def print_errors(self):
 536         print('Errors:')
 537         for error in self.errors:
 538             print(error)