]>
Commit | Line | Data |
---|---|---|
00243d9a ML |
1 | #!/usr/bin/env python3 |
2 | ||
83ffe9cd | 3 | # Copyright (C) 2020-2023 Free Software Foundation, Inc. |
00243d9a ML |
4 | # |
5 | # This file is part of GCC. | |
6 | # | |
7 | # GCC is free software; you can redistribute it and/or modify | |
8 | # it under the terms of the GNU General Public License as published by | |
9 | # the Free Software Foundation; either version 3, or (at your option) | |
10 | # any later version. | |
11 | # | |
12 | # GCC is distributed in the hope that it will be useful, | |
13 | # but WITHOUT ANY WARRANTY; without even the implied warranty of | |
14 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the | |
15 | # GNU General Public License for more details. | |
16 | # | |
17 | # You should have received a copy of the GNU General Public License | |
18 | # along with GCC; see the file COPYING. If not, write to | |
19 | # the Free Software Foundation, 51 Franklin Street, Fifth Floor, | |
20 | # Boston, MA 02110-1301, USA. | |
21 | ||
22 | # This script parses a .diff file generated with 'diff -up' or 'diff -cp' | |
23 | # and adds a skeleton ChangeLog file to the file. It does not try to be | |
24 | # too smart when parsing function names, but it produces a reasonable | |
25 | # approximation. | |
26 | # | |
27 | # Author: Martin Liska <mliska@suse.cz> | |
28 | ||
29 | import argparse | |
cf76bbf8 | 30 | import datetime |
18ef76d3 | 31 | import json |
00243d9a ML |
32 | import os |
33 | import re | |
cf76bbf8 | 34 | import subprocess |
00243d9a | 35 | import sys |
757dbb59 | 36 | from itertools import takewhile |
00243d9a | 37 | |
4f85a52c ML |
38 | import requests |
39 | ||
00243d9a ML |
40 | from unidiff import PatchSet |
41 | ||
b838641b ML |
42 | LINE_LIMIT = 100 |
43 | TAB_WIDTH = 8 | |
0684c8d3 | 44 | CO_AUTHORED_BY_PREFIX = 'co-authored-by: ' |
b838641b | 45 | |
00243d9a | 46 | pr_regex = re.compile(r'(\/(\/|\*)|[Cc*!])\s+(?P<pr>PR [a-z+-]+\/[0-9]+)') |
58e3b17f | 47 | prnum_regex = re.compile(r'PR (?P<comp>[a-z+-]+)/(?P<num>[0-9]+)') |
e7c7cdc5 | 48 | dr_regex = re.compile(r'(\/(\/|\*)|[Cc*!])\s+(?P<dr>DR [0-9]+)') |
ed5ae55e | 49 | dg_regex = re.compile(r'{\s+dg-(error|warning)') |
edf0c3ff | 50 | pr_filename_regex = re.compile(r'(^|[\W_])[Pp][Rr](?P<pr>\d{4,})') |
00243d9a ML |
51 | identifier_regex = re.compile(r'^([a-zA-Z0-9_#].*)') |
52 | comment_regex = re.compile(r'^\/\*') | |
4f85a52c ML |
53 | struct_regex = re.compile(r'^(class|struct|union|enum)\s+' |
54 | r'(GTY\(.*\)\s+)?([a-zA-Z0-9_]+)') | |
00243d9a ML |
55 | macro_regex = re.compile(r'#\s*(define|undef)\s+([a-zA-Z0-9_]+)') |
56 | super_macro_regex = re.compile(r'^DEF[A-Z0-9_]+\s*\(([a-zA-Z0-9_]+)') | |
57 | fn_regex = re.compile(r'([a-zA-Z_][^()\s]*)\s*\([^*]') | |
58 | template_and_param_regex = re.compile(r'<[^<>]*>') | |
a2d04f3d | 59 | md_def_regex = re.compile(r'\(define.*\s+"(.*)"') |
25c284f1 | 60 | bugzilla_url = 'https://gcc.gnu.org/bugzilla/rest.cgi/bug?id=%s&' \ |
edf0c3ff | 61 | 'include_fields=summary,component' |
00243d9a | 62 | |
a2d04f3d | 63 | function_extensions = {'.c', '.cpp', '.C', '.cc', '.h', '.inc', '.def', '.md'} |
00243d9a | 64 | |
432b9f61 MF |
65 | # NB: Makefile.in isn't listed as it's not always generated. |
66 | generated_files = {'aclocal.m4', 'config.h.in', 'configure'} | |
67 | ||
00243d9a ML |
68 | help_message = """\ |
69 | Generate ChangeLog template for PATCH. | |
70 | PATCH must be generated using diff(1)'s -up or -cp options | |
71 | (or their equivalent in git). | |
72 | """ | |
73 | ||
74 | script_folder = os.path.realpath(__file__) | |
e54da1b6 | 75 | root = os.path.dirname(os.path.dirname(script_folder)) |
00243d9a ML |
76 | |
77 | ||
78 | def find_changelog(path): | |
79 | folder = os.path.split(path)[0] | |
80 | while True: | |
e54da1b6 | 81 | if os.path.exists(os.path.join(root, folder, 'ChangeLog')): |
00243d9a ML |
82 | return folder |
83 | folder = os.path.dirname(folder) | |
84 | if folder == '': | |
85 | return folder | |
86 | raise AssertionError() | |
87 | ||
88 | ||
89 | def extract_function_name(line): | |
90 | if comment_regex.match(line): | |
91 | return None | |
92 | m = struct_regex.search(line) | |
93 | if m: | |
94 | # Struct declaration | |
4f85a52c | 95 | return m.group(1) + ' ' + m.group(3) |
00243d9a ML |
96 | m = macro_regex.search(line) |
97 | if m: | |
98 | # Macro definition | |
99 | return m.group(2) | |
100 | m = super_macro_regex.search(line) | |
101 | if m: | |
102 | # Supermacro | |
103 | return m.group(1) | |
104 | m = fn_regex.search(line) | |
105 | if m: | |
106 | # Discard template and function parameters. | |
107 | fn = m.group(1) | |
108 | fn = re.sub(template_and_param_regex, '', fn) | |
109 | return fn.rstrip() | |
110 | return None | |
111 | ||
112 | ||
113 | def try_add_function(functions, line): | |
114 | fn = extract_function_name(line) | |
115 | if fn and fn not in functions: | |
116 | functions.append(fn) | |
117 | return bool(fn) | |
118 | ||
119 | ||
120 | def sort_changelog_files(changed_file): | |
121 | return (changed_file.is_added_file, changed_file.is_removed_file) | |
122 | ||
123 | ||
124 | def get_pr_titles(prs): | |
edf0c3ff TB |
125 | output = [] |
126 | for idx, pr in enumerate(prs): | |
d6d9561c ML |
127 | pr_id = pr.split('/')[-1] |
128 | r = requests.get(bugzilla_url % pr_id) | |
25c284f1 ML |
129 | bugs = r.json()['bugs'] |
130 | if len(bugs) == 1: | |
edf0c3ff TB |
131 | prs[idx] = 'PR %s/%s' % (bugs[0]['component'], pr_id) |
132 | out = '%s - %s\n' % (prs[idx], bugs[0]['summary']) | |
133 | if out not in output: | |
134 | output.append(out) | |
25c284f1 | 135 | if output: |
edf0c3ff TB |
136 | output.append('') |
137 | return '\n'.join(output) | |
00243d9a | 138 | |
4f85a52c | 139 | |
b838641b ML |
140 | def append_changelog_line(out, relative_path, text): |
141 | line = f'\t* {relative_path}:' | |
142 | if len(line.replace('\t', ' ' * TAB_WIDTH) + ' ' + text) <= LINE_LIMIT: | |
143 | out += f'{line} {text}\n' | |
144 | else: | |
145 | out += f'{line}\n' | |
146 | out += f'\t{text}\n' | |
147 | return out | |
148 | ||
149 | ||
e370a248 ML |
150 | def get_rel_path_if_prefixed(path, folder): |
151 | if path.startswith(folder): | |
152 | return path[len(folder):].lstrip('/') | |
153 | else: | |
154 | return path | |
155 | ||
156 | ||
edf0c3ff TB |
157 | def generate_changelog(data, no_functions=False, fill_pr_titles=False, |
158 | additional_prs=None): | |
03c0b064 ML |
159 | global prs |
160 | prs = [] | |
161 | ||
00243d9a ML |
162 | changelogs = {} |
163 | changelog_list = [] | |
00243d9a ML |
164 | out = '' |
165 | diff = PatchSet(data) | |
166 | ||
edf0c3ff | 167 | if additional_prs: |
84f906df ML |
168 | for apr in additional_prs: |
169 | if not apr.startswith('PR ') and '/' in apr: | |
170 | apr = 'PR ' + apr | |
171 | if apr not in prs: | |
172 | prs.append(apr) | |
00243d9a | 173 | for file in diff: |
236d6a33 ML |
174 | # skip files that can't be parsed |
175 | if file.path == '/dev/null': | |
176 | continue | |
00243d9a ML |
177 | changelog = find_changelog(file.path) |
178 | if changelog not in changelogs: | |
179 | changelogs[changelog] = [] | |
180 | changelog_list.append(changelog) | |
181 | changelogs[changelog].append(file) | |
182 | ||
183 | # Extract PR entries from newly added tests | |
184 | if 'testsuite' in file.path and file.is_added_file: | |
ed5ae55e TB |
185 | # Only search first ten lines as later lines may |
186 | # contains commented code which a note that it | |
187 | # has not been tested due to a certain PR or DR. | |
edf0c3ff | 188 | this_file_prs = [] |
e1652147 ML |
189 | hunks = list(file) |
190 | if hunks: | |
191 | for line in hunks[0][0:10]: | |
14d0f82c | 192 | m = pr_regex.search(line.value) |
e7c7cdc5 | 193 | if m: |
14d0f82c ML |
194 | pr = m.group('pr') |
195 | if pr not in prs: | |
196 | prs.append(pr) | |
197 | this_file_prs.append(pr.split('/')[-1]) | |
198 | else: | |
199 | m = dr_regex.search(line.value) | |
200 | if m: | |
201 | dr = m.group('dr') | |
202 | if dr not in prs: | |
203 | prs.append(dr) | |
204 | this_file_prs.append(dr.split('/')[-1]) | |
205 | elif dg_regex.search(line.value): | |
206 | # Found dg-warning/dg-error line | |
207 | break | |
208 | ||
edf0c3ff TB |
209 | # PR number in the file name |
210 | fname = os.path.basename(file.path) | |
211 | m = pr_filename_regex.search(fname) | |
212 | if m: | |
213 | pr = m.group('pr') | |
214 | pr2 = 'PR ' + pr | |
215 | if pr not in this_file_prs and pr2 not in prs: | |
216 | prs.append(pr2) | |
00243d9a ML |
217 | |
218 | if fill_pr_titles: | |
219 | out += get_pr_titles(prs) | |
220 | ||
fef084dc ML |
221 | # print list of PR entries before ChangeLog entries |
222 | if prs: | |
223 | if not out: | |
224 | out += '\n' | |
225 | for pr in prs: | |
226 | out += '\t%s\n' % pr | |
227 | out += '\n' | |
228 | ||
00243d9a ML |
229 | # sort ChangeLog so that 'testsuite' is at the end |
230 | for changelog in sorted(changelog_list, key=lambda x: 'testsuite' in x): | |
231 | files = changelogs[changelog] | |
232 | out += '%s:\n' % os.path.join(changelog, 'ChangeLog') | |
233 | out += '\n' | |
00243d9a ML |
234 | # new and deleted files should be at the end |
235 | for file in sorted(files, key=sort_changelog_files): | |
236 | assert file.path.startswith(changelog) | |
237 | in_tests = 'testsuite' in changelog or 'testsuite' in file.path | |
e370a248 | 238 | relative_path = get_rel_path_if_prefixed(file.path, changelog) |
00243d9a ML |
239 | functions = [] |
240 | if file.is_added_file: | |
b838641b ML |
241 | msg = 'New test.' if in_tests else 'New file.' |
242 | out = append_changelog_line(out, relative_path, msg) | |
00243d9a | 243 | elif file.is_removed_file: |
b838641b | 244 | out = append_changelog_line(out, relative_path, 'Removed.') |
eb78da45 | 245 | elif hasattr(file, 'is_rename') and file.is_rename: |
eb78da45 ML |
246 | # A file can be theoretically moved to a location that |
247 | # belongs to a different ChangeLog. Let user fix it. | |
e370a248 ML |
248 | # |
249 | # Since unidiff 0.7.0, path.file == path.target_file[2:], | |
250 | # it used to be path.source_file[2:] | |
251 | relative_path = get_rel_path_if_prefixed(file.source_file[2:], | |
252 | changelog) | |
253 | out = append_changelog_line(out, relative_path, 'Moved to...') | |
254 | new_path = get_rel_path_if_prefixed(file.target_file[2:], | |
255 | changelog) | |
256 | out += f'\t* {new_path}: ...here.\n' | |
432b9f61 MF |
257 | elif os.path.basename(file.path) in generated_files: |
258 | out += '\t* %s: Regenerate.\n' % (relative_path) | |
b838641b | 259 | append_changelog_line(out, relative_path, 'Regenerate.') |
00243d9a ML |
260 | else: |
261 | if not no_functions: | |
262 | for hunk in file: | |
263 | # Do not add function names for testsuite files | |
264 | extension = os.path.splitext(relative_path)[1] | |
265 | if not in_tests and extension in function_extensions: | |
266 | last_fn = None | |
267 | modified_visited = False | |
268 | success = False | |
269 | for line in hunk: | |
270 | m = identifier_regex.match(line.value) | |
271 | if line.is_added or line.is_removed: | |
a2d04f3d ML |
272 | # special-case definition in .md files |
273 | m2 = md_def_regex.match(line.value) | |
274 | if extension == '.md' and m2: | |
275 | fn = m2.group(1) | |
276 | if fn not in functions: | |
277 | functions.append(fn) | |
278 | last_fn = None | |
279 | success = True | |
280 | ||
00243d9a ML |
281 | if not line.value.strip(): |
282 | continue | |
283 | modified_visited = True | |
284 | if m and try_add_function(functions, | |
285 | m.group(1)): | |
286 | last_fn = None | |
287 | success = True | |
288 | elif line.is_context: | |
289 | if last_fn and modified_visited: | |
290 | try_add_function(functions, last_fn) | |
291 | last_fn = None | |
292 | modified_visited = False | |
293 | success = True | |
294 | elif m: | |
295 | last_fn = m.group(1) | |
296 | modified_visited = False | |
297 | if not success: | |
298 | try_add_function(functions, | |
299 | hunk.section_header) | |
300 | if functions: | |
301 | out += '\t* %s (%s):\n' % (relative_path, functions[0]) | |
302 | for fn in functions[1:]: | |
303 | out += '\t(%s):\n' % fn | |
304 | else: | |
305 | out += '\t* %s:\n' % relative_path | |
306 | out += '\n' | |
307 | return out | |
308 | ||
309 | ||
cf76bbf8 ML |
310 | def update_copyright(data): |
311 | current_timestamp = datetime.datetime.now().strftime('%Y-%m-%d') | |
312 | username = subprocess.check_output('git config user.name', shell=True, | |
313 | encoding='utf8').strip() | |
314 | email = subprocess.check_output('git config user.email', shell=True, | |
315 | encoding='utf8').strip() | |
316 | ||
317 | changelogs = set() | |
318 | diff = PatchSet(data) | |
319 | ||
320 | for file in diff: | |
321 | changelog = os.path.join(find_changelog(file.path), 'ChangeLog') | |
322 | if changelog not in changelogs: | |
323 | changelogs.add(changelog) | |
324 | with open(changelog) as f: | |
325 | content = f.read() | |
326 | with open(changelog, 'w+') as f: | |
327 | f.write(f'{current_timestamp} {username} <{email}>\n\n') | |
328 | f.write('\tUpdate copyright years.\n\n') | |
329 | f.write(content) | |
330 | ||
331 | ||
0684c8d3 ML |
332 | def skip_line_in_changelog(line): |
333 | if line.lower().startswith(CO_AUTHORED_BY_PREFIX) or line.startswith('#'): | |
334 | return False | |
335 | return True | |
336 | ||
337 | ||
00243d9a | 338 | if __name__ == '__main__': |
18ef76d3 ML |
339 | extra_args = os.getenv('GCC_MKLOG_ARGS') |
340 | if extra_args: | |
341 | sys.argv += json.loads(extra_args) | |
342 | ||
00243d9a ML |
343 | parser = argparse.ArgumentParser(description=help_message) |
344 | parser.add_argument('input', nargs='?', | |
345 | help='Patch file (or missing, read standard input)') | |
edf0c3ff | 346 | parser.add_argument('-b', '--pr-numbers', action='store', |
48b312b4 | 347 | type=lambda arg: arg.split(','), nargs='?', |
edf0c3ff | 348 | help='Add the specified PRs (comma separated)') |
00243d9a ML |
349 | parser.add_argument('-s', '--no-functions', action='store_true', |
350 | help='Do not generate function names in ChangeLogs') | |
351 | parser.add_argument('-p', '--fill-up-bug-titles', action='store_true', | |
352 | help='Download title of mentioned PRs') | |
e54da1b6 | 353 | parser.add_argument('-d', '--directory', |
c38f679e ML |
354 | help='Root directory where to search for ChangeLog ' |
355 | 'files') | |
757dbb59 JM |
356 | parser.add_argument('-c', '--changelog', |
357 | help='Append the ChangeLog to a git commit message ' | |
358 | 'file') | |
cf76bbf8 ML |
359 | parser.add_argument('--update-copyright', action='store_true', |
360 | help='Update copyright in ChangeLog files') | |
00243d9a ML |
361 | args = parser.parse_args() |
362 | if args.input == '-': | |
363 | args.input = None | |
e54da1b6 ML |
364 | if args.directory: |
365 | root = args.directory | |
00243d9a | 366 | |
b0451799 | 367 | data = open(args.input, newline='\n') if args.input else sys.stdin |
cf76bbf8 ML |
368 | if args.update_copyright: |
369 | update_copyright(data) | |
757dbb59 | 370 | else: |
cf76bbf8 | 371 | output = generate_changelog(data, args.no_functions, |
edf0c3ff | 372 | args.fill_up_bug_titles, args.pr_numbers) |
cf76bbf8 ML |
373 | if args.changelog: |
374 | lines = open(args.changelog).read().split('\n') | |
0684c8d3 | 375 | start = list(takewhile(skip_line_in_changelog, lines)) |
cf76bbf8 ML |
376 | end = lines[len(start):] |
377 | with open(args.changelog, 'w') as f: | |
58e3b17f | 378 | if not start or not start[0]: |
03c0b064 ML |
379 | if len(prs) == 1: |
380 | # initial commit subject line 'component: [PRnnnnn]' | |
381 | m = prnum_regex.match(prs[0]) | |
382 | if m: | |
383 | title = f'{m.group("comp")}: [PR{m.group("num")}]' | |
384 | start.insert(0, title) | |
cf76bbf8 | 385 | if start: |
58e3b17f | 386 | # append empty line |
cf76bbf8 ML |
387 | if start[-1] != '': |
388 | start.append('') | |
389 | else: | |
390 | # append 2 empty lines | |
391 | start = 2 * [''] | |
392 | f.write('\n'.join(start)) | |
393 | f.write('\n') | |
394 | f.write(output) | |
395 | f.write('\n'.join(end)) | |
396 | else: | |
397 | print(output, end='') |