]>
Commit | Line | Data |
---|---|---|
00243d9a ML |
1 | #!/usr/bin/env python3 |
2 | ||
3 | # Copyright (C) 2020 Free Software Foundation, Inc. | |
4 | # | |
5 | # This file is part of GCC. | |
6 | # | |
7 | # GCC is free software; you can redistribute it and/or modify | |
8 | # it under the terms of the GNU General Public License as published by | |
9 | # the Free Software Foundation; either version 3, or (at your option) | |
10 | # any later version. | |
11 | # | |
12 | # GCC is distributed in the hope that it will be useful, | |
13 | # but WITHOUT ANY WARRANTY; without even the implied warranty of | |
14 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the | |
15 | # GNU General Public License for more details. | |
16 | # | |
17 | # You should have received a copy of the GNU General Public License | |
18 | # along with GCC; see the file COPYING. If not, write to | |
19 | # the Free Software Foundation, 51 Franklin Street, Fifth Floor, | |
20 | # Boston, MA 02110-1301, USA. | |
21 | ||
22 | # This script parses a .diff file generated with 'diff -up' or 'diff -cp' | |
23 | # and adds a skeleton ChangeLog file to the file. It does not try to be | |
24 | # too smart when parsing function names, but it produces a reasonable | |
25 | # approximation. | |
26 | # | |
27 | # Author: Martin Liska <mliska@suse.cz> | |
28 | ||
29 | import argparse | |
cf76bbf8 | 30 | import datetime |
00243d9a ML |
31 | import os |
32 | import re | |
cf76bbf8 | 33 | import subprocess |
00243d9a | 34 | import sys |
757dbb59 | 35 | from itertools import takewhile |
00243d9a | 36 | |
4f85a52c ML |
37 | import requests |
38 | ||
00243d9a ML |
39 | from unidiff import PatchSet |
40 | ||
b838641b ML |
41 | LINE_LIMIT = 100 |
42 | TAB_WIDTH = 8 | |
0684c8d3 | 43 | CO_AUTHORED_BY_PREFIX = 'co-authored-by: ' |
b838641b | 44 | |
00243d9a | 45 | pr_regex = re.compile(r'(\/(\/|\*)|[Cc*!])\s+(?P<pr>PR [a-z+-]+\/[0-9]+)') |
58e3b17f | 46 | prnum_regex = re.compile(r'PR (?P<comp>[a-z+-]+)/(?P<num>[0-9]+)') |
e7c7cdc5 | 47 | dr_regex = re.compile(r'(\/(\/|\*)|[Cc*!])\s+(?P<dr>DR [0-9]+)') |
ed5ae55e | 48 | dg_regex = re.compile(r'{\s+dg-(error|warning)') |
edf0c3ff | 49 | pr_filename_regex = re.compile(r'(^|[\W_])[Pp][Rr](?P<pr>\d{4,})') |
00243d9a ML |
50 | identifier_regex = re.compile(r'^([a-zA-Z0-9_#].*)') |
51 | comment_regex = re.compile(r'^\/\*') | |
4f85a52c ML |
52 | struct_regex = re.compile(r'^(class|struct|union|enum)\s+' |
53 | r'(GTY\(.*\)\s+)?([a-zA-Z0-9_]+)') | |
00243d9a ML |
54 | macro_regex = re.compile(r'#\s*(define|undef)\s+([a-zA-Z0-9_]+)') |
55 | super_macro_regex = re.compile(r'^DEF[A-Z0-9_]+\s*\(([a-zA-Z0-9_]+)') | |
56 | fn_regex = re.compile(r'([a-zA-Z_][^()\s]*)\s*\([^*]') | |
57 | template_and_param_regex = re.compile(r'<[^<>]*>') | |
a2d04f3d | 58 | md_def_regex = re.compile(r'\(define.*\s+"(.*)"') |
25c284f1 | 59 | bugzilla_url = 'https://gcc.gnu.org/bugzilla/rest.cgi/bug?id=%s&' \ |
edf0c3ff | 60 | 'include_fields=summary,component' |
00243d9a | 61 | |
a2d04f3d | 62 | function_extensions = {'.c', '.cpp', '.C', '.cc', '.h', '.inc', '.def', '.md'} |
00243d9a | 63 | |
432b9f61 MF |
64 | # NB: Makefile.in isn't listed as it's not always generated. |
65 | generated_files = {'aclocal.m4', 'config.h.in', 'configure'} | |
66 | ||
00243d9a ML |
67 | help_message = """\ |
68 | Generate ChangeLog template for PATCH. | |
69 | PATCH must be generated using diff(1)'s -up or -cp options | |
70 | (or their equivalent in git). | |
71 | """ | |
72 | ||
73 | script_folder = os.path.realpath(__file__) | |
e54da1b6 | 74 | root = os.path.dirname(os.path.dirname(script_folder)) |
00243d9a | 75 | |
58e3b17f JM |
76 | firstpr = '' |
77 | ||
00243d9a ML |
78 | |
79 | def find_changelog(path): | |
80 | folder = os.path.split(path)[0] | |
81 | while True: | |
e54da1b6 | 82 | if os.path.exists(os.path.join(root, folder, 'ChangeLog')): |
00243d9a ML |
83 | return folder |
84 | folder = os.path.dirname(folder) | |
85 | if folder == '': | |
86 | return folder | |
87 | raise AssertionError() | |
88 | ||
89 | ||
90 | def extract_function_name(line): | |
91 | if comment_regex.match(line): | |
92 | return None | |
93 | m = struct_regex.search(line) | |
94 | if m: | |
95 | # Struct declaration | |
4f85a52c | 96 | return m.group(1) + ' ' + m.group(3) |
00243d9a ML |
97 | m = macro_regex.search(line) |
98 | if m: | |
99 | # Macro definition | |
100 | return m.group(2) | |
101 | m = super_macro_regex.search(line) | |
102 | if m: | |
103 | # Supermacro | |
104 | return m.group(1) | |
105 | m = fn_regex.search(line) | |
106 | if m: | |
107 | # Discard template and function parameters. | |
108 | fn = m.group(1) | |
109 | fn = re.sub(template_and_param_regex, '', fn) | |
110 | return fn.rstrip() | |
111 | return None | |
112 | ||
113 | ||
114 | def try_add_function(functions, line): | |
115 | fn = extract_function_name(line) | |
116 | if fn and fn not in functions: | |
117 | functions.append(fn) | |
118 | return bool(fn) | |
119 | ||
120 | ||
121 | def sort_changelog_files(changed_file): | |
122 | return (changed_file.is_added_file, changed_file.is_removed_file) | |
123 | ||
124 | ||
125 | def get_pr_titles(prs): | |
edf0c3ff TB |
126 | output = [] |
127 | for idx, pr in enumerate(prs): | |
d6d9561c ML |
128 | pr_id = pr.split('/')[-1] |
129 | r = requests.get(bugzilla_url % pr_id) | |
25c284f1 ML |
130 | bugs = r.json()['bugs'] |
131 | if len(bugs) == 1: | |
edf0c3ff TB |
132 | prs[idx] = 'PR %s/%s' % (bugs[0]['component'], pr_id) |
133 | out = '%s - %s\n' % (prs[idx], bugs[0]['summary']) | |
134 | if out not in output: | |
135 | output.append(out) | |
25c284f1 | 136 | if output: |
edf0c3ff TB |
137 | output.append('') |
138 | return '\n'.join(output) | |
00243d9a | 139 | |
4f85a52c | 140 | |
b838641b ML |
141 | def append_changelog_line(out, relative_path, text): |
142 | line = f'\t* {relative_path}:' | |
143 | if len(line.replace('\t', ' ' * TAB_WIDTH) + ' ' + text) <= LINE_LIMIT: | |
144 | out += f'{line} {text}\n' | |
145 | else: | |
146 | out += f'{line}\n' | |
147 | out += f'\t{text}\n' | |
148 | return out | |
149 | ||
150 | ||
e370a248 ML |
151 | def get_rel_path_if_prefixed(path, folder): |
152 | if path.startswith(folder): | |
153 | return path[len(folder):].lstrip('/') | |
154 | else: | |
155 | return path | |
156 | ||
157 | ||
edf0c3ff TB |
158 | def generate_changelog(data, no_functions=False, fill_pr_titles=False, |
159 | additional_prs=None): | |
00243d9a ML |
160 | changelogs = {} |
161 | changelog_list = [] | |
162 | prs = [] | |
163 | out = '' | |
164 | diff = PatchSet(data) | |
58e3b17f | 165 | global firstpr |
00243d9a | 166 | |
edf0c3ff | 167 | if additional_prs: |
84f906df ML |
168 | for apr in additional_prs: |
169 | if not apr.startswith('PR ') and '/' in apr: | |
170 | apr = 'PR ' + apr | |
171 | if apr not in prs: | |
172 | prs.append(apr) | |
00243d9a | 173 | for file in diff: |
236d6a33 ML |
174 | # skip files that can't be parsed |
175 | if file.path == '/dev/null': | |
176 | continue | |
00243d9a ML |
177 | changelog = find_changelog(file.path) |
178 | if changelog not in changelogs: | |
179 | changelogs[changelog] = [] | |
180 | changelog_list.append(changelog) | |
181 | changelogs[changelog].append(file) | |
182 | ||
183 | # Extract PR entries from newly added tests | |
184 | if 'testsuite' in file.path and file.is_added_file: | |
ed5ae55e TB |
185 | # Only search first ten lines as later lines may |
186 | # contains commented code which a note that it | |
187 | # has not been tested due to a certain PR or DR. | |
edf0c3ff | 188 | this_file_prs = [] |
ed5ae55e | 189 | for line in list(file)[0][0:10]: |
00243d9a ML |
190 | m = pr_regex.search(line.value) |
191 | if m: | |
192 | pr = m.group('pr') | |
193 | if pr not in prs: | |
194 | prs.append(pr) | |
edf0c3ff | 195 | this_file_prs.append(pr.split('/')[-1]) |
00243d9a | 196 | else: |
e7c7cdc5 ML |
197 | m = dr_regex.search(line.value) |
198 | if m: | |
199 | dr = m.group('dr') | |
200 | if dr not in prs: | |
201 | prs.append(dr) | |
edf0c3ff | 202 | this_file_prs.append(dr.split('/')[-1]) |
ed5ae55e TB |
203 | elif dg_regex.search(line.value): |
204 | # Found dg-warning/dg-error line | |
e7c7cdc5 | 205 | break |
edf0c3ff TB |
206 | # PR number in the file name |
207 | fname = os.path.basename(file.path) | |
208 | m = pr_filename_regex.search(fname) | |
209 | if m: | |
210 | pr = m.group('pr') | |
211 | pr2 = 'PR ' + pr | |
212 | if pr not in this_file_prs and pr2 not in prs: | |
213 | prs.append(pr2) | |
00243d9a | 214 | |
58e3b17f JM |
215 | if prs: |
216 | firstpr = prs[0] | |
217 | ||
00243d9a ML |
218 | if fill_pr_titles: |
219 | out += get_pr_titles(prs) | |
220 | ||
fef084dc ML |
221 | # print list of PR entries before ChangeLog entries |
222 | if prs: | |
223 | if not out: | |
224 | out += '\n' | |
225 | for pr in prs: | |
226 | out += '\t%s\n' % pr | |
227 | out += '\n' | |
228 | ||
00243d9a ML |
229 | # sort ChangeLog so that 'testsuite' is at the end |
230 | for changelog in sorted(changelog_list, key=lambda x: 'testsuite' in x): | |
231 | files = changelogs[changelog] | |
232 | out += '%s:\n' % os.path.join(changelog, 'ChangeLog') | |
233 | out += '\n' | |
00243d9a ML |
234 | # new and deleted files should be at the end |
235 | for file in sorted(files, key=sort_changelog_files): | |
236 | assert file.path.startswith(changelog) | |
237 | in_tests = 'testsuite' in changelog or 'testsuite' in file.path | |
e370a248 | 238 | relative_path = get_rel_path_if_prefixed(file.path, changelog) |
00243d9a ML |
239 | functions = [] |
240 | if file.is_added_file: | |
b838641b ML |
241 | msg = 'New test.' if in_tests else 'New file.' |
242 | out = append_changelog_line(out, relative_path, msg) | |
00243d9a | 243 | elif file.is_removed_file: |
b838641b | 244 | out = append_changelog_line(out, relative_path, 'Removed.') |
eb78da45 | 245 | elif hasattr(file, 'is_rename') and file.is_rename: |
eb78da45 ML |
246 | # A file can be theoretically moved to a location that |
247 | # belongs to a different ChangeLog. Let user fix it. | |
e370a248 ML |
248 | # |
249 | # Since unidiff 0.7.0, path.file == path.target_file[2:], | |
250 | # it used to be path.source_file[2:] | |
251 | relative_path = get_rel_path_if_prefixed(file.source_file[2:], | |
252 | changelog) | |
253 | out = append_changelog_line(out, relative_path, 'Moved to...') | |
254 | new_path = get_rel_path_if_prefixed(file.target_file[2:], | |
255 | changelog) | |
256 | out += f'\t* {new_path}: ...here.\n' | |
432b9f61 MF |
257 | elif os.path.basename(file.path) in generated_files: |
258 | out += '\t* %s: Regenerate.\n' % (relative_path) | |
b838641b | 259 | append_changelog_line(out, relative_path, 'Regenerate.') |
00243d9a ML |
260 | else: |
261 | if not no_functions: | |
262 | for hunk in file: | |
263 | # Do not add function names for testsuite files | |
264 | extension = os.path.splitext(relative_path)[1] | |
265 | if not in_tests and extension in function_extensions: | |
266 | last_fn = None | |
267 | modified_visited = False | |
268 | success = False | |
269 | for line in hunk: | |
270 | m = identifier_regex.match(line.value) | |
271 | if line.is_added or line.is_removed: | |
a2d04f3d ML |
272 | # special-case definition in .md files |
273 | m2 = md_def_regex.match(line.value) | |
274 | if extension == '.md' and m2: | |
275 | fn = m2.group(1) | |
276 | if fn not in functions: | |
277 | functions.append(fn) | |
278 | last_fn = None | |
279 | success = True | |
280 | ||
00243d9a ML |
281 | if not line.value.strip(): |
282 | continue | |
283 | modified_visited = True | |
284 | if m and try_add_function(functions, | |
285 | m.group(1)): | |
286 | last_fn = None | |
287 | success = True | |
288 | elif line.is_context: | |
289 | if last_fn and modified_visited: | |
290 | try_add_function(functions, last_fn) | |
291 | last_fn = None | |
292 | modified_visited = False | |
293 | success = True | |
294 | elif m: | |
295 | last_fn = m.group(1) | |
296 | modified_visited = False | |
297 | if not success: | |
298 | try_add_function(functions, | |
299 | hunk.section_header) | |
300 | if functions: | |
301 | out += '\t* %s (%s):\n' % (relative_path, functions[0]) | |
302 | for fn in functions[1:]: | |
303 | out += '\t(%s):\n' % fn | |
304 | else: | |
305 | out += '\t* %s:\n' % relative_path | |
306 | out += '\n' | |
307 | return out | |
308 | ||
309 | ||
cf76bbf8 ML |
310 | def update_copyright(data): |
311 | current_timestamp = datetime.datetime.now().strftime('%Y-%m-%d') | |
312 | username = subprocess.check_output('git config user.name', shell=True, | |
313 | encoding='utf8').strip() | |
314 | email = subprocess.check_output('git config user.email', shell=True, | |
315 | encoding='utf8').strip() | |
316 | ||
317 | changelogs = set() | |
318 | diff = PatchSet(data) | |
319 | ||
320 | for file in diff: | |
321 | changelog = os.path.join(find_changelog(file.path), 'ChangeLog') | |
322 | if changelog not in changelogs: | |
323 | changelogs.add(changelog) | |
324 | with open(changelog) as f: | |
325 | content = f.read() | |
326 | with open(changelog, 'w+') as f: | |
327 | f.write(f'{current_timestamp} {username} <{email}>\n\n') | |
328 | f.write('\tUpdate copyright years.\n\n') | |
329 | f.write(content) | |
330 | ||
331 | ||
0684c8d3 ML |
332 | def skip_line_in_changelog(line): |
333 | if line.lower().startswith(CO_AUTHORED_BY_PREFIX) or line.startswith('#'): | |
334 | return False | |
335 | return True | |
336 | ||
337 | ||
00243d9a ML |
338 | if __name__ == '__main__': |
339 | parser = argparse.ArgumentParser(description=help_message) | |
340 | parser.add_argument('input', nargs='?', | |
341 | help='Patch file (or missing, read standard input)') | |
edf0c3ff | 342 | parser.add_argument('-b', '--pr-numbers', action='store', |
48b312b4 | 343 | type=lambda arg: arg.split(','), nargs='?', |
edf0c3ff | 344 | help='Add the specified PRs (comma separated)') |
00243d9a ML |
345 | parser.add_argument('-s', '--no-functions', action='store_true', |
346 | help='Do not generate function names in ChangeLogs') | |
347 | parser.add_argument('-p', '--fill-up-bug-titles', action='store_true', | |
348 | help='Download title of mentioned PRs') | |
e54da1b6 | 349 | parser.add_argument('-d', '--directory', |
c38f679e ML |
350 | help='Root directory where to search for ChangeLog ' |
351 | 'files') | |
757dbb59 JM |
352 | parser.add_argument('-c', '--changelog', |
353 | help='Append the ChangeLog to a git commit message ' | |
354 | 'file') | |
cf76bbf8 ML |
355 | parser.add_argument('--update-copyright', action='store_true', |
356 | help='Update copyright in ChangeLog files') | |
00243d9a ML |
357 | args = parser.parse_args() |
358 | if args.input == '-': | |
359 | args.input = None | |
e54da1b6 ML |
360 | if args.directory: |
361 | root = args.directory | |
00243d9a | 362 | |
d6d9561c | 363 | data = open(args.input) if args.input else sys.stdin |
cf76bbf8 ML |
364 | if args.update_copyright: |
365 | update_copyright(data) | |
757dbb59 | 366 | else: |
cf76bbf8 | 367 | output = generate_changelog(data, args.no_functions, |
edf0c3ff | 368 | args.fill_up_bug_titles, args.pr_numbers) |
cf76bbf8 ML |
369 | if args.changelog: |
370 | lines = open(args.changelog).read().split('\n') | |
0684c8d3 | 371 | start = list(takewhile(skip_line_in_changelog, lines)) |
cf76bbf8 ML |
372 | end = lines[len(start):] |
373 | with open(args.changelog, 'w') as f: | |
58e3b17f JM |
374 | if not start or not start[0]: |
375 | # initial commit subject line 'component: [PRnnnnn]' | |
376 | m = prnum_regex.match(firstpr) | |
377 | if m: | |
378 | title = f'{m.group("comp")}: [PR{m.group("num")}]' | |
379 | start.insert(0, title) | |
cf76bbf8 | 380 | if start: |
58e3b17f | 381 | # append empty line |
cf76bbf8 ML |
382 | if start[-1] != '': |
383 | start.append('') | |
384 | else: | |
385 | # append 2 empty lines | |
386 | start = 2 * [''] | |
387 | f.write('\n'.join(start)) | |
388 | f.write('\n') | |
389 | f.write(output) | |
390 | f.write('\n'.join(end)) | |
391 | else: | |
392 | print(output, end='') |