]>
Commit | Line | Data |
---|---|---|
00243d9a ML |
1 | #!/usr/bin/env python3 |
2 | ||
3 | # Copyright (C) 2020 Free Software Foundation, Inc. | |
4 | # | |
5 | # This file is part of GCC. | |
6 | # | |
7 | # GCC is free software; you can redistribute it and/or modify | |
8 | # it under the terms of the GNU General Public License as published by | |
9 | # the Free Software Foundation; either version 3, or (at your option) | |
10 | # any later version. | |
11 | # | |
12 | # GCC is distributed in the hope that it will be useful, | |
13 | # but WITHOUT ANY WARRANTY; without even the implied warranty of | |
14 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the | |
15 | # GNU General Public License for more details. | |
16 | # | |
17 | # You should have received a copy of the GNU General Public License | |
18 | # along with GCC; see the file COPYING. If not, write to | |
19 | # the Free Software Foundation, 51 Franklin Street, Fifth Floor, | |
20 | # Boston, MA 02110-1301, USA. | |
21 | ||
22 | # This script parses a .diff file generated with 'diff -up' or 'diff -cp' | |
23 | # and adds a skeleton ChangeLog file to the file. It does not try to be | |
24 | # too smart when parsing function names, but it produces a reasonable | |
25 | # approximation. | |
26 | # | |
27 | # Author: Martin Liska <mliska@suse.cz> | |
28 | ||
29 | import argparse | |
cf76bbf8 | 30 | import datetime |
00243d9a ML |
31 | import os |
32 | import re | |
cf76bbf8 | 33 | import subprocess |
00243d9a | 34 | import sys |
757dbb59 | 35 | from itertools import takewhile |
00243d9a | 36 | |
4f85a52c ML |
37 | import requests |
38 | ||
00243d9a ML |
39 | from unidiff import PatchSet |
40 | ||
b838641b ML |
41 | LINE_LIMIT = 100 |
42 | TAB_WIDTH = 8 | |
43 | ||
00243d9a | 44 | pr_regex = re.compile(r'(\/(\/|\*)|[Cc*!])\s+(?P<pr>PR [a-z+-]+\/[0-9]+)') |
58e3b17f | 45 | prnum_regex = re.compile(r'PR (?P<comp>[a-z+-]+)/(?P<num>[0-9]+)') |
e7c7cdc5 | 46 | dr_regex = re.compile(r'(\/(\/|\*)|[Cc*!])\s+(?P<dr>DR [0-9]+)') |
ed5ae55e | 47 | dg_regex = re.compile(r'{\s+dg-(error|warning)') |
edf0c3ff | 48 | pr_filename_regex = re.compile(r'(^|[\W_])[Pp][Rr](?P<pr>\d{4,})') |
00243d9a ML |
49 | identifier_regex = re.compile(r'^([a-zA-Z0-9_#].*)') |
50 | comment_regex = re.compile(r'^\/\*') | |
4f85a52c ML |
51 | struct_regex = re.compile(r'^(class|struct|union|enum)\s+' |
52 | r'(GTY\(.*\)\s+)?([a-zA-Z0-9_]+)') | |
00243d9a ML |
53 | macro_regex = re.compile(r'#\s*(define|undef)\s+([a-zA-Z0-9_]+)') |
54 | super_macro_regex = re.compile(r'^DEF[A-Z0-9_]+\s*\(([a-zA-Z0-9_]+)') | |
55 | fn_regex = re.compile(r'([a-zA-Z_][^()\s]*)\s*\([^*]') | |
56 | template_and_param_regex = re.compile(r'<[^<>]*>') | |
a2d04f3d | 57 | md_def_regex = re.compile(r'\(define.*\s+"(.*)"') |
25c284f1 | 58 | bugzilla_url = 'https://gcc.gnu.org/bugzilla/rest.cgi/bug?id=%s&' \ |
edf0c3ff | 59 | 'include_fields=summary,component' |
00243d9a | 60 | |
a2d04f3d | 61 | function_extensions = {'.c', '.cpp', '.C', '.cc', '.h', '.inc', '.def', '.md'} |
00243d9a | 62 | |
432b9f61 MF |
63 | # NB: Makefile.in isn't listed as it's not always generated. |
64 | generated_files = {'aclocal.m4', 'config.h.in', 'configure'} | |
65 | ||
00243d9a ML |
66 | help_message = """\ |
67 | Generate ChangeLog template for PATCH. | |
68 | PATCH must be generated using diff(1)'s -up or -cp options | |
69 | (or their equivalent in git). | |
70 | """ | |
71 | ||
72 | script_folder = os.path.realpath(__file__) | |
e54da1b6 | 73 | root = os.path.dirname(os.path.dirname(script_folder)) |
00243d9a | 74 | |
58e3b17f JM |
75 | firstpr = '' |
76 | ||
00243d9a ML |
77 | |
78 | def find_changelog(path): | |
79 | folder = os.path.split(path)[0] | |
80 | while True: | |
e54da1b6 | 81 | if os.path.exists(os.path.join(root, folder, 'ChangeLog')): |
00243d9a ML |
82 | return folder |
83 | folder = os.path.dirname(folder) | |
84 | if folder == '': | |
85 | return folder | |
86 | raise AssertionError() | |
87 | ||
88 | ||
89 | def extract_function_name(line): | |
90 | if comment_regex.match(line): | |
91 | return None | |
92 | m = struct_regex.search(line) | |
93 | if m: | |
94 | # Struct declaration | |
4f85a52c | 95 | return m.group(1) + ' ' + m.group(3) |
00243d9a ML |
96 | m = macro_regex.search(line) |
97 | if m: | |
98 | # Macro definition | |
99 | return m.group(2) | |
100 | m = super_macro_regex.search(line) | |
101 | if m: | |
102 | # Supermacro | |
103 | return m.group(1) | |
104 | m = fn_regex.search(line) | |
105 | if m: | |
106 | # Discard template and function parameters. | |
107 | fn = m.group(1) | |
108 | fn = re.sub(template_and_param_regex, '', fn) | |
109 | return fn.rstrip() | |
110 | return None | |
111 | ||
112 | ||
113 | def try_add_function(functions, line): | |
114 | fn = extract_function_name(line) | |
115 | if fn and fn not in functions: | |
116 | functions.append(fn) | |
117 | return bool(fn) | |
118 | ||
119 | ||
120 | def sort_changelog_files(changed_file): | |
121 | return (changed_file.is_added_file, changed_file.is_removed_file) | |
122 | ||
123 | ||
124 | def get_pr_titles(prs): | |
edf0c3ff TB |
125 | output = [] |
126 | for idx, pr in enumerate(prs): | |
d6d9561c ML |
127 | pr_id = pr.split('/')[-1] |
128 | r = requests.get(bugzilla_url % pr_id) | |
25c284f1 ML |
129 | bugs = r.json()['bugs'] |
130 | if len(bugs) == 1: | |
edf0c3ff TB |
131 | prs[idx] = 'PR %s/%s' % (bugs[0]['component'], pr_id) |
132 | out = '%s - %s\n' % (prs[idx], bugs[0]['summary']) | |
133 | if out not in output: | |
134 | output.append(out) | |
25c284f1 | 135 | if output: |
edf0c3ff TB |
136 | output.append('') |
137 | return '\n'.join(output) | |
00243d9a | 138 | |
4f85a52c | 139 | |
b838641b ML |
140 | def append_changelog_line(out, relative_path, text): |
141 | line = f'\t* {relative_path}:' | |
142 | if len(line.replace('\t', ' ' * TAB_WIDTH) + ' ' + text) <= LINE_LIMIT: | |
143 | out += f'{line} {text}\n' | |
144 | else: | |
145 | out += f'{line}\n' | |
146 | out += f'\t{text}\n' | |
147 | return out | |
148 | ||
149 | ||
edf0c3ff TB |
150 | def generate_changelog(data, no_functions=False, fill_pr_titles=False, |
151 | additional_prs=None): | |
00243d9a ML |
152 | changelogs = {} |
153 | changelog_list = [] | |
154 | prs = [] | |
155 | out = '' | |
156 | diff = PatchSet(data) | |
58e3b17f | 157 | global firstpr |
00243d9a | 158 | |
edf0c3ff TB |
159 | if additional_prs: |
160 | prs = [pr for pr in additional_prs if pr not in prs] | |
00243d9a | 161 | for file in diff: |
236d6a33 ML |
162 | # skip files that can't be parsed |
163 | if file.path == '/dev/null': | |
164 | continue | |
00243d9a ML |
165 | changelog = find_changelog(file.path) |
166 | if changelog not in changelogs: | |
167 | changelogs[changelog] = [] | |
168 | changelog_list.append(changelog) | |
169 | changelogs[changelog].append(file) | |
170 | ||
171 | # Extract PR entries from newly added tests | |
172 | if 'testsuite' in file.path and file.is_added_file: | |
ed5ae55e TB |
173 | # Only search first ten lines as later lines may |
174 | # contains commented code which a note that it | |
175 | # has not been tested due to a certain PR or DR. | |
edf0c3ff | 176 | this_file_prs = [] |
ed5ae55e | 177 | for line in list(file)[0][0:10]: |
00243d9a ML |
178 | m = pr_regex.search(line.value) |
179 | if m: | |
180 | pr = m.group('pr') | |
181 | if pr not in prs: | |
182 | prs.append(pr) | |
edf0c3ff | 183 | this_file_prs.append(pr.split('/')[-1]) |
00243d9a | 184 | else: |
e7c7cdc5 ML |
185 | m = dr_regex.search(line.value) |
186 | if m: | |
187 | dr = m.group('dr') | |
188 | if dr not in prs: | |
189 | prs.append(dr) | |
edf0c3ff | 190 | this_file_prs.append(dr.split('/')[-1]) |
ed5ae55e TB |
191 | elif dg_regex.search(line.value): |
192 | # Found dg-warning/dg-error line | |
e7c7cdc5 | 193 | break |
edf0c3ff TB |
194 | # PR number in the file name |
195 | fname = os.path.basename(file.path) | |
196 | m = pr_filename_regex.search(fname) | |
197 | if m: | |
198 | pr = m.group('pr') | |
199 | pr2 = 'PR ' + pr | |
200 | if pr not in this_file_prs and pr2 not in prs: | |
201 | prs.append(pr2) | |
00243d9a | 202 | |
58e3b17f JM |
203 | if prs: |
204 | firstpr = prs[0] | |
205 | ||
00243d9a ML |
206 | if fill_pr_titles: |
207 | out += get_pr_titles(prs) | |
208 | ||
fef084dc ML |
209 | # print list of PR entries before ChangeLog entries |
210 | if prs: | |
211 | if not out: | |
212 | out += '\n' | |
213 | for pr in prs: | |
214 | out += '\t%s\n' % pr | |
215 | out += '\n' | |
216 | ||
00243d9a ML |
217 | # sort ChangeLog so that 'testsuite' is at the end |
218 | for changelog in sorted(changelog_list, key=lambda x: 'testsuite' in x): | |
219 | files = changelogs[changelog] | |
220 | out += '%s:\n' % os.path.join(changelog, 'ChangeLog') | |
221 | out += '\n' | |
00243d9a ML |
222 | # new and deleted files should be at the end |
223 | for file in sorted(files, key=sort_changelog_files): | |
224 | assert file.path.startswith(changelog) | |
225 | in_tests = 'testsuite' in changelog or 'testsuite' in file.path | |
226 | relative_path = file.path[len(changelog):].lstrip('/') | |
227 | functions = [] | |
228 | if file.is_added_file: | |
b838641b ML |
229 | msg = 'New test.' if in_tests else 'New file.' |
230 | out = append_changelog_line(out, relative_path, msg) | |
00243d9a | 231 | elif file.is_removed_file: |
b838641b | 232 | out = append_changelog_line(out, relative_path, 'Removed.') |
eb78da45 | 233 | elif hasattr(file, 'is_rename') and file.is_rename: |
b838641b | 234 | out = append_changelog_line(out, relative_path, 'Moved to...') |
eb78da45 ML |
235 | new_path = file.target_file[2:] |
236 | # A file can be theoretically moved to a location that | |
237 | # belongs to a different ChangeLog. Let user fix it. | |
238 | if new_path.startswith(changelog): | |
239 | new_path = new_path[len(changelog):].lstrip('/') | |
240 | out += '\t* %s: ...here.\n' % (new_path) | |
432b9f61 MF |
241 | elif os.path.basename(file.path) in generated_files: |
242 | out += '\t* %s: Regenerate.\n' % (relative_path) | |
b838641b | 243 | append_changelog_line(out, relative_path, 'Regenerate.') |
00243d9a ML |
244 | else: |
245 | if not no_functions: | |
246 | for hunk in file: | |
247 | # Do not add function names for testsuite files | |
248 | extension = os.path.splitext(relative_path)[1] | |
249 | if not in_tests and extension in function_extensions: | |
250 | last_fn = None | |
251 | modified_visited = False | |
252 | success = False | |
253 | for line in hunk: | |
254 | m = identifier_regex.match(line.value) | |
255 | if line.is_added or line.is_removed: | |
a2d04f3d ML |
256 | # special-case definition in .md files |
257 | m2 = md_def_regex.match(line.value) | |
258 | if extension == '.md' and m2: | |
259 | fn = m2.group(1) | |
260 | if fn not in functions: | |
261 | functions.append(fn) | |
262 | last_fn = None | |
263 | success = True | |
264 | ||
00243d9a ML |
265 | if not line.value.strip(): |
266 | continue | |
267 | modified_visited = True | |
268 | if m and try_add_function(functions, | |
269 | m.group(1)): | |
270 | last_fn = None | |
271 | success = True | |
272 | elif line.is_context: | |
273 | if last_fn and modified_visited: | |
274 | try_add_function(functions, last_fn) | |
275 | last_fn = None | |
276 | modified_visited = False | |
277 | success = True | |
278 | elif m: | |
279 | last_fn = m.group(1) | |
280 | modified_visited = False | |
281 | if not success: | |
282 | try_add_function(functions, | |
283 | hunk.section_header) | |
284 | if functions: | |
285 | out += '\t* %s (%s):\n' % (relative_path, functions[0]) | |
286 | for fn in functions[1:]: | |
287 | out += '\t(%s):\n' % fn | |
288 | else: | |
289 | out += '\t* %s:\n' % relative_path | |
290 | out += '\n' | |
291 | return out | |
292 | ||
293 | ||
cf76bbf8 ML |
294 | def update_copyright(data): |
295 | current_timestamp = datetime.datetime.now().strftime('%Y-%m-%d') | |
296 | username = subprocess.check_output('git config user.name', shell=True, | |
297 | encoding='utf8').strip() | |
298 | email = subprocess.check_output('git config user.email', shell=True, | |
299 | encoding='utf8').strip() | |
300 | ||
301 | changelogs = set() | |
302 | diff = PatchSet(data) | |
303 | ||
304 | for file in diff: | |
305 | changelog = os.path.join(find_changelog(file.path), 'ChangeLog') | |
306 | if changelog not in changelogs: | |
307 | changelogs.add(changelog) | |
308 | with open(changelog) as f: | |
309 | content = f.read() | |
310 | with open(changelog, 'w+') as f: | |
311 | f.write(f'{current_timestamp} {username} <{email}>\n\n') | |
312 | f.write('\tUpdate copyright years.\n\n') | |
313 | f.write(content) | |
314 | ||
315 | ||
00243d9a ML |
316 | if __name__ == '__main__': |
317 | parser = argparse.ArgumentParser(description=help_message) | |
318 | parser.add_argument('input', nargs='?', | |
319 | help='Patch file (or missing, read standard input)') | |
edf0c3ff | 320 | parser.add_argument('-b', '--pr-numbers', action='store', |
48b312b4 | 321 | type=lambda arg: arg.split(','), nargs='?', |
edf0c3ff | 322 | help='Add the specified PRs (comma separated)') |
00243d9a ML |
323 | parser.add_argument('-s', '--no-functions', action='store_true', |
324 | help='Do not generate function names in ChangeLogs') | |
325 | parser.add_argument('-p', '--fill-up-bug-titles', action='store_true', | |
326 | help='Download title of mentioned PRs') | |
e54da1b6 | 327 | parser.add_argument('-d', '--directory', |
c38f679e ML |
328 | help='Root directory where to search for ChangeLog ' |
329 | 'files') | |
757dbb59 JM |
330 | parser.add_argument('-c', '--changelog', |
331 | help='Append the ChangeLog to a git commit message ' | |
332 | 'file') | |
cf76bbf8 ML |
333 | parser.add_argument('--update-copyright', action='store_true', |
334 | help='Update copyright in ChangeLog files') | |
00243d9a ML |
335 | args = parser.parse_args() |
336 | if args.input == '-': | |
337 | args.input = None | |
e54da1b6 ML |
338 | if args.directory: |
339 | root = args.directory | |
00243d9a | 340 | |
d6d9561c | 341 | data = open(args.input) if args.input else sys.stdin |
cf76bbf8 ML |
342 | if args.update_copyright: |
343 | update_copyright(data) | |
757dbb59 | 344 | else: |
cf76bbf8 | 345 | output = generate_changelog(data, args.no_functions, |
edf0c3ff | 346 | args.fill_up_bug_titles, args.pr_numbers) |
cf76bbf8 ML |
347 | if args.changelog: |
348 | lines = open(args.changelog).read().split('\n') | |
349 | start = list(takewhile(lambda l: not l.startswith('#'), lines)) | |
350 | end = lines[len(start):] | |
351 | with open(args.changelog, 'w') as f: | |
58e3b17f JM |
352 | if not start or not start[0]: |
353 | # initial commit subject line 'component: [PRnnnnn]' | |
354 | m = prnum_regex.match(firstpr) | |
355 | if m: | |
356 | title = f'{m.group("comp")}: [PR{m.group("num")}]' | |
357 | start.insert(0, title) | |
cf76bbf8 | 358 | if start: |
58e3b17f | 359 | # append empty line |
cf76bbf8 ML |
360 | if start[-1] != '': |
361 | start.append('') | |
362 | else: | |
363 | # append 2 empty lines | |
364 | start = 2 * [''] | |
365 | f.write('\n'.join(start)) | |
366 | f.write('\n') | |
367 | f.write(output) | |
368 | f.write('\n'.join(end)) | |
369 | else: | |
370 | print(output, end='') |