]>
Commit | Line | Data |
---|---|---|
00243d9a ML |
1 | #!/usr/bin/env python3 |
2 | ||
3 | # Copyright (C) 2020 Free Software Foundation, Inc. | |
4 | # | |
5 | # This file is part of GCC. | |
6 | # | |
7 | # GCC is free software; you can redistribute it and/or modify | |
8 | # it under the terms of the GNU General Public License as published by | |
9 | # the Free Software Foundation; either version 3, or (at your option) | |
10 | # any later version. | |
11 | # | |
12 | # GCC is distributed in the hope that it will be useful, | |
13 | # but WITHOUT ANY WARRANTY; without even the implied warranty of | |
14 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the | |
15 | # GNU General Public License for more details. | |
16 | # | |
17 | # You should have received a copy of the GNU General Public License | |
18 | # along with GCC; see the file COPYING. If not, write to | |
19 | # the Free Software Foundation, 51 Franklin Street, Fifth Floor, | |
20 | # Boston, MA 02110-1301, USA. | |
21 | ||
22 | # This script parses a .diff file generated with 'diff -up' or 'diff -cp' | |
23 | # and adds a skeleton ChangeLog file to the file. It does not try to be | |
24 | # too smart when parsing function names, but it produces a reasonable | |
25 | # approximation. | |
26 | # | |
27 | # Author: Martin Liska <mliska@suse.cz> | |
28 | ||
29 | import argparse | |
cf76bbf8 | 30 | import datetime |
00243d9a ML |
31 | import os |
32 | import re | |
cf76bbf8 | 33 | import subprocess |
00243d9a | 34 | import sys |
757dbb59 | 35 | from itertools import takewhile |
00243d9a | 36 | |
4f85a52c ML |
37 | import requests |
38 | ||
00243d9a ML |
39 | from unidiff import PatchSet |
40 | ||
b838641b ML |
41 | LINE_LIMIT = 100 |
42 | TAB_WIDTH = 8 | |
43 | ||
00243d9a | 44 | pr_regex = re.compile(r'(\/(\/|\*)|[Cc*!])\s+(?P<pr>PR [a-z+-]+\/[0-9]+)') |
58e3b17f | 45 | prnum_regex = re.compile(r'PR (?P<comp>[a-z+-]+)/(?P<num>[0-9]+)') |
e7c7cdc5 | 46 | dr_regex = re.compile(r'(\/(\/|\*)|[Cc*!])\s+(?P<dr>DR [0-9]+)') |
ed5ae55e | 47 | dg_regex = re.compile(r'{\s+dg-(error|warning)') |
edf0c3ff | 48 | pr_filename_regex = re.compile(r'(^|[\W_])[Pp][Rr](?P<pr>\d{4,})') |
00243d9a ML |
49 | identifier_regex = re.compile(r'^([a-zA-Z0-9_#].*)') |
50 | comment_regex = re.compile(r'^\/\*') | |
4f85a52c ML |
51 | struct_regex = re.compile(r'^(class|struct|union|enum)\s+' |
52 | r'(GTY\(.*\)\s+)?([a-zA-Z0-9_]+)') | |
00243d9a ML |
53 | macro_regex = re.compile(r'#\s*(define|undef)\s+([a-zA-Z0-9_]+)') |
54 | super_macro_regex = re.compile(r'^DEF[A-Z0-9_]+\s*\(([a-zA-Z0-9_]+)') | |
55 | fn_regex = re.compile(r'([a-zA-Z_][^()\s]*)\s*\([^*]') | |
56 | template_and_param_regex = re.compile(r'<[^<>]*>') | |
a2d04f3d | 57 | md_def_regex = re.compile(r'\(define.*\s+"(.*)"') |
25c284f1 | 58 | bugzilla_url = 'https://gcc.gnu.org/bugzilla/rest.cgi/bug?id=%s&' \ |
edf0c3ff | 59 | 'include_fields=summary,component' |
00243d9a | 60 | |
a2d04f3d | 61 | function_extensions = {'.c', '.cpp', '.C', '.cc', '.h', '.inc', '.def', '.md'} |
00243d9a | 62 | |
432b9f61 MF |
63 | # NB: Makefile.in isn't listed as it's not always generated. |
64 | generated_files = {'aclocal.m4', 'config.h.in', 'configure'} | |
65 | ||
00243d9a ML |
66 | help_message = """\ |
67 | Generate ChangeLog template for PATCH. | |
68 | PATCH must be generated using diff(1)'s -up or -cp options | |
69 | (or their equivalent in git). | |
70 | """ | |
71 | ||
72 | script_folder = os.path.realpath(__file__) | |
e54da1b6 | 73 | root = os.path.dirname(os.path.dirname(script_folder)) |
00243d9a | 74 | |
58e3b17f JM |
75 | firstpr = '' |
76 | ||
00243d9a ML |
77 | |
78 | def find_changelog(path): | |
79 | folder = os.path.split(path)[0] | |
80 | while True: | |
e54da1b6 | 81 | if os.path.exists(os.path.join(root, folder, 'ChangeLog')): |
00243d9a ML |
82 | return folder |
83 | folder = os.path.dirname(folder) | |
84 | if folder == '': | |
85 | return folder | |
86 | raise AssertionError() | |
87 | ||
88 | ||
89 | def extract_function_name(line): | |
90 | if comment_regex.match(line): | |
91 | return None | |
92 | m = struct_regex.search(line) | |
93 | if m: | |
94 | # Struct declaration | |
4f85a52c | 95 | return m.group(1) + ' ' + m.group(3) |
00243d9a ML |
96 | m = macro_regex.search(line) |
97 | if m: | |
98 | # Macro definition | |
99 | return m.group(2) | |
100 | m = super_macro_regex.search(line) | |
101 | if m: | |
102 | # Supermacro | |
103 | return m.group(1) | |
104 | m = fn_regex.search(line) | |
105 | if m: | |
106 | # Discard template and function parameters. | |
107 | fn = m.group(1) | |
108 | fn = re.sub(template_and_param_regex, '', fn) | |
109 | return fn.rstrip() | |
110 | return None | |
111 | ||
112 | ||
113 | def try_add_function(functions, line): | |
114 | fn = extract_function_name(line) | |
115 | if fn and fn not in functions: | |
116 | functions.append(fn) | |
117 | return bool(fn) | |
118 | ||
119 | ||
120 | def sort_changelog_files(changed_file): | |
121 | return (changed_file.is_added_file, changed_file.is_removed_file) | |
122 | ||
123 | ||
124 | def get_pr_titles(prs): | |
edf0c3ff TB |
125 | output = [] |
126 | for idx, pr in enumerate(prs): | |
d6d9561c ML |
127 | pr_id = pr.split('/')[-1] |
128 | r = requests.get(bugzilla_url % pr_id) | |
25c284f1 ML |
129 | bugs = r.json()['bugs'] |
130 | if len(bugs) == 1: | |
edf0c3ff TB |
131 | prs[idx] = 'PR %s/%s' % (bugs[0]['component'], pr_id) |
132 | out = '%s - %s\n' % (prs[idx], bugs[0]['summary']) | |
133 | if out not in output: | |
134 | output.append(out) | |
25c284f1 | 135 | if output: |
edf0c3ff TB |
136 | output.append('') |
137 | return '\n'.join(output) | |
00243d9a | 138 | |
4f85a52c | 139 | |
b838641b ML |
140 | def append_changelog_line(out, relative_path, text): |
141 | line = f'\t* {relative_path}:' | |
142 | if len(line.replace('\t', ' ' * TAB_WIDTH) + ' ' + text) <= LINE_LIMIT: | |
143 | out += f'{line} {text}\n' | |
144 | else: | |
145 | out += f'{line}\n' | |
146 | out += f'\t{text}\n' | |
147 | return out | |
148 | ||
149 | ||
edf0c3ff TB |
150 | def generate_changelog(data, no_functions=False, fill_pr_titles=False, |
151 | additional_prs=None): | |
00243d9a ML |
152 | changelogs = {} |
153 | changelog_list = [] | |
154 | prs = [] | |
155 | out = '' | |
156 | diff = PatchSet(data) | |
58e3b17f | 157 | global firstpr |
00243d9a | 158 | |
edf0c3ff | 159 | if additional_prs: |
84f906df ML |
160 | for apr in additional_prs: |
161 | if not apr.startswith('PR ') and '/' in apr: | |
162 | apr = 'PR ' + apr | |
163 | if apr not in prs: | |
164 | prs.append(apr) | |
00243d9a | 165 | for file in diff: |
236d6a33 ML |
166 | # skip files that can't be parsed |
167 | if file.path == '/dev/null': | |
168 | continue | |
00243d9a ML |
169 | changelog = find_changelog(file.path) |
170 | if changelog not in changelogs: | |
171 | changelogs[changelog] = [] | |
172 | changelog_list.append(changelog) | |
173 | changelogs[changelog].append(file) | |
174 | ||
175 | # Extract PR entries from newly added tests | |
176 | if 'testsuite' in file.path and file.is_added_file: | |
ed5ae55e TB |
177 | # Only search first ten lines as later lines may |
178 | # contains commented code which a note that it | |
179 | # has not been tested due to a certain PR or DR. | |
edf0c3ff | 180 | this_file_prs = [] |
ed5ae55e | 181 | for line in list(file)[0][0:10]: |
00243d9a ML |
182 | m = pr_regex.search(line.value) |
183 | if m: | |
184 | pr = m.group('pr') | |
185 | if pr not in prs: | |
186 | prs.append(pr) | |
edf0c3ff | 187 | this_file_prs.append(pr.split('/')[-1]) |
00243d9a | 188 | else: |
e7c7cdc5 ML |
189 | m = dr_regex.search(line.value) |
190 | if m: | |
191 | dr = m.group('dr') | |
192 | if dr not in prs: | |
193 | prs.append(dr) | |
edf0c3ff | 194 | this_file_prs.append(dr.split('/')[-1]) |
ed5ae55e TB |
195 | elif dg_regex.search(line.value): |
196 | # Found dg-warning/dg-error line | |
e7c7cdc5 | 197 | break |
edf0c3ff TB |
198 | # PR number in the file name |
199 | fname = os.path.basename(file.path) | |
200 | m = pr_filename_regex.search(fname) | |
201 | if m: | |
202 | pr = m.group('pr') | |
203 | pr2 = 'PR ' + pr | |
204 | if pr not in this_file_prs and pr2 not in prs: | |
205 | prs.append(pr2) | |
00243d9a | 206 | |
58e3b17f JM |
207 | if prs: |
208 | firstpr = prs[0] | |
209 | ||
00243d9a ML |
210 | if fill_pr_titles: |
211 | out += get_pr_titles(prs) | |
212 | ||
fef084dc ML |
213 | # print list of PR entries before ChangeLog entries |
214 | if prs: | |
215 | if not out: | |
216 | out += '\n' | |
217 | for pr in prs: | |
218 | out += '\t%s\n' % pr | |
219 | out += '\n' | |
220 | ||
00243d9a ML |
221 | # sort ChangeLog so that 'testsuite' is at the end |
222 | for changelog in sorted(changelog_list, key=lambda x: 'testsuite' in x): | |
223 | files = changelogs[changelog] | |
224 | out += '%s:\n' % os.path.join(changelog, 'ChangeLog') | |
225 | out += '\n' | |
00243d9a ML |
226 | # new and deleted files should be at the end |
227 | for file in sorted(files, key=sort_changelog_files): | |
228 | assert file.path.startswith(changelog) | |
229 | in_tests = 'testsuite' in changelog or 'testsuite' in file.path | |
230 | relative_path = file.path[len(changelog):].lstrip('/') | |
231 | functions = [] | |
232 | if file.is_added_file: | |
b838641b ML |
233 | msg = 'New test.' if in_tests else 'New file.' |
234 | out = append_changelog_line(out, relative_path, msg) | |
00243d9a | 235 | elif file.is_removed_file: |
b838641b | 236 | out = append_changelog_line(out, relative_path, 'Removed.') |
eb78da45 | 237 | elif hasattr(file, 'is_rename') and file.is_rename: |
b838641b | 238 | out = append_changelog_line(out, relative_path, 'Moved to...') |
eb78da45 ML |
239 | new_path = file.target_file[2:] |
240 | # A file can be theoretically moved to a location that | |
241 | # belongs to a different ChangeLog. Let user fix it. | |
242 | if new_path.startswith(changelog): | |
243 | new_path = new_path[len(changelog):].lstrip('/') | |
244 | out += '\t* %s: ...here.\n' % (new_path) | |
432b9f61 MF |
245 | elif os.path.basename(file.path) in generated_files: |
246 | out += '\t* %s: Regenerate.\n' % (relative_path) | |
b838641b | 247 | append_changelog_line(out, relative_path, 'Regenerate.') |
00243d9a ML |
248 | else: |
249 | if not no_functions: | |
250 | for hunk in file: | |
251 | # Do not add function names for testsuite files | |
252 | extension = os.path.splitext(relative_path)[1] | |
253 | if not in_tests and extension in function_extensions: | |
254 | last_fn = None | |
255 | modified_visited = False | |
256 | success = False | |
257 | for line in hunk: | |
258 | m = identifier_regex.match(line.value) | |
259 | if line.is_added or line.is_removed: | |
a2d04f3d ML |
260 | # special-case definition in .md files |
261 | m2 = md_def_regex.match(line.value) | |
262 | if extension == '.md' and m2: | |
263 | fn = m2.group(1) | |
264 | if fn not in functions: | |
265 | functions.append(fn) | |
266 | last_fn = None | |
267 | success = True | |
268 | ||
00243d9a ML |
269 | if not line.value.strip(): |
270 | continue | |
271 | modified_visited = True | |
272 | if m and try_add_function(functions, | |
273 | m.group(1)): | |
274 | last_fn = None | |
275 | success = True | |
276 | elif line.is_context: | |
277 | if last_fn and modified_visited: | |
278 | try_add_function(functions, last_fn) | |
279 | last_fn = None | |
280 | modified_visited = False | |
281 | success = True | |
282 | elif m: | |
283 | last_fn = m.group(1) | |
284 | modified_visited = False | |
285 | if not success: | |
286 | try_add_function(functions, | |
287 | hunk.section_header) | |
288 | if functions: | |
289 | out += '\t* %s (%s):\n' % (relative_path, functions[0]) | |
290 | for fn in functions[1:]: | |
291 | out += '\t(%s):\n' % fn | |
292 | else: | |
293 | out += '\t* %s:\n' % relative_path | |
294 | out += '\n' | |
295 | return out | |
296 | ||
297 | ||
cf76bbf8 ML |
298 | def update_copyright(data): |
299 | current_timestamp = datetime.datetime.now().strftime('%Y-%m-%d') | |
300 | username = subprocess.check_output('git config user.name', shell=True, | |
301 | encoding='utf8').strip() | |
302 | email = subprocess.check_output('git config user.email', shell=True, | |
303 | encoding='utf8').strip() | |
304 | ||
305 | changelogs = set() | |
306 | diff = PatchSet(data) | |
307 | ||
308 | for file in diff: | |
309 | changelog = os.path.join(find_changelog(file.path), 'ChangeLog') | |
310 | if changelog not in changelogs: | |
311 | changelogs.add(changelog) | |
312 | with open(changelog) as f: | |
313 | content = f.read() | |
314 | with open(changelog, 'w+') as f: | |
315 | f.write(f'{current_timestamp} {username} <{email}>\n\n') | |
316 | f.write('\tUpdate copyright years.\n\n') | |
317 | f.write(content) | |
318 | ||
319 | ||
00243d9a ML |
320 | if __name__ == '__main__': |
321 | parser = argparse.ArgumentParser(description=help_message) | |
322 | parser.add_argument('input', nargs='?', | |
323 | help='Patch file (or missing, read standard input)') | |
edf0c3ff | 324 | parser.add_argument('-b', '--pr-numbers', action='store', |
48b312b4 | 325 | type=lambda arg: arg.split(','), nargs='?', |
edf0c3ff | 326 | help='Add the specified PRs (comma separated)') |
00243d9a ML |
327 | parser.add_argument('-s', '--no-functions', action='store_true', |
328 | help='Do not generate function names in ChangeLogs') | |
329 | parser.add_argument('-p', '--fill-up-bug-titles', action='store_true', | |
330 | help='Download title of mentioned PRs') | |
e54da1b6 | 331 | parser.add_argument('-d', '--directory', |
c38f679e ML |
332 | help='Root directory where to search for ChangeLog ' |
333 | 'files') | |
757dbb59 JM |
334 | parser.add_argument('-c', '--changelog', |
335 | help='Append the ChangeLog to a git commit message ' | |
336 | 'file') | |
cf76bbf8 ML |
337 | parser.add_argument('--update-copyright', action='store_true', |
338 | help='Update copyright in ChangeLog files') | |
00243d9a ML |
339 | args = parser.parse_args() |
340 | if args.input == '-': | |
341 | args.input = None | |
e54da1b6 ML |
342 | if args.directory: |
343 | root = args.directory | |
00243d9a | 344 | |
d6d9561c | 345 | data = open(args.input) if args.input else sys.stdin |
cf76bbf8 ML |
346 | if args.update_copyright: |
347 | update_copyright(data) | |
757dbb59 | 348 | else: |
cf76bbf8 | 349 | output = generate_changelog(data, args.no_functions, |
edf0c3ff | 350 | args.fill_up_bug_titles, args.pr_numbers) |
cf76bbf8 ML |
351 | if args.changelog: |
352 | lines = open(args.changelog).read().split('\n') | |
353 | start = list(takewhile(lambda l: not l.startswith('#'), lines)) | |
354 | end = lines[len(start):] | |
355 | with open(args.changelog, 'w') as f: | |
58e3b17f JM |
356 | if not start or not start[0]: |
357 | # initial commit subject line 'component: [PRnnnnn]' | |
358 | m = prnum_regex.match(firstpr) | |
359 | if m: | |
360 | title = f'{m.group("comp")}: [PR{m.group("num")}]' | |
361 | start.insert(0, title) | |
cf76bbf8 | 362 | if start: |
58e3b17f | 363 | # append empty line |
cf76bbf8 ML |
364 | if start[-1] != '': |
365 | start.append('') | |
366 | else: | |
367 | # append 2 empty lines | |
368 | start = 2 * [''] | |
369 | f.write('\n'.join(start)) | |
370 | f.write('\n') | |
371 | f.write(output) | |
372 | f.write('\n'.join(end)) | |
373 | else: | |
374 | print(output, end='') |