]>
Commit | Line | Data |
---|---|---|
4b6fda0b | 1 | #!/usr/bin/env python2 |
24fe1f03 | 2 | |
b1a3f243 | 3 | """Find Kconfig symbols that are referenced but not defined.""" |
24fe1f03 | 4 | |
c7455663 | 5 | # (c) 2014-2015 Valentin Rothberg <valentinrothberg@gmail.com> |
cc641d55 | 6 | # (c) 2014 Stefan Hengelein <stefan.hengelein@fau.de> |
24fe1f03 | 7 | # |
cc641d55 | 8 | # Licensed under the terms of the GNU GPL License version 2 |
24fe1f03 VR |
9 | |
10 | ||
11 | import os | |
12 | import re | |
e2042a8a | 13 | import signal |
b1a3f243 | 14 | import sys |
e2042a8a | 15 | from multiprocessing import Pool, cpu_count |
b1a3f243 | 16 | from optparse import OptionParser |
e2042a8a | 17 | from subprocess import Popen, PIPE, STDOUT |
24fe1f03 | 18 | |
cc641d55 VR |
19 | |
20 | # regex expressions | |
24fe1f03 | 21 | OPERATORS = r"&|\(|\)|\||\!" |
cc641d55 VR |
22 | FEATURE = r"(?:\w*[A-Z0-9]\w*){2,}" |
23 | DEF = r"^\s*(?:menu){,1}config\s+(" + FEATURE + r")\s*" | |
24fe1f03 | 24 | EXPR = r"(?:" + OPERATORS + r"|\s|" + FEATURE + r")+" |
0bd38ae3 VR |
25 | DEFAULT = r"default\s+.*?(?:if\s.+){,1}" |
26 | STMT = r"^\s*(?:if|select|depends\s+on|(?:" + DEFAULT + r"))\s+" + EXPR | |
cc641d55 | 27 | SOURCE_FEATURE = r"(?:\W|\b)+[D]{,1}CONFIG_(" + FEATURE + r")" |
24fe1f03 | 28 | |
cc641d55 | 29 | # regex objects |
24fe1f03 | 30 | REGEX_FILE_KCONFIG = re.compile(r".*Kconfig[\.\w+\-]*$") |
e2042a8a | 31 | REGEX_FEATURE = re.compile(r'(?!\B)' + FEATURE + r'(?!\B)') |
cc641d55 VR |
32 | REGEX_SOURCE_FEATURE = re.compile(SOURCE_FEATURE) |
33 | REGEX_KCONFIG_DEF = re.compile(DEF) | |
24fe1f03 VR |
34 | REGEX_KCONFIG_EXPR = re.compile(EXPR) |
35 | REGEX_KCONFIG_STMT = re.compile(STMT) | |
36 | REGEX_KCONFIG_HELP = re.compile(r"^\s+(help|---help---)\s*$") | |
37 | REGEX_FILTER_FEATURES = re.compile(r"[A-Za-z0-9]$") | |
0bd38ae3 | 38 | REGEX_NUMERIC = re.compile(r"0[xX][0-9a-fA-F]+|[0-9]+") |
e2042a8a | 39 | REGEX_QUOTES = re.compile("(\"(.*?)\")") |
24fe1f03 VR |
40 | |
41 | ||
b1a3f243 VR |
42 | def parse_options(): |
43 | """The user interface of this module.""" | |
44 | usage = "%prog [options]\n\n" \ | |
45 | "Run this tool to detect Kconfig symbols that are referenced but " \ | |
46 | "not defined in\nKconfig. The output of this tool has the " \ | |
47 | "format \'Undefined symbol\\tFile list\'\n\n" \ | |
48 | "If no option is specified, %prog will default to check your\n" \ | |
49 | "current tree. Please note that specifying commits will " \ | |
50 | "\'git reset --hard\'\nyour current tree! You may save " \ | |
51 | "uncommitted changes to avoid losing data." | |
52 | ||
53 | parser = OptionParser(usage=usage) | |
54 | ||
55 | parser.add_option('-c', '--commit', dest='commit', action='store', | |
56 | default="", | |
57 | help="Check if the specified commit (hash) introduces " | |
58 | "undefined Kconfig symbols.") | |
59 | ||
60 | parser.add_option('-d', '--diff', dest='diff', action='store', | |
61 | default="", | |
62 | help="Diff undefined symbols between two commits. The " | |
63 | "input format bases on Git log's " | |
64 | "\'commmit1..commit2\'.") | |
65 | ||
a42fa92c VR |
66 | parser.add_option('-f', '--find', dest='find', action='store_true', |
67 | default=False, | |
68 | help="Find and show commits that may cause symbols to be " | |
69 | "missing. Required to run with --diff.") | |
70 | ||
cf132e4a VR |
71 | parser.add_option('-i', '--ignore', dest='ignore', action='store', |
72 | default="", | |
73 | help="Ignore files matching this pattern. Note that " | |
74 | "the pattern needs to be a Python regex. To " | |
75 | "ignore defconfigs, specify -i '.*defconfig'.") | |
76 | ||
b1a3f243 VR |
77 | parser.add_option('', '--force', dest='force', action='store_true', |
78 | default=False, | |
79 | help="Reset current Git tree even when it's dirty.") | |
80 | ||
81 | (opts, _) = parser.parse_args() | |
82 | ||
83 | if opts.commit and opts.diff: | |
84 | sys.exit("Please specify only one option at once.") | |
85 | ||
86 | if opts.diff and not re.match(r"^[\w\-\.]+\.\.[\w\-\.]+$", opts.diff): | |
87 | sys.exit("Please specify valid input in the following format: " | |
88 | "\'commmit1..commit2\'") | |
89 | ||
90 | if opts.commit or opts.diff: | |
91 | if not opts.force and tree_is_dirty(): | |
92 | sys.exit("The current Git tree is dirty (see 'git status'). " | |
93 | "Running this script may\ndelete important data since it " | |
94 | "calls 'git reset --hard' for some performance\nreasons. " | |
95 | " Please run this script in a clean Git tree or pass " | |
96 | "'--force' if you\nwant to ignore this warning and " | |
97 | "continue.") | |
98 | ||
a42fa92c VR |
99 | if opts.commit: |
100 | opts.find = False | |
101 | ||
cf132e4a VR |
102 | if opts.ignore: |
103 | try: | |
104 | re.match(opts.ignore, "this/is/just/a/test.c") | |
105 | except: | |
106 | sys.exit("Please specify a valid Python regex.") | |
107 | ||
b1a3f243 VR |
108 | return opts |
109 | ||
110 | ||
24fe1f03 VR |
111 | def main(): |
112 | """Main function of this module.""" | |
b1a3f243 VR |
113 | opts = parse_options() |
114 | ||
115 | if opts.commit or opts.diff: | |
116 | head = get_head() | |
117 | ||
118 | # get commit range | |
119 | commit_a = None | |
120 | commit_b = None | |
121 | if opts.commit: | |
122 | commit_a = opts.commit + "~" | |
123 | commit_b = opts.commit | |
124 | elif opts.diff: | |
125 | split = opts.diff.split("..") | |
126 | commit_a = split[0] | |
127 | commit_b = split[1] | |
128 | undefined_a = {} | |
129 | undefined_b = {} | |
130 | ||
131 | # get undefined items before the commit | |
132 | execute("git reset --hard %s" % commit_a) | |
cf132e4a | 133 | undefined_a = check_symbols(opts.ignore) |
b1a3f243 VR |
134 | |
135 | # get undefined items for the commit | |
136 | execute("git reset --hard %s" % commit_b) | |
cf132e4a | 137 | undefined_b = check_symbols(opts.ignore) |
b1a3f243 VR |
138 | |
139 | # report cases that are present for the commit but not before | |
e9533ae5 | 140 | for feature in sorted(undefined_b): |
b1a3f243 VR |
141 | # feature has not been undefined before |
142 | if not feature in undefined_a: | |
e9533ae5 | 143 | files = sorted(undefined_b.get(feature)) |
c7455663 | 144 | print "%s\t%s" % (yel(feature), ", ".join(files)) |
a42fa92c VR |
145 | if opts.find: |
146 | commits = find_commits(feature, opts.diff) | |
c7455663 | 147 | print red(commits) |
b1a3f243 VR |
148 | # check if there are new files that reference the undefined feature |
149 | else: | |
e9533ae5 VR |
150 | files = sorted(undefined_b.get(feature) - |
151 | undefined_a.get(feature)) | |
b1a3f243 | 152 | if files: |
c7455663 | 153 | print "%s\t%s" % (yel(feature), ", ".join(files)) |
a42fa92c VR |
154 | if opts.find: |
155 | commits = find_commits(feature, opts.diff) | |
c7455663 | 156 | print red(commits) |
b1a3f243 VR |
157 | |
158 | # reset to head | |
159 | execute("git reset --hard %s" % head) | |
160 | ||
161 | # default to check the entire tree | |
162 | else: | |
cf132e4a | 163 | undefined = check_symbols(opts.ignore) |
e9533ae5 VR |
164 | for feature in sorted(undefined): |
165 | files = sorted(undefined.get(feature)) | |
c7455663 VR |
166 | print "%s\t%s" % (yel(feature), ", ".join(files)) |
167 | ||
168 | ||
169 | def yel(string): | |
170 | """ | |
171 | Color %string yellow. | |
172 | """ | |
173 | return "\033[33m%s\033[0m" % string | |
174 | ||
175 | ||
176 | def red(string): | |
177 | """ | |
178 | Color %string red. | |
179 | """ | |
180 | return "\033[31m%s\033[0m" % string | |
b1a3f243 VR |
181 | |
182 | ||
183 | def execute(cmd): | |
184 | """Execute %cmd and return stdout. Exit in case of error.""" | |
185 | pop = Popen(cmd, stdout=PIPE, stderr=STDOUT, shell=True) | |
186 | (stdout, _) = pop.communicate() # wait until finished | |
187 | if pop.returncode != 0: | |
188 | sys.exit(stdout) | |
189 | return stdout | |
190 | ||
191 | ||
a42fa92c VR |
192 | def find_commits(symbol, diff): |
193 | """Find commits changing %symbol in the given range of %diff.""" | |
194 | commits = execute("git log --pretty=oneline --abbrev-commit -G %s %s" | |
195 | % (symbol, diff)) | |
196 | return commits | |
197 | ||
198 | ||
b1a3f243 VR |
199 | def tree_is_dirty(): |
200 | """Return true if the current working tree is dirty (i.e., if any file has | |
201 | been added, deleted, modified, renamed or copied but not committed).""" | |
202 | stdout = execute("git status --porcelain") | |
203 | for line in stdout: | |
204 | if re.findall(r"[URMADC]{1}", line[:2]): | |
205 | return True | |
206 | return False | |
207 | ||
208 | ||
209 | def get_head(): | |
210 | """Return commit hash of current HEAD.""" | |
211 | stdout = execute("git rev-parse HEAD") | |
212 | return stdout.strip('\n') | |
213 | ||
214 | ||
e2042a8a VR |
215 | def partition(lst, size): |
216 | """Partition list @lst into eveni-sized lists of size @size.""" | |
217 | return [lst[i::size] for i in xrange(size)] | |
218 | ||
219 | ||
220 | def init_worker(): | |
221 | """Set signal handler to ignore SIGINT.""" | |
222 | signal.signal(signal.SIGINT, signal.SIG_IGN) | |
223 | ||
224 | ||
cf132e4a | 225 | def check_symbols(ignore): |
b1a3f243 | 226 | """Find undefined Kconfig symbols and return a dict with the symbol as key |
cf132e4a VR |
227 | and a list of referencing files as value. Files matching %ignore are not |
228 | checked for undefined symbols.""" | |
e2042a8a VR |
229 | pool = Pool(cpu_count(), init_worker) |
230 | try: | |
231 | return check_symbols_helper(pool, ignore) | |
232 | except KeyboardInterrupt: | |
233 | pool.terminate() | |
234 | pool.join() | |
235 | sys.exit(1) | |
236 | ||
237 | ||
238 | def check_symbols_helper(pool, ignore): | |
239 | """Helper method for check_symbols(). Used to catch keyboard interrupts in | |
240 | check_symbols() in order to properly terminate running worker processes.""" | |
24fe1f03 VR |
241 | source_files = [] |
242 | kconfig_files = [] | |
e2042a8a VR |
243 | defined_features = [] |
244 | referenced_features = dict() # {file: [features]} | |
24fe1f03 VR |
245 | |
246 | # use 'git ls-files' to get the worklist | |
b1a3f243 | 247 | stdout = execute("git ls-files") |
24fe1f03 VR |
248 | if len(stdout) > 0 and stdout[-1] == "\n": |
249 | stdout = stdout[:-1] | |
250 | ||
251 | for gitfile in stdout.rsplit("\n"): | |
208d5115 VR |
252 | if ".git" in gitfile or "ChangeLog" in gitfile or \ |
253 | ".log" in gitfile or os.path.isdir(gitfile) or \ | |
254 | gitfile.startswith("tools/"): | |
24fe1f03 VR |
255 | continue |
256 | if REGEX_FILE_KCONFIG.match(gitfile): | |
257 | kconfig_files.append(gitfile) | |
258 | else: | |
e2042a8a VR |
259 | if ignore and not re.match(ignore, gitfile): |
260 | continue | |
261 | # add source files that do not match the ignore pattern | |
24fe1f03 VR |
262 | source_files.append(gitfile) |
263 | ||
e2042a8a VR |
264 | # parse source files |
265 | arglist = partition(source_files, cpu_count()) | |
266 | for res in pool.map(parse_source_files, arglist): | |
267 | referenced_features.update(res) | |
24fe1f03 | 268 | |
e2042a8a VR |
269 | |
270 | # parse kconfig files | |
271 | arglist = [] | |
272 | for part in partition(kconfig_files, cpu_count()): | |
273 | arglist.append((part, ignore)) | |
274 | for res in pool.map(parse_kconfig_files, arglist): | |
275 | defined_features.extend(res[0]) | |
276 | referenced_features.update(res[1]) | |
277 | defined_features = set(defined_features) | |
278 | ||
279 | # inverse mapping of referenced_features to dict(feature: [files]) | |
280 | inv_map = dict() | |
281 | for _file, features in referenced_features.iteritems(): | |
282 | for feature in features: | |
283 | inv_map[feature] = inv_map.get(feature, set()) | |
284 | inv_map[feature].add(_file) | |
285 | referenced_features = inv_map | |
24fe1f03 | 286 | |
b1a3f243 | 287 | undefined = {} # {feature: [files]} |
24fe1f03 | 288 | for feature in sorted(referenced_features): |
cc641d55 VR |
289 | # filter some false positives |
290 | if feature == "FOO" or feature == "BAR" or \ | |
291 | feature == "FOO_BAR" or feature == "XXX": | |
292 | continue | |
24fe1f03 VR |
293 | if feature not in defined_features: |
294 | if feature.endswith("_MODULE"): | |
cc641d55 | 295 | # avoid false positives for kernel modules |
24fe1f03 VR |
296 | if feature[:-len("_MODULE")] in defined_features: |
297 | continue | |
b1a3f243 VR |
298 | undefined[feature] = referenced_features.get(feature) |
299 | return undefined | |
24fe1f03 VR |
300 | |
301 | ||
e2042a8a VR |
302 | def parse_source_files(source_files): |
303 | """Parse each source file in @source_files and return dictionary with source | |
304 | files as keys and lists of references Kconfig symbols as values.""" | |
305 | referenced_features = dict() | |
306 | for sfile in source_files: | |
307 | referenced_features[sfile] = parse_source_file(sfile) | |
308 | return referenced_features | |
309 | ||
310 | ||
311 | def parse_source_file(sfile): | |
312 | """Parse @sfile and return a list of referenced Kconfig features.""" | |
24fe1f03 | 313 | lines = [] |
e2042a8a VR |
314 | references = [] |
315 | ||
316 | if not os.path.exists(sfile): | |
317 | return references | |
318 | ||
24fe1f03 VR |
319 | with open(sfile, "r") as stream: |
320 | lines = stream.readlines() | |
321 | ||
322 | for line in lines: | |
323 | if not "CONFIG_" in line: | |
324 | continue | |
325 | features = REGEX_SOURCE_FEATURE.findall(line) | |
326 | for feature in features: | |
327 | if not REGEX_FILTER_FEATURES.search(feature): | |
328 | continue | |
e2042a8a VR |
329 | references.append(feature) |
330 | ||
331 | return references | |
24fe1f03 VR |
332 | |
333 | ||
334 | def get_features_in_line(line): | |
335 | """Return mentioned Kconfig features in @line.""" | |
336 | return REGEX_FEATURE.findall(line) | |
337 | ||
338 | ||
e2042a8a VR |
339 | def parse_kconfig_files(args): |
340 | """Parse kconfig files and return tuple of defined and references Kconfig | |
341 | symbols. Note, @args is a tuple of a list of files and the @ignore | |
342 | pattern.""" | |
343 | kconfig_files = args[0] | |
344 | ignore = args[1] | |
345 | defined_features = [] | |
346 | referenced_features = dict() | |
347 | ||
348 | for kfile in kconfig_files: | |
349 | defined, references = parse_kconfig_file(kfile) | |
350 | defined_features.extend(defined) | |
351 | if ignore and re.match(ignore, kfile): | |
352 | # do not collect references for files that match the ignore pattern | |
353 | continue | |
354 | referenced_features[kfile] = references | |
355 | return (defined_features, referenced_features) | |
356 | ||
357 | ||
358 | def parse_kconfig_file(kfile): | |
24fe1f03 VR |
359 | """Parse @kfile and update feature definitions and references.""" |
360 | lines = [] | |
e2042a8a VR |
361 | defined = [] |
362 | references = [] | |
24fe1f03 VR |
363 | skip = False |
364 | ||
e2042a8a VR |
365 | if not os.path.exists(kfile): |
366 | return defined, references | |
367 | ||
24fe1f03 VR |
368 | with open(kfile, "r") as stream: |
369 | lines = stream.readlines() | |
370 | ||
371 | for i in range(len(lines)): | |
372 | line = lines[i] | |
373 | line = line.strip('\n') | |
cc641d55 | 374 | line = line.split("#")[0] # ignore comments |
24fe1f03 VR |
375 | |
376 | if REGEX_KCONFIG_DEF.match(line): | |
377 | feature_def = REGEX_KCONFIG_DEF.findall(line) | |
e2042a8a | 378 | defined.append(feature_def[0]) |
24fe1f03 VR |
379 | skip = False |
380 | elif REGEX_KCONFIG_HELP.match(line): | |
381 | skip = True | |
382 | elif skip: | |
cc641d55 | 383 | # ignore content of help messages |
24fe1f03 VR |
384 | pass |
385 | elif REGEX_KCONFIG_STMT.match(line): | |
e2042a8a | 386 | line = REGEX_QUOTES.sub("", line) |
24fe1f03 | 387 | features = get_features_in_line(line) |
cc641d55 | 388 | # multi-line statements |
24fe1f03 VR |
389 | while line.endswith("\\"): |
390 | i += 1 | |
391 | line = lines[i] | |
392 | line = line.strip('\n') | |
393 | features.extend(get_features_in_line(line)) | |
394 | for feature in set(features): | |
0bd38ae3 VR |
395 | if REGEX_NUMERIC.match(feature): |
396 | # ignore numeric values | |
397 | continue | |
e2042a8a VR |
398 | references.append(feature) |
399 | ||
400 | return defined, references | |
24fe1f03 VR |
401 | |
402 | ||
403 | if __name__ == "__main__": | |
404 | main() |