]>
Commit | Line | Data |
---|---|---|
0d24de9d SG |
1 | # Copyright (c) 2011 The Chromium OS Authors. |
2 | # | |
1a459660 | 3 | # SPDX-License-Identifier: GPL-2.0+ |
0d24de9d SG |
4 | # |
5 | ||
35ce2dc4 | 6 | import math |
0d24de9d SG |
7 | import os |
8 | import re | |
9 | import shutil | |
10 | import tempfile | |
11 | ||
12 | import command | |
13 | import commit | |
14 | import gitutil | |
15 | from series import Series | |
16 | ||
17 | # Tags that we detect and remove | |
619dd5de | 18 | re_remove = re.compile('^BUG=|^TEST=|^BRANCH=|^Change-Id:|^Review URL:' |
3fefd5ef | 19 | '|Reviewed-on:|Commit-\w*:') |
0d24de9d SG |
20 | |
21 | # Lines which are allowed after a TEST= line | |
22 | re_allowed_after_test = re.compile('^Signed-off-by:') | |
23 | ||
05e5b735 | 24 | # Signoffs |
102061bd | 25 | re_signoff = re.compile('^Signed-off-by: *(.*)') |
05e5b735 | 26 | |
0d24de9d SG |
27 | # The start of the cover letter |
28 | re_cover = re.compile('^Cover-letter:') | |
29 | ||
fe2f8d9e SG |
30 | # A cover letter Cc |
31 | re_cover_cc = re.compile('^Cover-letter-cc: *(.*)') | |
32 | ||
0d24de9d | 33 | # Patch series tag |
5c8fdd91 AA |
34 | re_series_tag = re.compile('^Series-([a-z-]*): *(.*)') |
35 | ||
36 | # Commit series tag | |
37 | re_commit_tag = re.compile('^Commit-([a-z-]*): *(.*)') | |
0d24de9d SG |
38 | |
39 | # Commit tags that we want to collect and keep | |
659c89da | 40 | re_tag = re.compile('^(Tested-by|Acked-by|Reviewed-by|Patch-cc): (.*)') |
0d24de9d SG |
41 | |
42 | # The start of a new commit in the git log | |
68618281 | 43 | re_commit = re.compile('^commit ([0-9a-f]*)$') |
0d24de9d SG |
44 | |
45 | # We detect these since checkpatch doesn't always do it | |
46 | re_space_before_tab = re.compile('^[+].* \t') | |
47 | ||
48 | # States we can be in - can we use range() and still have comments? | |
49 | STATE_MSG_HEADER = 0 # Still in the message header | |
50 | STATE_PATCH_SUBJECT = 1 # In patch subject (first line of log for a commit) | |
51 | STATE_PATCH_HEADER = 2 # In patch header (after the subject) | |
52 | STATE_DIFFS = 3 # In the diff part (past --- line) | |
53 | ||
54 | class PatchStream: | |
55 | """Class for detecting/injecting tags in a patch or series of patches | |
56 | ||
57 | We support processing the output of 'git log' to read out the tags we | |
58 | are interested in. We can also process a patch file in order to remove | |
59 | unwanted tags or inject additional ones. These correspond to the two | |
60 | phases of processing. | |
61 | """ | |
62 | def __init__(self, series, name=None, is_log=False): | |
63 | self.skip_blank = False # True to skip a single blank line | |
64 | self.found_test = False # Found a TEST= line | |
65 | self.lines_after_test = 0 # MNumber of lines found after TEST= | |
66 | self.warn = [] # List of warnings we have collected | |
67 | self.linenum = 1 # Output line number we are up to | |
68 | self.in_section = None # Name of start...END section we are in | |
69 | self.notes = [] # Series notes | |
70 | self.section = [] # The current section...END section | |
71 | self.series = series # Info about the patch series | |
72 | self.is_log = is_log # True if indent like git log | |
73 | self.in_change = 0 # Non-zero if we are in a change list | |
74 | self.blank_count = 0 # Number of blank lines stored up | |
75 | self.state = STATE_MSG_HEADER # What state are we in? | |
0d24de9d SG |
76 | self.signoff = [] # Contents of signoff line |
77 | self.commit = None # Current commit | |
78 | ||
79 | def AddToSeries(self, line, name, value): | |
80 | """Add a new Series-xxx tag. | |
81 | ||
82 | When a Series-xxx tag is detected, we come here to record it, if we | |
83 | are scanning a 'git log'. | |
84 | ||
85 | Args: | |
86 | line: Source line containing tag (useful for debug/error messages) | |
87 | name: Tag name (part after 'Series-') | |
88 | value: Tag value (part after 'Series-xxx: ') | |
89 | """ | |
90 | if name == 'notes': | |
91 | self.in_section = name | |
92 | self.skip_blank = False | |
93 | if self.is_log: | |
94 | self.series.AddTag(self.commit, line, name, value) | |
95 | ||
5c8fdd91 AA |
96 | def AddToCommit(self, line, name, value): |
97 | """Add a new Commit-xxx tag. | |
98 | ||
99 | When a Commit-xxx tag is detected, we come here to record it. | |
100 | ||
101 | Args: | |
102 | line: Source line containing tag (useful for debug/error messages) | |
103 | name: Tag name (part after 'Commit-') | |
104 | value: Tag value (part after 'Commit-xxx: ') | |
105 | """ | |
106 | if name == 'notes': | |
107 | self.in_section = 'commit-' + name | |
108 | self.skip_blank = False | |
109 | ||
0d24de9d SG |
110 | def CloseCommit(self): |
111 | """Save the current commit into our commit list, and reset our state""" | |
112 | if self.commit and self.is_log: | |
113 | self.series.AddCommit(self.commit) | |
114 | self.commit = None | |
0d577187 BM |
115 | # If 'END' is missing in a 'Cover-letter' section, and that section |
116 | # happens to show up at the very end of the commit message, this is | |
117 | # the chance for us to fix it up. | |
118 | if self.in_section == 'cover' and self.is_log: | |
119 | self.series.cover = self.section | |
120 | self.in_section = None | |
121 | self.skip_blank = True | |
122 | self.section = [] | |
0d24de9d | 123 | |
0d24de9d SG |
124 | def ProcessLine(self, line): |
125 | """Process a single line of a patch file or commit log | |
126 | ||
127 | This process a line and returns a list of lines to output. The list | |
128 | may be empty or may contain multiple output lines. | |
129 | ||
130 | This is where all the complicated logic is located. The class's | |
131 | state is used to move between different states and detect things | |
132 | properly. | |
133 | ||
134 | We can be in one of two modes: | |
135 | self.is_log == True: This is 'git log' mode, where most output is | |
136 | indented by 4 characters and we are scanning for tags | |
137 | ||
138 | self.is_log == False: This is 'patch' mode, where we already have | |
139 | all the tags, and are processing patches to remove junk we | |
140 | don't want, and add things we think are required. | |
141 | ||
142 | Args: | |
143 | line: text line to process | |
144 | ||
145 | Returns: | |
146 | list of output lines, or [] if nothing should be output | |
147 | """ | |
148 | # Initially we have no output. Prepare the input line string | |
149 | out = [] | |
150 | line = line.rstrip('\n') | |
4b89b813 SW |
151 | |
152 | commit_match = re_commit.match(line) if self.is_log else None | |
153 | ||
0d24de9d SG |
154 | if self.is_log: |
155 | if line[:4] == ' ': | |
156 | line = line[4:] | |
157 | ||
158 | # Handle state transition and skipping blank lines | |
5c8fdd91 AA |
159 | series_tag_match = re_series_tag.match(line) |
160 | commit_tag_match = re_commit_tag.match(line) | |
e7df218c | 161 | cover_match = re_cover.match(line) |
fe2f8d9e | 162 | cover_cc_match = re_cover_cc.match(line) |
102061bd | 163 | signoff_match = re_signoff.match(line) |
0d24de9d SG |
164 | tag_match = None |
165 | if self.state == STATE_PATCH_HEADER: | |
166 | tag_match = re_tag.match(line) | |
167 | is_blank = not line.strip() | |
168 | if is_blank: | |
169 | if (self.state == STATE_MSG_HEADER | |
170 | or self.state == STATE_PATCH_SUBJECT): | |
171 | self.state += 1 | |
172 | ||
173 | # We don't have a subject in the text stream of patch files | |
174 | # It has its own line with a Subject: tag | |
175 | if not self.is_log and self.state == STATE_PATCH_SUBJECT: | |
176 | self.state += 1 | |
177 | elif commit_match: | |
178 | self.state = STATE_MSG_HEADER | |
179 | ||
94fbd3e3 | 180 | # If a tag is detected, or a new commit starts |
13b98d95 | 181 | if series_tag_match or commit_tag_match or \ |
94fbd3e3 BM |
182 | cover_match or cover_cc_match or signoff_match or \ |
183 | self.state == STATE_MSG_HEADER: | |
57b6b190 BM |
184 | # but we are already in a section, this means 'END' is missing |
185 | # for that section, fix it up. | |
13b98d95 BM |
186 | if self.in_section: |
187 | self.warn.append("Missing 'END' in section '%s'" % self.in_section) | |
188 | if self.in_section == 'cover': | |
189 | self.series.cover = self.section | |
190 | elif self.in_section == 'notes': | |
191 | if self.is_log: | |
192 | self.series.notes += self.section | |
193 | elif self.in_section == 'commit-notes': | |
194 | if self.is_log: | |
195 | self.commit.notes += self.section | |
196 | else: | |
197 | self.warn.append("Unknown section '%s'" % self.in_section) | |
198 | self.in_section = None | |
199 | self.skip_blank = True | |
200 | self.section = [] | |
57b6b190 BM |
201 | # but we are already in a change list, that means a blank line |
202 | # is missing, fix it up. | |
203 | if self.in_change: | |
204 | self.warn.append("Missing 'blank line' in section 'Series-changes'") | |
205 | self.in_change = 0 | |
13b98d95 | 206 | |
0d24de9d SG |
207 | # If we are in a section, keep collecting lines until we see END |
208 | if self.in_section: | |
209 | if line == 'END': | |
210 | if self.in_section == 'cover': | |
211 | self.series.cover = self.section | |
212 | elif self.in_section == 'notes': | |
213 | if self.is_log: | |
214 | self.series.notes += self.section | |
5c8fdd91 AA |
215 | elif self.in_section == 'commit-notes': |
216 | if self.is_log: | |
217 | self.commit.notes += self.section | |
0d24de9d SG |
218 | else: |
219 | self.warn.append("Unknown section '%s'" % self.in_section) | |
220 | self.in_section = None | |
221 | self.skip_blank = True | |
222 | self.section = [] | |
223 | else: | |
224 | self.section.append(line) | |
225 | ||
226 | # Detect the commit subject | |
227 | elif not is_blank and self.state == STATE_PATCH_SUBJECT: | |
228 | self.commit.subject = line | |
229 | ||
230 | # Detect the tags we want to remove, and skip blank lines | |
5c8fdd91 | 231 | elif re_remove.match(line) and not commit_tag_match: |
0d24de9d SG |
232 | self.skip_blank = True |
233 | ||
234 | # TEST= should be the last thing in the commit, so remove | |
235 | # everything after it | |
236 | if line.startswith('TEST='): | |
237 | self.found_test = True | |
238 | elif self.skip_blank and is_blank: | |
239 | self.skip_blank = False | |
240 | ||
241 | # Detect the start of a cover letter section | |
e7df218c | 242 | elif cover_match: |
0d24de9d SG |
243 | self.in_section = 'cover' |
244 | self.skip_blank = False | |
245 | ||
fe2f8d9e SG |
246 | elif cover_cc_match: |
247 | value = cover_cc_match.group(1) | |
248 | self.AddToSeries(line, 'cover-cc', value) | |
249 | ||
0d24de9d SG |
250 | # If we are in a change list, key collected lines until a blank one |
251 | elif self.in_change: | |
252 | if is_blank: | |
253 | # Blank line ends this change list | |
254 | self.in_change = 0 | |
102061bd | 255 | elif line == '---': |
05e5b735 IY |
256 | self.in_change = 0 |
257 | out = self.ProcessLine(line) | |
0d24de9d | 258 | else: |
a8840cb2 IY |
259 | if self.is_log: |
260 | self.series.AddChange(self.in_change, self.commit, line) | |
0d24de9d SG |
261 | self.skip_blank = False |
262 | ||
263 | # Detect Series-xxx tags | |
5c8fdd91 AA |
264 | elif series_tag_match: |
265 | name = series_tag_match.group(1) | |
266 | value = series_tag_match.group(2) | |
0d24de9d SG |
267 | if name == 'changes': |
268 | # value is the version number: e.g. 1, or 2 | |
269 | try: | |
270 | value = int(value) | |
271 | except ValueError as str: | |
272 | raise ValueError("%s: Cannot decode version info '%s'" % | |
273 | (self.commit.hash, line)) | |
274 | self.in_change = int(value) | |
275 | else: | |
276 | self.AddToSeries(line, name, value) | |
277 | self.skip_blank = True | |
278 | ||
5c8fdd91 AA |
279 | # Detect Commit-xxx tags |
280 | elif commit_tag_match: | |
281 | name = commit_tag_match.group(1) | |
282 | value = commit_tag_match.group(2) | |
283 | if name == 'notes': | |
284 | self.AddToCommit(line, name, value) | |
285 | self.skip_blank = True | |
286 | ||
0d24de9d SG |
287 | # Detect the start of a new commit |
288 | elif commit_match: | |
289 | self.CloseCommit() | |
0b5b409a | 290 | self.commit = commit.Commit(commit_match.group(1)) |
0d24de9d SG |
291 | |
292 | # Detect tags in the commit message | |
293 | elif tag_match: | |
0d24de9d | 294 | # Remove Tested-by self, since few will take much notice |
c7379149 | 295 | if (tag_match.group(1) == 'Tested-by' and |
0d24de9d SG |
296 | tag_match.group(2).find(os.getenv('USER') + '@') != -1): |
297 | self.warn.append("Ignoring %s" % line) | |
659c89da | 298 | elif tag_match.group(1) == 'Patch-cc': |
0d24de9d SG |
299 | self.commit.AddCc(tag_match.group(2).split(',')) |
300 | else: | |
d0c5719d | 301 | out = [line] |
0d24de9d | 302 | |
102061bd SG |
303 | # Suppress duplicate signoffs |
304 | elif signoff_match: | |
e752edcb | 305 | if (self.is_log or not self.commit or |
6be6b6bc | 306 | self.commit.CheckDuplicateSignoff(signoff_match.group(1))): |
102061bd SG |
307 | out = [line] |
308 | ||
0d24de9d SG |
309 | # Well that means this is an ordinary line |
310 | else: | |
0d24de9d SG |
311 | # Look for space before tab |
312 | m = re_space_before_tab.match(line) | |
313 | if m: | |
314 | self.warn.append('Line %d/%d has space before tab' % | |
315 | (self.linenum, m.start())) | |
316 | ||
317 | # OK, we have a valid non-blank line | |
318 | out = [line] | |
319 | self.linenum += 1 | |
320 | self.skip_blank = False | |
321 | if self.state == STATE_DIFFS: | |
322 | pass | |
323 | ||
324 | # If this is the start of the diffs section, emit our tags and | |
325 | # change log | |
326 | elif line == '---': | |
327 | self.state = STATE_DIFFS | |
328 | ||
329 | # Output the tags (signeoff first), then change list | |
330 | out = [] | |
0d24de9d | 331 | log = self.series.MakeChangeLog(self.commit) |
e752edcb SG |
332 | out += [line] |
333 | if self.commit: | |
334 | out += self.commit.notes | |
335 | out += [''] + log | |
0d24de9d SG |
336 | elif self.found_test: |
337 | if not re_allowed_after_test.match(line): | |
338 | self.lines_after_test += 1 | |
339 | ||
340 | return out | |
341 | ||
342 | def Finalize(self): | |
343 | """Close out processing of this patch stream""" | |
344 | self.CloseCommit() | |
345 | if self.lines_after_test: | |
346 | self.warn.append('Found %d lines after TEST=' % | |
347 | self.lines_after_test) | |
348 | ||
349 | def ProcessStream(self, infd, outfd): | |
350 | """Copy a stream from infd to outfd, filtering out unwanting things. | |
351 | ||
352 | This is used to process patch files one at a time. | |
353 | ||
354 | Args: | |
355 | infd: Input stream file object | |
356 | outfd: Output stream file object | |
357 | """ | |
358 | # Extract the filename from each diff, for nice warnings | |
359 | fname = None | |
360 | last_fname = None | |
361 | re_fname = re.compile('diff --git a/(.*) b/.*') | |
362 | while True: | |
363 | line = infd.readline() | |
364 | if not line: | |
365 | break | |
366 | out = self.ProcessLine(line) | |
367 | ||
368 | # Try to detect blank lines at EOF | |
369 | for line in out: | |
370 | match = re_fname.match(line) | |
371 | if match: | |
372 | last_fname = fname | |
373 | fname = match.group(1) | |
374 | if line == '+': | |
375 | self.blank_count += 1 | |
376 | else: | |
377 | if self.blank_count and (line == '-- ' or match): | |
378 | self.warn.append("Found possible blank line(s) at " | |
379 | "end of file '%s'" % last_fname) | |
380 | outfd.write('+\n' * self.blank_count) | |
381 | outfd.write(line + '\n') | |
382 | self.blank_count = 0 | |
383 | self.Finalize() | |
384 | ||
385 | ||
e62f905e | 386 | def GetMetaDataForList(commit_range, git_dir=None, count=None, |
950a2313 | 387 | series = None, allow_overwrite=False): |
0d24de9d SG |
388 | """Reads out patch series metadata from the commits |
389 | ||
390 | This does a 'git log' on the relevant commits and pulls out the tags we | |
391 | are interested in. | |
392 | ||
393 | Args: | |
e62f905e SG |
394 | commit_range: Range of commits to count (e.g. 'HEAD..base') |
395 | git_dir: Path to git repositiory (None to use default) | |
396 | count: Number of commits to list, or None for no limit | |
397 | series: Series object to add information into. By default a new series | |
398 | is started. | |
950a2313 | 399 | allow_overwrite: Allow tags to overwrite an existing tag |
e62f905e SG |
400 | Returns: |
401 | A Series object containing information about the commits. | |
0d24de9d | 402 | """ |
891b7a07 SG |
403 | if not series: |
404 | series = Series() | |
950a2313 | 405 | series.allow_overwrite = allow_overwrite |
9ad96982 | 406 | params = gitutil.LogCmd(commit_range, reverse=True, count=count, |
cda2a611 SG |
407 | git_dir=git_dir) |
408 | stdout = command.RunPipe([params], capture=True).stdout | |
0d24de9d SG |
409 | ps = PatchStream(series, is_log=True) |
410 | for line in stdout.splitlines(): | |
411 | ps.ProcessLine(line) | |
412 | ps.Finalize() | |
413 | return series | |
414 | ||
e62f905e SG |
415 | def GetMetaData(start, count): |
416 | """Reads out patch series metadata from the commits | |
417 | ||
418 | This does a 'git log' on the relevant commits and pulls out the tags we | |
419 | are interested in. | |
420 | ||
421 | Args: | |
422 | start: Commit to start from: 0=HEAD, 1=next one, etc. | |
423 | count: Number of commits to list | |
424 | """ | |
425 | return GetMetaDataForList('HEAD~%d' % start, None, count) | |
426 | ||
6e87ae1c SG |
427 | def GetMetaDataForTest(text): |
428 | """Process metadata from a file containing a git log. Used for tests | |
429 | ||
430 | Args: | |
431 | text: | |
432 | """ | |
433 | series = Series() | |
434 | ps = PatchStream(series, is_log=True) | |
435 | for line in text.splitlines(): | |
436 | ps.ProcessLine(line) | |
437 | ps.Finalize() | |
438 | return series | |
439 | ||
0d24de9d SG |
440 | def FixPatch(backup_dir, fname, series, commit): |
441 | """Fix up a patch file, by adding/removing as required. | |
442 | ||
443 | We remove our tags from the patch file, insert changes lists, etc. | |
444 | The patch file is processed in place, and overwritten. | |
445 | ||
446 | A backup file is put into backup_dir (if not None). | |
447 | ||
448 | Args: | |
449 | fname: Filename to patch file to process | |
450 | series: Series information about this patch set | |
451 | commit: Commit object for this patch file | |
452 | Return: | |
453 | A list of errors, or [] if all ok. | |
454 | """ | |
455 | handle, tmpname = tempfile.mkstemp() | |
456 | outfd = os.fdopen(handle, 'w') | |
457 | infd = open(fname, 'r') | |
458 | ps = PatchStream(series) | |
459 | ps.commit = commit | |
460 | ps.ProcessStream(infd, outfd) | |
461 | infd.close() | |
462 | outfd.close() | |
463 | ||
464 | # Create a backup file if required | |
465 | if backup_dir: | |
466 | shutil.copy(fname, os.path.join(backup_dir, os.path.basename(fname))) | |
467 | shutil.move(tmpname, fname) | |
468 | return ps.warn | |
469 | ||
470 | def FixPatches(series, fnames): | |
471 | """Fix up a list of patches identified by filenames | |
472 | ||
473 | The patch files are processed in place, and overwritten. | |
474 | ||
475 | Args: | |
476 | series: The series object | |
477 | fnames: List of patch files to process | |
478 | """ | |
479 | # Current workflow creates patches, so we shouldn't need a backup | |
480 | backup_dir = None #tempfile.mkdtemp('clean-patch') | |
481 | count = 0 | |
482 | for fname in fnames: | |
483 | commit = series.commits[count] | |
484 | commit.patch = fname | |
485 | result = FixPatch(backup_dir, fname, series, commit) | |
486 | if result: | |
a920a17b | 487 | print('%d warnings for %s:' % (len(result), fname)) |
0d24de9d | 488 | for warn in result: |
a920a17b | 489 | print('\t', warn) |
0d24de9d SG |
490 | |
491 | count += 1 | |
a920a17b | 492 | print('Cleaned %d patches' % count) |
0d24de9d SG |
493 | |
494 | def InsertCoverLetter(fname, series, count): | |
495 | """Inserts a cover letter with the required info into patch 0 | |
496 | ||
497 | Args: | |
498 | fname: Input / output filename of the cover letter file | |
499 | series: Series object | |
500 | count: Number of patches in the series | |
501 | """ | |
502 | fd = open(fname, 'r') | |
503 | lines = fd.readlines() | |
504 | fd.close() | |
505 | ||
506 | fd = open(fname, 'w') | |
507 | text = series.cover | |
508 | prefix = series.GetPatchPrefix() | |
509 | for line in lines: | |
510 | if line.startswith('Subject:'): | |
35ce2dc4 WJ |
511 | # if more than 10 or 100 patches, it should say 00/xx, 000/xxx, etc |
512 | zero_repeat = int(math.log10(count)) + 1 | |
513 | zero = '0' * zero_repeat | |
514 | line = 'Subject: [%s %s/%d] %s\n' % (prefix, zero, count, text[0]) | |
0d24de9d SG |
515 | |
516 | # Insert our cover letter | |
517 | elif line.startswith('*** BLURB HERE ***'): | |
518 | # First the blurb test | |
519 | line = '\n'.join(text[1:]) + '\n' | |
520 | if series.get('notes'): | |
521 | line += '\n'.join(series.notes) + '\n' | |
522 | ||
523 | # Now the change list | |
524 | out = series.MakeChangeLog(None) | |
525 | line += '\n' + '\n'.join(out) | |
526 | fd.write(line) | |
527 | fd.close() |