]>
Commit | Line | Data |
---|---|---|
0d24de9d SG |
1 | # Copyright (c) 2011 The Chromium OS Authors. |
2 | # | |
3 | # See file CREDITS for list of people who contributed to this | |
4 | # project. | |
5 | # | |
6 | # This program is free software; you can redistribute it and/or | |
7 | # modify it under the terms of the GNU General Public License as | |
8 | # published by the Free Software Foundation; either version 2 of | |
9 | # the License, or (at your option) any later version. | |
10 | # | |
11 | # This program is distributed in the hope that it will be useful, | |
12 | # but WITHOUT ANY WARRANTY; without even the implied warranty of | |
13 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the | |
14 | # GNU General Public License for more details. | |
15 | # | |
16 | # You should have received a copy of the GNU General Public License | |
17 | # along with this program; if not, write to the Free Software | |
18 | # Foundation, Inc., 59 Temple Place, Suite 330, Boston, | |
19 | # MA 02111-1307 USA | |
20 | # | |
21 | ||
22 | import os | |
23 | import re | |
24 | import shutil | |
25 | import tempfile | |
26 | ||
27 | import command | |
28 | import commit | |
29 | import gitutil | |
30 | from series import Series | |
31 | ||
32 | # Tags that we detect and remove | |
33 | re_remove = re.compile('^BUG=|^TEST=|^Change-Id:|^Review URL:' | |
34 | '|Reviewed-on:|Reviewed-by:') | |
35 | ||
36 | # Lines which are allowed after a TEST= line | |
37 | re_allowed_after_test = re.compile('^Signed-off-by:') | |
38 | ||
05e5b735 IY |
39 | # Signoffs |
40 | re_signoff = re.compile('^Signed-off-by:') | |
41 | ||
0d24de9d SG |
42 | # The start of the cover letter |
43 | re_cover = re.compile('^Cover-letter:') | |
44 | ||
45 | # Patch series tag | |
46 | re_series = re.compile('^Series-(\w*): *(.*)') | |
47 | ||
48 | # Commit tags that we want to collect and keep | |
49 | re_tag = re.compile('^(Tested-by|Acked-by|Signed-off-by|Cc): (.*)') | |
50 | ||
51 | # The start of a new commit in the git log | |
52 | re_commit = re.compile('^commit (.*)') | |
53 | ||
54 | # We detect these since checkpatch doesn't always do it | |
55 | re_space_before_tab = re.compile('^[+].* \t') | |
56 | ||
57 | # States we can be in - can we use range() and still have comments? | |
58 | STATE_MSG_HEADER = 0 # Still in the message header | |
59 | STATE_PATCH_SUBJECT = 1 # In patch subject (first line of log for a commit) | |
60 | STATE_PATCH_HEADER = 2 # In patch header (after the subject) | |
61 | STATE_DIFFS = 3 # In the diff part (past --- line) | |
62 | ||
63 | class PatchStream: | |
64 | """Class for detecting/injecting tags in a patch or series of patches | |
65 | ||
66 | We support processing the output of 'git log' to read out the tags we | |
67 | are interested in. We can also process a patch file in order to remove | |
68 | unwanted tags or inject additional ones. These correspond to the two | |
69 | phases of processing. | |
70 | """ | |
71 | def __init__(self, series, name=None, is_log=False): | |
72 | self.skip_blank = False # True to skip a single blank line | |
73 | self.found_test = False # Found a TEST= line | |
74 | self.lines_after_test = 0 # MNumber of lines found after TEST= | |
75 | self.warn = [] # List of warnings we have collected | |
76 | self.linenum = 1 # Output line number we are up to | |
77 | self.in_section = None # Name of start...END section we are in | |
78 | self.notes = [] # Series notes | |
79 | self.section = [] # The current section...END section | |
80 | self.series = series # Info about the patch series | |
81 | self.is_log = is_log # True if indent like git log | |
82 | self.in_change = 0 # Non-zero if we are in a change list | |
83 | self.blank_count = 0 # Number of blank lines stored up | |
84 | self.state = STATE_MSG_HEADER # What state are we in? | |
85 | self.tags = [] # Tags collected, like Tested-by... | |
86 | self.signoff = [] # Contents of signoff line | |
87 | self.commit = None # Current commit | |
88 | ||
89 | def AddToSeries(self, line, name, value): | |
90 | """Add a new Series-xxx tag. | |
91 | ||
92 | When a Series-xxx tag is detected, we come here to record it, if we | |
93 | are scanning a 'git log'. | |
94 | ||
95 | Args: | |
96 | line: Source line containing tag (useful for debug/error messages) | |
97 | name: Tag name (part after 'Series-') | |
98 | value: Tag value (part after 'Series-xxx: ') | |
99 | """ | |
100 | if name == 'notes': | |
101 | self.in_section = name | |
102 | self.skip_blank = False | |
103 | if self.is_log: | |
104 | self.series.AddTag(self.commit, line, name, value) | |
105 | ||
106 | def CloseCommit(self): | |
107 | """Save the current commit into our commit list, and reset our state""" | |
108 | if self.commit and self.is_log: | |
109 | self.series.AddCommit(self.commit) | |
110 | self.commit = None | |
111 | ||
112 | def FormatTags(self, tags): | |
113 | out_list = [] | |
114 | for tag in sorted(tags): | |
115 | if tag.startswith('Cc:'): | |
116 | tag_list = tag[4:].split(',') | |
117 | out_list += gitutil.BuildEmailList(tag_list, 'Cc:') | |
118 | else: | |
119 | out_list.append(tag) | |
120 | return out_list | |
121 | ||
122 | def ProcessLine(self, line): | |
123 | """Process a single line of a patch file or commit log | |
124 | ||
125 | This process a line and returns a list of lines to output. The list | |
126 | may be empty or may contain multiple output lines. | |
127 | ||
128 | This is where all the complicated logic is located. The class's | |
129 | state is used to move between different states and detect things | |
130 | properly. | |
131 | ||
132 | We can be in one of two modes: | |
133 | self.is_log == True: This is 'git log' mode, where most output is | |
134 | indented by 4 characters and we are scanning for tags | |
135 | ||
136 | self.is_log == False: This is 'patch' mode, where we already have | |
137 | all the tags, and are processing patches to remove junk we | |
138 | don't want, and add things we think are required. | |
139 | ||
140 | Args: | |
141 | line: text line to process | |
142 | ||
143 | Returns: | |
144 | list of output lines, or [] if nothing should be output | |
145 | """ | |
146 | # Initially we have no output. Prepare the input line string | |
147 | out = [] | |
148 | line = line.rstrip('\n') | |
149 | if self.is_log: | |
150 | if line[:4] == ' ': | |
151 | line = line[4:] | |
152 | ||
153 | # Handle state transition and skipping blank lines | |
154 | series_match = re_series.match(line) | |
155 | commit_match = re_commit.match(line) if self.is_log else None | |
156 | tag_match = None | |
157 | if self.state == STATE_PATCH_HEADER: | |
158 | tag_match = re_tag.match(line) | |
159 | is_blank = not line.strip() | |
160 | if is_blank: | |
161 | if (self.state == STATE_MSG_HEADER | |
162 | or self.state == STATE_PATCH_SUBJECT): | |
163 | self.state += 1 | |
164 | ||
165 | # We don't have a subject in the text stream of patch files | |
166 | # It has its own line with a Subject: tag | |
167 | if not self.is_log and self.state == STATE_PATCH_SUBJECT: | |
168 | self.state += 1 | |
169 | elif commit_match: | |
170 | self.state = STATE_MSG_HEADER | |
171 | ||
172 | # If we are in a section, keep collecting lines until we see END | |
173 | if self.in_section: | |
174 | if line == 'END': | |
175 | if self.in_section == 'cover': | |
176 | self.series.cover = self.section | |
177 | elif self.in_section == 'notes': | |
178 | if self.is_log: | |
179 | self.series.notes += self.section | |
180 | else: | |
181 | self.warn.append("Unknown section '%s'" % self.in_section) | |
182 | self.in_section = None | |
183 | self.skip_blank = True | |
184 | self.section = [] | |
185 | else: | |
186 | self.section.append(line) | |
187 | ||
188 | # Detect the commit subject | |
189 | elif not is_blank and self.state == STATE_PATCH_SUBJECT: | |
190 | self.commit.subject = line | |
191 | ||
192 | # Detect the tags we want to remove, and skip blank lines | |
193 | elif re_remove.match(line): | |
194 | self.skip_blank = True | |
195 | ||
196 | # TEST= should be the last thing in the commit, so remove | |
197 | # everything after it | |
198 | if line.startswith('TEST='): | |
199 | self.found_test = True | |
200 | elif self.skip_blank and is_blank: | |
201 | self.skip_blank = False | |
202 | ||
203 | # Detect the start of a cover letter section | |
204 | elif re_cover.match(line): | |
205 | self.in_section = 'cover' | |
206 | self.skip_blank = False | |
207 | ||
208 | # If we are in a change list, key collected lines until a blank one | |
209 | elif self.in_change: | |
210 | if is_blank: | |
211 | # Blank line ends this change list | |
212 | self.in_change = 0 | |
05e5b735 IY |
213 | elif line == '---' or re_signoff.match(line): |
214 | self.in_change = 0 | |
215 | out = self.ProcessLine(line) | |
0d24de9d SG |
216 | else: |
217 | self.series.AddChange(self.in_change, self.commit, line) | |
218 | self.skip_blank = False | |
219 | ||
220 | # Detect Series-xxx tags | |
221 | elif series_match: | |
222 | name = series_match.group(1) | |
223 | value = series_match.group(2) | |
224 | if name == 'changes': | |
225 | # value is the version number: e.g. 1, or 2 | |
226 | try: | |
227 | value = int(value) | |
228 | except ValueError as str: | |
229 | raise ValueError("%s: Cannot decode version info '%s'" % | |
230 | (self.commit.hash, line)) | |
231 | self.in_change = int(value) | |
232 | else: | |
233 | self.AddToSeries(line, name, value) | |
234 | self.skip_blank = True | |
235 | ||
236 | # Detect the start of a new commit | |
237 | elif commit_match: | |
238 | self.CloseCommit() | |
239 | self.commit = commit.Commit(commit_match.group(1)[:7]) | |
240 | ||
241 | # Detect tags in the commit message | |
242 | elif tag_match: | |
243 | # Onlly allow a single signoff tag | |
244 | if tag_match.group(1) == 'Signed-off-by': | |
245 | if self.signoff: | |
246 | self.warn.append('Patch has more than one Signed-off-by ' | |
247 | 'tag') | |
248 | self.signoff += [line] | |
249 | ||
250 | # Remove Tested-by self, since few will take much notice | |
251 | elif (tag_match.group(1) == 'Tested-by' and | |
252 | tag_match.group(2).find(os.getenv('USER') + '@') != -1): | |
253 | self.warn.append("Ignoring %s" % line) | |
254 | elif tag_match.group(1) == 'Cc': | |
255 | self.commit.AddCc(tag_match.group(2).split(',')) | |
256 | else: | |
257 | self.tags.append(line); | |
258 | ||
259 | # Well that means this is an ordinary line | |
260 | else: | |
261 | pos = 1 | |
262 | # Look for ugly ASCII characters | |
263 | for ch in line: | |
264 | # TODO: Would be nicer to report source filename and line | |
265 | if ord(ch) > 0x80: | |
266 | self.warn.append("Line %d/%d ('%s') has funny ascii char" % | |
267 | (self.linenum, pos, line)) | |
268 | pos += 1 | |
269 | ||
270 | # Look for space before tab | |
271 | m = re_space_before_tab.match(line) | |
272 | if m: | |
273 | self.warn.append('Line %d/%d has space before tab' % | |
274 | (self.linenum, m.start())) | |
275 | ||
276 | # OK, we have a valid non-blank line | |
277 | out = [line] | |
278 | self.linenum += 1 | |
279 | self.skip_blank = False | |
280 | if self.state == STATE_DIFFS: | |
281 | pass | |
282 | ||
283 | # If this is the start of the diffs section, emit our tags and | |
284 | # change log | |
285 | elif line == '---': | |
286 | self.state = STATE_DIFFS | |
287 | ||
288 | # Output the tags (signeoff first), then change list | |
289 | out = [] | |
290 | if self.signoff: | |
291 | out += self.signoff | |
292 | log = self.series.MakeChangeLog(self.commit) | |
293 | out += self.FormatTags(self.tags) | |
294 | out += [line] + log | |
295 | elif self.found_test: | |
296 | if not re_allowed_after_test.match(line): | |
297 | self.lines_after_test += 1 | |
298 | ||
299 | return out | |
300 | ||
301 | def Finalize(self): | |
302 | """Close out processing of this patch stream""" | |
303 | self.CloseCommit() | |
304 | if self.lines_after_test: | |
305 | self.warn.append('Found %d lines after TEST=' % | |
306 | self.lines_after_test) | |
307 | ||
308 | def ProcessStream(self, infd, outfd): | |
309 | """Copy a stream from infd to outfd, filtering out unwanting things. | |
310 | ||
311 | This is used to process patch files one at a time. | |
312 | ||
313 | Args: | |
314 | infd: Input stream file object | |
315 | outfd: Output stream file object | |
316 | """ | |
317 | # Extract the filename from each diff, for nice warnings | |
318 | fname = None | |
319 | last_fname = None | |
320 | re_fname = re.compile('diff --git a/(.*) b/.*') | |
321 | while True: | |
322 | line = infd.readline() | |
323 | if not line: | |
324 | break | |
325 | out = self.ProcessLine(line) | |
326 | ||
327 | # Try to detect blank lines at EOF | |
328 | for line in out: | |
329 | match = re_fname.match(line) | |
330 | if match: | |
331 | last_fname = fname | |
332 | fname = match.group(1) | |
333 | if line == '+': | |
334 | self.blank_count += 1 | |
335 | else: | |
336 | if self.blank_count and (line == '-- ' or match): | |
337 | self.warn.append("Found possible blank line(s) at " | |
338 | "end of file '%s'" % last_fname) | |
339 | outfd.write('+\n' * self.blank_count) | |
340 | outfd.write(line + '\n') | |
341 | self.blank_count = 0 | |
342 | self.Finalize() | |
343 | ||
344 | ||
345 | def GetMetaData(start, count): | |
346 | """Reads out patch series metadata from the commits | |
347 | ||
348 | This does a 'git log' on the relevant commits and pulls out the tags we | |
349 | are interested in. | |
350 | ||
351 | Args: | |
352 | start: Commit to start from: 0=HEAD, 1=next one, etc. | |
353 | count: Number of commits to list | |
354 | """ | |
355 | pipe = [['git', 'log', '--reverse', 'HEAD~%d' % start, '-n%d' % count]] | |
356 | stdout = command.RunPipe(pipe, capture=True) | |
357 | series = Series() | |
358 | ps = PatchStream(series, is_log=True) | |
359 | for line in stdout.splitlines(): | |
360 | ps.ProcessLine(line) | |
361 | ps.Finalize() | |
362 | return series | |
363 | ||
364 | def FixPatch(backup_dir, fname, series, commit): | |
365 | """Fix up a patch file, by adding/removing as required. | |
366 | ||
367 | We remove our tags from the patch file, insert changes lists, etc. | |
368 | The patch file is processed in place, and overwritten. | |
369 | ||
370 | A backup file is put into backup_dir (if not None). | |
371 | ||
372 | Args: | |
373 | fname: Filename to patch file to process | |
374 | series: Series information about this patch set | |
375 | commit: Commit object for this patch file | |
376 | Return: | |
377 | A list of errors, or [] if all ok. | |
378 | """ | |
379 | handle, tmpname = tempfile.mkstemp() | |
380 | outfd = os.fdopen(handle, 'w') | |
381 | infd = open(fname, 'r') | |
382 | ps = PatchStream(series) | |
383 | ps.commit = commit | |
384 | ps.ProcessStream(infd, outfd) | |
385 | infd.close() | |
386 | outfd.close() | |
387 | ||
388 | # Create a backup file if required | |
389 | if backup_dir: | |
390 | shutil.copy(fname, os.path.join(backup_dir, os.path.basename(fname))) | |
391 | shutil.move(tmpname, fname) | |
392 | return ps.warn | |
393 | ||
394 | def FixPatches(series, fnames): | |
395 | """Fix up a list of patches identified by filenames | |
396 | ||
397 | The patch files are processed in place, and overwritten. | |
398 | ||
399 | Args: | |
400 | series: The series object | |
401 | fnames: List of patch files to process | |
402 | """ | |
403 | # Current workflow creates patches, so we shouldn't need a backup | |
404 | backup_dir = None #tempfile.mkdtemp('clean-patch') | |
405 | count = 0 | |
406 | for fname in fnames: | |
407 | commit = series.commits[count] | |
408 | commit.patch = fname | |
409 | result = FixPatch(backup_dir, fname, series, commit) | |
410 | if result: | |
411 | print '%d warnings for %s:' % (len(result), fname) | |
412 | for warn in result: | |
413 | print '\t', warn | |
414 | ||
415 | count += 1 | |
416 | print 'Cleaned %d patches' % count | |
417 | return series | |
418 | ||
419 | def InsertCoverLetter(fname, series, count): | |
420 | """Inserts a cover letter with the required info into patch 0 | |
421 | ||
422 | Args: | |
423 | fname: Input / output filename of the cover letter file | |
424 | series: Series object | |
425 | count: Number of patches in the series | |
426 | """ | |
427 | fd = open(fname, 'r') | |
428 | lines = fd.readlines() | |
429 | fd.close() | |
430 | ||
431 | fd = open(fname, 'w') | |
432 | text = series.cover | |
433 | prefix = series.GetPatchPrefix() | |
434 | for line in lines: | |
435 | if line.startswith('Subject:'): | |
436 | # TODO: if more than 10 patches this should save 00/xx, not 0/xx | |
437 | line = 'Subject: [%s 0/%d] %s\n' % (prefix, count, text[0]) | |
438 | ||
439 | # Insert our cover letter | |
440 | elif line.startswith('*** BLURB HERE ***'): | |
441 | # First the blurb test | |
442 | line = '\n'.join(text[1:]) + '\n' | |
443 | if series.get('notes'): | |
444 | line += '\n'.join(series.notes) + '\n' | |
445 | ||
446 | # Now the change list | |
447 | out = series.MakeChangeLog(None) | |
448 | line += '\n' + '\n'.join(out) | |
449 | fd.write(line) | |
450 | fd.close() |