]>
Commit | Line | Data |
---|---|---|
f8c9c511 | 1 | # |
bb731d1f RP |
2 | # Copyright OpenEmbedded Contributors |
3 | # | |
f8c9c511 RP |
4 | # SPDX-License-Identifier: GPL-2.0-only |
5 | # | |
6 | ||
5f33c7b9 ON |
7 | import os |
8 | import shlex | |
9 | import subprocess | |
1fbcd2ca | 10 | import oe.path |
26ff9d28 | 11 | import oe.types |
1fbcd2ca | 12 | |
ce6c80a1 | 13 | class NotFoundError(bb.BBHandledException): |
ac023d77 JL |
14 | def __init__(self, path): |
15 | self.path = path | |
16 | ||
17 | def __str__(self): | |
18 | return "Error: %s not found." % self.path | |
19 | ||
ce6c80a1 | 20 | class CmdError(bb.BBHandledException): |
8e9c03df ML |
21 | def __init__(self, command, exitstatus, output): |
22 | self.command = command | |
ac023d77 JL |
23 | self.status = exitstatus |
24 | self.output = output | |
25 | ||
26 | def __str__(self): | |
8e9c03df ML |
27 | return "Command Error: '%s' exited with %d Output:\n%s" % \ |
28 | (self.command, self.status, self.output) | |
ac023d77 JL |
29 | |
30 | ||
31 | def runcmd(args, dir = None): | |
ac023d77 JL |
32 | if dir: |
33 | olddir = os.path.abspath(os.curdir) | |
34 | if not os.path.exists(dir): | |
35 | raise NotFoundError(dir) | |
36 | os.chdir(dir) | |
37 | # print("cwd: %s -> %s" % (olddir, dir)) | |
38 | ||
39 | try: | |
5f33c7b9 | 40 | args = [ shlex.quote(str(arg)) for arg in args ] |
ac023d77 JL |
41 | cmd = " ".join(args) |
42 | # print("cmd: %s" % cmd) | |
482589e2 TD |
43 | proc = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=True) |
44 | stdout, stderr = proc.communicate() | |
45 | stdout = stdout.decode('utf-8') | |
46 | stderr = stderr.decode('utf-8') | |
47 | exitstatus = proc.returncode | |
ac023d77 | 48 | if exitstatus != 0: |
482589e2 TD |
49 | raise CmdError(cmd, exitstatus >> 8, "stdout: %s\nstderr: %s" % (stdout, stderr)) |
50 | if " fuzz " in stdout and "Hunk " in stdout: | |
c762c0be AM |
51 | # Drop patch fuzz info with header and footer to log file so |
52 | # insane.bbclass can handle to throw error/warning | |
482589e2 | 53 | bb.note("--- Patch fuzz start ---\n%s\n--- Patch fuzz end ---" % format(stdout)) |
5133fd46 | 54 | |
482589e2 | 55 | return stdout |
ac023d77 JL |
56 | |
57 | finally: | |
58 | if dir: | |
59 | os.chdir(olddir) | |
60 | ||
6184b56a | 61 | |
ac023d77 JL |
62 | class PatchError(Exception): |
63 | def __init__(self, msg): | |
64 | self.msg = msg | |
65 | ||
66 | def __str__(self): | |
67 | return "Patch Error: %s" % self.msg | |
68 | ||
69 | class PatchSet(object): | |
70 | defaults = { | |
71 | "strippath": 1 | |
72 | } | |
73 | ||
74 | def __init__(self, dir, d): | |
75 | self.dir = dir | |
76 | self.d = d | |
77 | self.patches = [] | |
78 | self._current = None | |
79 | ||
80 | def current(self): | |
81 | return self._current | |
82 | ||
83 | def Clean(self): | |
84 | """ | |
85 | Clean out the patch set. Generally includes unapplying all | |
86 | patches and wiping out all associated metadata. | |
87 | """ | |
88 | raise NotImplementedError() | |
89 | ||
90 | def Import(self, patch, force): | |
91 | if not patch.get("file"): | |
92 | if not patch.get("remote"): | |
93 | raise PatchError("Patch file must be specified in patch import.") | |
94 | else: | |
984e90f4 | 95 | patch["file"] = bb.fetch2.localpath(patch["remote"], self.d) |
ac023d77 JL |
96 | |
97 | for param in PatchSet.defaults: | |
98 | if not patch.get(param): | |
99 | patch[param] = PatchSet.defaults[param] | |
100 | ||
101 | if patch.get("remote"): | |
a361babe | 102 | patch["file"] = self.d.expand(bb.fetch2.localpath(patch["remote"], self.d)) |
ac023d77 JL |
103 | |
104 | patch["filemd5"] = bb.utils.md5_file(patch["file"]) | |
105 | ||
106 | def Push(self, force): | |
107 | raise NotImplementedError() | |
108 | ||
109 | def Pop(self, force): | |
110 | raise NotImplementedError() | |
111 | ||
112 | def Refresh(self, remote = None, all = None): | |
113 | raise NotImplementedError() | |
114 | ||
dd2aa93b PE |
115 | @staticmethod |
116 | def getPatchedFiles(patchfile, striplevel, srcdir=None): | |
117 | """ | |
118 | Read a patch file and determine which files it will modify. | |
119 | Params: | |
120 | patchfile: the patch file to read | |
121 | striplevel: the strip level at which the patch is going to be applied | |
122 | srcdir: optional path to join onto the patched file paths | |
123 | Returns: | |
124 | A list of tuples of file path and change mode ('A' for add, | |
125 | 'D' for delete or 'M' for modify) | |
126 | """ | |
127 | ||
128 | def patchedpath(patchline): | |
129 | filepth = patchline.split()[1] | |
130 | if filepth.endswith('/dev/null'): | |
131 | return '/dev/null' | |
132 | filesplit = filepth.split(os.sep) | |
133 | if striplevel > len(filesplit): | |
134 | bb.error('Patch %s has invalid strip level %d' % (patchfile, striplevel)) | |
135 | return None | |
136 | return os.sep.join(filesplit[striplevel:]) | |
137 | ||
7f4d7a6f PE |
138 | for encoding in ['utf-8', 'latin-1']: |
139 | try: | |
140 | copiedmode = False | |
141 | filelist = [] | |
142 | with open(patchfile) as f: | |
143 | for line in f: | |
144 | if line.startswith('--- '): | |
145 | patchpth = patchedpath(line) | |
146 | if not patchpth: | |
147 | break | |
148 | if copiedmode: | |
149 | addedfile = patchpth | |
150 | else: | |
151 | removedfile = patchpth | |
152 | elif line.startswith('+++ '): | |
153 | addedfile = patchedpath(line) | |
154 | if not addedfile: | |
155 | break | |
156 | elif line.startswith('*** '): | |
157 | copiedmode = True | |
158 | removedfile = patchedpath(line) | |
159 | if not removedfile: | |
160 | break | |
161 | else: | |
162 | removedfile = None | |
163 | addedfile = None | |
164 | ||
165 | if addedfile and removedfile: | |
166 | if removedfile == '/dev/null': | |
167 | mode = 'A' | |
168 | elif addedfile == '/dev/null': | |
169 | mode = 'D' | |
170 | else: | |
171 | mode = 'M' | |
172 | if srcdir: | |
173 | fullpath = os.path.abspath(os.path.join(srcdir, addedfile)) | |
174 | else: | |
175 | fullpath = addedfile | |
176 | filelist.append((fullpath, mode)) | |
177 | except UnicodeDecodeError: | |
178 | continue | |
179 | break | |
180 | else: | |
181 | raise PatchError('Unable to decode %s' % patchfile) | |
dd2aa93b PE |
182 | |
183 | return filelist | |
184 | ||
ac023d77 JL |
185 | |
186 | class PatchTree(PatchSet): | |
187 | def __init__(self, dir, d): | |
188 | PatchSet.__init__(self, dir, d) | |
f0fc47ae RP |
189 | self.patchdir = os.path.join(self.dir, 'patches') |
190 | self.seriespath = os.path.join(self.dir, 'patches', 'series') | |
191 | bb.utils.mkdirhier(self.patchdir) | |
192 | ||
193 | def _appendPatchFile(self, patch, strippath): | |
194 | with open(self.seriespath, 'a') as f: | |
195 | f.write(os.path.basename(patch) + "," + strippath + "\n") | |
196 | shellcmd = ["cat", patch, ">" , self.patchdir + "/" + os.path.basename(patch)] | |
197 | runcmd(["sh", "-c", " ".join(shellcmd)], self.dir) | |
198 | ||
199 | def _removePatch(self, p): | |
200 | patch = {} | |
201 | patch['file'] = p.split(",")[0] | |
202 | patch['strippath'] = p.split(",")[1] | |
203 | self._applypatch(patch, False, True) | |
204 | ||
205 | def _removePatchFile(self, all = False): | |
206 | if not os.path.exists(self.seriespath): | |
207 | return | |
99ac382d PE |
208 | with open(self.seriespath, 'r+') as f: |
209 | patches = f.readlines() | |
f0fc47ae RP |
210 | if all: |
211 | for p in reversed(patches): | |
212 | self._removePatch(os.path.join(self.patchdir, p.strip())) | |
213 | patches = [] | |
214 | else: | |
215 | self._removePatch(os.path.join(self.patchdir, patches[-1].strip())) | |
216 | patches.pop() | |
217 | with open(self.seriespath, 'w') as f: | |
218 | for p in patches: | |
219 | f.write(p) | |
5f33c7b9 | 220 | |
ac023d77 JL |
221 | def Import(self, patch, force = None): |
222 | """""" | |
223 | PatchSet.Import(self, patch, force) | |
224 | ||
225 | if self._current is not None: | |
226 | i = self._current + 1 | |
227 | else: | |
228 | i = 0 | |
229 | self.patches.insert(i, patch) | |
230 | ||
231 | def _applypatch(self, patch, force = False, reverse = False, run = True): | |
f4f3406c | 232 | shellcmd = ["cat", patch['file'], "|", "patch", "--no-backup-if-mismatch", "-p", patch['strippath']] |
ac023d77 JL |
233 | if reverse: |
234 | shellcmd.append('-R') | |
235 | ||
236 | if not run: | |
237 | return "sh" + "-c" + " ".join(shellcmd) | |
238 | ||
239 | if not force: | |
240 | shellcmd.append('--dry-run') | |
241 | ||
574405a9 ML |
242 | try: |
243 | output = runcmd(["sh", "-c", " ".join(shellcmd)], self.dir) | |
ac023d77 | 244 | |
574405a9 ML |
245 | if force: |
246 | return | |
ac023d77 | 247 | |
574405a9 ML |
248 | shellcmd.pop(len(shellcmd) - 1) |
249 | output = runcmd(["sh", "-c", " ".join(shellcmd)], self.dir) | |
250 | except CmdError as err: | |
251 | raise bb.BBHandledException("Applying '%s' failed:\n%s" % | |
252 | (os.path.basename(patch['file']), err.output)) | |
f0fc47ae RP |
253 | |
254 | if not reverse: | |
255 | self._appendPatchFile(patch['file'], patch['strippath']) | |
256 | ||
ac023d77 JL |
257 | return output |
258 | ||
259 | def Push(self, force = False, all = False, run = True): | |
260 | bb.note("self._current is %s" % self._current) | |
261 | bb.note("patches is %s" % self.patches) | |
262 | if all: | |
263 | for i in self.patches: | |
ac023d77 JL |
264 | bb.note("applying patch %s" % i) |
265 | self._applypatch(i, force) | |
f0fc47ae | 266 | self._current = i |
ac023d77 JL |
267 | else: |
268 | if self._current is not None: | |
f0fc47ae | 269 | next = self._current + 1 |
ac023d77 | 270 | else: |
f0fc47ae | 271 | next = 0 |
ac023d77 | 272 | |
f0fc47ae RP |
273 | bb.note("applying patch %s" % self.patches[next]) |
274 | ret = self._applypatch(self.patches[next], force) | |
275 | ||
276 | self._current = next | |
277 | return ret | |
ac023d77 JL |
278 | |
279 | def Pop(self, force = None, all = None): | |
280 | if all: | |
f0fc47ae RP |
281 | self._removePatchFile(True) |
282 | self._current = None | |
ac023d77 | 283 | else: |
f0fc47ae RP |
284 | self._removePatchFile(False) |
285 | ||
286 | if self._current == 0: | |
287 | self._current = None | |
288 | ||
289 | if self._current is not None: | |
290 | self._current = self._current - 1 | |
ac023d77 JL |
291 | |
292 | def Clean(self): | |
293 | """""" | |
f0fc47ae | 294 | self.Pop(all=True) |
ac023d77 JL |
295 | |
296 | class GitApplyTree(PatchTree): | |
f5e6183b PK |
297 | notes_ref = "refs/notes/devtool" |
298 | original_patch = 'original patch' | |
299 | ignore_commit = 'ignore' | |
765b7bad | 300 | |
ac023d77 JL |
301 | def __init__(self, dir, d): |
302 | PatchTree.__init__(self, dir, d) | |
7c552996 JL |
303 | self.commituser = d.getVar('PATCH_GIT_USER_NAME') |
304 | self.commitemail = d.getVar('PATCH_GIT_USER_EMAIL') | |
9cca53a2 | 305 | if not self._isInitialized(d): |
6184b56a PZ |
306 | self._initRepo() |
307 | ||
9cca53a2 | 308 | def _isInitialized(self, d): |
6184b56a | 309 | cmd = "git rev-parse --show-toplevel" |
80500ecd PZ |
310 | try: |
311 | output = runcmd(cmd.split(), self.dir).strip() | |
312 | except CmdError as err: | |
313 | ## runcmd returned non-zero which most likely means 128 | |
314 | ## Not a git directory | |
315 | return False | |
9cca53a2 PE |
316 | ## Make sure repo is in builddir to not break top-level git repos, or under workdir |
317 | return os.path.samefile(output, self.dir) or oe.path.is_path_parent(d.getVar('WORKDIR'), output) | |
6184b56a PZ |
318 | |
319 | def _initRepo(self): | |
320 | runcmd("git init".split(), self.dir) | |
321 | runcmd("git add .".split(), self.dir) | |
80500ecd | 322 | runcmd("git commit -a --allow-empty -m bitbake_patching_started".split(), self.dir) |
ac023d77 | 323 | |
8c522846 PE |
324 | @staticmethod |
325 | def extractPatchHeader(patchfile): | |
326 | """ | |
327 | Extract just the header lines from the top of a patch file | |
328 | """ | |
7f4d7a6f PE |
329 | for encoding in ['utf-8', 'latin-1']: |
330 | lines = [] | |
331 | try: | |
332 | with open(patchfile, 'r', encoding=encoding) as f: | |
333 | for line in f: | |
334 | if line.startswith('Index: ') or line.startswith('diff -') or line.startswith('---'): | |
335 | break | |
336 | lines.append(line) | |
337 | except UnicodeDecodeError: | |
338 | continue | |
339 | break | |
340 | else: | |
341 | raise PatchError('Unable to find a character encoding to decode %s' % patchfile) | |
8c522846 PE |
342 | return lines |
343 | ||
344 | @staticmethod | |
13ec296b PE |
345 | def decodeAuthor(line): |
346 | from email.header import decode_header | |
347 | authorval = line.split(':', 1)[1].strip().replace('"', '') | |
30d02e2a EB |
348 | result = decode_header(authorval)[0][0] |
349 | if hasattr(result, 'decode'): | |
350 | result = result.decode('utf-8') | |
351 | return result | |
13ec296b PE |
352 | |
353 | @staticmethod | |
354 | def interpretPatchHeader(headerlines): | |
8c522846 | 355 | import re |
4b1c0c7d RP |
356 | author_re = re.compile(r'[\S ]+ <\S+@\S+\.\S+>') |
357 | from_commit_re = re.compile(r'^From [a-z0-9]{40} .*') | |
8c522846 PE |
358 | outlines = [] |
359 | author = None | |
360 | date = None | |
13ec296b PE |
361 | subject = None |
362 | for line in headerlines: | |
8c522846 PE |
363 | if line.startswith('Subject: '): |
364 | subject = line.split(':', 1)[1] | |
365 | # Remove any [PATCH][oe-core] etc. | |
366 | subject = re.sub(r'\[.+?\]\s*', '', subject) | |
8c522846 | 367 | continue |
13ec296b PE |
368 | elif line.startswith('From: ') or line.startswith('Author: '): |
369 | authorval = GitApplyTree.decodeAuthor(line) | |
8c522846 PE |
370 | # git is fussy about author formatting i.e. it must be Name <email@domain> |
371 | if author_re.match(authorval): | |
372 | author = authorval | |
373 | continue | |
13ec296b | 374 | elif line.startswith('Date: '): |
8c522846 PE |
375 | if date is None: |
376 | dateval = line.split(':', 1)[1].strip() | |
377 | # Very crude check for date format, since git will blow up if it's not in the right | |
378 | # format. Without e.g. a python-dateutils dependency we can't do a whole lot more | |
379 | if len(dateval) > 12: | |
380 | date = dateval | |
381 | continue | |
13ec296b PE |
382 | elif not author and line.lower().startswith('signed-off-by: '): |
383 | authorval = GitApplyTree.decodeAuthor(line) | |
8c522846 PE |
384 | # git is fussy about author formatting i.e. it must be Name <email@domain> |
385 | if author_re.match(authorval): | |
386 | author = authorval | |
19a6b18a PE |
387 | elif from_commit_re.match(line): |
388 | # We don't want the From <commit> line - if it's present it will break rebasing | |
389 | continue | |
8c522846 | 390 | outlines.append(line) |
896cfb10 PE |
391 | |
392 | if not subject: | |
393 | firstline = None | |
394 | for line in headerlines: | |
395 | line = line.strip() | |
396 | if firstline: | |
397 | if line: | |
398 | # Second line is not blank, the first line probably isn't usable | |
399 | firstline = None | |
400 | break | |
401 | elif line: | |
402 | firstline = line | |
403 | if firstline and not firstline.startswith(('#', 'Index:', 'Upstream-Status:')) and len(firstline) < 100: | |
404 | subject = firstline | |
405 | ||
13ec296b PE |
406 | return outlines, author, date, subject |
407 | ||
408 | @staticmethod | |
765a9017 PE |
409 | def gitCommandUserOptions(cmd, commituser=None, commitemail=None, d=None): |
410 | if d: | |
7c552996 JL |
411 | commituser = d.getVar('PATCH_GIT_USER_NAME') |
412 | commitemail = d.getVar('PATCH_GIT_USER_EMAIL') | |
765a9017 PE |
413 | if commituser: |
414 | cmd += ['-c', 'user.name="%s"' % commituser] | |
415 | if commitemail: | |
416 | cmd += ['-c', 'user.email="%s"' % commitemail] | |
417 | ||
418 | @staticmethod | |
419 | def prepareCommit(patchfile, commituser=None, commitemail=None): | |
13ec296b PE |
420 | """ |
421 | Prepare a git commit command line based on the header from a patch file | |
422 | (typically this is useful for patches that cannot be applied with "git am" due to formatting) | |
423 | """ | |
424 | import tempfile | |
425 | # Process patch header and extract useful information | |
426 | lines = GitApplyTree.extractPatchHeader(patchfile) | |
427 | outlines, author, date, subject = GitApplyTree.interpretPatchHeader(lines) | |
896cfb10 | 428 | if not author or not subject or not date: |
13ec296b | 429 | try: |
896cfb10 | 430 | shellcmd = ["git", "log", "--format=email", "--follow", "--diff-filter=A", "--", patchfile] |
13ec296b PE |
431 | out = runcmd(["sh", "-c", " ".join(shellcmd)], os.path.dirname(patchfile)) |
432 | except CmdError: | |
433 | out = None | |
434 | if out: | |
435 | _, newauthor, newdate, newsubject = GitApplyTree.interpretPatchHeader(out.splitlines()) | |
896cfb10 PE |
436 | if not author: |
437 | # If we're setting the author then the date should be set as well | |
13ec296b PE |
438 | author = newauthor |
439 | date = newdate | |
896cfb10 PE |
440 | elif not date: |
441 | # If we don't do this we'll get the current date, at least this will be closer | |
442 | date = newdate | |
13ec296b PE |
443 | if not subject: |
444 | subject = newsubject | |
0a8252f0 | 445 | if subject and not (outlines and outlines[0].strip() == subject): |
13ec296b PE |
446 | outlines.insert(0, '%s\n\n' % subject.strip()) |
447 | ||
8c522846 PE |
448 | # Write out commit message to a file |
449 | with tempfile.NamedTemporaryFile('w', delete=False) as tf: | |
450 | tmpfile = tf.name | |
451 | for line in outlines: | |
452 | tf.write(line) | |
453 | # Prepare git command | |
765a9017 PE |
454 | cmd = ["git"] |
455 | GitApplyTree.gitCommandUserOptions(cmd, commituser, commitemail) | |
f5e6183b | 456 | cmd += ["commit", "-F", tmpfile, "--no-verify"] |
8c522846 PE |
457 | # git doesn't like plain email addresses as authors |
458 | if author and '<' in author: | |
459 | cmd.append('--author="%s"' % author) | |
460 | if date: | |
461 | cmd.append('--date="%s"' % date) | |
462 | return (tmpfile, cmd) | |
463 | ||
f5e6183b PK |
464 | @staticmethod |
465 | def addNote(repo, ref, key, value=None): | |
466 | note = key + (": %s" % value if value else "") | |
467 | notes_ref = GitApplyTree.notes_ref | |
468 | runcmd(["git", "config", "notes.rewriteMode", "ignore"], repo) | |
469 | runcmd(["git", "config", "notes.displayRef", notes_ref, notes_ref], repo) | |
470 | runcmd(["git", "config", "notes.rewriteRef", notes_ref, notes_ref], repo) | |
471 | runcmd(["git", "notes", "--ref", notes_ref, "append", "-m", note, ref], repo) | |
472 | ||
473 | @staticmethod | |
474 | def removeNote(repo, ref, key): | |
475 | notes = GitApplyTree.getNotes(repo, ref) | |
476 | notes = {k: v for k, v in notes.items() if k != key and not k.startswith(key + ":")} | |
477 | runcmd(["git", "notes", "--ref", GitApplyTree.notes_ref, "remove", "--ignore-missing", ref], repo) | |
478 | for note, value in notes.items(): | |
479 | GitApplyTree.addNote(repo, ref, note, value) | |
480 | ||
481 | @staticmethod | |
482 | def getNotes(repo, ref): | |
483 | import re | |
484 | ||
485 | note = None | |
486 | try: | |
487 | note = runcmd(["git", "notes", "--ref", GitApplyTree.notes_ref, "show", ref], repo) | |
488 | prefix = "" | |
489 | except CmdError: | |
490 | note = runcmd(['git', 'show', '-s', '--format=%B', ref], repo) | |
491 | prefix = "%% " | |
492 | ||
493 | note_re = re.compile(r'^%s(.*?)(?::\s*(.*))?$' % prefix) | |
494 | notes = dict() | |
495 | for line in note.splitlines(): | |
496 | m = note_re.match(line) | |
497 | if m: | |
498 | notes[m.group(1)] = m.group(2) | |
499 | ||
500 | return notes | |
501 | ||
94f0838b PK |
502 | @staticmethod |
503 | def commitIgnored(subject, dir=None, files=None, d=None): | |
504 | if files: | |
505 | runcmd(['git', 'add'] + files, dir) | |
94f0838b PK |
506 | cmd = ["git"] |
507 | GitApplyTree.gitCommandUserOptions(cmd, d=d) | |
f5e6183b | 508 | cmd += ["commit", "-m", subject, "--no-verify"] |
94f0838b | 509 | runcmd(cmd, dir) |
f5e6183b | 510 | GitApplyTree.addNote(dir, "HEAD", GitApplyTree.ignore_commit) |
94f0838b | 511 | |
765b7bad | 512 | @staticmethod |
900129cb | 513 | def extractPatches(tree, startcommits, outdir, paths=None): |
765b7bad PE |
514 | import tempfile |
515 | import shutil | |
516 | tempdir = tempfile.mkdtemp(prefix='oepatch') | |
517 | try: | |
900129cb JS |
518 | for name, rev in startcommits.items(): |
519 | shellcmd = ["git", "format-patch", "--no-signature", "--no-numbered", rev, "-o", tempdir] | |
520 | if paths: | |
521 | shellcmd.append('--') | |
522 | shellcmd.extend(paths) | |
523 | out = runcmd(["sh", "-c", " ".join(shellcmd)], os.path.join(tree, name)) | |
524 | if out: | |
525 | for srcfile in out.split(): | |
f5e6183b PK |
526 | # This loop, which is used to remove any line that |
527 | # starts with "%% original patch", is kept for backwards | |
528 | # compatibility. If/when that compatibility is dropped, | |
529 | # it can be replaced with code to just read the first | |
530 | # line of the patch file to get the SHA-1, and the code | |
531 | # below that writes the modified patch file can be | |
532 | # replaced with a simple file move. | |
900129cb JS |
533 | for encoding in ['utf-8', 'latin-1']: |
534 | patchlines = [] | |
900129cb | 535 | try: |
58f84549 | 536 | with open(srcfile, 'r', encoding=encoding, newline='') as f: |
900129cb | 537 | for line in f: |
f5e6183b | 538 | if line.startswith("%% " + GitApplyTree.original_patch): |
900129cb | 539 | continue |
900129cb JS |
540 | patchlines.append(line) |
541 | except UnicodeDecodeError: | |
542 | continue | |
543 | break | |
544 | else: | |
545 | raise PatchError('Unable to find a character encoding to decode %s' % srcfile) | |
546 | ||
f5e6183b PK |
547 | sha1 = patchlines[0].split()[1] |
548 | notes = GitApplyTree.getNotes(os.path.join(tree, name), sha1) | |
549 | if GitApplyTree.ignore_commit in notes: | |
550 | continue | |
551 | outfile = notes.get(GitApplyTree.original_patch, os.path.basename(srcfile)) | |
552 | ||
553 | bb.utils.mkdirhier(os.path.join(outdir, name)) | |
554 | with open(os.path.join(outdir, name, outfile), 'w') as of: | |
555 | for line in patchlines: | |
556 | of.write(line) | |
765b7bad PE |
557 | finally: |
558 | shutil.rmtree(tempdir) | |
559 | ||
d86cac27 FM |
560 | def _need_dirty_check(self): |
561 | fetch = bb.fetch2.Fetch([], self.d) | |
562 | check_dirtyness = False | |
563 | for url in fetch.urls: | |
564 | url_data = fetch.ud[url] | |
565 | parm = url_data.parm | |
566 | # a git url with subpath param will surely be dirty | |
567 | # since the git tree from which we clone will be emptied | |
568 | # from all files that are not in the subpath | |
569 | if url_data.type == 'git' and parm.get('subpath'): | |
570 | check_dirtyness = True | |
571 | return check_dirtyness | |
572 | ||
573 | def _commitpatch(self, patch, patchfilevar): | |
574 | output = "" | |
575 | # Add all files | |
576 | shellcmd = ["git", "add", "-f", "-A", "."] | |
577 | output += runcmd(["sh", "-c", " ".join(shellcmd)], self.dir) | |
578 | # Exclude the patches directory | |
579 | shellcmd = ["git", "reset", "HEAD", self.patchdir] | |
580 | output += runcmd(["sh", "-c", " ".join(shellcmd)], self.dir) | |
581 | # Commit the result | |
582 | (tmpfile, shellcmd) = self.prepareCommit(patch['file'], self.commituser, self.commitemail) | |
583 | try: | |
584 | shellcmd.insert(0, patchfilevar) | |
585 | output += runcmd(["sh", "-c", " ".join(shellcmd)], self.dir) | |
586 | finally: | |
587 | os.remove(tmpfile) | |
588 | return output | |
589 | ||
ac023d77 | 590 | def _applypatch(self, patch, force = False, reverse = False, run = True): |
765b7bad PE |
591 | import shutil |
592 | ||
3a14b094 LP |
593 | def _applypatchhelper(shellcmd, patch, force = False, reverse = False, run = True): |
594 | if reverse: | |
595 | shellcmd.append('-R') | |
ac023d77 | 596 | |
3a14b094 | 597 | shellcmd.append(patch['file']) |
ac023d77 | 598 | |
3a14b094 LP |
599 | if not run: |
600 | return "sh" + "-c" + " ".join(shellcmd) | |
ac023d77 | 601 | |
3a14b094 | 602 | return runcmd(["sh", "-c", " ".join(shellcmd)], self.dir) |
ac023d77 | 603 | |
d820303f PE |
604 | reporoot = (runcmd("git rev-parse --show-toplevel".split(), self.dir) or '').strip() |
605 | if not reporoot: | |
606 | raise Exception("Cannot get repository root for directory %s" % self.dir) | |
f5e6183b PK |
607 | |
608 | patch_applied = True | |
3a14b094 | 609 | try: |
765b7bad | 610 | patchfilevar = 'PATCHFILE="%s"' % os.path.basename(patch['file']) |
d86cac27 FM |
611 | if self._need_dirty_check(): |
612 | # Check dirtyness of the tree | |
613 | try: | |
614 | output = runcmd(["git", "--work-tree=%s" % reporoot, "status", "--short"]) | |
615 | except CmdError: | |
616 | pass | |
617 | else: | |
618 | if output: | |
f5e6183b | 619 | # The tree is dirty, no need to try to apply patches with git anymore |
d86cac27 FM |
620 | # since they fail, fallback directly to patch |
621 | output = PatchTree._applypatch(self, patch, force, reverse, run) | |
622 | output += self._commitpatch(patch, patchfilevar) | |
623 | return output | |
a8143f33 | 624 | try: |
765a9017 PE |
625 | shellcmd = [patchfilevar, "git", "--work-tree=%s" % reporoot] |
626 | self.gitCommandUserOptions(shellcmd, self.commituser, self.commitemail) | |
13ea33fb | 627 | shellcmd += ["am", "-3", "--keep-cr", "--no-scissors", "-p%s" % patch['strippath']] |
765b7bad | 628 | return _applypatchhelper(shellcmd, patch, force, reverse, run) |
a8143f33 | 629 | except CmdError: |
765b7bad PE |
630 | # Need to abort the git am, or we'll still be within it at the end |
631 | try: | |
91d76e63 | 632 | shellcmd = ["git", "--work-tree=%s" % reporoot, "am", "--abort"] |
765b7bad PE |
633 | runcmd(["sh", "-c", " ".join(shellcmd)], self.dir) |
634 | except CmdError: | |
635 | pass | |
21fdbd76 PE |
636 | # git am won't always clean up after itself, sadly, so... |
637 | shellcmd = ["git", "--work-tree=%s" % reporoot, "reset", "--hard", "HEAD"] | |
638 | runcmd(["sh", "-c", " ".join(shellcmd)], self.dir) | |
639 | # Also need to take care of any stray untracked files | |
640 | shellcmd = ["git", "--work-tree=%s" % reporoot, "clean", "-f"] | |
641 | runcmd(["sh", "-c", " ".join(shellcmd)], self.dir) | |
642 | ||
765b7bad | 643 | # Fall back to git apply |
91d76e63 | 644 | shellcmd = ["git", "--git-dir=%s" % reporoot, "apply", "-p%s" % patch['strippath']] |
765b7bad PE |
645 | try: |
646 | output = _applypatchhelper(shellcmd, patch, force, reverse, run) | |
647 | except CmdError: | |
648 | # Fall back to patch | |
649 | output = PatchTree._applypatch(self, patch, force, reverse, run) | |
d86cac27 | 650 | output += self._commitpatch(patch, patchfilevar) |
765b7bad | 651 | return output |
f5e6183b PK |
652 | except: |
653 | patch_applied = False | |
654 | raise | |
765b7bad | 655 | finally: |
f5e6183b PK |
656 | if patch_applied: |
657 | GitApplyTree.addNote(self.dir, "HEAD", GitApplyTree.original_patch, os.path.basename(patch['file'])) | |
ac023d77 JL |
658 | |
659 | ||
660 | class QuiltTree(PatchSet): | |
661 | def _runcmd(self, args, run = True): | |
7c552996 | 662 | quiltrc = self.d.getVar('QUILTRCFILE') |
ac023d77 JL |
663 | if not run: |
664 | return ["quilt"] + ["--quiltrc"] + [quiltrc] + args | |
665 | runcmd(["quilt"] + ["--quiltrc"] + [quiltrc] + args, self.dir) | |
666 | ||
667 | def _quiltpatchpath(self, file): | |
668 | return os.path.join(self.dir, "patches", os.path.basename(file)) | |
669 | ||
670 | ||
671 | def __init__(self, dir, d): | |
672 | PatchSet.__init__(self, dir, d) | |
673 | self.initialized = False | |
674 | p = os.path.join(self.dir, 'patches') | |
675 | if not os.path.exists(p): | |
676 | os.makedirs(p) | |
677 | ||
678 | def Clean(self): | |
679 | try: | |
c9d36882 MJ |
680 | # make sure that patches/series file exists before quilt pop to keep quilt-0.67 happy |
681 | open(os.path.join(self.dir, "patches","series"), 'a').close() | |
ac023d77 | 682 | self._runcmd(["pop", "-a", "-f"]) |
fd07744a | 683 | oe.path.remove(os.path.join(self.dir, "patches","series")) |
ac023d77 JL |
684 | except Exception: |
685 | pass | |
686 | self.initialized = True | |
687 | ||
688 | def InitFromDir(self): | |
689 | # read series -> self.patches | |
690 | seriespath = os.path.join(self.dir, 'patches', 'series') | |
691 | if not os.path.exists(self.dir): | |
ce6c80a1 | 692 | raise NotFoundError(self.dir) |
ac023d77 | 693 | if os.path.exists(seriespath): |
99ac382d PE |
694 | with open(seriespath, 'r') as f: |
695 | for line in f.readlines(): | |
696 | patch = {} | |
697 | parts = line.strip().split() | |
698 | patch["quiltfile"] = self._quiltpatchpath(parts[0]) | |
699 | patch["quiltfilemd5"] = bb.utils.md5_file(patch["quiltfile"]) | |
700 | if len(parts) > 1: | |
701 | patch["strippath"] = parts[1][2:] | |
702 | self.patches.append(patch) | |
ac023d77 JL |
703 | |
704 | # determine which patches are applied -> self._current | |
705 | try: | |
706 | output = runcmd(["quilt", "applied"], self.dir) | |
707 | except CmdError: | |
708 | import sys | |
709 | if sys.exc_value.output.strip() == "No patches applied": | |
710 | return | |
711 | else: | |
ce6c80a1 | 712 | raise |
ac023d77 JL |
713 | output = [val for val in output.split('\n') if not val.startswith('#')] |
714 | for patch in self.patches: | |
715 | if os.path.basename(patch["quiltfile"]) == output[-1]: | |
716 | self._current = self.patches.index(patch) | |
717 | self.initialized = True | |
718 | ||
719 | def Import(self, patch, force = None): | |
720 | if not self.initialized: | |
721 | self.InitFromDir() | |
722 | PatchSet.Import(self, patch, force) | |
4741b90b | 723 | oe.path.symlink(patch["file"], self._quiltpatchpath(patch["file"]), force=True) |
99ac382d PE |
724 | with open(os.path.join(self.dir, "patches", "series"), "a") as f: |
725 | f.write(os.path.basename(patch["file"]) + " -p" + patch["strippath"] + "\n") | |
ac023d77 JL |
726 | patch["quiltfile"] = self._quiltpatchpath(patch["file"]) |
727 | patch["quiltfilemd5"] = bb.utils.md5_file(patch["quiltfile"]) | |
728 | ||
729 | # TODO: determine if the file being imported: | |
730 | # 1) is already imported, and is the same | |
731 | # 2) is already imported, but differs | |
732 | ||
733 | self.patches.insert(self._current or 0, patch) | |
734 | ||
735 | ||
736 | def Push(self, force = False, all = False, run = True): | |
737 | # quilt push [-f] | |
738 | ||
739 | args = ["push"] | |
740 | if force: | |
741 | args.append("-f") | |
742 | if all: | |
743 | args.append("-a") | |
744 | if not run: | |
745 | return self._runcmd(args, run) | |
746 | ||
747 | self._runcmd(args) | |
748 | ||
749 | if self._current is not None: | |
750 | self._current = self._current + 1 | |
751 | else: | |
752 | self._current = 0 | |
753 | ||
754 | def Pop(self, force = None, all = None): | |
755 | # quilt pop [-f] | |
756 | args = ["pop"] | |
757 | if force: | |
758 | args.append("-f") | |
759 | if all: | |
760 | args.append("-a") | |
761 | ||
762 | self._runcmd(args) | |
763 | ||
764 | if self._current == 0: | |
765 | self._current = None | |
766 | ||
767 | if self._current is not None: | |
768 | self._current = self._current - 1 | |
769 | ||
770 | def Refresh(self, **kwargs): | |
771 | if kwargs.get("remote"): | |
772 | patch = self.patches[kwargs["patch"]] | |
773 | if not patch: | |
774 | raise PatchError("No patch found at index %s in patchset." % kwargs["patch"]) | |
6a39835a | 775 | (type, host, path, user, pswd, parm) = bb.fetch.decodeurl(patch["remote"]) |
ac023d77 JL |
776 | if type == "file": |
777 | import shutil | |
778 | if not patch.get("file") and patch.get("remote"): | |
984e90f4 | 779 | patch["file"] = bb.fetch2.localpath(patch["remote"], self.d) |
ac023d77 JL |
780 | |
781 | shutil.copyfile(patch["quiltfile"], patch["file"]) | |
782 | else: | |
783 | raise PatchError("Unable to do a remote refresh of %s, unsupported remote url scheme %s." % (os.path.basename(patch["quiltfile"]), type)) | |
784 | else: | |
785 | # quilt refresh | |
786 | args = ["refresh"] | |
787 | if kwargs.get("quiltfile"): | |
788 | args.append(os.path.basename(kwargs["quiltfile"])) | |
789 | elif kwargs.get("patch"): | |
790 | args.append(os.path.basename(self.patches[kwargs["patch"]]["quiltfile"])) | |
791 | self._runcmd(args) | |
792 | ||
793 | class Resolver(object): | |
9e0a21dd | 794 | def __init__(self, patchset, terminal): |
ac023d77 JL |
795 | raise NotImplementedError() |
796 | ||
797 | def Resolve(self): | |
798 | raise NotImplementedError() | |
799 | ||
800 | def Revert(self): | |
801 | raise NotImplementedError() | |
802 | ||
803 | def Finalize(self): | |
804 | raise NotImplementedError() | |
805 | ||
806 | class NOOPResolver(Resolver): | |
9e0a21dd | 807 | def __init__(self, patchset, terminal): |
ac023d77 | 808 | self.patchset = patchset |
9e0a21dd | 809 | self.terminal = terminal |
ac023d77 JL |
810 | |
811 | def Resolve(self): | |
812 | olddir = os.path.abspath(os.curdir) | |
813 | os.chdir(self.patchset.dir) | |
814 | try: | |
815 | self.patchset.Push() | |
816 | except Exception: | |
817 | import sys | |
ce6c80a1 | 818 | raise |
cdc40292 RB |
819 | finally: |
820 | os.chdir(olddir) | |
ac023d77 JL |
821 | |
822 | # Patch resolver which relies on the user doing all the work involved in the | |
823 | # resolution, with the exception of refreshing the remote copy of the patch | |
824 | # files (the urls). | |
825 | class UserResolver(Resolver): | |
9e0a21dd | 826 | def __init__(self, patchset, terminal): |
ac023d77 | 827 | self.patchset = patchset |
9e0a21dd | 828 | self.terminal = terminal |
ac023d77 JL |
829 | |
830 | # Force a push in the patchset, then drop to a shell for the user to | |
831 | # resolve any rejected hunks | |
832 | def Resolve(self): | |
ac023d77 JL |
833 | olddir = os.path.abspath(os.curdir) |
834 | os.chdir(self.patchset.dir) | |
835 | try: | |
836 | self.patchset.Push(False) | |
b010501c | 837 | except CmdError as v: |
ac023d77 JL |
838 | # Patch application failed |
839 | patchcmd = self.patchset.Push(True, False, False) | |
840 | ||
7c552996 | 841 | t = self.patchset.d.getVar('T') |
ac023d77 | 842 | if not t: |
2383e06c | 843 | bb.msg.fatal("Build", "T not set") |
cd28d5f5 | 844 | bb.utils.mkdirhier(t) |
ac023d77 JL |
845 | import random |
846 | rcfile = "%s/bashrc.%s.%s" % (t, str(os.getpid()), random.random()) | |
99ac382d PE |
847 | with open(rcfile, "w") as f: |
848 | f.write("echo '*** Manual patch resolution mode ***'\n") | |
849 | f.write("echo 'Dropping to a shell, so patch rejects can be fixed manually.'\n") | |
850 | f.write("echo 'Run \"quilt refresh\" when patch is corrected, press CTRL+D to exit.'\n") | |
851 | f.write("echo ''\n") | |
852 | f.write(" ".join(patchcmd) + "\n") | |
737a095f | 853 | os.chmod(rcfile, 0o775) |
ac023d77 | 854 | |
9e0a21dd | 855 | self.terminal("bash --rcfile " + rcfile, 'Patch Rejects: Please fix patch rejects manually', self.patchset.d) |
ac023d77 JL |
856 | |
857 | # Construct a new PatchSet after the user's changes, compare the | |
858 | # sets, checking patches for modifications, and doing a remote | |
859 | # refresh on each. | |
860 | oldpatchset = self.patchset | |
861 | self.patchset = oldpatchset.__class__(self.patchset.dir, self.patchset.d) | |
862 | ||
863 | for patch in self.patchset.patches: | |
864 | oldpatch = None | |
865 | for opatch in oldpatchset.patches: | |
866 | if opatch["quiltfile"] == patch["quiltfile"]: | |
867 | oldpatch = opatch | |
868 | ||
869 | if oldpatch: | |
870 | patch["remote"] = oldpatch["remote"] | |
871 | if patch["quiltfile"] == oldpatch["quiltfile"]: | |
872 | if patch["quiltfilemd5"] != oldpatch["quiltfilemd5"]: | |
873 | bb.note("Patch %s has changed, updating remote url %s" % (os.path.basename(patch["quiltfile"]), patch["remote"])) | |
874 | # user change? remote refresh | |
875 | self.patchset.Refresh(remote=True, patch=self.patchset.patches.index(patch)) | |
876 | else: | |
877 | # User did not fix the problem. Abort. | |
878 | raise PatchError("Patch application failed, and user did not fix and refresh the patch.") | |
879 | except Exception: | |
ac023d77 | 880 | raise |
cdc40292 RB |
881 | finally: |
882 | os.chdir(olddir) | |
2724511e PE |
883 | |
884 | ||
86fec41b | 885 | def patch_path(url, fetch, unpackdir, expand=True): |
cc4a75c7 | 886 | """Return the local path of a patch, or return nothing if this isn't a patch""" |
2724511e PE |
887 | |
888 | local = fetch.localpath(url) | |
1e38d74a TD |
889 | if os.path.isdir(local): |
890 | return | |
2724511e | 891 | base, ext = os.path.splitext(os.path.basename(local)) |
f50fd7f2 | 892 | if ext in ('.gz', '.bz2', '.xz', '.Z'): |
2724511e | 893 | if expand: |
86fec41b | 894 | local = os.path.join(unpackdir, base) |
2724511e PE |
895 | ext = os.path.splitext(base)[1] |
896 | ||
897 | urldata = fetch.ud[url] | |
898 | if "apply" in urldata.parm: | |
899 | apply = oe.types.boolean(urldata.parm["apply"]) | |
900 | if not apply: | |
901 | return | |
902 | elif ext not in (".diff", ".patch"): | |
903 | return | |
904 | ||
905 | return local | |
906 | ||
907 | def src_patches(d, all=False, expand=True): | |
86fec41b | 908 | unpackdir = d.getVar('UNPACKDIR') |
2724511e PE |
909 | fetch = bb.fetch2.Fetch([], d) |
910 | patches = [] | |
911 | sources = [] | |
912 | for url in fetch.urls: | |
86fec41b | 913 | local = patch_path(url, fetch, unpackdir, expand) |
2724511e PE |
914 | if not local: |
915 | if all: | |
916 | local = fetch.localpath(url) | |
917 | sources.append(local) | |
918 | continue | |
919 | ||
920 | urldata = fetch.ud[url] | |
921 | parm = urldata.parm | |
922 | patchname = parm.get('pname') or os.path.basename(local) | |
923 | ||
924 | apply, reason = should_apply(parm, d) | |
925 | if not apply: | |
926 | if reason: | |
927 | bb.note("Patch %s %s" % (patchname, reason)) | |
928 | continue | |
929 | ||
930 | patchparm = {'patchname': patchname} | |
931 | if "striplevel" in parm: | |
932 | striplevel = parm["striplevel"] | |
933 | elif "pnum" in parm: | |
934 | #bb.msg.warn(None, "Deprecated usage of 'pnum' url parameter in '%s', please use 'striplevel'" % url) | |
935 | striplevel = parm["pnum"] | |
936 | else: | |
937 | striplevel = '1' | |
938 | patchparm['striplevel'] = striplevel | |
939 | ||
940 | patchdir = parm.get('patchdir') | |
941 | if patchdir: | |
942 | patchparm['patchdir'] = patchdir | |
943 | ||
944 | localurl = bb.fetch.encodeurl(('file', '', local, '', '', patchparm)) | |
945 | patches.append(localurl) | |
946 | ||
947 | if all: | |
948 | return sources | |
949 | ||
950 | return patches | |
951 | ||
952 | ||
953 | def should_apply(parm, d): | |
3fac9f88 | 954 | import bb.utils |
2724511e | 955 | if "mindate" in parm or "maxdate" in parm: |
7c552996 JL |
956 | pn = d.getVar('PN') |
957 | srcdate = d.getVar('SRCDATE_%s' % pn) | |
2724511e | 958 | if not srcdate: |
7c552996 | 959 | srcdate = d.getVar('SRCDATE') |
2724511e PE |
960 | |
961 | if srcdate == "now": | |
7c552996 | 962 | srcdate = d.getVar('DATE') |
2724511e PE |
963 | |
964 | if "maxdate" in parm and parm["maxdate"] < srcdate: | |
965 | return False, 'is outdated' | |
966 | ||
967 | if "mindate" in parm and parm["mindate"] > srcdate: | |
968 | return False, 'is predated' | |
969 | ||
970 | ||
971 | if "minrev" in parm: | |
7c552996 | 972 | srcrev = d.getVar('SRCREV') |
2724511e PE |
973 | if srcrev and srcrev < parm["minrev"]: |
974 | return False, 'applies to later revisions' | |
975 | ||
976 | if "maxrev" in parm: | |
7c552996 | 977 | srcrev = d.getVar('SRCREV') |
2724511e PE |
978 | if srcrev and srcrev > parm["maxrev"]: |
979 | return False, 'applies to earlier revisions' | |
980 | ||
981 | if "rev" in parm: | |
7c552996 | 982 | srcrev = d.getVar('SRCREV') |
2724511e PE |
983 | if srcrev and parm["rev"] not in srcrev: |
984 | return False, "doesn't apply to revision" | |
985 | ||
986 | if "notrev" in parm: | |
7c552996 | 987 | srcrev = d.getVar('SRCREV') |
2724511e PE |
988 | if srcrev and parm["notrev"] in srcrev: |
989 | return False, "doesn't apply to revision" | |
990 | ||
3fac9f88 RB |
991 | if "maxver" in parm: |
992 | pv = d.getVar('PV') | |
993 | if bb.utils.vercmp_string_op(pv, parm["maxver"], ">"): | |
994 | return False, "applies to earlier version" | |
995 | ||
996 | if "minver" in parm: | |
997 | pv = d.getVar('PV') | |
998 | if bb.utils.vercmp_string_op(pv, parm["minver"], "<"): | |
999 | return False, "applies to later version" | |
1000 | ||
2724511e | 1001 | return True, None |