]>
Commit | Line | Data |
---|---|---|
4020f5d9 LS |
1 | # Development tool - upgrade command plugin |
2 | # | |
802829f1 | 3 | # Copyright (C) 2014-2017 Intel Corporation |
4020f5d9 | 4 | # |
f8c9c511 | 5 | # SPDX-License-Identifier: GPL-2.0-only |
4020f5d9 LS |
6 | # |
7 | """Devtool upgrade plugin""" | |
8 | ||
9 | import os | |
10 | import sys | |
11 | import re | |
12 | import shutil | |
13 | import tempfile | |
14 | import logging | |
15 | import argparse | |
16 | import scriptutils | |
17 | import errno | |
18 | import bb | |
5f140359 LH |
19 | |
20 | devtool_path = os.path.dirname(os.path.realpath(__file__)) + '/../../../meta/lib' | |
21 | sys.path = sys.path + [devtool_path] | |
22 | ||
4020f5d9 LS |
23 | import oe.recipeutils |
24 | from devtool import standard | |
92c4d9be | 25 | from devtool import exec_build_env_command, setup_tinfoil, DevtoolError, parse_recipe, use_external_build, update_unlockedsigs, check_prerelease_version |
4020f5d9 LS |
26 | |
27 | logger = logging.getLogger('devtool') | |
28 | ||
4020f5d9 LS |
29 | def _run(cmd, cwd=''): |
30 | logger.debug("Running command %s> %s" % (cwd,cmd)) | |
31 | return bb.process.run('%s' % cmd, cwd=cwd) | |
32 | ||
33 | def _get_srctree(tmpdir): | |
34 | srctree = tmpdir | |
0d642861 | 35 | dirs = scriptutils.filter_src_subdirs(tmpdir) |
4020f5d9 LS |
36 | if len(dirs) == 1: |
37 | srctree = os.path.join(tmpdir, dirs[0]) | |
38 | return srctree | |
39 | ||
40 | def _copy_source_code(orig, dest): | |
41 | for path in standard._ls_tree(orig): | |
42 | dest_dir = os.path.join(dest, os.path.dirname(path)) | |
43 | bb.utils.mkdirhier(dest_dir) | |
44 | dest_path = os.path.join(dest, path) | |
479c8eb6 | 45 | shutil.move(os.path.join(orig, path), dest_path) |
4020f5d9 | 46 | |
4020f5d9 LS |
47 | def _remove_patch_dirs(recipefolder): |
48 | for root, dirs, files in os.walk(recipefolder): | |
49 | for d in dirs: | |
50 | shutil.rmtree(os.path.join(root,d)) | |
51 | ||
8b8f0422 | 52 | def _recipe_contains(rd, var): |
0a36bd96 | 53 | rf = rd.getVar('FILE') |
8b8f0422 | 54 | varfiles = oe.recipeutils.get_var_files(rf, [var], rd) |
25d4d827 | 55 | for var, fn in varfiles.items(): |
8b8f0422 PE |
56 | if fn and fn.startswith(os.path.dirname(rf) + os.sep): |
57 | return True | |
58 | return False | |
4020f5d9 LS |
59 | |
60 | def _rename_recipe_dirs(oldpv, newpv, path): | |
61 | for root, dirs, files in os.walk(path): | |
3c623aac | 62 | # Rename directories with the version in their name |
4020f5d9 LS |
63 | for olddir in dirs: |
64 | if olddir.find(oldpv) != -1: | |
65 | newdir = olddir.replace(oldpv, newpv) | |
66 | if olddir != newdir: | |
60454a0b | 67 | shutil.move(os.path.join(path, olddir), os.path.join(path, newdir)) |
3c623aac PE |
68 | # Rename any inc files with the version in their name (unusual, but possible) |
69 | for oldfile in files: | |
70 | if oldfile.endswith('.inc'): | |
71 | if oldfile.find(oldpv) != -1: | |
72 | newfile = oldfile.replace(oldpv, newpv) | |
73 | if oldfile != newfile: | |
656a65b2 DT |
74 | bb.utils.rename(os.path.join(path, oldfile), |
75 | os.path.join(path, newfile)) | |
4020f5d9 | 76 | |
e144db36 PE |
77 | def _rename_recipe_file(oldrecipe, bpn, oldpv, newpv, path): |
78 | oldrecipe = os.path.basename(oldrecipe) | |
79 | if oldrecipe.endswith('_%s.bb' % oldpv): | |
80 | newrecipe = '%s_%s.bb' % (bpn, newpv) | |
4020f5d9 | 81 | if oldrecipe != newrecipe: |
e144db36 | 82 | shutil.move(os.path.join(path, oldrecipe), os.path.join(path, newrecipe)) |
4020f5d9 | 83 | else: |
e144db36 | 84 | newrecipe = oldrecipe |
4020f5d9 LS |
85 | return os.path.join(path, newrecipe) |
86 | ||
e144db36 | 87 | def _rename_recipe_files(oldrecipe, bpn, oldpv, newpv, path): |
4020f5d9 | 88 | _rename_recipe_dirs(oldpv, newpv, path) |
e144db36 | 89 | return _rename_recipe_file(oldrecipe, bpn, oldpv, newpv, path) |
4020f5d9 | 90 | |
92eb42c3 | 91 | def _write_append(rc, srctree, same_dir, no_same_dir, rev, copied, workspace, d): |
4020f5d9 LS |
92 | """Writes an append file""" |
93 | if not os.path.exists(rc): | |
94 | raise DevtoolError("bbappend not created because %s does not exist" % rc) | |
95 | ||
96 | appendpath = os.path.join(workspace, 'appends') | |
97 | if not os.path.exists(appendpath): | |
98 | bb.utils.mkdirhier(appendpath) | |
99 | ||
100 | brf = os.path.basename(os.path.splitext(rc)[0]) # rc basename | |
101 | ||
102 | srctree = os.path.abspath(srctree) | |
688f7a64 | 103 | pn = d.getVar('PN') |
4020f5d9 LS |
104 | af = os.path.join(appendpath, '%s.bbappend' % brf) |
105 | with open(af, 'w') as f: | |
42344347 | 106 | f.write('FILESEXTRAPATHS:prepend := "${THISDIR}/${PN}:"\n\n') |
4020f5d9 LS |
107 | f.write('inherit externalsrc\n') |
108 | f.write(('# NOTE: We use pn- overrides here to avoid affecting' | |
109 | 'multiple variants in the case where the recipe uses BBCLASSEXTEND\n')) | |
42344347 | 110 | f.write('EXTERNALSRC:pn-%s = "%s"\n' % (pn, srctree)) |
320585b7 PE |
111 | b_is_s = use_external_build(same_dir, no_same_dir, d) |
112 | if b_is_s: | |
42344347 | 113 | f.write('EXTERNALSRC_BUILD:pn-%s = "%s"\n' % (pn, srctree)) |
92eb42c3 | 114 | f.write('\n') |
4020f5d9 | 115 | if rev: |
92eb42c3 PE |
116 | f.write('# initial_rev: %s\n' % rev) |
117 | if copied: | |
0a36bd96 | 118 | f.write('# original_path: %s\n' % os.path.dirname(d.getVar('FILE'))) |
92eb42c3 | 119 | f.write('# original_files: %s\n' % ' '.join(copied)) |
4020f5d9 LS |
120 | return af |
121 | ||
122 | def _cleanup_on_error(rf, srctree): | |
123 | rfp = os.path.split(rf)[0] # recipe folder | |
124 | rfpp = os.path.split(rfp)[0] # recipes folder | |
125 | if os.path.exists(rfp): | |
c519ac36 | 126 | shutil.rmtree(rfp) |
4020f5d9 LS |
127 | if not len(os.listdir(rfpp)): |
128 | os.rmdir(rfpp) | |
129 | srctree = os.path.abspath(srctree) | |
130 | if os.path.exists(srctree): | |
131 | shutil.rmtree(srctree) | |
132 | ||
c519ac36 PE |
133 | def _upgrade_error(e, rf, srctree, keep_failure=False, extramsg=None): |
134 | if rf and not keep_failure: | |
135 | _cleanup_on_error(rf, srctree) | |
4020f5d9 | 136 | logger.error(e) |
c519ac36 PE |
137 | if extramsg: |
138 | logger.error(extramsg) | |
139 | if keep_failure: | |
140 | logger.info('Preserving failed upgrade files (--keep-failure)') | |
141 | sys.exit(1) | |
4020f5d9 LS |
142 | |
143 | def _get_uri(rd): | |
0a36bd96 | 144 | srcuris = rd.getVar('SRC_URI').split() |
4020f5d9 LS |
145 | if not len(srcuris): |
146 | raise DevtoolError('SRC_URI not found on recipe') | |
c868198c PE |
147 | # Get first non-local entry in SRC_URI - usually by convention it's |
148 | # the first entry, but not always! | |
149 | srcuri = None | |
150 | for entry in srcuris: | |
151 | if not entry.startswith('file://'): | |
152 | srcuri = entry | |
153 | break | |
154 | if not srcuri: | |
155 | raise DevtoolError('Unable to find non-local entry in SRC_URI') | |
4020f5d9 LS |
156 | srcrev = '${AUTOREV}' |
157 | if '://' in srcuri: | |
158 | # Fetch a URL | |
159 | rev_re = re.compile(';rev=([^;]+)') | |
160 | res = rev_re.search(srcuri) | |
161 | if res: | |
162 | srcrev = res.group(1) | |
163 | srcuri = rev_re.sub('', srcuri) | |
164 | return srcuri, srcrev | |
165 | ||
29ced738 | 166 | def _extract_new_source(newpv, srctree, no_patch, srcrev, srcbranch, branch, keep_temp, tinfoil, rd): |
4020f5d9 LS |
167 | """Extract sources of a recipe with a new version""" |
168 | ||
169 | def __run(cmd): | |
170 | """Simple wrapper which calls _run with srctree as cwd""" | |
171 | return _run(cmd, srctree) | |
172 | ||
173 | crd = rd.createCopy() | |
174 | ||
0a36bd96 | 175 | pv = crd.getVar('PV') |
4020f5d9 LS |
176 | crd.setVar('PV', newpv) |
177 | ||
178 | tmpsrctree = None | |
179 | uri, rev = _get_uri(crd) | |
180 | if srcrev: | |
181 | rev = srcrev | |
e7076f17 | 182 | if uri.startswith('git://') or uri.startswith('gitsm://'): |
c4daebf3 | 183 | __run('git fetch') |
4020f5d9 LS |
184 | __run('git checkout %s' % rev) |
185 | __run('git tag -f devtool-base-new') | |
186 | md5 = None | |
187 | sha256 = None | |
d29881a6 PE |
188 | _, _, _, _, _, params = bb.fetch2.decodeurl(uri) |
189 | srcsubdir_rel = params.get('destsuffix', 'git') | |
29ced738 CRSF |
190 | if not srcbranch: |
191 | check_branch, check_branch_err = __run('git branch -r --contains %s' % srcrev) | |
192 | get_branch = [x.strip() for x in check_branch.splitlines()] | |
193 | # Remove HEAD reference point and drop remote prefix | |
194 | get_branch = [x.split('/', 1)[1] for x in get_branch if not x.startswith('origin/HEAD')] | |
e2f27f05 AK |
195 | if len(get_branch) == 1: |
196 | # If srcrev is on only ONE branch, then use that branch | |
29ced738 | 197 | srcbranch = get_branch[0] |
e2f27f05 AK |
198 | elif 'main' in get_branch: |
199 | # If srcrev is on multiple branches, then choose 'main' if it is one of them | |
200 | srcbranch = 'main' | |
201 | elif 'master' in get_branch: | |
202 | # Otherwise choose 'master' if it is one of the branches | |
203 | srcbranch = 'master' | |
29ced738 CRSF |
204 | else: |
205 | # If get_branch contains more than one objects, then display error and exit. | |
206 | mbrch = '\n ' + '\n '.join(get_branch) | |
207 | raise DevtoolError('Revision %s was found on multiple branches: %s\nPlease provide the correct branch in the devtool command with "--srcbranch" or "-B" option.' % (srcrev, mbrch)) | |
4020f5d9 | 208 | else: |
a4f3e6d5 | 209 | __run('git checkout devtool-base -b devtool-%s' % newpv) |
4020f5d9 LS |
210 | |
211 | tmpdir = tempfile.mkdtemp(prefix='devtool') | |
212 | try: | |
9a47a669 PE |
213 | checksums, ftmpdir = scriptutils.fetch_url(tinfoil, uri, rev, tmpdir, logger, preserve_tmp=keep_temp) |
214 | except scriptutils.FetchUrlFailure as e: | |
4020f5d9 LS |
215 | raise DevtoolError(e) |
216 | ||
9a47a669 PE |
217 | if ftmpdir and keep_temp: |
218 | logger.info('Fetch temp directory is %s' % ftmpdir) | |
219 | ||
220 | md5 = checksums['md5sum'] | |
221 | sha256 = checksums['sha256sum'] | |
222 | ||
4020f5d9 | 223 | tmpsrctree = _get_srctree(tmpdir) |
4f7595a5 | 224 | srctree = os.path.abspath(srctree) |
d29881a6 | 225 | srcsubdir_rel = os.path.relpath(tmpsrctree, tmpdir) |
4f7595a5 PE |
226 | |
227 | # Delete all sources so we ensure no stray files are left over | |
228 | for item in os.listdir(srctree): | |
229 | if item in ['.git', 'oe-local-files']: | |
230 | continue | |
231 | itempath = os.path.join(srctree, item) | |
232 | if os.path.isdir(itempath): | |
233 | shutil.rmtree(itempath) | |
234 | else: | |
235 | os.remove(itempath) | |
236 | ||
237 | # Copy in new ones | |
4020f5d9 LS |
238 | _copy_source_code(tmpsrctree, srctree) |
239 | ||
06a24a61 | 240 | (stdout,_) = __run('git ls-files --modified --others') |
8b184f6c PE |
241 | filelist = stdout.splitlines() |
242 | pbar = bb.ui.knotty.BBProgress('Adding changed files', len(filelist)) | |
243 | pbar.start() | |
244 | batchsize = 100 | |
245 | for i in range(0, len(filelist), batchsize): | |
246 | batch = filelist[i:i+batchsize] | |
06a24a61 | 247 | __run('git add -f -A %s' % ' '.join(['"%s"' % item for item in batch])) |
8b184f6c PE |
248 | pbar.update(i) |
249 | pbar.finish() | |
4020f5d9 | 250 | |
765a9017 PE |
251 | useroptions = [] |
252 | oe.patch.GitApplyTree.gitCommandUserOptions(useroptions, d=rd) | |
253 | __run('git %s commit -q -m "Commit of upstream changes at version %s" --allow-empty' % (' '.join(useroptions), newpv)) | |
4020f5d9 LS |
254 | __run('git tag -f devtool-base-%s' % newpv) |
255 | ||
256 | (stdout, _) = __run('git rev-parse HEAD') | |
257 | rev = stdout.rstrip() | |
258 | ||
259 | if no_patch: | |
260 | patches = oe.recipeutils.get_recipe_patches(crd) | |
2d11e9e6 | 261 | if patches: |
f467fd27 | 262 | logger.warning('By user choice, the following patches will NOT be applied to the new source tree:\n %s' % '\n '.join([os.path.basename(patch) for patch in patches])) |
4020f5d9 | 263 | else: |
2a76be95 | 264 | __run('git checkout devtool-patched -b %s' % branch) |
588d0474 AK |
265 | (stdout, _) = __run('git branch --list devtool-override-*') |
266 | branches_to_rebase = [branch] + stdout.split() | |
267 | for b in branches_to_rebase: | |
268 | logger.info("Rebasing {} onto {}".format(b, rev)) | |
269 | __run('git checkout %s' % b) | |
270 | try: | |
271 | __run('git rebase %s' % rev) | |
272 | except bb.process.ExecutionError as e: | |
273 | if 'conflict' in e.stdout: | |
274 | logger.warning('Command \'%s\' failed:\n%s\n\nYou will need to resolve conflicts in order to complete the upgrade.' % (e.command, e.stdout.rstrip())) | |
275 | __run('git rebase --abort') | |
276 | else: | |
277 | logger.warning('Command \'%s\' failed:\n%s' % (e.command, e.stdout)) | |
278 | __run('git checkout %s' % branch) | |
4020f5d9 LS |
279 | |
280 | if tmpsrctree: | |
281 | if keep_temp: | |
282 | logger.info('Preserving temporary directory %s' % tmpsrctree) | |
283 | else: | |
284 | shutil.rmtree(tmpsrctree) | |
0d642861 PE |
285 | if tmpdir != tmpsrctree: |
286 | shutil.rmtree(tmpdir) | |
4020f5d9 | 287 | |
d29881a6 | 288 | return (rev, md5, sha256, srcbranch, srcsubdir_rel) |
4020f5d9 | 289 | |
ccb37f9e AK |
290 | def _add_license_diff_to_recipe(path, diff): |
291 | notice_text = """# FIXME: the LIC_FILES_CHKSUM values have been updated by 'devtool upgrade'. | |
292 | # The following is the difference between the old and the new license text. | |
293 | # Please update the LICENSE value if needed, and summarize the changes in | |
3905ae20 AK |
294 | # the commit message via 'License-Update:' tag. |
295 | # (example: 'License-Update: copyright years updated.') | |
ccb37f9e AK |
296 | # |
297 | # The changes: | |
298 | # | |
299 | """ | |
300 | commented_diff = "\n".join(["# {}".format(l) for l in diff.split('\n')]) | |
301 | with open(path, 'rb') as f: | |
302 | orig_content = f.read() | |
303 | with open(path, 'wb') as f: | |
304 | f.write(notice_text.encode()) | |
305 | f.write(commented_diff.encode()) | |
306 | f.write("\n#\n\n".encode()) | |
307 | f.write(orig_content) | |
308 | ||
c519ac36 | 309 | def _create_new_recipe(newpv, md5, sha256, srcrev, srcbranch, srcsubdir_old, srcsubdir_new, workspace, tinfoil, rd, license_diff, new_licenses, srctree, keep_failure): |
4020f5d9 | 310 | """Creates the new recipe under workspace""" |
4020f5d9 | 311 | |
0a36bd96 | 312 | bpn = rd.getVar('BPN') |
4020f5d9 LS |
313 | path = os.path.join(workspace, 'recipes', bpn) |
314 | bb.utils.mkdirhier(path) | |
56bf5e93 | 315 | copied, _ = oe.recipeutils.copy_recipe_files(rd, path, all_variants=True) |
3861486a PE |
316 | if not copied: |
317 | raise DevtoolError('Internal error - no files were copied for recipe %s' % bpn) | |
318 | logger.debug('Copied %s to %s' % (copied, path)) | |
4020f5d9 | 319 | |
0a36bd96 | 320 | oldpv = rd.getVar('PV') |
4020f5d9 LS |
321 | if not newpv: |
322 | newpv = oldpv | |
0a36bd96 | 323 | origpath = rd.getVar('FILE') |
e144db36 PE |
324 | fullpath = _rename_recipe_files(origpath, bpn, oldpv, newpv, path) |
325 | logger.debug('Upgraded %s => %s' % (origpath, fullpath)) | |
4020f5d9 | 326 | |
8b8f0422 PE |
327 | newvalues = {} |
328 | if _recipe_contains(rd, 'PV') and newpv != oldpv: | |
329 | newvalues['PV'] = newpv | |
330 | ||
331 | if srcrev: | |
332 | newvalues['SRCREV'] = srcrev | |
333 | ||
e49a66fd PE |
334 | if srcbranch: |
335 | src_uri = oe.recipeutils.split_var_value(rd.getVar('SRC_URI', False) or '') | |
336 | changed = False | |
337 | replacing = True | |
338 | new_src_uri = [] | |
339 | for entry in src_uri: | |
340 | scheme, network, path, user, passwd, params = bb.fetch2.decodeurl(entry) | |
341 | if replacing and scheme in ['git', 'gitsm']: | |
342 | branch = params.get('branch', 'master') | |
343 | if rd.expand(branch) != srcbranch: | |
344 | # Handle case where branch is set through a variable | |
345 | res = re.match(r'\$\{([^}@]+)\}', branch) | |
346 | if res: | |
347 | newvalues[res.group(1)] = srcbranch | |
348 | # We know we won't change SRC_URI now, so break out | |
349 | break | |
350 | else: | |
351 | params['branch'] = srcbranch | |
352 | entry = bb.fetch2.encodeurl((scheme, network, path, user, passwd, params)) | |
353 | changed = True | |
354 | replacing = False | |
355 | new_src_uri.append(entry) | |
356 | if changed: | |
357 | newvalues['SRC_URI'] = ' '.join(new_src_uri) | |
358 | ||
4acf8a69 PE |
359 | newvalues['PR'] = None |
360 | ||
c914a5e1 PE |
361 | # Work out which SRC_URI entries have changed in case the entry uses a name |
362 | crd = rd.createCopy() | |
363 | crd.setVar('PV', newpv) | |
364 | for var, value in newvalues.items(): | |
365 | crd.setVar(var, value) | |
366 | old_src_uri = (rd.getVar('SRC_URI') or '').split() | |
367 | new_src_uri = (crd.getVar('SRC_URI') or '').split() | |
368 | newnames = [] | |
369 | addnames = [] | |
370 | for newentry in new_src_uri: | |
371 | _, _, _, _, _, params = bb.fetch2.decodeurl(newentry) | |
372 | if 'name' in params: | |
373 | newnames.append(params['name']) | |
374 | if newentry not in old_src_uri: | |
375 | addnames.append(params['name']) | |
376 | # Find what's been set in the original recipe | |
377 | oldnames = [] | |
378 | noname = False | |
379 | for varflag in rd.getVarFlags('SRC_URI'): | |
380 | if varflag.endswith(('.md5sum', '.sha256sum')): | |
381 | name = varflag.rsplit('.', 1)[0] | |
382 | if name not in oldnames: | |
383 | oldnames.append(name) | |
384 | elif varflag in ['md5sum', 'sha256sum']: | |
385 | noname = True | |
386 | # Even if SRC_URI has named entries it doesn't have to actually use the name | |
387 | if noname and addnames and addnames[0] not in oldnames: | |
388 | addnames = [] | |
389 | # Drop any old names (the name actually might include ${PV}) | |
390 | for name in oldnames: | |
391 | if name not in newnames: | |
392 | newvalues['SRC_URI[%s.md5sum]' % name] = None | |
393 | newvalues['SRC_URI[%s.sha256sum]' % name] = None | |
394 | ||
1609e77a | 395 | if sha256: |
c914a5e1 PE |
396 | if addnames: |
397 | nameprefix = '%s.' % addnames[0] | |
398 | else: | |
399 | nameprefix = '' | |
1609e77a | 400 | newvalues['SRC_URI[%smd5sum]' % nameprefix] = None |
c914a5e1 | 401 | newvalues['SRC_URI[%ssha256sum]' % nameprefix] = sha256 |
0f81b83f | 402 | |
d29881a6 PE |
403 | if srcsubdir_new != srcsubdir_old: |
404 | s_subdir_old = os.path.relpath(os.path.abspath(rd.getVar('S')), rd.getVar('WORKDIR')) | |
405 | s_subdir_new = os.path.relpath(os.path.abspath(crd.getVar('S')), crd.getVar('WORKDIR')) | |
406 | if srcsubdir_old == s_subdir_old and srcsubdir_new != s_subdir_new: | |
407 | # Subdir for old extracted source matches what S points to (it should!) | |
408 | # but subdir for new extracted source doesn't match what S will be | |
409 | newvalues['S'] = '${WORKDIR}/%s' % srcsubdir_new.replace(newpv, '${PV}') | |
410 | if crd.expand(newvalues['S']) == crd.expand('${WORKDIR}/${BP}'): | |
411 | # It's the default, drop it | |
412 | # FIXME what if S is being set in a .inc? | |
413 | newvalues['S'] = None | |
414 | logger.info('Source subdirectory has changed, dropping S value since it now matches the default ("${WORKDIR}/${BP}")') | |
415 | else: | |
416 | logger.info('Source subdirectory has changed, updating S value') | |
417 | ||
ccb37f9e | 418 | if license_diff: |
6c5cc1b2 CQ |
419 | newlicchksum = " ".join(["file://{}".format(l['path']) + |
420 | (";beginline={}".format(l['beginline']) if l['beginline'] else "") + | |
421 | (";endline={}".format(l['endline']) if l['endline'] else "") + | |
422 | (";md5={}".format(l['actual_md5'])) for l in new_licenses]) | |
ccb37f9e AK |
423 | newvalues["LIC_FILES_CHKSUM"] = newlicchksum |
424 | _add_license_diff_to_recipe(fullpath, license_diff) | |
425 | ||
c519ac36 PE |
426 | try: |
427 | rd = tinfoil.parse_recipe_file(fullpath, False) | |
428 | except bb.tinfoil.TinfoilCommandFailed as e: | |
429 | _upgrade_error(e, fullpath, srctree, keep_failure, 'Parsing of upgraded recipe failed') | |
4acf8a69 | 430 | oe.recipeutils.patch_recipe(rd, fullpath, newvalues) |
4020f5d9 | 431 | |
92eb42c3 | 432 | return fullpath, copied |
4020f5d9 | 433 | |
802829f1 PE |
434 | |
435 | def _check_git_config(): | |
436 | def getconfig(name): | |
437 | try: | |
438 | value = bb.process.run('git config --global %s' % name)[0].strip() | |
439 | except bb.process.ExecutionError as e: | |
440 | if e.exitcode == 1: | |
441 | value = None | |
442 | else: | |
443 | raise | |
444 | return value | |
445 | ||
446 | username = getconfig('user.name') | |
447 | useremail = getconfig('user.email') | |
448 | configerr = [] | |
449 | if not username: | |
450 | configerr.append('Please set your name using:\n git config --global user.name') | |
451 | if not useremail: | |
452 | configerr.append('Please set your email using:\n git config --global user.email') | |
453 | if configerr: | |
454 | raise DevtoolError('Your git configuration is incomplete which will prevent rebases from working:\n' + '\n'.join(configerr)) | |
455 | ||
ccb37f9e AK |
456 | def _extract_licenses(srcpath, recipe_licenses): |
457 | licenses = [] | |
458 | for url in recipe_licenses.split(): | |
459 | license = {} | |
460 | (type, host, path, user, pswd, parm) = bb.fetch.decodeurl(url) | |
461 | license['path'] = path | |
462 | license['md5'] = parm.get('md5', '') | |
463 | license['beginline'], license['endline'] = 0, 0 | |
464 | if 'beginline' in parm: | |
465 | license['beginline'] = int(parm['beginline']) | |
466 | if 'endline' in parm: | |
467 | license['endline'] = int(parm['endline']) | |
468 | license['text'] = [] | |
469 | with open(os.path.join(srcpath, path), 'rb') as f: | |
470 | import hashlib | |
471 | actual_md5 = hashlib.md5() | |
472 | lineno = 0 | |
473 | for line in f: | |
474 | lineno += 1 | |
475 | if (lineno >= license['beginline']) and ((lineno <= license['endline']) or not license['endline']): | |
476 | license['text'].append(line.decode(errors='ignore')) | |
477 | actual_md5.update(line) | |
478 | license['actual_md5'] = actual_md5.hexdigest() | |
479 | licenses.append(license) | |
480 | return licenses | |
481 | ||
482 | def _generate_license_diff(old_licenses, new_licenses): | |
483 | need_diff = False | |
484 | for l in new_licenses: | |
485 | if l['md5'] != l['actual_md5']: | |
486 | need_diff = True | |
487 | break | |
488 | if need_diff == False: | |
489 | return None | |
490 | ||
491 | import difflib | |
492 | diff = '' | |
493 | for old, new in zip(old_licenses, new_licenses): | |
494 | for line in difflib.unified_diff(old['text'], new['text'], old['path'], new['path']): | |
495 | diff = diff + line | |
496 | return diff | |
802829f1 | 497 | |
4020f5d9 LS |
498 | def upgrade(args, config, basepath, workspace): |
499 | """Entry point for the devtool 'upgrade' subcommand""" | |
500 | ||
501 | if args.recipename in workspace: | |
502 | raise DevtoolError("recipe %s is already in your workspace" % args.recipename) | |
e49a66fd PE |
503 | if args.srcbranch and not args.srcrev: |
504 | raise DevtoolError("If you specify --srcbranch/-B then you must use --srcrev/-S to specify the revision" % args.recipename) | |
4020f5d9 | 505 | |
802829f1 PE |
506 | _check_git_config() |
507 | ||
8b8f0422 | 508 | tinfoil = setup_tinfoil(basepath=basepath, tracking=True) |
4020f5d9 | 509 | try: |
5ec6d9ef PE |
510 | rd = parse_recipe(config, tinfoil, args.recipename, True) |
511 | if not rd: | |
512 | return 1 | |
513 | ||
0a36bd96 | 514 | pn = rd.getVar('PN') |
5ec6d9ef PE |
515 | if pn != args.recipename: |
516 | logger.info('Mapping %s to %s' % (args.recipename, pn)) | |
517 | if pn in workspace: | |
518 | raise DevtoolError("recipe %s is already in your workspace" % pn) | |
519 | ||
520 | if args.srctree: | |
521 | srctree = os.path.abspath(args.srctree) | |
522 | else: | |
523 | srctree = standard.get_default_srctree(config, pn) | |
524 | ||
fd77e356 AK |
525 | # Check that recipe isn't using a shared workdir |
526 | s = os.path.abspath(rd.getVar('S')) | |
527 | workdir = os.path.abspath(rd.getVar('WORKDIR')) | |
528 | srctree_s = srctree | |
529 | if s.startswith(workdir) and s != workdir and os.path.dirname(s) != workdir: | |
530 | # Handle if S is set to a subdirectory of the source | |
531 | srcsubdir = os.path.relpath(s, workdir).split(os.sep, 1)[1] | |
532 | srctree_s = os.path.join(srctree, srcsubdir) | |
533 | ||
45b4242b AK |
534 | # try to automatically discover latest version and revision if not provided on command line |
535 | if not args.version and not args.srcrev: | |
536 | version_info = oe.recipeutils.get_recipe_upstream_version(rd) | |
537 | if version_info['version'] and not version_info['version'].endswith("new-commits-available"): | |
538 | args.version = version_info['version'] | |
539 | if version_info['revision']: | |
540 | args.srcrev = version_info['revision'] | |
541 | if not args.version and not args.srcrev: | |
542 | raise DevtoolError("Automatic discovery of latest version/revision failed - you must provide a version using the --version/-V option, or for recipes that fetch from an SCM such as git, the --srcrev/-S option.") | |
543 | ||
5ec6d9ef | 544 | standard._check_compatible_recipe(pn, rd) |
0a36bd96 | 545 | old_srcrev = rd.getVar('SRCREV') |
5ec6d9ef PE |
546 | if old_srcrev == 'INVALID': |
547 | old_srcrev = None | |
548 | if old_srcrev and not args.srcrev: | |
549 | raise DevtoolError("Recipe specifies a SRCREV value; you must specify a new one when upgrading") | |
92c4d9be PE |
550 | old_ver = rd.getVar('PV') |
551 | if old_ver == args.version and old_srcrev == args.srcrev: | |
5ec6d9ef | 552 | raise DevtoolError("Current and upgrade versions are the same version") |
92c4d9be PE |
553 | if args.version: |
554 | if bb.utils.vercmp_string(args.version, old_ver) < 0: | |
555 | logger.warning('Upgrade version %s compares as less than the current version %s. If you are using a package feed for on-target upgrades or providing this recipe for general consumption, then you should increment PE in the recipe (or if there is no current PE value set, set it to "1")' % (args.version, old_ver)) | |
556 | check_prerelease_version(args.version, 'devtool upgrade') | |
5ec6d9ef PE |
557 | |
558 | rf = None | |
ccb37f9e | 559 | license_diff = None |
5ec6d9ef | 560 | try: |
9cf2089b | 561 | logger.info('Extracting current version source...') |
aa87603d | 562 | rev1, srcsubdir1 = standard._extract_source(srctree, False, 'devtool-orig', False, config, basepath, workspace, args.fixed_setup, rd, tinfoil, no_overrides=args.no_overrides) |
fd77e356 | 563 | old_licenses = _extract_licenses(srctree_s, (rd.getVar('LIC_FILES_CHKSUM') or "")) |
9cf2089b | 564 | logger.info('Extracting upgraded version source...') |
d29881a6 | 565 | rev2, md5, sha256, srcbranch, srcsubdir2 = _extract_new_source(args.version, srctree, args.no_patch, |
29ced738 | 566 | args.srcrev, args.srcbranch, args.branch, args.keep_temp, |
5ec6d9ef | 567 | tinfoil, rd) |
fd77e356 | 568 | new_licenses = _extract_licenses(srctree_s, (rd.getVar('LIC_FILES_CHKSUM') or "")) |
ccb37f9e | 569 | license_diff = _generate_license_diff(old_licenses, new_licenses) |
c519ac36 | 570 | rf, copied = _create_new_recipe(args.version, md5, sha256, args.srcrev, srcbranch, srcsubdir1, srcsubdir2, config.workspace_path, tinfoil, rd, license_diff, new_licenses, srctree, args.keep_failure) |
5ec6d9ef | 571 | except bb.process.CmdError as e: |
c519ac36 | 572 | _upgrade_error(e, rf, srctree, args.keep_failure) |
5ec6d9ef | 573 | except DevtoolError as e: |
c519ac36 | 574 | _upgrade_error(e, rf, srctree, args.keep_failure) |
5ec6d9ef PE |
575 | standard._add_md5(config, pn, os.path.dirname(rf)) |
576 | ||
fd77e356 | 577 | af = _write_append(rf, srctree_s, args.same_dir, args.no_same_dir, rev2, |
5ec6d9ef PE |
578 | copied, config.workspace_path, rd) |
579 | standard._add_md5(config, pn, af) | |
4e9a0be3 | 580 | |
05e2c4ad | 581 | update_unlockedsigs(basepath, workspace, args.fixed_setup, [pn]) |
4e9a0be3 | 582 | |
5ec6d9ef PE |
583 | logger.info('Upgraded source extracted to %s' % srctree) |
584 | logger.info('New recipe is %s' % rf) | |
ccb37f9e AK |
585 | if license_diff: |
586 | logger.info('License checksums have been updated in the new recipe; please refer to it for the difference between the old and the new license texts.') | |
0b0f53ee AK |
587 | preferred_version = rd.getVar('PREFERRED_VERSION_%s' % rd.getVar('PN')) |
588 | if preferred_version: | |
589 | logger.warning('Version is pinned to %s via PREFERRED_VERSION; it may need adjustment to match the new version before any further steps are taken' % preferred_version) | |
5ec6d9ef PE |
590 | finally: |
591 | tinfoil.shutdown() | |
4020f5d9 LS |
592 | return 0 |
593 | ||
e8f5b5cc AK |
594 | def latest_version(args, config, basepath, workspace): |
595 | """Entry point for the devtool 'latest_version' subcommand""" | |
596 | tinfoil = setup_tinfoil(basepath=basepath, tracking=True) | |
597 | try: | |
598 | rd = parse_recipe(config, tinfoil, args.recipename, True) | |
599 | if not rd: | |
600 | return 1 | |
601 | version_info = oe.recipeutils.get_recipe_upstream_version(rd) | |
602 | # "new-commits-available" is an indication that upstream never issues version tags | |
603 | if not version_info['version'].endswith("new-commits-available"): | |
604 | logger.info("Current version: {}".format(version_info['current_version'])) | |
605 | logger.info("Latest version: {}".format(version_info['version'])) | |
606 | if version_info['revision']: | |
607 | logger.info("Latest version's commit: {}".format(version_info['revision'])) | |
608 | else: | |
609 | logger.info("Latest commit: {}".format(version_info['revision'])) | |
610 | finally: | |
611 | tinfoil.shutdown() | |
612 | return 0 | |
613 | ||
02284423 AK |
614 | def check_upgrade_status(args, config, basepath, workspace): |
615 | if not args.recipe: | |
616 | logger.info("Checking the upstream status for all recipes may take a few minutes") | |
617 | results = oe.recipeutils.get_recipe_upgrade_status(args.recipe) | |
618 | for result in results: | |
619 | # pn, update_status, current, latest, maintainer, latest_commit, no_update_reason | |
29e064ac | 620 | if args.all or result[1] != 'MATCH': |
02284423 AK |
621 | logger.info("{:25} {:15} {:15} {} {} {}".format( result[0], |
622 | result[2], | |
623 | result[1] if result[1] != 'UPDATE' else (result[3] if not result[3].endswith("new-commits-available") else "new commits"), | |
624 | result[4], | |
625 | result[5] if result[5] != 'N/A' else "", | |
626 | "cannot be updated due to: %s" %(result[6]) if result[6] else "")) | |
627 | ||
4020f5d9 LS |
628 | def register_commands(subparsers, context): |
629 | """Register devtool subcommands from this plugin""" | |
8a952407 PE |
630 | |
631 | defsrctree = standard.get_default_srctree(context.config) | |
632 | ||
4020f5d9 | 633 | parser_upgrade = subparsers.add_parser('upgrade', help='Upgrade an existing recipe', |
e1b9d31e PE |
634 | description='Upgrades an existing recipe to a new upstream version. Puts the upgraded recipe file into the workspace along with any associated files, and extracts the source tree to a specified location (in case patches need rebasing or adding to as a result of the upgrade).', |
635 | group='starting') | |
a90ffea3 | 636 | parser_upgrade.add_argument('recipename', help='Name of recipe to upgrade (just name - no version, path or extension)') |
8a952407 | 637 | parser_upgrade.add_argument('srctree', nargs='?', help='Path to where to extract the source tree. If not specified, a subdirectory of %s will be used.' % defsrctree) |
45b4242b AK |
638 | parser_upgrade.add_argument('--version', '-V', help='Version to upgrade to (PV). If omitted, latest upstream version will be determined and used, if possible.') |
639 | parser_upgrade.add_argument('--srcrev', '-S', help='Source revision to upgrade to (useful when fetching from an SCM such as git)') | |
e49a66fd | 640 | parser_upgrade.add_argument('--srcbranch', '-B', help='Branch in source repository containing the revision to use (if fetching from an SCM such as git)') |
4020f5d9 LS |
641 | parser_upgrade.add_argument('--branch', '-b', default="devtool", help='Name for new development branch to checkout (default "%(default)s")') |
642 | parser_upgrade.add_argument('--no-patch', action="store_true", help='Do not apply patches from the recipe to the new source code') | |
aa87603d | 643 | parser_upgrade.add_argument('--no-overrides', '-O', action="store_true", help='Do not create branches for other override configurations') |
4020f5d9 LS |
644 | group = parser_upgrade.add_mutually_exclusive_group() |
645 | group.add_argument('--same-dir', '-s', help='Build in same directory as source', action="store_true") | |
646 | group.add_argument('--no-same-dir', help='Force build in a separate build directory', action="store_true") | |
647 | parser_upgrade.add_argument('--keep-temp', action="store_true", help='Keep temporary directory (for debugging)') | |
c519ac36 | 648 | parser_upgrade.add_argument('--keep-failure', action="store_true", help='Keep failed upgrade recipe and associated files (for debugging)') |
4e9a0be3 | 649 | parser_upgrade.set_defaults(func=upgrade, fixed_setup=context.fixed_setup) |
e8f5b5cc AK |
650 | |
651 | parser_latest_version = subparsers.add_parser('latest-version', help='Report the latest version of an existing recipe', | |
652 | description='Queries the upstream server for what the latest upstream release is (for git, tags are checked, for tarballs, a list of them is obtained, and one with the highest version number is reported)', | |
653 | group='info') | |
654 | parser_latest_version.add_argument('recipename', help='Name of recipe to query (just name - no version, path or extension)') | |
655 | parser_latest_version.set_defaults(func=latest_version) | |
02284423 AK |
656 | |
657 | parser_check_upgrade_status = subparsers.add_parser('check-upgrade-status', help="Report upgradability for multiple (or all) recipes", | |
658 | description="Prints a table of recipes together with versions currently provided by recipes, and latest upstream versions, when there is a later version available", | |
659 | group='info') | |
660 | parser_check_upgrade_status.add_argument('recipe', help='Name of the recipe to report (omit to report upgrade info for all recipes)', nargs='*') | |
29e064ac | 661 | parser_check_upgrade_status.add_argument('--all', '-a', help='Show all recipes, not just recipes needing upgrade', action="store_true") |
02284423 | 662 | parser_check_upgrade_status.set_defaults(func=check_upgrade_status) |