]>
Commit | Line | Data |
---|---|---|
4b544ff3 | 1 | #!/usr/bin/env python3 |
f8c9c511 | 2 | # |
deb3ccec RP |
3 | # |
4 | # Copyright OpenEmbedded Contributors | |
5 | # | |
f8c9c511 RP |
6 | # SPDX-License-Identifier: GPL-2.0-only |
7 | # | |
884d4fa3 RP |
8 | |
9 | import os | |
10 | import sys | |
884d4fa3 RP |
11 | import shutil |
12 | import errno | |
ad36335b | 13 | import time |
884d4fa3 RP |
14 | |
15 | def mkdir(d): | |
16 | try: | |
17 | os.makedirs(d) | |
18 | except OSError as e: | |
19 | if e.errno != errno.EEXIST: | |
20 | raise e | |
21 | ||
ad36335b KS |
22 | # extract the hash from past the last colon to last underscore |
23 | def extract_sha(filename): | |
24 | return filename.split(':')[7].split('_')[0] | |
25 | ||
26 | # get all files in a directory, extract hash and make | |
27 | # a map from hash to list of file with that hash | |
28 | def map_sha_to_files(dir_, prefix, sha_map): | |
29 | sstate_prefix_path = dir_ + '/' + prefix + '/' | |
8ffe6ca9 RP |
30 | if not os.path.exists(sstate_prefix_path): |
31 | return | |
ad36335b KS |
32 | sstate_files = os.listdir(sstate_prefix_path) |
33 | for f in sstate_files: | |
34 | try: | |
35 | sha = extract_sha(f) | |
36 | if sha not in sha_map: | |
37 | sha_map[sha] = [] | |
38 | sha_map[sha].append(sstate_prefix_path + f) | |
39 | except IndexError: | |
40 | continue | |
41 | ||
42 | # given a prefix build a map of hash to list of files | |
43 | def build_sha_cache(prefix): | |
44 | sha_map = {} | |
45 | ||
46 | sstate_dir = sys.argv[2] | |
47 | map_sha_to_files(sstate_dir, prefix, sha_map) | |
48 | ||
49 | native_sstate_dir = sys.argv[2] + '/' + sys.argv[4] | |
50 | map_sha_to_files(native_sstate_dir, prefix, sha_map) | |
51 | ||
52 | return sha_map | |
53 | ||
6c6baf6a | 54 | if len(sys.argv) < 5: |
884d4fa3 | 55 | print("Incorrect number of arguments specified") |
4b7b48fc | 56 | print("syntax: gen-lockedsig-cache <locked-sigs.inc> <input-cachedir> <output-cachedir> <nativelsbstring> [filterfile]") |
884d4fa3 RP |
57 | sys.exit(1) |
58 | ||
4b7b48fc PE |
59 | filterlist = [] |
60 | if len(sys.argv) > 5: | |
61 | print('Reading filter file %s' % sys.argv[5]) | |
62 | with open(sys.argv[5]) as f: | |
63 | for l in f.readlines(): | |
64 | if ":" in l: | |
65 | filterlist.append(l.rstrip()) | |
66 | ||
ac38d245 | 67 | print('Reading %s' % sys.argv[1]) |
884d4fa3 RP |
68 | sigs = [] |
69 | with open(sys.argv[1]) as f: | |
70 | for l in f.readlines(): | |
71 | if ":" in l: | |
4b7b48fc PE |
72 | task, sig = l.split()[0].rsplit(':', 1) |
73 | if filterlist and not task in filterlist: | |
74 | print('Filtering out %s' % task) | |
75 | else: | |
76 | sigs.append(sig) | |
884d4fa3 | 77 | |
ac38d245 | 78 | print('Gathering file list') |
ad36335b | 79 | start_time = time.perf_counter() |
884d4fa3 | 80 | files = set() |
ad36335b | 81 | sstate_content_cache = {} |
884d4fa3 | 82 | for s in sigs: |
ad36335b | 83 | prefix = s[:2] |
d05bde16 | 84 | prefix2 = s[2:4] |
ad36335b | 85 | if prefix not in sstate_content_cache: |
610b314d | 86 | sstate_content_cache[prefix] = {} |
d05bde16 RP |
87 | if prefix2 not in sstate_content_cache[prefix]: |
88 | sstate_content_cache[prefix][prefix2] = build_sha_cache(prefix + "/" + prefix2) | |
89 | ||
d05bde16 RP |
90 | if s in sstate_content_cache[prefix][prefix2]: |
91 | for f in sstate_content_cache[prefix][prefix2][s]: | |
92 | files.add(f) | |
ad36335b KS |
93 | |
94 | elapsed = time.perf_counter() - start_time | |
95 | print("Gathering file list took %.1fs" % elapsed) | |
884d4fa3 | 96 | |
ac38d245 | 97 | print('Processing files') |
884d4fa3 | 98 | for f in files: |
ac38d245 | 99 | sys.stdout.write('Processing %s... ' % f) |
f5a90df0 | 100 | if not f.endswith(('.tar.zst', '.siginfo', '.sig')): |
588de519 | 101 | # Most likely a temp file, skip it |
ac38d245 | 102 | print('skipping') |
588de519 | 103 | continue |
5eb8f15c | 104 | dst = os.path.join(sys.argv[3], os.path.relpath(f, sys.argv[2])) |
cf675896 RW |
105 | destdir = os.path.dirname(dst) |
106 | mkdir(destdir) | |
884d4fa3 | 107 | |
d65a6ee9 | 108 | src = os.path.realpath(f) |
cf675896 RW |
109 | if os.path.exists(dst): |
110 | os.remove(dst) | |
d65a6ee9 | 111 | if (os.stat(src).st_dev == os.stat(destdir).st_dev): |
ac38d245 | 112 | print('linking') |
fb9fdd7a | 113 | try: |
114 | os.link(src, dst) | |
115 | except OSError as e: | |
116 | print('hard linking failed, copying') | |
117 | shutil.copyfile(src, dst) | |
cf675896 | 118 | else: |
ac38d245 | 119 | print('copying') |
d65a6ee9 | 120 | shutil.copyfile(src, dst) |
ac38d245 PE |
121 | |
122 | print('Done!') |