]>
Commit | Line | Data |
---|---|---|
a7ede909 | 1 | # |
bb731d1f RP |
2 | # Copyright OpenEmbedded Contributors |
3 | # | |
a7ede909 JW |
4 | # SPDX-License-Identifier: GPL-2.0-only |
5 | # | |
6 | import os | |
7 | import subprocess | |
8 | import bb | |
9 | ||
abb0671d RP |
10 | # For reproducible builds, this code sets the default SOURCE_DATE_EPOCH in each |
11 | # component's build environment. The format is number of seconds since the | |
12 | # system epoch. | |
13 | # | |
14 | # Upstream components (generally) respect this environment variable, | |
15 | # using it in place of the "current" date and time. | |
16 | # See https://reproducible-builds.org/specs/source-date-epoch/ | |
17 | # | |
18 | # The default value of SOURCE_DATE_EPOCH comes from the function | |
19 | # get_source_date_epoch_value which reads from the SDE_FILE, or if the file | |
20 | # is not available will use the fallback of SOURCE_DATE_EPOCH_FALLBACK. | |
21 | # | |
22 | # The SDE_FILE is normally constructed from the function | |
23 | # create_source_date_epoch_stamp which is typically added as a postfuncs to | |
24 | # the do_unpack task. If a recipe does NOT have do_unpack, it should be added | |
25 | # to a task that runs after the source is available and before the | |
26 | # do_deploy_source_date_epoch task is executed. | |
27 | # | |
28 | # If a recipe wishes to override the default behavior it should set it's own | |
29 | # SOURCE_DATE_EPOCH or override the do_deploy_source_date_epoch_stamp task | |
30 | # with recipe-specific functionality to write the appropriate | |
31 | # SOURCE_DATE_EPOCH into the SDE_FILE. | |
32 | # | |
33 | # SOURCE_DATE_EPOCH is intended to be a reproducible value. This value should | |
34 | # be reproducible for anyone who builds the same revision from the same | |
35 | # sources. | |
36 | # | |
37 | # There are 4 ways the create_source_date_epoch_stamp function determines what | |
38 | # becomes SOURCE_DATE_EPOCH: | |
39 | # | |
40 | # 1. Use the value from __source_date_epoch.txt file if this file exists. | |
41 | # This file was most likely created in the previous build by one of the | |
42 | # following methods 2,3,4. | |
43 | # Alternatively, it can be provided by a recipe via SRC_URI. | |
44 | # | |
45 | # If the file does not exist: | |
46 | # | |
47 | # 2. If there is a git checkout, use the last git commit timestamp. | |
48 | # Git does not preserve file timestamps on checkout. | |
49 | # | |
50 | # 3. Use the mtime of "known" files such as NEWS, CHANGLELOG, ... | |
51 | # This works for well-kept repositories distributed via tarball. | |
52 | # | |
53 | # 4. Use the modification time of the youngest file in the source tree, if | |
54 | # there is one. | |
55 | # This will be the newest file from the distribution tarball, if any. | |
56 | # | |
57 | # 5. Fall back to a fixed timestamp (SOURCE_DATE_EPOCH_FALLBACK). | |
58 | # | |
59 | # Once the value is determined, it is stored in the recipe's SDE_FILE. | |
60 | ||
a7ede909 JW |
61 | def get_source_date_epoch_from_known_files(d, sourcedir): |
62 | source_date_epoch = None | |
63 | newest_file = None | |
64 | known_files = set(["NEWS", "ChangeLog", "Changelog", "CHANGES"]) | |
65 | for file in known_files: | |
66 | filepath = os.path.join(sourcedir, file) | |
67 | if os.path.isfile(filepath): | |
68 | mtime = int(os.lstat(filepath).st_mtime) | |
69 | # There may be more than one "known_file" present, if so, use the youngest one | |
70 | if not source_date_epoch or mtime > source_date_epoch: | |
71 | source_date_epoch = mtime | |
72 | newest_file = filepath | |
73 | if newest_file: | |
74 | bb.debug(1, "SOURCE_DATE_EPOCH taken from: %s" % newest_file) | |
75 | return source_date_epoch | |
76 | ||
77 | def find_git_folder(d, sourcedir): | |
78 | # First guess: WORKDIR/git | |
79 | # This is the default git fetcher unpack path | |
80 | workdir = d.getVar('WORKDIR') | |
81 | gitpath = os.path.join(workdir, "git/.git") | |
82 | if os.path.isdir(gitpath): | |
83 | return gitpath | |
84 | ||
85 | # Second guess: ${S} | |
86 | gitpath = os.path.join(sourcedir, ".git") | |
87 | if os.path.isdir(gitpath): | |
88 | return gitpath | |
89 | ||
90 | # Perhaps there was a subpath or destsuffix specified. | |
91 | # Go looking in the WORKDIR | |
92 | exclude = set(["build", "image", "license-destdir", "patches", "pseudo", | |
93 | "recipe-sysroot", "recipe-sysroot-native", "sysroot-destdir", "temp"]) | |
94 | for root, dirs, files in os.walk(workdir, topdown=True): | |
95 | dirs[:] = [d for d in dirs if d not in exclude] | |
96 | if '.git' in dirs: | |
ffdaa1a0 | 97 | return os.path.join(root, ".git") |
a7ede909 JW |
98 | |
99 | bb.warn("Failed to find a git repository in WORKDIR: %s" % workdir) | |
100 | return None | |
101 | ||
102 | def get_source_date_epoch_from_git(d, sourcedir): | |
4a293612 | 103 | if not "git://" in d.getVar('SRC_URI') and not "gitsm://" in d.getVar('SRC_URI'): |
a64caca5 JW |
104 | return None |
105 | ||
106 | gitpath = find_git_folder(d, sourcedir) | |
107 | if not gitpath: | |
108 | return None | |
109 | ||
110 | # Check that the repository has a valid HEAD; it may not if subdir is used | |
111 | # in SRC_URI | |
cdbd47dd | 112 | p = subprocess.run(['git', '--git-dir', gitpath, 'rev-parse', 'HEAD'], stdout=subprocess.PIPE, stderr=subprocess.STDOUT) |
a64caca5 JW |
113 | if p.returncode != 0: |
114 | bb.debug(1, "%s does not have a valid HEAD: %s" % (gitpath, p.stdout.decode('utf-8'))) | |
115 | return None | |
116 | ||
117 | bb.debug(1, "git repository: %s" % gitpath) | |
3bd6f78f BM |
118 | p = subprocess.run(['git', '-c', 'log.showSignature=false', '--git-dir', gitpath, 'log', '-1', '--pretty=%ct'], |
119 | check=True, stdout=subprocess.PIPE) | |
a64caca5 | 120 | return int(p.stdout.decode('utf-8')) |
a7ede909 JW |
121 | |
122 | def get_source_date_epoch_from_youngest_file(d, sourcedir): | |
123 | if sourcedir == d.getVar('WORKDIR'): | |
124 | # These sources are almost certainly not from a tarball | |
125 | return None | |
126 | ||
127 | # Do it the hard way: check all files and find the youngest one... | |
128 | source_date_epoch = None | |
129 | newest_file = None | |
130 | for root, dirs, files in os.walk(sourcedir, topdown=True): | |
131 | files = [f for f in files if not f[0] == '.'] | |
132 | ||
133 | for fname in files: | |
4bc0eb4b RP |
134 | if fname == "singletask.lock": |
135 | # Ignore externalsrc/devtool lockfile [YOCTO #14921] | |
136 | continue | |
a7ede909 JW |
137 | filename = os.path.join(root, fname) |
138 | try: | |
139 | mtime = int(os.lstat(filename).st_mtime) | |
140 | except ValueError: | |
141 | mtime = 0 | |
142 | if not source_date_epoch or mtime > source_date_epoch: | |
143 | source_date_epoch = mtime | |
144 | newest_file = filename | |
145 | ||
146 | if newest_file: | |
147 | bb.debug(1, "Newest file found: %s" % newest_file) | |
148 | return source_date_epoch | |
149 | ||
53ffffa3 | 150 | def fixed_source_date_epoch(d): |
a7ede909 | 151 | bb.debug(1, "No tarball or git repo found to determine SOURCE_DATE_EPOCH") |
53ffffa3 RP |
152 | source_date_epoch = d.getVar('SOURCE_DATE_EPOCH_FALLBACK') |
153 | if source_date_epoch: | |
154 | bb.debug(1, "Using SOURCE_DATE_EPOCH_FALLBACK") | |
155 | return int(source_date_epoch) | |
a7ede909 JW |
156 | return 0 |
157 | ||
158 | def get_source_date_epoch(d, sourcedir): | |
159 | return ( | |
160 | get_source_date_epoch_from_git(d, sourcedir) or | |
a7ede909 | 161 | get_source_date_epoch_from_youngest_file(d, sourcedir) or |
53ffffa3 | 162 | fixed_source_date_epoch(d) # Last resort |
a7ede909 JW |
163 | ) |
164 | ||
f2fd1c9d RP |
165 | def epochfile_read(epochfile, d): |
166 | cached, efile = d.getVar('__CACHED_SOURCE_DATE_EPOCH') or (None, None) | |
167 | if cached and efile == epochfile: | |
168 | return cached | |
169 | ||
170 | if cached and epochfile != efile: | |
171 | bb.debug(1, "Epoch file changed from %s to %s" % (efile, epochfile)) | |
172 | ||
173 | source_date_epoch = int(d.getVar('SOURCE_DATE_EPOCH_FALLBACK')) | |
174 | try: | |
175 | with open(epochfile, 'r') as f: | |
176 | s = f.read() | |
177 | try: | |
178 | source_date_epoch = int(s) | |
179 | except ValueError: | |
180 | bb.warn("SOURCE_DATE_EPOCH value '%s' is invalid. Reverting to SOURCE_DATE_EPOCH_FALLBACK" % s) | |
181 | source_date_epoch = int(d.getVar('SOURCE_DATE_EPOCH_FALLBACK')) | |
182 | bb.debug(1, "SOURCE_DATE_EPOCH: %d" % source_date_epoch) | |
183 | except FileNotFoundError: | |
184 | bb.debug(1, "Cannot find %s. SOURCE_DATE_EPOCH will default to %d" % (epochfile, source_date_epoch)) | |
185 | ||
186 | d.setVar('__CACHED_SOURCE_DATE_EPOCH', (str(source_date_epoch), epochfile)) | |
187 | return str(source_date_epoch) | |
188 | ||
189 | def epochfile_write(source_date_epoch, epochfile, d): | |
190 | ||
191 | bb.debug(1, "SOURCE_DATE_EPOCH: %d" % source_date_epoch) | |
192 | bb.utils.mkdirhier(os.path.dirname(epochfile)) | |
193 | ||
194 | tmp_file = "%s.new" % epochfile | |
195 | with open(tmp_file, 'w') as f: | |
196 | f.write(str(source_date_epoch)) | |
197 | os.rename(tmp_file, epochfile) |