]>
Commit | Line | Data |
---|---|---|
657343a6 LS |
1 | # Library of functions shared by all CI scripts |
2 | ||
a4761b60 | 3 | if test true = "$GITHUB_ACTIONS" |
dab73aeb | 4 | then |
dab73aeb JS |
5 | begin_group () { |
6 | need_to_end_group=t | |
7 | echo "::group::$1" >&2 | |
8 | set -x | |
9 | } | |
10 | ||
11 | end_group () { | |
12 | test -n "$need_to_end_group" || return 0 | |
13 | set +x | |
14 | need_to_end_group= | |
15 | echo '::endgroup::' >&2 | |
16 | } | |
0e3b67e2 PS |
17 | elif test true = "$GITLAB_CI" |
18 | then | |
19 | begin_group () { | |
20 | need_to_end_group=t | |
21 | printf "\e[0Ksection_start:$(date +%s):$(echo "$1" | tr ' ' _)\r\e[0K$1\n" | |
22 | trap "end_group '$1'" EXIT | |
23 | set -x | |
24 | } | |
dab73aeb | 25 | |
0e3b67e2 PS |
26 | end_group () { |
27 | test -n "$need_to_end_group" || return 0 | |
dab73aeb | 28 | set +x |
0e3b67e2 PS |
29 | need_to_end_group= |
30 | printf "\e[0Ksection_end:$(date +%s):$(echo "$1" | tr ' ' _)\r\e[0K\n" | |
31 | trap - EXIT | |
dab73aeb | 32 | } |
a4761b60 PS |
33 | else |
34 | begin_group () { :; } | |
35 | end_group () { :; } | |
dab73aeb | 36 | |
a4761b60 | 37 | set -x |
dab73aeb JS |
38 | fi |
39 | ||
a7d499cb PS |
40 | group () { |
41 | group="$1" | |
42 | shift | |
43 | begin_group "$group" | |
44 | ||
45 | # work around `dash` not supporting `set -o pipefail` | |
46 | ( | |
47 | "$@" 2>&1 | |
48 | echo $? >exit.status | |
49 | ) | | |
50 | sed 's/^\(\([^ ]*\):\([0-9]*\):\([0-9]*:\) \)\(error\|warning\): /::\5 file=\2,line=\3::\1/' | |
51 | res=$(cat exit.status) | |
52 | rm exit.status | |
53 | ||
54 | end_group "$group" | |
55 | return $res | |
56 | } | |
57 | ||
58 | begin_group "CI setup" | |
59 | trap "end_group 'CI setup'" EXIT | |
60 | ||
dab73aeb JS |
61 | # Set 'exit on error' for all CI scripts to let the caller know that |
62 | # something went wrong. | |
63 | # | |
64 | # We already enabled tracing executed commands earlier. This helps by showing | |
65 | # how # environment variables are set and and dependencies are installed. | |
66 | set -e | |
67 | ||
09f5e974 LS |
68 | skip_branch_tip_with_tag () { |
69 | # Sometimes, a branch is pushed at the same time the tag that points | |
70 | # at the same commit as the tip of the branch is pushed, and building | |
71 | # both at the same time is a waste. | |
72 | # | |
4096a98d JS |
73 | # When the build is triggered by a push to a tag, $CI_BRANCH will |
74 | # have that tagname, e.g. v2.14.0. Let's see if $CI_BRANCH is | |
75 | # exactly at a tag, and if so, if it is different from $CI_BRANCH. | |
76 | # That way, we can tell if we are building the tip of a branch that | |
77 | # is tagged and we can skip the build because we won't be skipping a | |
78 | # build of a tag. | |
09f5e974 | 79 | |
4096a98d JS |
80 | if TAG=$(git describe --exact-match "$CI_BRANCH" 2>/dev/null) && |
81 | test "$TAG" != "$CI_BRANCH" | |
09f5e974 | 82 | then |
4096a98d | 83 | echo "$(tput setaf 2)Tip of $CI_BRANCH is exactly at $TAG$(tput sgr0)" |
09f5e974 LS |
84 | exit 0 |
85 | fi | |
86 | } | |
87 | ||
dd02c3b6 PS |
88 | # Check whether we can use the path passed via the first argument as Git |
89 | # repository. | |
90 | is_usable_git_repository () { | |
91 | # We require Git in our PATH, otherwise we cannot access repositories | |
92 | # at all. | |
93 | if ! command -v git >/dev/null | |
94 | then | |
95 | return 1 | |
96 | fi | |
97 | ||
98 | # And the target directory needs to be a proper Git repository. | |
99 | if ! git -C "$1" rev-parse 2>/dev/null | |
100 | then | |
101 | return 1 | |
102 | fi | |
103 | } | |
104 | ||
9cc2c76f SG |
105 | # Save some info about the current commit's tree, so we can skip the build |
106 | # job if we encounter the same tree again and can provide a useful info | |
107 | # message. | |
108 | save_good_tree () { | |
dd02c3b6 PS |
109 | if ! is_usable_git_repository . |
110 | then | |
111 | return | |
112 | fi | |
113 | ||
b011fabd | 114 | echo "$(git rev-parse $CI_COMMIT^{tree}) $CI_COMMIT $CI_JOB_NUMBER $CI_JOB_ID" >>"$good_trees_file" |
9cc2c76f SG |
115 | # limit the file size |
116 | tail -1000 "$good_trees_file" >"$good_trees_file".tmp | |
117 | mv "$good_trees_file".tmp "$good_trees_file" | |
118 | } | |
119 | ||
120 | # Skip the build job if the same tree has already been built and tested | |
121 | # successfully before (e.g. because the branch got rebased, changing only | |
122 | # the commit messages). | |
123 | skip_good_tree () { | |
4a6e4b96 | 124 | if test true = "$GITHUB_ACTIONS" |
c46ebc24 SG |
125 | then |
126 | return | |
127 | fi | |
128 | ||
dd02c3b6 PS |
129 | if ! is_usable_git_repository . |
130 | then | |
131 | return | |
132 | fi | |
133 | ||
b011fabd | 134 | if ! good_tree_info="$(grep "^$(git rev-parse $CI_COMMIT^{tree}) " "$good_trees_file")" |
9cc2c76f SG |
135 | then |
136 | # Haven't seen this tree yet, or no cached good trees file yet. | |
137 | # Continue the build job. | |
138 | return | |
139 | fi | |
140 | ||
141 | echo "$good_tree_info" | { | |
142 | read tree prev_good_commit prev_good_job_number prev_good_job_id | |
143 | ||
b011fabd | 144 | if test "$CI_JOB_ID" = "$prev_good_job_id" |
9cc2c76f SG |
145 | then |
146 | cat <<-EOF | |
b011fabd | 147 | $(tput setaf 2)Skipping build job for commit $CI_COMMIT.$(tput sgr0) |
9cc2c76f SG |
148 | This commit has already been built and tested successfully by this build job. |
149 | To force a re-build delete the branch's cache and then hit 'Restart job'. | |
150 | EOF | |
151 | else | |
152 | cat <<-EOF | |
b011fabd | 153 | $(tput setaf 2)Skipping build job for commit $CI_COMMIT.$(tput sgr0) |
9cc2c76f | 154 | This commit's tree has already been built and tested successfully in build job $prev_good_job_number for commit $prev_good_commit. |
4a6e4b96 | 155 | The log of that build job is available at $SYSTEM_TASKDEFINITIONSURI$SYSTEM_TEAMPROJECT/_build/results?buildId=$prev_good_job_id |
9cc2c76f SG |
156 | To force a re-build delete the branch's cache and then hit 'Restart job'. |
157 | EOF | |
158 | fi | |
159 | } | |
160 | ||
161 | exit 0 | |
162 | } | |
163 | ||
863d6ceb | 164 | check_unignored_build_artifacts () { |
dd02c3b6 PS |
165 | if ! is_usable_git_repository . |
166 | then | |
167 | return | |
168 | fi | |
169 | ||
b92cb86e SG |
170 | ! git ls-files --other --exclude-standard --error-unmatch \ |
171 | -- ':/*' 2>/dev/null || | |
172 | { | |
173 | echo "$(tput setaf 1)error: found unignored build artifacts$(tput sgr0)" | |
174 | false | |
175 | } | |
176 | } | |
177 | ||
08dccc8f JS |
178 | handle_failed_tests () { |
179 | return 1 | |
180 | } | |
181 | ||
e624f206 PS |
182 | create_failed_test_artifacts () { |
183 | mkdir -p t/failed-test-artifacts | |
184 | ||
185 | for test_exit in t/test-results/*.exit | |
186 | do | |
187 | test 0 != "$(cat "$test_exit")" || continue | |
188 | ||
189 | test_name="${test_exit%.exit}" | |
190 | test_name="${test_name##*/}" | |
191 | printf "\\e[33m\\e[1m=== Failed test: ${test_name} ===\\e[m\\n" | |
192 | echo "The full logs are in the 'print test failures' step below." | |
193 | echo "See also the 'failed-tests-*' artifacts attached to this run." | |
194 | cat "t/test-results/$test_name.markup" | |
195 | ||
196 | trash_dir="t/trash directory.$test_name" | |
197 | cp "t/test-results/$test_name.out" t/failed-test-artifacts/ | |
198 | tar czf t/failed-test-artifacts/"$test_name".trash.tar.gz "$trash_dir" | |
199 | done | |
200 | } | |
201 | ||
855c158e ĐTCD |
202 | # GitHub Action doesn't set TERM, which is required by tput |
203 | export TERM=${TERM:-dumb} | |
204 | ||
a8c51f77 JH |
205 | # Clear MAKEFLAGS that may come from the outside world. |
206 | export MAKEFLAGS= | |
207 | ||
4a6e4b96 | 208 | if test -n "$SYSTEM_COLLECTIONURI" || test -n "$SYSTEM_TASKDEFINITIONSURI" |
6141a2ed JS |
209 | then |
210 | CI_TYPE=azure-pipelines | |
211 | # We are running in Azure Pipelines | |
212 | CI_BRANCH="$BUILD_SOURCEBRANCH" | |
213 | CI_COMMIT="$BUILD_SOURCEVERSION" | |
214 | CI_JOB_ID="$BUILD_BUILDID" | |
215 | CI_JOB_NUMBER="$BUILD_BUILDNUMBER" | |
216 | CI_OS_NAME="$(echo "$AGENT_OS" | tr A-Z a-z)" | |
217 | test darwin != "$CI_OS_NAME" || CI_OS_NAME=osx | |
218 | CI_REPO_SLUG="$(expr "$BUILD_REPOSITORY_URI" : '.*/\([^/]*/[^/]*\)$')" | |
219 | CC="${CC:-gcc}" | |
220 | ||
221 | # use a subdirectory of the cache dir (because the file share is shared | |
222 | # among *all* phases) | |
223 | cache_dir="$HOME/test-cache/$SYSTEM_PHASENAME" | |
224 | ||
9f17bef9 PS |
225 | GIT_TEST_OPTS="--write-junit-xml" |
226 | JOBS=10 | |
a3f2eec8 JS |
227 | elif test true = "$GITHUB_ACTIONS" |
228 | then | |
229 | CI_TYPE=github-actions | |
230 | CI_BRANCH="$GITHUB_REF" | |
231 | CI_COMMIT="$GITHUB_SHA" | |
232 | CI_OS_NAME="$(echo "$RUNNER_OS" | tr A-Z a-z)" | |
233 | test macos != "$CI_OS_NAME" || CI_OS_NAME=osx | |
234 | CI_REPO_SLUG="$GITHUB_REPOSITORY" | |
235 | CI_JOB_ID="$GITHUB_RUN_ID" | |
3506cae0 | 236 | CC="${CC_PACKAGE:-${CC:-gcc}}" |
4463ce75 | 237 | DONT_SKIP_TAGS=t |
08dccc8f | 238 | handle_failed_tests () { |
08dccc8f | 239 | echo "FAILED_TEST_ARTIFACTS=t/failed-test-artifacts" >>$GITHUB_ENV |
e624f206 | 240 | create_failed_test_artifacts |
08dccc8f JS |
241 | return 1 |
242 | } | |
08dccc8f | 243 | |
a3f2eec8 JS |
244 | cache_dir="$HOME/none" |
245 | ||
9f17bef9 PS |
246 | GIT_TEST_OPTS="--github-workflow-markup" |
247 | JOBS=10 | |
0e3b67e2 PS |
248 | elif test true = "$GITLAB_CI" |
249 | then | |
250 | CI_TYPE=gitlab-ci | |
251 | CI_BRANCH="$CI_COMMIT_REF_NAME" | |
252 | CI_COMMIT="$CI_COMMIT_SHA" | |
253 | case "$CI_JOB_IMAGE" in | |
254 | macos-*) | |
255 | CI_OS_NAME=osx;; | |
256 | alpine:*|fedora:*|ubuntu:*) | |
257 | CI_OS_NAME=linux;; | |
258 | *) | |
259 | echo "Could not identify OS image" >&2 | |
260 | env >&2 | |
261 | exit 1 | |
262 | ;; | |
263 | esac | |
264 | CI_REPO_SLUG="$CI_PROJECT_PATH" | |
265 | CI_JOB_ID="$CI_JOB_ID" | |
266 | CC="${CC_PACKAGE:-${CC:-gcc}}" | |
267 | DONT_SKIP_TAGS=t | |
268 | handle_failed_tests () { | |
269 | create_failed_test_artifacts | |
08dccc8f JS |
270 | return 1 |
271 | } | |
a3f2eec8 JS |
272 | |
273 | cache_dir="$HOME/none" | |
274 | ||
0e3b67e2 PS |
275 | runs_on_pool=$(echo "$CI_JOB_IMAGE" | tr : -) |
276 | JOBS=$(nproc) | |
b011fabd JS |
277 | else |
278 | echo "Could not identify CI type" >&2 | |
5127e8cf | 279 | env >&2 |
b011fabd JS |
280 | exit 1 |
281 | fi | |
4096a98d | 282 | |
9f17bef9 | 283 | MAKEFLAGS="$MAKEFLAGS --jobs=$JOBS" |
e7e03ef9 | 284 | GIT_PROVE_OPTS="--timer --jobs $JOBS" |
9f17bef9 PS |
285 | |
286 | GIT_TEST_OPTS="$GIT_TEST_OPTS --verbose-log -x" | |
287 | case "$CI_OS_NAME" in | |
288 | windows|windows_nt) | |
289 | GIT_TEST_OPTS="$GIT_TEST_OPTS --no-chain-lint --no-bin-wrappers" | |
290 | ;; | |
291 | esac | |
292 | ||
293 | export GIT_TEST_OPTS | |
294 | export GIT_PROVE_OPTS | |
295 | ||
b2cbaa09 SG |
296 | good_trees_file="$cache_dir/good-trees" |
297 | ||
298 | mkdir -p "$cache_dir" | |
b4a2fdc9 | 299 | |
4463ce75 | 300 | test -n "${DONT_SKIP_TAGS-}" || |
09f5e974 | 301 | skip_branch_tip_with_tag |
9cc2c76f | 302 | skip_good_tree |
83d1efe5 | 303 | |
bf427a94 SG |
304 | if test -z "$jobname" |
305 | then | |
b011fabd | 306 | jobname="$CI_OS_NAME-$CC" |
bf427a94 SG |
307 | fi |
308 | ||
e3371e92 SG |
309 | export DEVELOPER=1 |
310 | export DEFAULT_TEST_TARGET=prove | |
a85efb59 | 311 | export GIT_TEST_CLONE_2GB=true |
ef60e9f7 | 312 | export SKIP_DASHED_BUILT_INS=YesPlease |
e3371e92 | 313 | |
707d2f2f | 314 | case "$runs_on_pool" in |
0178420b | 315 | ubuntu-*) |
707d2f2f ÆAB |
316 | if test "$jobname" = "linux-gcc-default" |
317 | then | |
318 | break | |
319 | fi | |
320 | ||
0d3507f3 JX |
321 | PYTHON_PACKAGE=python2 |
322 | if test "$jobname" = linux-gcc | |
2c8921db | 323 | then |
0d3507f3 | 324 | PYTHON_PACKAGE=python3 |
2c8921db | 325 | fi |
0d3507f3 | 326 | MAKEFLAGS="$MAKEFLAGS PYTHON_PATH=/usr/bin/$PYTHON_PACKAGE" |
2c8921db | 327 | |
39602906 | 328 | export GIT_TEST_HTTPD=true |
a1157b76 | 329 | |
e3371e92 | 330 | # The Linux build installs the defined dependency versions below. |
37a2e353 SG |
331 | # The OS X build installs much more recent versions, whichever |
332 | # were recorded in the Homebrew database upon creating the OS X | |
333 | # image. | |
334 | # Keep that in mind when you encounter a broken OS X build! | |
e3371e92 SG |
335 | export LINUX_GIT_LFS_VERSION="1.5.2" |
336 | ||
88e00b70 SG |
337 | P4_PATH="$HOME/custom/p4" |
338 | GIT_LFS_PATH="$HOME/custom/git-lfs" | |
83d1efe5 SG |
339 | export PATH="$GIT_LFS_PATH:$P4_PATH:$PATH" |
340 | ;; | |
d1119225 | 341 | macos-*) |
682a868f JS |
342 | MAKEFLAGS="$MAKEFLAGS PYTHON_PATH=$(which python3)" |
343 | if [ "$jobname" != osx-gcc ] | |
2c8921db | 344 | then |
35898ad2 | 345 | MAKEFLAGS="$MAKEFLAGS APPLE_COMMON_CRYPTO_SHA1=Yes" |
2c8921db | 346 | fi |
e3371e92 | 347 | ;; |
707d2f2f ÆAB |
348 | esac |
349 | ||
350 | case "$jobname" in | |
c08bb260 | 351 | linux32) |
d2fae19e SG |
352 | CC=gcc |
353 | ;; | |
e0f8690d ĐTCD |
354 | linux-musl) |
355 | CC=gcc | |
356 | MAKEFLAGS="$MAKEFLAGS PYTHON_PATH=/usr/bin/python3 USE_LIBPCRE2=Yes" | |
357 | MAKEFLAGS="$MAKEFLAGS NO_REGEX=Yes ICONV_OMITS_BOM=Yes" | |
482c962d | 358 | MAKEFLAGS="$MAKEFLAGS GIT_TEST_UTF8_LOCALE=C.UTF-8" |
e0f8690d | 359 | ;; |
956d2e46 ÆAB |
360 | linux-leaks) |
361 | export SANITIZE=leak | |
362 | export GIT_TEST_PASSING_SANITIZE_LEAK=true | |
c24feabc | 363 | export GIT_TEST_SANITIZE_LEAK_LOG=true |
956d2e46 | 364 | ;; |
ec691526 JK |
365 | linux-asan-ubsan) |
366 | export SANITIZE=address,undefined | |
6ba91362 JS |
367 | export NO_SVN_TESTS=LetsSaveSomeTime |
368 | MAKEFLAGS="$MAKEFLAGS NO_PYTHON=YepBecauseP4FlakesTooOften" | |
1c0962c0 | 369 | ;; |
956d2e46 ÆAB |
370 | esac |
371 | ||
a8c51f77 | 372 | MAKEFLAGS="$MAKEFLAGS CC=${CC:-cc}" |
dab73aeb | 373 | |
a7d499cb | 374 | end_group "CI setup" |
dab73aeb | 375 | set -x |