]>
Commit | Line | Data |
---|---|---|
99bb3530 PD |
1 | from invoke import task |
2 | from invoke.exceptions import Failure, UnexpectedExit | |
3 | ||
4ccb0f78 | 4 | import json |
7ec6fb65 | 5 | import os |
99bb3530 PD |
6 | import sys |
7 | import time | |
8 | ||
7d862cb3 AR |
9 | auth_backend_ip_addr = os.getenv('AUTH_BACKEND_IP_ADDR', '127.0.0.1') |
10 | ||
11 | clang_version = os.getenv('CLANG_VERSION', '13') | |
91cfe4c9 | 12 | repo_home = os.getenv('REPO_HOME', '.') |
7d862cb3 | 13 | |
99bb3530 | 14 | all_build_deps = [ |
d3cb00f9 | 15 | 'ccache', |
99bb3530 PD |
16 | 'libboost-all-dev', |
17 | 'libluajit-5.1-dev', | |
18 | 'libsodium-dev', | |
b69425da | 19 | 'libssl-dev', # This will install libssl 1.1 on Debian 11 and libssl3 on Debian 12 |
99bb3530 PD |
20 | 'libsystemd-dev', |
21 | 'libtool', | |
22 | 'make', | |
23 | 'pkg-config', | |
24 | 'python3-venv', | |
25 | 'systemd', | |
26 | ] | |
27 | git_build_deps = [ | |
28 | 'autoconf', | |
29 | 'automake', | |
30 | 'bison', | |
31 | 'bzip2', | |
32 | 'curl', | |
33 | 'flex', | |
34 | 'git', | |
35 | 'ragel' | |
36 | ] | |
37 | auth_build_deps = [ # FIXME: perhaps we should be stealing these from the debian (Ubuntu) control file | |
38 | 'default-libmysqlclient-dev', | |
39 | 'libcdb-dev', | |
40 | 'libcurl4-openssl-dev', | |
41 | 'libgeoip-dev', | |
42 | 'libkrb5-dev', | |
43 | 'libldap2-dev', | |
44 | 'liblmdb-dev', | |
45 | 'libmaxminddb-dev', | |
46 | 'libp11-kit-dev', | |
47 | 'libpq-dev', | |
48 | 'libsqlite3-dev', | |
49 | 'libyaml-cpp-dev', | |
50 | 'libzmq3-dev', | |
bfa34df3 | 51 | 'python3-venv', |
99bb3530 | 52 | 'sqlite3', |
9f930bd7 | 53 | 'unixodbc-dev', |
26cf02ca | 54 | 'cmake', |
99bb3530 PD |
55 | ] |
56 | rec_build_deps = [ | |
57 | 'libcap-dev', | |
58 | 'libfstrm-dev', | |
59 | 'libsnmp-dev', | |
60 | ] | |
4467dd85 O |
61 | rec_bulk_deps = [ |
62 | 'curl', | |
4467dd85 O |
63 | 'libboost-all-dev', |
64 | 'libcap2', | |
2b219e37 O |
65 | 'libfstrm0', |
66 | 'libluajit-5.1-2', | |
7d862cb3 | 67 | '"libsnmp[1-9]+"', |
2b219e37 | 68 | 'libsodium23', |
4467dd85 | 69 | 'libsystemd0', |
2b219e37 O |
70 | 'moreutils', |
71 | 'pdns-tools', | |
869fc2b5 | 72 | 'unzip', |
4467dd85 | 73 | ] |
99bb3530 PD |
74 | dnsdist_build_deps = [ |
75 | 'libcap-dev', | |
76 | 'libcdb-dev', | |
77 | 'libedit-dev', | |
78 | 'libfstrm-dev', | |
79e3404d | 79 | 'libgnutls28-dev', |
99bb3530 PD |
80 | 'libh2o-evloop-dev', |
81 | 'liblmdb-dev', | |
ff4c1303 | 82 | 'libnghttp2-dev', |
99bb3530 PD |
83 | 'libre2-dev', |
84 | 'libsnmp-dev', | |
85 | ] | |
c184b26a RG |
86 | dnsdist_xdp_build_deps = [ |
87 | 'libbpf-dev', | |
88 | 'libxdp-dev', | |
89 | ] | |
99bb3530 PD |
90 | auth_test_deps = [ # FIXME: we should be generating some of these from shlibdeps in build |
91 | 'authbind', | |
92 | 'bc', | |
93 | 'bind9utils', | |
94 | 'curl', | |
95 | 'default-jre-headless', | |
96 | 'dnsutils', | |
222d17e2 | 97 | 'faketime', |
99bb3530 | 98 | 'gawk', |
bb4f68fd | 99 | 'krb5-user', |
99bb3530 | 100 | 'ldnsutils', |
7d862cb3 | 101 | '"libboost-serialization1.7[1-9]+"', |
99bb3530 PD |
102 | 'libcdb1', |
103 | 'libcurl4', | |
104 | 'libgeoip1', | |
105 | 'libkrb5-3', | |
b69425da | 106 | '"libldap-2.[1-9]+"', |
99bb3530 PD |
107 | 'liblmdb0', |
108 | 'libluajit-5.1-2', | |
109 | 'libmaxminddb0', | |
110 | 'libnet-dns-perl', | |
111 | 'libp11-kit0', | |
112 | 'libpq5', | |
113 | 'libsodium23', | |
114 | 'libsqlite3-dev', | |
99bb3530 | 115 | 'libsystemd0', |
b69425da | 116 | '"libyaml-cpp0.[1-9]+"', |
99bb3530 | 117 | 'libzmq3-dev', |
3a52d52f | 118 | 'lmdb-utils', |
7c05901b | 119 | 'prometheus', |
bfa34df3 | 120 | 'python3-venv', |
99bb3530 PD |
121 | 'socat', |
122 | 'softhsm2', | |
123 | 'unbound-host', | |
124 | 'unixodbc', | |
869fc2b5 | 125 | 'wget', |
99bb3530 | 126 | ] |
e8d83f88 FM |
127 | doc_deps = [ |
128 | 'autoconf', | |
129 | 'automake', | |
130 | 'bison', | |
131 | 'curl', | |
132 | 'flex', | |
133 | 'g++', | |
134 | 'git', | |
135 | 'latexmk', | |
136 | 'libboost-all-dev', | |
137 | 'libedit-dev', | |
138 | 'libluajit-5.1-dev', | |
139 | 'libssl-dev', | |
140 | 'make', | |
141 | 'pkg-config', | |
142 | 'python3-venv', | |
143 | 'ragel', | |
144 | 'rsync', | |
145 | ] | |
146 | doc_deps_pdf = [ | |
147 | 'texlive-binaries', | |
148 | 'texlive-formats-extra', | |
149 | 'texlive-latex-extra', | |
150 | ] | |
99bb3530 PD |
151 | |
152 | @task | |
153 | def apt_fresh(c): | |
154 | c.sudo('apt-get update') | |
699d088a | 155 | c.sudo('apt-get -y --allow-downgrades dist-upgrade') |
99bb3530 PD |
156 | |
157 | @task | |
158 | def install_clang(c): | |
159 | """ | |
7d862cb3 | 160 | install clang and llvm |
99bb3530 | 161 | """ |
7d862cb3 | 162 | c.sudo(f'apt-get -y --no-install-recommends install clang-{clang_version} llvm-{clang_version}') |
99bb3530 | 163 | |
fae3e64c FM |
164 | @task |
165 | def install_clang_tidy_tools(c): | |
7d862cb3 | 166 | c.sudo(f'apt-get -y --no-install-recommends install clang-tidy-{clang_version} clang-tools-{clang_version} bear python3-yaml') |
fae3e64c | 167 | |
99bb3530 PD |
168 | @task |
169 | def install_clang_runtime(c): | |
170 | # this gives us the symbolizer, for symbols in asan/ubsan traces | |
7d862cb3 | 171 | c.sudo(f'apt-get -y --no-install-recommends install clang-{clang_version}') |
99bb3530 | 172 | |
9883d3f9 OM |
173 | @task |
174 | def ci_install_rust(c, repo): | |
4ccb0f78 RG |
175 | with c.cd(f'{repo}/builder-support/helpers/'): |
176 | c.run('sudo sh install_rust.sh') | |
9883d3f9 | 177 | |
d1c1159f | 178 | def install_libdecaf(c, product): |
c344e4b2 | 179 | c.run('rm -rf /tmp/libdecaf && git clone https://git.code.sf.net/p/ed448goldilocks/code /tmp/libdecaf') |
d1c1159f FM |
180 | with c.cd('/tmp/libdecaf'): |
181 | c.run('git checkout 41f349') | |
2ef73ca0 | 182 | c.run(f'CC={get_c_compiler()} CXX={get_cxx_compiler()} ' |
9466b8e6 | 183 | 'cmake -B build ' |
d1c1159f FM |
184 | '-DCMAKE_INSTALL_PREFIX=/usr/local ' |
185 | '-DCMAKE_INSTALL_LIBDIR=lib ' | |
186 | '-DENABLE_STATIC=OFF ' | |
187 | '-DENABLE_TESTS=OFF ' | |
188 | '-DCMAKE_C_FLAGS="-Wno-sizeof-array-div -Wno-array-parameter" .') | |
189 | c.run('make -C build') | |
190 | c.run('sudo make -C build install') | |
191 | c.sudo(f'mkdir -p /opt/{product}/libdecaf') | |
192 | c.sudo(f'cp /usr/local/lib/libdecaf.so* /opt/{product}/libdecaf/.') | |
193 | ||
e8d83f88 FM |
194 | @task |
195 | def install_doc_deps(c): | |
699d088a | 196 | c.sudo('apt-get install -y ' + ' '.join(doc_deps)) |
e8d83f88 FM |
197 | |
198 | @task | |
199 | def install_doc_deps_pdf(c): | |
699d088a | 200 | c.sudo('apt-get install -y ' + ' '.join(doc_deps_pdf)) |
e8d83f88 | 201 | |
c344e4b2 | 202 | def install_meson(c): |
203 | c.run(f'python3 -m venv {repo_home}/.venv') | |
204 | c.run(f'. {repo_home}/.venv/bin/activate && pip install meson pyyaml ninja') | |
205 | ||
99bb3530 PD |
206 | @task |
207 | def install_auth_build_deps(c): | |
699d088a | 208 | c.sudo('apt-get install -y --no-install-recommends ' + ' '.join(all_build_deps + git_build_deps + auth_build_deps)) |
c344e4b2 | 209 | install_meson(c) |
825560a1 RG |
210 | if os.getenv('DECAF_SUPPORT', 'no') == 'yes': |
211 | install_libdecaf(c, 'pdns-auth') | |
99bb3530 | 212 | |
628a1dec RG |
213 | def is_coverage_enabled(): |
214 | sanitizers = os.getenv('SANITIZERS') | |
215 | if sanitizers: | |
216 | sanitizers = sanitizers.split('+') | |
217 | if 'tsan' in sanitizers: | |
218 | return False | |
219 | return os.getenv('COVERAGE') == 'yes' | |
220 | ||
91cfe4c9 | 221 | def get_coverage(meson=False): |
222 | if meson: | |
223 | return '-D b_coverage=true' if os.getenv('COVERAGE') == 'yes' else '' | |
7146a45a RG |
224 | return '--enable-coverage=clang' if is_coverage_enabled() else '' |
225 | ||
628a1dec RG |
226 | @task |
227 | def install_coverage_deps(c): | |
228 | if is_coverage_enabled(): | |
229 | c.sudo(f'apt-get install -y --no-install-recommends llvm-{clang_version}') | |
230 | ||
231 | @task | |
232 | def generate_coverage_info(c, binary, outputDir): | |
233 | if is_coverage_enabled(): | |
234 | version = os.getenv('BUILDER_VERSION') | |
235 | c.run(f'llvm-profdata-{clang_version} merge -sparse -o {outputDir}/temp.profdata /tmp/code-*.profraw') | |
236 | c.run(f'llvm-cov-{clang_version} export --format=lcov --ignore-filename-regex=\'^/usr/\' -instr-profile={outputDir}/temp.profdata -object {binary} > {outputDir}/coverage.lcov') | |
237 | c.run(f'{outputDir}/.github/scripts/normalize_paths_in_coverage.py {outputDir} {version} {outputDir}/coverage.lcov {outputDir}/normalized_coverage.lcov') | |
238 | c.run(f'mv {outputDir}/normalized_coverage.lcov {outputDir}/coverage.lcov') | |
239 | ||
99bb3530 PD |
240 | def setup_authbind(c): |
241 | c.sudo('touch /etc/authbind/byport/53') | |
242 | c.sudo('chmod 755 /etc/authbind/byport/53') | |
243 | ||
244 | auth_backend_test_deps = dict( | |
245 | gsqlite3=['sqlite3'], | |
246 | gmysql=['default-libmysqlclient-dev'], | |
247 | gpgsql=['libpq-dev'], | |
0e77de07 | 248 | lmdb=[], |
b33a88da PD |
249 | remote=[], |
250 | bind=[], | |
251 | geoip=[], | |
252 | lua2=[], | |
222d17e2 | 253 | tinydns=[], |
8af54cc6 AR |
254 | authpy=[], |
255 | godbc_sqlite3=['libsqliteodbc'], | |
c4a7e1df AR |
256 | godbc_mssql=['freetds-bin','tdsodbc'], |
257 | ldap=[], | |
258 | geoip_mmdb=[] | |
99bb3530 PD |
259 | ) |
260 | ||
261 | @task(help={'backend': 'Backend to install test deps for, e.g. gsqlite3; can be repeated'}, iterable=['backend'], optional=['backend']) | |
262 | def install_auth_test_deps(c, backend): # FIXME: rename this, we do way more than apt-get | |
263 | extra=[] | |
264 | for b in backend: | |
265 | extra.extend(auth_backend_test_deps[b]) | |
7d862cb3 | 266 | c.sudo('DEBIAN_FRONTEND=noninteractive apt-get -y install ' + ' '.join(extra+auth_test_deps)) |
99bb3530 PD |
267 | |
268 | c.run('chmod +x /opt/pdns-auth/bin/* /opt/pdns-auth/sbin/*') | |
269 | # c.run('''if [ ! -e $HOME/bin/jdnssec-verifyzone ]; then | |
270 | # wget https://github.com/dblacka/jdnssec-tools/releases/download/0.14/jdnssec-tools-0.14.tar.gz | |
271 | # tar xfz jdnssec-tools-0.14.tar.gz -C $HOME | |
272 | # rm jdnssec-tools-0.14.tar.gz | |
273 | # fi | |
274 | # echo 'export PATH=$HOME/jdnssec-tools-0.14/bin:$PATH' >> $BASH_ENV''') # FIXME: why did this fail with no error? | |
222d17e2 PD |
275 | c.run('touch regression-tests/tests/verify-dnssec-zone/allow-missing regression-tests.nobackend/rectify-axfr/allow-missing') # FIXME: can this go? |
276 | # FIXME we may want to start a background recursor here to make ALIAS tests more robust | |
99bb3530 PD |
277 | setup_authbind(c) |
278 | ||
825560a1 RG |
279 | if os.getenv('DECAF_SUPPORT', 'no') == 'yes': |
280 | # Copy libdecaf out | |
281 | c.sudo('mkdir -p /usr/local/lib') | |
282 | c.sudo('cp /opt/pdns-auth/libdecaf/libdecaf.so* /usr/local/lib/.') | |
d1c1159f | 283 | |
4467dd85 O |
284 | @task |
285 | def install_rec_bulk_deps(c): # FIXME: rename this, we do way more than apt-get | |
699d088a | 286 | c.sudo('apt-get --no-install-recommends -y install ' + ' '.join(rec_bulk_deps)) |
4467dd85 O |
287 | c.run('chmod +x /opt/pdns-recursor/bin/* /opt/pdns-recursor/sbin/*') |
288 | ||
99bb3530 PD |
289 | @task |
290 | def install_rec_test_deps(c): # FIXME: rename this, we do way more than apt-get | |
699d088a | 291 | c.sudo('apt-get --no-install-recommends install -y ' + ' '.join(rec_bulk_deps) + ' \ |
4467dd85 O |
292 | pdns-server pdns-backend-bind daemontools \ |
293 | jq libfaketime lua-posix lua-socket bc authbind \ | |
6b45d67b | 294 | python3-venv python3-dev default-libmysqlclient-dev libpq-dev \ |
4467dd85 | 295 | protobuf-compiler snmpd prometheus') |
99bb3530 PD |
296 | |
297 | c.run('chmod +x /opt/pdns-recursor/bin/* /opt/pdns-recursor/sbin/*') | |
298 | ||
299 | setup_authbind(c) | |
300 | ||
6b45d67b | 301 | c.run('sed "s/agentxperms 0700 0755 recursor/agentxperms 0777 0755/g" regression-tests.recursor-dnssec/snmpd.conf | sudo tee /etc/snmp/snmpd.conf') |
7d862cb3 | 302 | c.sudo('/etc/init.d/snmpd restart') |
6b45d67b O |
303 | time.sleep(5) |
304 | c.sudo('chmod 755 /var/agentx') | |
305 | ||
f941004d RG |
306 | @task(optional=['skipXDP']) |
307 | def install_dnsdist_test_deps(c, skipXDP=False): # FIXME: rename this, we do way more than apt-get | |
c184b26a RG |
308 | deps = 'libluajit-5.1-2 \ |
309 | libboost-all-dev \ | |
310 | libcap2 \ | |
311 | libcdb1 \ | |
312 | libcurl4-openssl-dev \ | |
313 | libfstrm0 \ | |
314 | libgnutls30 \ | |
315 | libh2o-evloop0.13 \ | |
316 | liblmdb0 \ | |
317 | libnghttp2-14 \ | |
318 | "libre2-[1-9]+" \ | |
319 | libssl-dev \ | |
320 | libsystemd0 \ | |
321 | libsodium23 \ | |
322 | lua-socket \ | |
323 | patch \ | |
324 | protobuf-compiler \ | |
325 | python3-venv snmpd prometheus' | |
f941004d RG |
326 | if not skipXDP: |
327 | deps = deps + '\ | |
328 | libbpf1 \ | |
c184b26a RG |
329 | libxdp1' |
330 | ||
331 | c.sudo(f'apt-get install -y {deps}') | |
99bb3530 | 332 | c.run('sed "s/agentxperms 0700 0755 dnsdist/agentxperms 0777 0755/g" regression-tests.dnsdist/snmpd.conf | sudo tee /etc/snmp/snmpd.conf') |
7d862cb3 | 333 | c.sudo('/etc/init.d/snmpd restart') |
99bb3530 PD |
334 | time.sleep(5) |
335 | c.sudo('chmod 755 /var/agentx') | |
336 | ||
337 | @task | |
338 | def install_rec_build_deps(c): | |
699d088a | 339 | c.sudo('apt-get install -y --no-install-recommends ' + ' '.join(all_build_deps + git_build_deps + rec_build_deps)) |
99bb3530 | 340 | |
c184b26a RG |
341 | @task(optional=['skipXDP']) |
342 | def install_dnsdist_build_deps(c, skipXDP=False): | |
f941004d | 343 | c.sudo('apt-get install -y --no-install-recommends ' + ' '.join(all_build_deps + git_build_deps + dnsdist_build_deps + (dnsdist_xdp_build_deps if not skipXDP else []))) |
99bb3530 PD |
344 | |
345 | @task | |
c344e4b2 | 346 | def ci_autoconf(c, meson=False): |
347 | if not meson: | |
348 | c.run('autoreconf -vfi') | |
99bb3530 | 349 | |
fdcc46e4 OM |
350 | @task |
351 | def ci_docs_rec_generate(c): | |
352 | c.run('python3 generate.py') | |
353 | ||
e8d83f88 FM |
354 | @task |
355 | def ci_docs_build(c): | |
356 | c.run('make -f Makefile.sphinx -C docs html') | |
357 | ||
358 | @task | |
359 | def ci_docs_build_pdf(c): | |
360 | c.run('make -f Makefile.sphinx -C docs latexpdf') | |
361 | ||
362 | @task | |
e0ec64f1 | 363 | def ci_docs_upload_master(c, docs_host, pdf, username, product, directory=""): |
5d9b131b FM |
364 | rsync_cmd = " ".join([ |
365 | "rsync", | |
366 | "--checksum", | |
367 | "--recursive", | |
368 | "--verbose", | |
369 | "--no-p", | |
370 | "--chmod=g=rwX", | |
371 | "--exclude '*~'", | |
372 | ]) | |
373 | c.run(f"{rsync_cmd} --delete ./docs/_build/{product}-html-docs/ {username}@{docs_host}:{directory}") | |
374 | c.run(f"{rsync_cmd} ./docs/_build/{product}-html-docs.tar.bz2 {username}@{docs_host}:{directory}/html-docs.tar.bz2") | |
375 | c.run(f"{rsync_cmd} ./docs/_build/latex/{pdf} {username}@{docs_host}:{directory}") | |
e8d83f88 FM |
376 | |
377 | @task | |
378 | def ci_docs_add_ssh(c, ssh_key, host_key): | |
379 | c.run('mkdir -m 700 -p ~/.ssh') | |
380 | c.run(f'echo "{ssh_key}" > ~/.ssh/id_ed25519') | |
381 | c.run('chmod 600 ~/.ssh/id_ed25519') | |
382 | c.run(f'echo "{host_key}" > ~/.ssh/known_hosts') | |
383 | ||
8804bc1d | 384 | |
91cfe4c9 | 385 | def get_sanitizers(meson=False): |
35859c0b | 386 | sanitizers = os.getenv('SANITIZERS', '') |
91cfe4c9 | 387 | if meson: |
388 | return f'-D b_sanitize={sanitizers}' if sanitizers != '' else '' | |
8804bc1d FM |
389 | if sanitizers != '': |
390 | sanitizers = sanitizers.split('+') | |
391 | sanitizers = ['--enable-' + sanitizer for sanitizer in sanitizers] | |
392 | sanitizers = ' '.join(sanitizers) | |
393 | return sanitizers | |
394 | ||
91cfe4c9 | 395 | def get_unit_tests(meson=False, auth=False): |
7146a45a RG |
396 | if os.getenv('UNIT_TESTS') != 'yes': |
397 | return '' | |
91cfe4c9 | 398 | if meson: |
399 | return '-D unit-tests=true -D unit-tests-backends=true' if auth else '-D unit-tests=true' | |
7146a45a RG |
400 | return '--enable-unit-tests --enable-backend-unit-tests' if auth else '--enable-unit-tests' |
401 | ||
402 | def get_build_concurrency(default=8): | |
403 | return os.getenv('CONCURRENCY', default) | |
404 | ||
91cfe4c9 | 405 | def get_fuzzing_targets(meson=False): |
406 | if meson: | |
407 | return '-D fuzz-targets=true' if os.getenv('FUZZING_TARGETS') == 'yes' else '' | |
7146a45a RG |
408 | return '--enable-fuzz-targets' if os.getenv('FUZZING_TARGETS') == 'yes' else '' |
409 | ||
410 | def is_compiler_clang(): | |
c6034e81 | 411 | compiler = os.getenv('COMPILER', 'clang') |
7146a45a RG |
412 | return compiler == 'clang' |
413 | ||
414 | def get_c_compiler(): | |
415 | return f'clang-{clang_version}' if is_compiler_clang() else 'gcc' | |
2ef73ca0 RG |
416 | |
417 | def get_cxx_compiler(): | |
7146a45a | 418 | return f'clang++-{clang_version}' if is_compiler_clang() else 'g++' |
2ef73ca0 RG |
419 | |
420 | def get_optimizations(): | |
472404b9 RG |
421 | optimizations = os.getenv('OPTIMIZATIONS', 'yes') |
422 | return '-O1' if optimizations == 'yes' else '-O0' | |
8804bc1d FM |
423 | |
424 | def get_cflags(): | |
425 | return " ".join([ | |
2ef73ca0 | 426 | get_optimizations(), |
8804bc1d FM |
427 | "-Werror=vla", |
428 | "-Werror=shadow", | |
429 | "-Wformat=2", | |
430 | "-Werror=format-security", | |
11c5018a RG |
431 | "-fstack-clash-protection", |
432 | "-fstack-protector-strong", | |
433 | "-fcf-protection=full", | |
7146a45a | 434 | "-Werror=string-plus-int" if is_compiler_clang() else '', |
8804bc1d FM |
435 | ]) |
436 | ||
437 | ||
438 | def get_cxxflags(): | |
439 | return " ".join([ | |
440 | get_cflags(), | |
441 | "-Wp,-D_GLIBCXX_ASSERTIONS", | |
442 | ]) | |
443 | ||
444 | ||
7146a45a RG |
445 | def get_base_configure_cmd(additional_c_flags='', additional_cxx_flags='', enable_systemd=True, enable_sodium=True): |
446 | cflags = " ".join([get_cflags(), additional_c_flags]) | |
447 | cxxflags = " ".join([get_cxxflags(), additional_cxx_flags]) | |
8804bc1d | 448 | return " ".join([ |
7146a45a RG |
449 | f'CFLAGS="{cflags}"', |
450 | f'CXXFLAGS="{cxxflags}"', | |
8804bc1d | 451 | './configure', |
2ef73ca0 RG |
452 | f"CC='{get_c_compiler()}'", |
453 | f"CXX='{get_cxx_compiler()}'", | |
8804bc1d | 454 | "--enable-option-checking=fatal", |
7146a45a RG |
455 | "--enable-systemd" if enable_systemd else '', |
456 | "--with-libsodium" if enable_sodium else '', | |
8804bc1d FM |
457 | "--enable-fortify-source=auto", |
458 | "--enable-auto-var-init=pattern", | |
7146a45a RG |
459 | get_coverage(), |
460 | get_sanitizers() | |
8804bc1d FM |
461 | ]) |
462 | ||
91cfe4c9 | 463 | def get_base_configure_cmd_meson(build_dir, additional_c_flags='', additional_cxx_flags='', enable_systemd=True, enable_sodium=True): |
464 | cflags = " ".join([get_cflags(), additional_c_flags]) | |
465 | cxxflags = " ".join([get_cxxflags(), additional_cxx_flags]) | |
466 | return " ".join([ | |
467 | f'CFLAGS="{cflags}"', | |
468 | f'CXXFLAGS="{cxxflags}"', | |
469 | f"CC='{get_c_compiler()}'", | |
470 | f"CXX='{get_cxx_compiler()}'", | |
471 | f'. {repo_home}/.venv/bin/activate && meson setup {build_dir}', | |
69a82b58 | 472 | "-D systemd={}".format("enabled" if enable_systemd else "disabled"), |
91cfe4c9 | 473 | "-D signers-libsodium={}".format("enabled" if enable_sodium else "disabled"), |
474 | "-D hardening-fortify-source=auto", | |
475 | "-D auto-var-init=pattern", | |
476 | get_coverage(meson=True), | |
477 | get_sanitizers(meson=True) | |
478 | ]) | |
8804bc1d | 479 | |
c344e4b2 | 480 | def ci_auth_configure_autotools(c): |
91cfe4c9 | 481 | unittests = get_unit_tests(auth=True) |
7146a45a | 482 | fuzz_targets = get_fuzzing_targets() |
8804bc1d FM |
483 | modules = " ".join([ |
484 | "bind", | |
485 | "geoip", | |
486 | "gmysql", | |
487 | "godbc", | |
488 | "gpgsql", | |
489 | "gsqlite3", | |
490 | "ldap", | |
491 | "lmdb", | |
492 | "lua2", | |
493 | "pipe", | |
494 | "remote", | |
495 | "tinydns", | |
496 | ]) | |
497 | configure_cmd = " ".join([ | |
498 | get_base_configure_cmd(), | |
499 | "LDFLAGS='-L/usr/local/lib -Wl,-rpath,/usr/local/lib'", | |
500 | f"--with-modules='{modules}'", | |
501 | "--enable-tools", | |
2ef73ca0 | 502 | "--enable-dns-over-tls", |
8804bc1d FM |
503 | "--enable-experimental-pkcs11", |
504 | "--enable-experimental-gss-tsig", | |
505 | "--enable-remotebackend-zeromq", | |
9a299f9d | 506 | "--enable-verbose-logging", |
8804bc1d | 507 | "--with-lmdb=/usr", |
825560a1 | 508 | "--with-libdecaf" if os.getenv('DECAF_SUPPORT', 'no') == 'yes' else '', |
8804bc1d FM |
509 | "--prefix=/opt/pdns-auth", |
510 | "--enable-ixfrdist", | |
8804bc1d | 511 | unittests, |
7146a45a | 512 | fuzz_targets |
8804bc1d FM |
513 | ]) |
514 | res = c.run(configure_cmd, warn=True) | |
99bb3530 PD |
515 | if res.exited != 0: |
516 | c.run('cat config.log') | |
517 | raise UnexpectedExit(res) | |
8804bc1d | 518 | |
91cfe4c9 | 519 | def ci_auth_configure_meson(c, build_dir): |
520 | unittests = get_unit_tests(meson=True, auth=True) | |
521 | fuzz_targets = get_fuzzing_targets(meson=True) | |
522 | configure_cmd = " ".join([ | |
523 | "LDFLAGS='-L/usr/local/lib -Wl,-rpath,/usr/local/lib'", | |
524 | get_base_configure_cmd_meson(build_dir), | |
525 | "-D module-bind=static", | |
526 | "-D module-geoip=static", | |
527 | "-D module-gmysql=static", | |
528 | "-D module-godbc=static", | |
529 | "-D module-gpgsql=static", | |
530 | "-D module-gsqlite3=static", | |
531 | "-D module-ldap=static", | |
532 | "-D module-lmdb=static", | |
533 | "-D module-lua2=static", | |
534 | "-D module-pipe=static", | |
535 | "-D module-remote=static", | |
536 | "-D module-remote-zeromq=true", | |
537 | "-D module-tinydns=static", | |
538 | "-D tools=true", | |
539 | "-D dns-over-tls=true", | |
540 | "-D experimental-pkcs11=enabled", | |
541 | "-D experimental-gss-tsig=enabled", | |
542 | "-D signers-libdecaf=enabled" if os.getenv('DECAF_SUPPORT', 'no') == 'yes' else '', | |
543 | "-D prefix=/opt/pdns-auth", | |
544 | "-D tools-ixfrdist=true", | |
545 | unittests, | |
546 | fuzz_targets | |
547 | ]) | |
548 | res = c.run(configure_cmd, warn=True) | |
549 | if res.exited != 0: | |
550 | c.run(f'cat {build_dir}/meson-logs/meson-log.txt') | |
551 | raise UnexpectedExit(res) | |
8804bc1d | 552 | |
c344e4b2 | 553 | @task |
554 | def ci_auth_configure(c, build_dir=None, meson=False): | |
555 | if meson: | |
556 | ci_auth_configure_meson(c, build_dir) | |
557 | else: | |
558 | ci_auth_configure_autotools(c) | |
559 | if build_dir: | |
560 | ci_make_distdir(c) | |
561 | with c.cd(f'{build_dir}'): | |
562 | ci_auth_configure_autotools(c) | |
563 | ||
99bb3530 | 564 | @task |
dc46a028 | 565 | def ci_rec_configure(c, features): |
7146a45a | 566 | unittests = get_unit_tests() |
8804bc1d | 567 | |
dc46a028 OM |
568 | if features == 'full': |
569 | configure_cmd = " ".join([ | |
570 | get_base_configure_cmd(), | |
571 | "--prefix=/opt/pdns-recursor", | |
572 | "--enable-option-checking", | |
573 | "--enable-verbose-logging", | |
574 | "--enable-dns-over-tls", | |
575 | "--enable-nod", | |
576 | "--with-libcap", | |
577 | "--with-lua=luajit", | |
578 | "--with-net-snmp", | |
579 | unittests, | |
580 | ]) | |
581 | else: | |
582 | configure_cmd = " ".join([ | |
583 | get_base_configure_cmd(), | |
584 | "--prefix=/opt/pdns-recursor", | |
585 | "--enable-option-checking", | |
586 | "--enable-verbose-logging", | |
587 | "--disable-dns-over-tls", | |
588 | "--disable-dnstap", | |
589 | "--disable-nod", | |
590 | "--disable-systemd", | |
591 | "--with-lua=luajit", | |
592 | "--without-libcap", | |
593 | "--without-libcurl", | |
594 | "--without-libdecaf", | |
595 | "--without-libsodium", | |
596 | "--without-net-snmp", | |
597 | unittests, | |
598 | ]) | |
8804bc1d | 599 | res = c.run(configure_cmd, warn=True) |
99bb3530 PD |
600 | if res.exited != 0: |
601 | c.run('cat config.log') | |
602 | raise UnexpectedExit(res) | |
603 | ||
8804bc1d | 604 | |
99bb3530 | 605 | @task |
e3d6cf05 RG |
606 | def ci_dnsdist_configure(c, features): |
607 | additional_flags = '' | |
608 | if features == 'full': | |
609 | features_set = '--enable-dnstap \ | |
610 | --enable-dnscrypt \ | |
611 | --enable-dns-over-tls \ | |
612 | --enable-dns-over-https \ | |
3e5c7a76 | 613 | --enable-dns-over-quic \ |
d1f77ae6 | 614 | --enable-dns-over-http3 \ |
e3d6cf05 RG |
615 | --enable-systemd \ |
616 | --prefix=/opt/dnsdist \ | |
617 | --with-gnutls \ | |
f31d8bad | 618 | --with-h2o \ |
e3d6cf05 RG |
619 | --with-libsodium \ |
620 | --with-lua=luajit \ | |
621 | --with-libcap \ | |
d5d26f84 | 622 | --with-net-snmp \ |
e3d6cf05 | 623 | --with-nghttp2 \ |
7146a45a | 624 | --with-re2' |
e3d6cf05 RG |
625 | else: |
626 | features_set = '--disable-dnstap \ | |
627 | --disable-dnscrypt \ | |
628 | --disable-ipcipher \ | |
629 | --disable-systemd \ | |
630 | --without-cdb \ | |
631 | --without-ebpf \ | |
632 | --without-gnutls \ | |
f31d8bad | 633 | --without-h2o \ |
e3d6cf05 RG |
634 | --without-libedit \ |
635 | --without-libsodium \ | |
636 | --without-lmdb \ | |
637 | --without-net-snmp \ | |
6135a84e | 638 | --without-nghttp2 \ |
7146a45a | 639 | --without-re2' |
e3d6cf05 | 640 | additional_flags = '-DDISABLE_COMPLETION \ |
6b6f0aa6 RG |
641 | -DDISABLE_DELAY_PIPE \ |
642 | -DDISABLE_DYNBLOCKS \ | |
e3d6cf05 RG |
643 | -DDISABLE_PROMETHEUS \ |
644 | -DDISABLE_PROTOBUF \ | |
645 | -DDISABLE_BUILTIN_HTML \ | |
646 | -DDISABLE_CARBON \ | |
647 | -DDISABLE_SECPOLL \ | |
648 | -DDISABLE_DEPRECATED_DYNBLOCK \ | |
649 | -DDISABLE_LUA_WEB_HANDLERS \ | |
650 | -DDISABLE_NON_FFI_DQ_BINDINGS \ | |
651 | -DDISABLE_POLICIES_BINDINGS \ | |
652 | -DDISABLE_PACKETCACHE_BINDINGS \ | |
653 | -DDISABLE_DOWNSTREAM_BINDINGS \ | |
654 | -DDISABLE_COMBO_ADDR_BINDINGS \ | |
655 | -DDISABLE_CLIENT_STATE_BINDINGS \ | |
656 | -DDISABLE_QPS_LIMITER_BINDINGS \ | |
657 | -DDISABLE_SUFFIX_MATCH_BINDINGS \ | |
658 | -DDISABLE_NETMASK_BINDINGS \ | |
659 | -DDISABLE_DNSNAME_BINDINGS \ | |
660 | -DDISABLE_DNSHEADER_BINDINGS \ | |
661 | -DDISABLE_RECVMMSG \ | |
85241b78 | 662 | -DDISABLE_WEB_CACHE_MANAGEMENT \ |
e3d6cf05 RG |
663 | -DDISABLE_WEB_CONFIG \ |
664 | -DDISABLE_RULES_ALTERING_QUERIES \ | |
665 | -DDISABLE_ECS_ACTIONS \ | |
dbefe674 RG |
666 | -DDISABLE_TOP_N_BINDINGS \ |
667 | -DDISABLE_OCSP_STAPLING \ | |
668 | -DDISABLE_HASHED_CREDENTIALS \ | |
669 | -DDISABLE_FALSE_SHARING_PADDING \ | |
670 | -DDISABLE_NPN' | |
7146a45a RG |
671 | unittests = get_unit_tests() |
672 | fuzztargets = get_fuzzing_targets() | |
673 | tools = f'''AR=llvm-ar-{clang_version} RANLIB=llvm-ranlib-{clang_version}''' if is_compiler_clang() else '' | |
674 | configure_cmd = " ".join([ | |
675 | tools, | |
676 | get_base_configure_cmd(additional_c_flags='', additional_cxx_flags=additional_flags, enable_systemd=False, enable_sodium=False), | |
677 | features_set, | |
678 | unittests, | |
679 | fuzztargets, | |
6e8e1c43 | 680 | '--enable-lto=thin', |
7146a45a RG |
681 | '--prefix=/opt/dnsdist' |
682 | ]) | |
683 | ||
684 | res = c.run(configure_cmd, warn=True) | |
99bb3530 PD |
685 | if res.exited != 0: |
686 | c.run('cat config.log') | |
687 | raise UnexpectedExit(res) | |
688 | ||
689 | @task | |
690 | def ci_auth_make(c): | |
7146a45a | 691 | c.run(f'make -j{get_build_concurrency()} -k V=1') |
99bb3530 | 692 | |
fae3e64c FM |
693 | @task |
694 | def ci_auth_make_bear(c): | |
7146a45a | 695 | c.run(f'bear --append -- make -j{get_build_concurrency()} -k V=1') |
fae3e64c | 696 | |
c344e4b2 | 697 | def run_ninja(c): |
698 | c.run(f'. {repo_home}/.venv/bin/activate && ninja -j{get_build_concurrency()} --verbose') | |
699 | ||
700 | @task | |
701 | def ci_auth_build(c, meson=False): | |
702 | if meson: | |
703 | run_ninja(c) | |
704 | else: | |
705 | ci_auth_make_bear(c) | |
706 | ||
99bb3530 PD |
707 | @task |
708 | def ci_rec_make(c): | |
7146a45a | 709 | c.run(f'make -j{get_build_concurrency()} -k V=1') |
99bb3530 | 710 | |
f01e3a4a FM |
711 | @task |
712 | def ci_rec_make_bear(c): | |
713 | # Assumed to be running under ./pdns/recursordist/ | |
7146a45a | 714 | c.run(f'bear --append -- make -j{get_build_concurrency()} -k V=1') |
f01e3a4a | 715 | |
99bb3530 PD |
716 | @task |
717 | def ci_dnsdist_make(c): | |
7146a45a | 718 | c.run(f'make -j{get_build_concurrency(4)} -k V=1') |
99bb3530 | 719 | |
97145bb4 FM |
720 | @task |
721 | def ci_dnsdist_make_bear(c): | |
722 | # Assumed to be running under ./pdns/dnsdistdist/ | |
7146a45a | 723 | c.run(f'bear --append -- make -j{get_build_concurrency(4)} -k V=1') |
97145bb4 | 724 | |
99bb3530 | 725 | @task |
e55d3a4b | 726 | def ci_auth_install_remotebackend_test_deps(c): |
699d088a | 727 | c.sudo('apt-get install -y socat') |
99bb3530 PD |
728 | |
729 | @task | |
91cfe4c9 | 730 | def ci_auth_run_unit_tests(c, meson=False): |
731 | if meson: | |
732 | suite_timeout_sec = 120 | |
733 | logfile = 'meson-logs/testlog.txt' | |
734 | res = c.run(f'. {repo_home}/.venv/bin/activate && meson test --verbose -t {suite_timeout_sec}', warn=True) | |
735 | else: | |
736 | logfile = 'pdns/test-suite.log' | |
737 | res = c.run('make check', warn=True) | |
99bb3530 | 738 | if res.exited != 0: |
91cfe4c9 | 739 | c.run(f'cat {logfile}', warn=True) |
740 | c.run('cat ../modules/remotebackend/*.log', warn=True) | |
741 | raise UnexpectedExit(res) | |
99bb3530 PD |
742 | |
743 | @task | |
744 | def ci_rec_run_unit_tests(c): | |
745 | res = c.run('make check', warn=True) | |
746 | if res.exited != 0: | |
747 | c.run('cat test-suite.log') | |
748 | raise UnexpectedExit(res) | |
749 | ||
750 | @task | |
751 | def ci_dnsdist_run_unit_tests(c): | |
752 | res = c.run('make check', warn=True) | |
753 | if res.exited != 0: | |
754 | c.run('cat test-suite.log') | |
755 | raise UnexpectedExit(res) | |
756 | ||
e917c86b AR |
757 | @task |
758 | def ci_make_distdir(c): | |
91cfe4c9 | 759 | c.run('make distdir') |
760 | ||
761 | @task | |
c344e4b2 | 762 | def ci_auth_install(c, meson=False): |
763 | if not meson: | |
764 | c.run('make install') # FIXME: this builds auth docs - again | |
91cfe4c9 | 765 | |
766 | @task | |
c344e4b2 | 767 | def ci_make_install(c): |
768 | c.run('make install') | |
99bb3530 PD |
769 | |
770 | @task | |
7d862cb3 | 771 | def add_auth_repo(c, dist_name, dist_release_name, pdns_repo_version): |
699d088a | 772 | c.sudo('apt-get install -y curl gnupg2') |
7d862cb3 | 773 | if pdns_repo_version == 'master': |
99bb3530 PD |
774 | c.sudo('curl -s -o /etc/apt/trusted.gpg.d/pdns-repo.asc https://repo.powerdns.com/CBC8B383-pub.asc') |
775 | else: | |
776 | c.sudo('curl -s -o /etc/apt/trusted.gpg.d/pdns-repo.asc https://repo.powerdns.com/FD380FBB-pub.asc') | |
7d862cb3 | 777 | c.run(f"echo 'deb [arch=amd64] http://repo.powerdns.com/{dist_name} {dist_release_name}-auth-{pdns_repo_version} main' | sudo tee /etc/apt/sources.list.d/pdns.list") |
99bb3530 PD |
778 | c.run("echo 'Package: pdns-*' | sudo tee /etc/apt/preferences.d/pdns") |
779 | c.run("echo 'Pin: origin repo.powerdns.com' | sudo tee -a /etc/apt/preferences.d/pdns") | |
780 | c.run("echo 'Pin-Priority: 600' | sudo tee -a /etc/apt/preferences.d/pdns") | |
781 | c.sudo('apt-get update') | |
782 | ||
783 | @task | |
784 | def test_api(c, product, backend=''): | |
785 | if product == 'recursor': | |
786 | with c.cd('regression-tests.api'): | |
787 | c.run(f'PDNSRECURSOR=/opt/pdns-recursor/sbin/pdns_recursor ./runtests recursor {backend}') | |
788 | elif product == 'auth': | |
789 | with c.cd('regression-tests.api'): | |
7d862cb3 | 790 | c.run(f'PDNSSERVER=/opt/pdns-auth/sbin/pdns_server PDNSUTIL=/opt/pdns-auth/bin/pdnsutil SDIG=/opt/pdns-auth/bin/sdig MYSQL_HOST={auth_backend_ip_addr} PGHOST={auth_backend_ip_addr} PGPORT=5432 ./runtests authoritative {backend}') |
99bb3530 PD |
791 | else: |
792 | raise Failure('unknown product') | |
793 | ||
0e77de07 | 794 | backend_regress_tests = dict( |
b33a88da | 795 | bind = [ |
8af54cc6 AR |
796 | 'bind-both', |
797 | 'bind-dnssec-both', | |
798 | 'bind-dnssec-nsec3-both', | |
799 | 'bind-dnssec-nsec3-optout-both', | |
800 | 'bind-dnssec-nsec3-narrow', | |
57d9ffa8 | 801 | 'bind-dnssec-pkcs11' |
b33a88da PD |
802 | ], |
803 | geoip = [ | |
8af54cc6 AR |
804 | 'geoip', |
805 | 'geoip-nsec3-narrow' | |
b33a88da | 806 | ], |
8af54cc6 AR |
807 | lua2 = ['lua2', 'lua2-dnssec'], |
808 | tinydns = ['tinydns'], | |
b33a88da | 809 | remote = [ |
8af54cc6 AR |
810 | 'remotebackend-pipe', |
811 | 'remotebackend-unix', | |
812 | 'remotebackend-http', | |
813 | 'remotebackend-zeromq', | |
814 | 'remotebackend-pipe-dnssec', | |
815 | 'remotebackend-unix-dnssec', | |
816 | 'remotebackend-http-dnssec', | |
817 | 'remotebackend-zeromq-dnssec' | |
b33a88da PD |
818 | ], |
819 | lmdb = [ | |
8af54cc6 AR |
820 | 'lmdb-nodnssec-both', |
821 | 'lmdb-both', | |
822 | 'lmdb-nsec3-both', | |
823 | 'lmdb-nsec3-optout-both', | |
824 | 'lmdb-nsec3-narrow' | |
825 | ], | |
826 | gmysql = [ | |
827 | 'gmysql', | |
828 | 'gmysql-nodnssec-both', | |
829 | 'gmysql-nsec3-both', | |
830 | 'gmysql-nsec3-optout-both', | |
831 | 'gmysql-nsec3-narrow', | |
832 | 'gmysql_sp-both' | |
833 | ], | |
834 | gpgsql = [ | |
835 | 'gpgsql', | |
836 | 'gpgsql-nodnssec-both', | |
837 | 'gpgsql-nsec3-both', | |
838 | 'gpgsql-nsec3-optout-both', | |
839 | 'gpgsql-nsec3-narrow', | |
840 | 'gpgsql_sp-both' | |
841 | ], | |
842 | gsqlite3 = [ | |
843 | 'gsqlite3', | |
844 | 'gsqlite3-nodnssec-both', | |
845 | 'gsqlite3-nsec3-both', | |
846 | 'gsqlite3-nsec3-optout-both', | |
847 | 'gsqlite3-nsec3-narrow' | |
848 | ], | |
849 | godbc_sqlite3 = ['godbc_sqlite3-nodnssec'], | |
850 | godbc_mssql = [ | |
851 | 'godbc_mssql', | |
852 | 'godbc_mssql-nodnssec', | |
853 | 'godbc_mssql-nsec3', | |
854 | 'godbc_mssql-nsec3-optout', | |
855 | 'godbc_mssql-nsec3-narrow' | |
b33a88da | 856 | ], |
c4a7e1df AR |
857 | ldap = [ |
858 | 'ldap-tree', | |
859 | 'ldap-simple', | |
860 | 'ldap-strict' | |
861 | ], | |
862 | geoip_mmdb = ['geoip'], | |
0e77de07 PD |
863 | ) |
864 | ||
b47c41d1 | 865 | godbc_mssql_credentials = {"username": "sa", "password": "SAsa12%%-not-a-secret-password"} |
8af54cc6 | 866 | |
7d862cb3 | 867 | godbc_config = f''' |
8af54cc6 AR |
868 | [pdns-mssql-docker] |
869 | Driver=FreeTDS | |
870 | Trace=No | |
7d862cb3 | 871 | Server={auth_backend_ip_addr} |
8af54cc6 AR |
872 | Port=1433 |
873 | Database=pdns | |
874 | TDS_Version=7.1 | |
875 | ||
876 | [pdns-mssql-docker-nodb] | |
877 | Driver=FreeTDS | |
878 | Trace=No | |
7d862cb3 | 879 | Server={auth_backend_ip_addr} |
8af54cc6 AR |
880 | Port=1433 |
881 | TDS_Version=7.1 | |
882 | ||
883 | [pdns-sqlite3-1] | |
884 | Driver = SQLite3 | |
885 | Database = pdns.sqlite3 | |
886 | ||
887 | [pdns-sqlite3-2] | |
888 | Driver = SQLite3 | |
889 | Database = pdns.sqlite32 | |
890 | ''' | |
891 | ||
892 | def setup_godbc_mssql(c): | |
893 | with open(os.path.expanduser("~/.odbc.ini"), "a") as f: | |
894 | f.write(godbc_config) | |
895 | c.sudo('sh -c \'echo "Threading=1" | cat /usr/share/tdsodbc/odbcinst.ini - | tee -a /etc/odbcinst.ini\'') | |
896 | c.sudo('sed -i "s/libtdsodbc.so/\/usr\/lib\/x86_64-linux-gnu\/odbc\/libtdsodbc.so/g" /etc/odbcinst.ini') | |
897 | c.run(f'echo "create database pdns" | isql -v pdns-mssql-docker-nodb {godbc_mssql_credentials["username"]} {godbc_mssql_credentials["password"]}') | |
898 | # FIXME: Skip 8bit-txt-unescaped test | |
899 | c.run('touch ${PWD}/regression-tests/tests/8bit-txt-unescaped/skip') | |
900 | ||
901 | def setup_godbc_sqlite3(c): | |
902 | with open(os.path.expanduser("~/.odbc.ini"), "a") as f: | |
903 | f.write(godbc_config) | |
904 | c.sudo('sed -i "s/libsqlite3odbc.so/\/usr\/lib\/x86_64-linux-gnu\/odbc\/libsqlite3odbc.so/g" /etc/odbcinst.ini') | |
905 | ||
c4a7e1df | 906 | def setup_ldap_client(c): |
699d088a | 907 | c.sudo('DEBIAN_FRONTEND=noninteractive apt-get install -y ldap-utils') |
7d862cb3 | 908 | c.sudo(f'sh -c \'echo "{auth_backend_ip_addr} ldapserver" | tee -a /etc/hosts\'') |
c4a7e1df | 909 | |
57d9ffa8 | 910 | def setup_softhsm(c): |
911 | # Modify the location of the softhsm tokens and configuration directory. | |
912 | # Enables token generation by non-root users (runner) | |
913 | c.run('mkdir -p /opt/pdns-auth/softhsm/tokens') | |
914 | c.run('echo "directories.tokendir = /opt/pdns-auth/softhsm/tokens" > /opt/pdns-auth/softhsm/softhsm2.conf') | |
915 | ||
0e77de07 PD |
916 | @task |
917 | def test_auth_backend(c, backend): | |
7d862cb3 | 918 | pdns_auth_env_vars = f'PDNS=/opt/pdns-auth/sbin/pdns_server PDNS2=/opt/pdns-auth/sbin/pdns_server SDIG=/opt/pdns-auth/bin/sdig NOTIFY=/opt/pdns-auth/bin/pdns_notify NSEC3DIG=/opt/pdns-auth/bin/nsec3dig SAXFR=/opt/pdns-auth/bin/saxfr ZONE2SQL=/opt/pdns-auth/bin/zone2sql ZONE2LDAP=/opt/pdns-auth/bin/zone2ldap ZONE2JSON=/opt/pdns-auth/bin/zone2json PDNSUTIL=/opt/pdns-auth/bin/pdnsutil PDNSCONTROL=/opt/pdns-auth/bin/pdns_control PDNSSERVER=/opt/pdns-auth/sbin/pdns_server SDIG=/opt/pdns-auth/bin/sdig GMYSQLHOST={auth_backend_ip_addr} GMYSQL2HOST={auth_backend_ip_addr} MYSQL_HOST={auth_backend_ip_addr} PGHOST={auth_backend_ip_addr} PGPORT=5432' |
8af54cc6 | 919 | |
0e77de07 | 920 | if backend == 'remote': |
e55d3a4b | 921 | ci_auth_install_remotebackend_test_deps(c) |
0e77de07 | 922 | |
222d17e2 | 923 | if backend == 'authpy': |
7d862cb3 | 924 | c.sudo(f'sh -c \'echo "{auth_backend_ip_addr} kerberos-server" | tee -a /etc/hosts\'') |
222d17e2 | 925 | with c.cd('regression-tests.auth-py'): |
8af54cc6 AR |
926 | c.run(f'{pdns_auth_env_vars} WITHKERBEROS=YES ./runtests') |
927 | return | |
928 | ||
57d9ffa8 | 929 | if backend == 'bind': |
930 | setup_softhsm(c) | |
931 | with c.cd('regression-tests'): | |
932 | for variant in backend_regress_tests[backend]: | |
933 | c.run(f'{pdns_auth_env_vars} SOFTHSM2_CONF=/opt/pdns-auth/softhsm/softhsm2.conf ./start-test-stop 5300 {variant}') | |
934 | return | |
935 | ||
8af54cc6 AR |
936 | if backend == 'godbc_sqlite3': |
937 | setup_godbc_sqlite3(c) | |
938 | with c.cd('regression-tests'): | |
939 | for variant in backend_regress_tests[backend]: | |
940 | c.run(f'{pdns_auth_env_vars} GODBC_SQLITE3_DSN=pdns-sqlite3-1 ./start-test-stop 5300 {variant}') | |
941 | return | |
942 | ||
943 | if backend == 'godbc_mssql': | |
944 | setup_godbc_mssql(c) | |
945 | with c.cd('regression-tests'): | |
946 | for variant in backend_regress_tests[backend]: | |
947 | c.run(f'{pdns_auth_env_vars} GODBC_MSSQL_PASSWORD={godbc_mssql_credentials["password"]} GODBC_MSSQL_USERNAME={godbc_mssql_credentials["username"]} GODBC_MSSQL_DSN=pdns-mssql-docker GODBC_MSSQL2_PASSWORD={godbc_mssql_credentials["password"]} GODBC_MSSQL2_USERNAME={godbc_mssql_credentials["username"]} GODBC_MSSQL2_DSN=pdns-mssql-docker ./start-test-stop 5300 {variant}') | |
222d17e2 PD |
948 | return |
949 | ||
c4a7e1df AR |
950 | if backend == 'ldap': |
951 | setup_ldap_client(c) | |
952 | ||
953 | if backend == 'geoip_mmdb': | |
954 | with c.cd('regression-tests'): | |
955 | for variant in backend_regress_tests[backend]: | |
956 | c.run(f'{pdns_auth_env_vars} geoipdatabase=../modules/geoipbackend/regression-tests/GeoLiteCity.mmdb ./start-test-stop 5300 {variant}') | |
957 | return | |
958 | ||
0e77de07 | 959 | with c.cd('regression-tests'): |
b33a88da PD |
960 | if backend == 'lua2': |
961 | c.run('touch trustedkeys') # avoid silly error during cleanup | |
962 | for variant in backend_regress_tests[backend]: | |
8af54cc6 | 963 | c.run(f'{pdns_auth_env_vars} ./start-test-stop 5300 {variant}') |
222d17e2 PD |
964 | |
965 | if backend == 'gsqlite3': | |
7d862cb3 AR |
966 | if os.getenv('SKIP_IPV6_TESTS'): |
967 | pdns_auth_env_vars += ' context=noipv6' | |
222d17e2 | 968 | with c.cd('regression-tests.nobackend'): |
8af54cc6 | 969 | c.run(f'{pdns_auth_env_vars} ./runtests') |
222d17e2 PD |
970 | c.run('/opt/pdns-auth/bin/pdnsutil test-algorithms') |
971 | return | |
b33a88da PD |
972 | |
973 | @task | |
974 | def test_ixfrdist(c): | |
975 | with c.cd('regression-tests.ixfrdist'): | |
976 | c.run('IXFRDISTBIN=/opt/pdns-auth/bin/ixfrdist ./runtests') | |
0e77de07 | 977 | |
99bb3530 PD |
978 | @task |
979 | def test_dnsdist(c): | |
980 | c.run('chmod +x /opt/dnsdist/bin/*') | |
981 | c.run('ls -ald /var /var/agentx /var/agentx/master') | |
982 | c.run('ls -al /var/agentx/master') | |
983 | with c.cd('regression-tests.dnsdist'): | |
6e8e1c43 | 984 | c.run('DNSDISTBIN=/opt/dnsdist/bin/dnsdist LD_LIBRARY_PATH=/opt/dnsdist/lib/ ENABLE_SUDO_TESTS=1 ./runtests') |
d3cb00f9 | 985 | |
6b45d67b O |
986 | @task |
987 | def test_regression_recursor(c): | |
988 | c.run('/opt/pdns-recursor/sbin/pdns_recursor --version') | |
7d862cb3 | 989 | c.run('PDNSRECURSOR=/opt/pdns-recursor/sbin/pdns_recursor RECCONTROL=/opt/pdns-recursor/bin/rec_control ./build-scripts/test-recursor') |
6b45d67b O |
990 | |
991 | @task | |
375c8fd6 O |
992 | def test_bulk_recursor(c, threads, mthreads, shards): |
993 | # We run an extremely small version of the bulk test, as GH does not seem to be able to handle the UDP load | |
6b45d67b O |
994 | with c.cd('regression-tests'): |
995 | c.run('curl -LO http://s3-us-west-1.amazonaws.com/umbrella-static/top-1m.csv.zip') | |
996 | c.run('unzip top-1m.csv.zip -d .') | |
997 | c.run('chmod +x /opt/pdns-recursor/bin/* /opt/pdns-recursor/sbin/*') | |
30a57c3d | 998 | c.run(f'DNSBULKTEST=/usr/bin/dnsbulktest RECURSOR=/opt/pdns-recursor/sbin/pdns_recursor RECCONTROL=/opt/pdns-recursor/bin/rec_control THRESHOLD=95 TRACE=no ./recursor-test 5300 100 {threads} {mthreads} {shards}') |
6b45d67b | 999 | |
dab788a9 PD |
1000 | @task |
1001 | def install_swagger_tools(c): | |
1002 | c.run('npm install -g api-spec-converter') | |
1003 | ||
1004 | @task | |
1005 | def swagger_syntax_check(c): | |
1006 | c.run('api-spec-converter docs/http-api/swagger/authoritative-api-swagger.yaml -f swagger_2 -t openapi_3 -s json -c') | |
1007 | ||
66c07369 | 1008 | @task |
df23d4bf RG |
1009 | def install_coverity_tools(c, project): |
1010 | token = os.getenv('COVERITY_TOKEN') | |
1011 | c.run(f'curl -s https://scan.coverity.com/download/linux64 --data "token={token}&project={project}" | gunzip | sudo tar xvf /dev/stdin --strip-components=1 --no-same-owner -C /usr/local', hide=True) | |
66c07369 RG |
1012 | |
1013 | @task | |
1014 | def coverity_clang_configure(c): | |
7d862cb3 | 1015 | c.sudo(f'/usr/local/bin/cov-configure --template --comptype clangcc --compiler clang++-{clang_version}') |
66c07369 RG |
1016 | |
1017 | @task | |
1018 | def coverity_make(c): | |
1019 | c.run('/usr/local/bin/cov-build --dir cov-int make -j8 -k') | |
1020 | ||
1021 | @task | |
1022 | def coverity_tarball(c, tarball): | |
1023 | c.run(f'tar caf {tarball} cov-int') | |
1024 | ||
1025 | @task | |
df23d4bf RG |
1026 | def coverity_upload(c, email, project, tarball): |
1027 | token = os.getenv('COVERITY_TOKEN') | |
66c07369 RG |
1028 | c.run(f'curl --form token={token} \ |
1029 | --form email="{email}" \ | |
1030 | --form file=@{tarball} \ | |
1031 | --form version="$(./builder-support/gen-version)" \ | |
1032 | --form description="master build" \ | |
df23d4bf | 1033 | https://scan.coverity.com/builds?project={project}', hide=True) |
66c07369 | 1034 | |
3e5c7a76 | 1035 | @task |
4ccb0f78 RG |
1036 | def ci_build_and_install_quiche(c, repo): |
1037 | with open(f'{repo}/builder-support/helpers/quiche.json') as quiche_json: | |
1038 | quiche_data = json.load(quiche_json) | |
1039 | quiche_version = quiche_data['version'] | |
1040 | quiche_hash = quiche_data['SHA256SUM'] | |
1041 | ||
3e5c7a76 RG |
1042 | # we have to pass -L because GitHub will do a redirect, sadly |
1043 | c.run(f'curl -L -o quiche-{quiche_version}.tar.gz https://github.com/cloudflare/quiche/archive/{quiche_version}.tar.gz') | |
1044 | # Line below should echo two spaces between digest and name | |
1045 | c.run(f'echo {quiche_hash}" "quiche-{quiche_version}.tar.gz | sha256sum -c -') | |
1046 | c.run(f'tar xf quiche-{quiche_version}.tar.gz') | |
1047 | with c.cd(f'quiche-{quiche_version}'): | |
2ade4784 | 1048 | c.run('cargo build --release --no-default-features --features ffi,boringssl-boring-crate --package quiche') |
3e5c7a76 RG |
1049 | # cannot use c.sudo() inside a cd() context, see https://github.com/pyinvoke/invoke/issues/687 |
1050 | c.run('sudo install -Dm644 quiche/include/quiche.h /usr/include') | |
2ade4784 RG |
1051 | c.run('sudo install -Dm644 target/release/libquiche.so /usr/lib') |
1052 | c.run('install -D target/release/libquiche.so /opt/dnsdist/lib/libquiche.so') | |
1053 | c.run(f"""sudo install -Dm644 /dev/stdin /usr/lib/pkgconfig/quiche.pc <<PC | |
3e5c7a76 RG |
1054 | # quiche |
1055 | Name: quiche | |
1056 | Description: quiche library | |
1057 | URL: https://github.com/cloudflare/quiche | |
2ade4784 | 1058 | Version: {quiche_version} |
3e5c7a76 RG |
1059 | Libs: -lquiche |
1060 | PC""") | |
1061 | ||
d3cb00f9 PD |
1062 | # this is run always |
1063 | def setup(): | |
1064 | if '/usr/lib/ccache' not in os.environ['PATH']: | |
1065 | os.environ['PATH']='/usr/lib/ccache:'+os.environ['PATH'] | |
1066 | ||
1067 | setup() |