]>
Commit | Line | Data |
---|---|---|
99bb3530 PD |
1 | from invoke import task |
2 | from invoke.exceptions import Failure, UnexpectedExit | |
3 | ||
7ec6fb65 | 4 | import os |
99bb3530 PD |
5 | import sys |
6 | import time | |
7 | ||
7d862cb3 AR |
8 | auth_backend_ip_addr = os.getenv('AUTH_BACKEND_IP_ADDR', '127.0.0.1') |
9 | ||
10 | clang_version = os.getenv('CLANG_VERSION', '13') | |
11 | ||
99bb3530 | 12 | all_build_deps = [ |
d3cb00f9 | 13 | 'ccache', |
99bb3530 PD |
14 | 'libboost-all-dev', |
15 | 'libluajit-5.1-dev', | |
16 | 'libsodium-dev', | |
17 | 'libssl-dev', | |
18 | 'libsystemd-dev', | |
19 | 'libtool', | |
20 | 'make', | |
21 | 'pkg-config', | |
22 | 'python3-venv', | |
23 | 'systemd', | |
24 | ] | |
25 | git_build_deps = [ | |
26 | 'autoconf', | |
27 | 'automake', | |
28 | 'bison', | |
29 | 'bzip2', | |
30 | 'curl', | |
31 | 'flex', | |
32 | 'git', | |
33 | 'ragel' | |
34 | ] | |
35 | auth_build_deps = [ # FIXME: perhaps we should be stealing these from the debian (Ubuntu) control file | |
36 | 'default-libmysqlclient-dev', | |
37 | 'libcdb-dev', | |
38 | 'libcurl4-openssl-dev', | |
39 | 'libgeoip-dev', | |
40 | 'libkrb5-dev', | |
41 | 'libldap2-dev', | |
42 | 'liblmdb-dev', | |
43 | 'libmaxminddb-dev', | |
44 | 'libp11-kit-dev', | |
45 | 'libpq-dev', | |
46 | 'libsqlite3-dev', | |
47 | 'libyaml-cpp-dev', | |
48 | 'libzmq3-dev', | |
49 | 'ruby-bundler', | |
50 | 'ruby-dev', | |
51 | 'sqlite3', | |
9f930bd7 | 52 | 'unixodbc-dev', |
26cf02ca | 53 | 'cmake', |
99bb3530 PD |
54 | ] |
55 | rec_build_deps = [ | |
56 | 'libcap-dev', | |
57 | 'libfstrm-dev', | |
58 | 'libsnmp-dev', | |
59 | ] | |
4467dd85 O |
60 | rec_bulk_deps = [ |
61 | 'curl', | |
4467dd85 O |
62 | 'libboost-all-dev', |
63 | 'libcap2', | |
2b219e37 O |
64 | 'libfstrm0', |
65 | 'libluajit-5.1-2', | |
7d862cb3 | 66 | '"libsnmp[1-9]+"', |
2b219e37 | 67 | 'libsodium23', |
4467dd85 O |
68 | 'libssl1.1', |
69 | 'libsystemd0', | |
2b219e37 O |
70 | 'moreutils', |
71 | 'pdns-tools', | |
869fc2b5 | 72 | 'unzip', |
4467dd85 | 73 | ] |
99bb3530 PD |
74 | dnsdist_build_deps = [ |
75 | 'libcap-dev', | |
76 | 'libcdb-dev', | |
77 | 'libedit-dev', | |
78 | 'libfstrm-dev', | |
79e3404d | 79 | 'libgnutls28-dev', |
99bb3530 PD |
80 | 'libh2o-evloop-dev', |
81 | 'liblmdb-dev', | |
ff4c1303 | 82 | 'libnghttp2-dev', |
99bb3530 PD |
83 | 'libre2-dev', |
84 | 'libsnmp-dev', | |
85 | ] | |
86 | auth_test_deps = [ # FIXME: we should be generating some of these from shlibdeps in build | |
87 | 'authbind', | |
88 | 'bc', | |
89 | 'bind9utils', | |
90 | 'curl', | |
91 | 'default-jre-headless', | |
92 | 'dnsutils', | |
222d17e2 | 93 | 'faketime', |
99bb3530 | 94 | 'gawk', |
bb4f68fd | 95 | 'krb5-user', |
99bb3530 | 96 | 'ldnsutils', |
7d862cb3 | 97 | '"libboost-serialization1.7[1-9]+"', |
99bb3530 PD |
98 | 'libcdb1', |
99 | 'libcurl4', | |
100 | 'libgeoip1', | |
101 | 'libkrb5-3', | |
102 | 'libldap-2.4-2', | |
103 | 'liblmdb0', | |
104 | 'libluajit-5.1-2', | |
105 | 'libmaxminddb0', | |
106 | 'libnet-dns-perl', | |
107 | 'libp11-kit0', | |
108 | 'libpq5', | |
109 | 'libsodium23', | |
110 | 'libsqlite3-dev', | |
111 | 'libssl1.1', | |
112 | 'libsystemd0', | |
113 | 'libyaml-cpp0.6', | |
114 | 'libzmq3-dev', | |
3a52d52f | 115 | 'lmdb-utils', |
7c05901b | 116 | 'prometheus', |
0e77de07 PD |
117 | 'ruby-bundler', |
118 | 'ruby-dev', | |
99bb3530 PD |
119 | 'socat', |
120 | 'softhsm2', | |
121 | 'unbound-host', | |
122 | 'unixodbc', | |
869fc2b5 | 123 | 'wget', |
99bb3530 | 124 | ] |
e8d83f88 FM |
125 | doc_deps = [ |
126 | 'autoconf', | |
127 | 'automake', | |
128 | 'bison', | |
129 | 'curl', | |
130 | 'flex', | |
131 | 'g++', | |
132 | 'git', | |
133 | 'latexmk', | |
134 | 'libboost-all-dev', | |
135 | 'libedit-dev', | |
136 | 'libluajit-5.1-dev', | |
137 | 'libssl-dev', | |
138 | 'make', | |
139 | 'pkg-config', | |
140 | 'python3-venv', | |
141 | 'ragel', | |
142 | 'rsync', | |
143 | ] | |
144 | doc_deps_pdf = [ | |
145 | 'texlive-binaries', | |
146 | 'texlive-formats-extra', | |
147 | 'texlive-latex-extra', | |
148 | ] | |
99bb3530 PD |
149 | |
150 | @task | |
151 | def apt_fresh(c): | |
b1fac6f4 | 152 | c.sudo('sed -i \'s/azure\.//\' /etc/apt/sources.list') |
99bb3530 | 153 | c.sudo('apt-get update') |
699d088a | 154 | c.sudo('apt-get -y --allow-downgrades dist-upgrade') |
99bb3530 PD |
155 | |
156 | @task | |
157 | def install_clang(c): | |
158 | """ | |
7d862cb3 | 159 | install clang and llvm |
99bb3530 | 160 | """ |
7d862cb3 | 161 | c.sudo(f'apt-get -y --no-install-recommends install clang-{clang_version} llvm-{clang_version}') |
99bb3530 | 162 | |
fae3e64c FM |
163 | @task |
164 | def install_clang_tidy_tools(c): | |
7d862cb3 | 165 | c.sudo(f'apt-get -y --no-install-recommends install clang-tidy-{clang_version} clang-tools-{clang_version} bear python3-yaml') |
fae3e64c | 166 | |
99bb3530 PD |
167 | @task |
168 | def install_clang_runtime(c): | |
169 | # this gives us the symbolizer, for symbols in asan/ubsan traces | |
7d862cb3 | 170 | c.sudo(f'apt-get -y --no-install-recommends install clang-{clang_version}') |
99bb3530 | 171 | |
d1c1159f FM |
172 | def install_libdecaf(c, product): |
173 | c.run('git clone https://git.code.sf.net/p/ed448goldilocks/code /tmp/libdecaf') | |
174 | with c.cd('/tmp/libdecaf'): | |
175 | c.run('git checkout 41f349') | |
7d862cb3 | 176 | c.run(f'CC=clang-{clang_version} CXX=clang-{clang_version} ' |
9466b8e6 | 177 | 'cmake -B build ' |
d1c1159f FM |
178 | '-DCMAKE_INSTALL_PREFIX=/usr/local ' |
179 | '-DCMAKE_INSTALL_LIBDIR=lib ' | |
180 | '-DENABLE_STATIC=OFF ' | |
181 | '-DENABLE_TESTS=OFF ' | |
182 | '-DCMAKE_C_FLAGS="-Wno-sizeof-array-div -Wno-array-parameter" .') | |
183 | c.run('make -C build') | |
184 | c.run('sudo make -C build install') | |
185 | c.sudo(f'mkdir -p /opt/{product}/libdecaf') | |
186 | c.sudo(f'cp /usr/local/lib/libdecaf.so* /opt/{product}/libdecaf/.') | |
187 | ||
e8d83f88 FM |
188 | @task |
189 | def install_doc_deps(c): | |
699d088a | 190 | c.sudo('apt-get install -y ' + ' '.join(doc_deps)) |
e8d83f88 FM |
191 | |
192 | @task | |
193 | def install_doc_deps_pdf(c): | |
699d088a | 194 | c.sudo('apt-get install -y ' + ' '.join(doc_deps_pdf)) |
e8d83f88 | 195 | |
99bb3530 PD |
196 | @task |
197 | def install_auth_build_deps(c): | |
699d088a | 198 | c.sudo('apt-get install -y --no-install-recommends ' + ' '.join(all_build_deps + git_build_deps + auth_build_deps)) |
d1c1159f | 199 | install_libdecaf(c, 'pdns-auth') |
99bb3530 PD |
200 | |
201 | def setup_authbind(c): | |
202 | c.sudo('touch /etc/authbind/byport/53') | |
203 | c.sudo('chmod 755 /etc/authbind/byport/53') | |
204 | ||
205 | auth_backend_test_deps = dict( | |
206 | gsqlite3=['sqlite3'], | |
207 | gmysql=['default-libmysqlclient-dev'], | |
208 | gpgsql=['libpq-dev'], | |
0e77de07 | 209 | lmdb=[], |
b33a88da PD |
210 | remote=[], |
211 | bind=[], | |
212 | geoip=[], | |
213 | lua2=[], | |
222d17e2 | 214 | tinydns=[], |
8af54cc6 AR |
215 | authpy=[], |
216 | godbc_sqlite3=['libsqliteodbc'], | |
c4a7e1df AR |
217 | godbc_mssql=['freetds-bin','tdsodbc'], |
218 | ldap=[], | |
219 | geoip_mmdb=[] | |
99bb3530 PD |
220 | ) |
221 | ||
222 | @task(help={'backend': 'Backend to install test deps for, e.g. gsqlite3; can be repeated'}, iterable=['backend'], optional=['backend']) | |
223 | def install_auth_test_deps(c, backend): # FIXME: rename this, we do way more than apt-get | |
224 | extra=[] | |
225 | for b in backend: | |
226 | extra.extend(auth_backend_test_deps[b]) | |
7d862cb3 | 227 | c.sudo('DEBIAN_FRONTEND=noninteractive apt-get -y install ' + ' '.join(extra+auth_test_deps)) |
99bb3530 PD |
228 | |
229 | c.run('chmod +x /opt/pdns-auth/bin/* /opt/pdns-auth/sbin/*') | |
230 | # c.run('''if [ ! -e $HOME/bin/jdnssec-verifyzone ]; then | |
231 | # wget https://github.com/dblacka/jdnssec-tools/releases/download/0.14/jdnssec-tools-0.14.tar.gz | |
232 | # tar xfz jdnssec-tools-0.14.tar.gz -C $HOME | |
233 | # rm jdnssec-tools-0.14.tar.gz | |
234 | # fi | |
235 | # echo 'export PATH=$HOME/jdnssec-tools-0.14/bin:$PATH' >> $BASH_ENV''') # FIXME: why did this fail with no error? | |
222d17e2 PD |
236 | c.run('touch regression-tests/tests/verify-dnssec-zone/allow-missing regression-tests.nobackend/rectify-axfr/allow-missing') # FIXME: can this go? |
237 | # FIXME we may want to start a background recursor here to make ALIAS tests more robust | |
99bb3530 PD |
238 | setup_authbind(c) |
239 | ||
d1c1159f FM |
240 | # Copy libdecaf out |
241 | c.sudo('mkdir -p /usr/local/lib') | |
242 | c.sudo('cp /opt/pdns-auth/libdecaf/libdecaf.so* /usr/local/lib/.') | |
243 | ||
4467dd85 O |
244 | @task |
245 | def install_rec_bulk_deps(c): # FIXME: rename this, we do way more than apt-get | |
699d088a | 246 | c.sudo('apt-get --no-install-recommends -y install ' + ' '.join(rec_bulk_deps)) |
4467dd85 O |
247 | c.run('chmod +x /opt/pdns-recursor/bin/* /opt/pdns-recursor/sbin/*') |
248 | ||
99bb3530 PD |
249 | @task |
250 | def install_rec_test_deps(c): # FIXME: rename this, we do way more than apt-get | |
699d088a | 251 | c.sudo('apt-get --no-install-recommends install -y ' + ' '.join(rec_bulk_deps) + ' \ |
4467dd85 O |
252 | pdns-server pdns-backend-bind daemontools \ |
253 | jq libfaketime lua-posix lua-socket bc authbind \ | |
6b45d67b | 254 | python3-venv python3-dev default-libmysqlclient-dev libpq-dev \ |
4467dd85 | 255 | protobuf-compiler snmpd prometheus') |
99bb3530 PD |
256 | |
257 | c.run('chmod +x /opt/pdns-recursor/bin/* /opt/pdns-recursor/sbin/*') | |
258 | ||
259 | setup_authbind(c) | |
260 | ||
6b45d67b | 261 | c.run('sed "s/agentxperms 0700 0755 recursor/agentxperms 0777 0755/g" regression-tests.recursor-dnssec/snmpd.conf | sudo tee /etc/snmp/snmpd.conf') |
7d862cb3 | 262 | c.sudo('/etc/init.d/snmpd restart') |
6b45d67b O |
263 | time.sleep(5) |
264 | c.sudo('chmod 755 /var/agentx') | |
265 | ||
99bb3530 PD |
266 | @task |
267 | def install_dnsdist_test_deps(c): # FIXME: rename this, we do way more than apt-get | |
699d088a | 268 | c.sudo('apt-get install -y \ |
99bb3530 PD |
269 | libluajit-5.1-2 \ |
270 | libboost-all-dev \ | |
271 | libcap2 \ | |
272 | libcdb1 \ | |
273 | libcurl4-openssl-dev \ | |
274 | libfstrm0 \ | |
79e3404d | 275 | libgnutls30 \ |
99bb3530 PD |
276 | libh2o-evloop0.13 \ |
277 | liblmdb0 \ | |
ff4c1303 | 278 | libnghttp2-14 \ |
7d862cb3 | 279 | "libre2-[1-9]+" \ |
99bb3530 PD |
280 | libssl-dev \ |
281 | libsystemd0 \ | |
282 | libsodium23 \ | |
74a2ea87 | 283 | lua-socket \ |
99bb3530 PD |
284 | patch \ |
285 | protobuf-compiler \ | |
286 | python3-venv snmpd prometheus') | |
287 | c.run('sed "s/agentxperms 0700 0755 dnsdist/agentxperms 0777 0755/g" regression-tests.dnsdist/snmpd.conf | sudo tee /etc/snmp/snmpd.conf') | |
7d862cb3 | 288 | c.sudo('/etc/init.d/snmpd restart') |
99bb3530 PD |
289 | time.sleep(5) |
290 | c.sudo('chmod 755 /var/agentx') | |
291 | ||
292 | @task | |
293 | def install_rec_build_deps(c): | |
699d088a | 294 | c.sudo('apt-get install -y --no-install-recommends ' + ' '.join(all_build_deps + git_build_deps + rec_build_deps)) |
99bb3530 PD |
295 | |
296 | @task | |
297 | def install_dnsdist_build_deps(c): | |
699d088a | 298 | c.sudo('apt-get install -y --no-install-recommends ' + ' '.join(all_build_deps + git_build_deps + dnsdist_build_deps)) |
99bb3530 PD |
299 | |
300 | @task | |
301 | def ci_autoconf(c): | |
302 | c.run('BUILDER_VERSION=0.0.0-git1 autoreconf -vfi') | |
303 | ||
e8d83f88 FM |
304 | @task |
305 | def ci_docs_build(c): | |
306 | c.run('make -f Makefile.sphinx -C docs html') | |
307 | ||
308 | @task | |
309 | def ci_docs_build_pdf(c): | |
310 | c.run('make -f Makefile.sphinx -C docs latexpdf') | |
311 | ||
312 | @task | |
e0ec64f1 | 313 | def ci_docs_upload_master(c, docs_host, pdf, username, product, directory=""): |
5d9b131b FM |
314 | rsync_cmd = " ".join([ |
315 | "rsync", | |
316 | "--checksum", | |
317 | "--recursive", | |
318 | "--verbose", | |
319 | "--no-p", | |
320 | "--chmod=g=rwX", | |
321 | "--exclude '*~'", | |
322 | ]) | |
323 | c.run(f"{rsync_cmd} --delete ./docs/_build/{product}-html-docs/ {username}@{docs_host}:{directory}") | |
324 | c.run(f"{rsync_cmd} ./docs/_build/{product}-html-docs.tar.bz2 {username}@{docs_host}:{directory}/html-docs.tar.bz2") | |
325 | c.run(f"{rsync_cmd} ./docs/_build/latex/{pdf} {username}@{docs_host}:{directory}") | |
e8d83f88 FM |
326 | |
327 | @task | |
328 | def ci_docs_add_ssh(c, ssh_key, host_key): | |
329 | c.run('mkdir -m 700 -p ~/.ssh') | |
330 | c.run(f'echo "{ssh_key}" > ~/.ssh/id_ed25519') | |
331 | c.run('chmod 600 ~/.ssh/id_ed25519') | |
332 | c.run(f'echo "{host_key}" > ~/.ssh/known_hosts') | |
333 | ||
8804bc1d FM |
334 | |
335 | def get_sanitizers(): | |
336 | sanitizers = os.getenv('SANITIZERS') | |
337 | if sanitizers != '': | |
338 | sanitizers = sanitizers.split('+') | |
339 | sanitizers = ['--enable-' + sanitizer for sanitizer in sanitizers] | |
340 | sanitizers = ' '.join(sanitizers) | |
341 | return sanitizers | |
342 | ||
343 | ||
344 | def get_cflags(): | |
345 | return " ".join([ | |
346 | "-O1", | |
347 | "-Werror=vla", | |
348 | "-Werror=shadow", | |
349 | "-Wformat=2", | |
350 | "-Werror=format-security", | |
351 | "-Werror=string-plus-int", | |
352 | ]) | |
353 | ||
354 | ||
355 | def get_cxxflags(): | |
356 | return " ".join([ | |
357 | get_cflags(), | |
358 | "-Wp,-D_GLIBCXX_ASSERTIONS", | |
359 | ]) | |
360 | ||
361 | ||
362 | def get_base_configure_cmd(): | |
363 | return " ".join([ | |
364 | f'CFLAGS="{get_cflags()}"', | |
365 | f'CXXFLAGS="{get_cxxflags()}"', | |
366 | './configure', | |
7d862cb3 AR |
367 | f"CC='clang-{clang_version}'", |
368 | f"CXX='clang++-{clang_version}'", | |
8804bc1d FM |
369 | "--enable-option-checking=fatal", |
370 | "--enable-systemd", | |
371 | "--with-libsodium", | |
372 | "--enable-fortify-source=auto", | |
373 | "--enable-auto-var-init=pattern", | |
374 | ]) | |
375 | ||
376 | ||
99bb3530 PD |
377 | @task |
378 | def ci_auth_configure(c): | |
8804bc1d FM |
379 | sanitizers = get_sanitizers() |
380 | ||
381 | unittests = os.getenv('UNIT_TESTS') | |
382 | if unittests == 'yes': | |
383 | unittests = '--enable-unit-tests --enable-backend-unit-tests' | |
384 | else: | |
385 | unittests = '' | |
386 | ||
387 | fuzz_targets = os.getenv('FUZZING_TARGETS') | |
388 | fuzz_targets = '--enable-fuzz-targets' if fuzz_targets == 'yes' else '' | |
389 | ||
390 | modules = " ".join([ | |
391 | "bind", | |
392 | "geoip", | |
393 | "gmysql", | |
394 | "godbc", | |
395 | "gpgsql", | |
396 | "gsqlite3", | |
397 | "ldap", | |
398 | "lmdb", | |
399 | "lua2", | |
400 | "pipe", | |
401 | "remote", | |
402 | "tinydns", | |
403 | ]) | |
404 | configure_cmd = " ".join([ | |
405 | get_base_configure_cmd(), | |
406 | "LDFLAGS='-L/usr/local/lib -Wl,-rpath,/usr/local/lib'", | |
407 | f"--with-modules='{modules}'", | |
408 | "--enable-tools", | |
409 | "--enable-experimental-pkcs11", | |
410 | "--enable-experimental-gss-tsig", | |
411 | "--enable-remotebackend-zeromq", | |
412 | "--with-lmdb=/usr", | |
413 | "--with-libdecaf", | |
414 | "--prefix=/opt/pdns-auth", | |
415 | "--enable-ixfrdist", | |
416 | sanitizers, | |
417 | unittests, | |
418 | fuzz_targets, | |
419 | ]) | |
420 | res = c.run(configure_cmd, warn=True) | |
99bb3530 PD |
421 | if res.exited != 0: |
422 | c.run('cat config.log') | |
423 | raise UnexpectedExit(res) | |
8804bc1d FM |
424 | |
425 | ||
99bb3530 PD |
426 | @task |
427 | def ci_rec_configure(c): | |
8804bc1d FM |
428 | sanitizers = get_sanitizers() |
429 | ||
430 | unittests = os.getenv('UNIT_TESTS') | |
431 | unittests = '--enable-unit-tests' if unittests == 'yes' else '' | |
432 | ||
433 | configure_cmd = " ".join([ | |
434 | get_base_configure_cmd(), | |
435 | "--enable-nod", | |
436 | "--prefix=/opt/pdns-recursor", | |
437 | "--with-lua=luajit", | |
438 | "--with-libcap", | |
439 | "--with-net-snmp", | |
440 | "--enable-dns-over-tls", | |
441 | sanitizers, | |
442 | unittests, | |
443 | ]) | |
444 | res = c.run(configure_cmd, warn=True) | |
99bb3530 PD |
445 | if res.exited != 0: |
446 | c.run('cat config.log') | |
447 | raise UnexpectedExit(res) | |
448 | ||
8804bc1d | 449 | |
99bb3530 | 450 | @task |
e3d6cf05 RG |
451 | def ci_dnsdist_configure(c, features): |
452 | additional_flags = '' | |
453 | if features == 'full': | |
454 | features_set = '--enable-dnstap \ | |
455 | --enable-dnscrypt \ | |
456 | --enable-dns-over-tls \ | |
457 | --enable-dns-over-https \ | |
458 | --enable-systemd \ | |
459 | --prefix=/opt/dnsdist \ | |
460 | --with-gnutls \ | |
461 | --with-libsodium \ | |
462 | --with-lua=luajit \ | |
463 | --with-libcap \ | |
d5d26f84 | 464 | --with-net-snmp \ |
e3d6cf05 RG |
465 | --with-nghttp2 \ |
466 | --with-re2 ' | |
467 | else: | |
468 | features_set = '--disable-dnstap \ | |
469 | --disable-dnscrypt \ | |
470 | --disable-ipcipher \ | |
471 | --disable-systemd \ | |
472 | --without-cdb \ | |
473 | --without-ebpf \ | |
474 | --without-gnutls \ | |
475 | --without-libedit \ | |
476 | --without-libsodium \ | |
477 | --without-lmdb \ | |
478 | --without-net-snmp \ | |
6135a84e | 479 | --without-nghttp2 \ |
e3d6cf05 RG |
480 | --without-re2 ' |
481 | additional_flags = '-DDISABLE_COMPLETION \ | |
6b6f0aa6 RG |
482 | -DDISABLE_DELAY_PIPE \ |
483 | -DDISABLE_DYNBLOCKS \ | |
e3d6cf05 RG |
484 | -DDISABLE_PROMETHEUS \ |
485 | -DDISABLE_PROTOBUF \ | |
486 | -DDISABLE_BUILTIN_HTML \ | |
487 | -DDISABLE_CARBON \ | |
488 | -DDISABLE_SECPOLL \ | |
489 | -DDISABLE_DEPRECATED_DYNBLOCK \ | |
490 | -DDISABLE_LUA_WEB_HANDLERS \ | |
491 | -DDISABLE_NON_FFI_DQ_BINDINGS \ | |
492 | -DDISABLE_POLICIES_BINDINGS \ | |
493 | -DDISABLE_PACKETCACHE_BINDINGS \ | |
494 | -DDISABLE_DOWNSTREAM_BINDINGS \ | |
495 | -DDISABLE_COMBO_ADDR_BINDINGS \ | |
496 | -DDISABLE_CLIENT_STATE_BINDINGS \ | |
497 | -DDISABLE_QPS_LIMITER_BINDINGS \ | |
498 | -DDISABLE_SUFFIX_MATCH_BINDINGS \ | |
499 | -DDISABLE_NETMASK_BINDINGS \ | |
500 | -DDISABLE_DNSNAME_BINDINGS \ | |
501 | -DDISABLE_DNSHEADER_BINDINGS \ | |
502 | -DDISABLE_RECVMMSG \ | |
85241b78 | 503 | -DDISABLE_WEB_CACHE_MANAGEMENT \ |
e3d6cf05 RG |
504 | -DDISABLE_WEB_CONFIG \ |
505 | -DDISABLE_RULES_ALTERING_QUERIES \ | |
506 | -DDISABLE_ECS_ACTIONS \ | |
dbefe674 RG |
507 | -DDISABLE_TOP_N_BINDINGS \ |
508 | -DDISABLE_OCSP_STAPLING \ | |
509 | -DDISABLE_HASHED_CREDENTIALS \ | |
510 | -DDISABLE_FALSE_SHARING_PADDING \ | |
511 | -DDISABLE_NPN' | |
66c07369 RG |
512 | unittests = ' --enable-unit-tests' if os.getenv('UNIT_TESTS') == 'yes' else '' |
513 | sanitizers = ' '.join('--enable-'+x for x in os.getenv('SANITIZERS').split('+')) if os.getenv('SANITIZERS') != '' else '' | |
e3d6cf05 RG |
514 | cflags = '-O1 -Werror=vla -Werror=shadow -Wformat=2 -Werror=format-security -Werror=string-plus-int' |
515 | cxxflags = cflags + ' -Wp,-D_GLIBCXX_ASSERTIONS ' + additional_flags | |
7d862cb3 | 516 | res = c.run(f'''CFLAGS="%s" \ |
e3d6cf05 | 517 | CXXFLAGS="%s" \ |
7d862cb3 AR |
518 | AR=llvm-ar-{clang_version} \ |
519 | RANLIB=llvm-ranlib-{clang_version} \ | |
99bb3530 | 520 | ./configure \ |
7d862cb3 AR |
521 | CC='clang-{clang_version}' \ |
522 | CXX='clang++-{clang_version}' \ | |
99bb3530 | 523 | --enable-option-checking=fatal \ |
d3757f1d | 524 | --enable-fortify-source=auto \ |
327d085f | 525 | --enable-auto-var-init=pattern \ |
f52b10f7 | 526 | --enable-lto=thin \ |
66c07369 | 527 | --prefix=/opt/dnsdist %s %s %s''' % (cflags, cxxflags, features_set, sanitizers, unittests), warn=True) |
99bb3530 PD |
528 | if res.exited != 0: |
529 | c.run('cat config.log') | |
530 | raise UnexpectedExit(res) | |
531 | ||
532 | @task | |
533 | def ci_auth_make(c): | |
534 | c.run('make -j8 -k V=1') | |
535 | ||
fae3e64c FM |
536 | @task |
537 | def ci_auth_make_bear(c): | |
538 | # Needed for clang-tidy -line-filter vs project structure shenanigans | |
539 | with c.cd('pdns'): | |
7d862cb3 | 540 | c.run('bear --append -- make -j8 -k V=1 -C ..') |
fae3e64c | 541 | |
99bb3530 PD |
542 | @task |
543 | def ci_rec_make(c): | |
544 | c.run('make -j8 -k V=1') | |
545 | ||
f01e3a4a FM |
546 | @task |
547 | def ci_rec_make_bear(c): | |
548 | # Assumed to be running under ./pdns/recursordist/ | |
7d862cb3 | 549 | c.run('bear --append -- make -j8 -k V=1') |
f01e3a4a | 550 | |
99bb3530 PD |
551 | @task |
552 | def ci_dnsdist_make(c): | |
553 | c.run('make -j4 -k V=1') | |
554 | ||
97145bb4 FM |
555 | @task |
556 | def ci_dnsdist_make_bear(c): | |
557 | # Assumed to be running under ./pdns/dnsdistdist/ | |
7d862cb3 | 558 | c.run('bear --append -- make -j4 -k V=1') |
97145bb4 | 559 | |
99bb3530 | 560 | @task |
e55d3a4b | 561 | def ci_auth_install_remotebackend_test_deps(c): |
99bb3530 | 562 | with c.cd('modules/remotebackend'): |
0e77de07 PD |
563 | # c.run('bundle config set path vendor/bundle') |
564 | c.run('sudo ruby -S bundle install') | |
699d088a | 565 | c.sudo('apt-get install -y socat') |
99bb3530 PD |
566 | |
567 | @task | |
568 | def ci_auth_run_unit_tests(c): | |
569 | res = c.run('make check', warn=True) | |
570 | if res.exited != 0: | |
222d17e2 PD |
571 | c.run('cat pdns/test-suite.log', warn=True) |
572 | c.run('cat modules/remotebackend/test-suite.log', warn=True) | |
99bb3530 PD |
573 | raise UnexpectedExit(res) |
574 | ||
575 | @task | |
576 | def ci_rec_run_unit_tests(c): | |
577 | res = c.run('make check', warn=True) | |
578 | if res.exited != 0: | |
579 | c.run('cat test-suite.log') | |
580 | raise UnexpectedExit(res) | |
581 | ||
582 | @task | |
583 | def ci_dnsdist_run_unit_tests(c): | |
584 | res = c.run('make check', warn=True) | |
585 | if res.exited != 0: | |
586 | c.run('cat test-suite.log') | |
587 | raise UnexpectedExit(res) | |
588 | ||
589 | @task | |
590 | def ci_make_install(c): | |
591 | res = c.run('make install') # FIXME: this builds auth docs - again | |
592 | ||
593 | @task | |
7d862cb3 | 594 | def add_auth_repo(c, dist_name, dist_release_name, pdns_repo_version): |
699d088a | 595 | c.sudo('apt-get install -y curl gnupg2') |
7d862cb3 | 596 | if pdns_repo_version == 'master': |
99bb3530 PD |
597 | c.sudo('curl -s -o /etc/apt/trusted.gpg.d/pdns-repo.asc https://repo.powerdns.com/CBC8B383-pub.asc') |
598 | else: | |
599 | c.sudo('curl -s -o /etc/apt/trusted.gpg.d/pdns-repo.asc https://repo.powerdns.com/FD380FBB-pub.asc') | |
7d862cb3 | 600 | c.run(f"echo 'deb [arch=amd64] http://repo.powerdns.com/{dist_name} {dist_release_name}-auth-{pdns_repo_version} main' | sudo tee /etc/apt/sources.list.d/pdns.list") |
99bb3530 PD |
601 | c.run("echo 'Package: pdns-*' | sudo tee /etc/apt/preferences.d/pdns") |
602 | c.run("echo 'Pin: origin repo.powerdns.com' | sudo tee -a /etc/apt/preferences.d/pdns") | |
603 | c.run("echo 'Pin-Priority: 600' | sudo tee -a /etc/apt/preferences.d/pdns") | |
604 | c.sudo('apt-get update') | |
605 | ||
606 | @task | |
607 | def test_api(c, product, backend=''): | |
608 | if product == 'recursor': | |
609 | with c.cd('regression-tests.api'): | |
610 | c.run(f'PDNSRECURSOR=/opt/pdns-recursor/sbin/pdns_recursor ./runtests recursor {backend}') | |
611 | elif product == 'auth': | |
612 | with c.cd('regression-tests.api'): | |
7d862cb3 | 613 | c.run(f'PDNSSERVER=/opt/pdns-auth/sbin/pdns_server PDNSUTIL=/opt/pdns-auth/bin/pdnsutil SDIG=/opt/pdns-auth/bin/sdig MYSQL_HOST={auth_backend_ip_addr} PGHOST={auth_backend_ip_addr} PGPORT=5432 ./runtests authoritative {backend}') |
99bb3530 PD |
614 | else: |
615 | raise Failure('unknown product') | |
616 | ||
0e77de07 | 617 | backend_regress_tests = dict( |
b33a88da | 618 | bind = [ |
8af54cc6 AR |
619 | 'bind-both', |
620 | 'bind-dnssec-both', | |
621 | 'bind-dnssec-nsec3-both', | |
622 | 'bind-dnssec-nsec3-optout-both', | |
623 | 'bind-dnssec-nsec3-narrow', | |
624 | # FIXME 'bind-dnssec-pkcs11' | |
b33a88da PD |
625 | ], |
626 | geoip = [ | |
8af54cc6 AR |
627 | 'geoip', |
628 | 'geoip-nsec3-narrow' | |
b33a88da | 629 | ], |
8af54cc6 AR |
630 | lua2 = ['lua2', 'lua2-dnssec'], |
631 | tinydns = ['tinydns'], | |
b33a88da | 632 | remote = [ |
8af54cc6 AR |
633 | 'remotebackend-pipe', |
634 | 'remotebackend-unix', | |
635 | 'remotebackend-http', | |
636 | 'remotebackend-zeromq', | |
637 | 'remotebackend-pipe-dnssec', | |
638 | 'remotebackend-unix-dnssec', | |
639 | 'remotebackend-http-dnssec', | |
640 | 'remotebackend-zeromq-dnssec' | |
b33a88da PD |
641 | ], |
642 | lmdb = [ | |
8af54cc6 AR |
643 | 'lmdb-nodnssec-both', |
644 | 'lmdb-both', | |
645 | 'lmdb-nsec3-both', | |
646 | 'lmdb-nsec3-optout-both', | |
647 | 'lmdb-nsec3-narrow' | |
648 | ], | |
649 | gmysql = [ | |
650 | 'gmysql', | |
651 | 'gmysql-nodnssec-both', | |
652 | 'gmysql-nsec3-both', | |
653 | 'gmysql-nsec3-optout-both', | |
654 | 'gmysql-nsec3-narrow', | |
655 | 'gmysql_sp-both' | |
656 | ], | |
657 | gpgsql = [ | |
658 | 'gpgsql', | |
659 | 'gpgsql-nodnssec-both', | |
660 | 'gpgsql-nsec3-both', | |
661 | 'gpgsql-nsec3-optout-both', | |
662 | 'gpgsql-nsec3-narrow', | |
663 | 'gpgsql_sp-both' | |
664 | ], | |
665 | gsqlite3 = [ | |
666 | 'gsqlite3', | |
667 | 'gsqlite3-nodnssec-both', | |
668 | 'gsqlite3-nsec3-both', | |
669 | 'gsqlite3-nsec3-optout-both', | |
670 | 'gsqlite3-nsec3-narrow' | |
671 | ], | |
672 | godbc_sqlite3 = ['godbc_sqlite3-nodnssec'], | |
673 | godbc_mssql = [ | |
674 | 'godbc_mssql', | |
675 | 'godbc_mssql-nodnssec', | |
676 | 'godbc_mssql-nsec3', | |
677 | 'godbc_mssql-nsec3-optout', | |
678 | 'godbc_mssql-nsec3-narrow' | |
b33a88da | 679 | ], |
c4a7e1df AR |
680 | ldap = [ |
681 | 'ldap-tree', | |
682 | 'ldap-simple', | |
683 | 'ldap-strict' | |
684 | ], | |
685 | geoip_mmdb = ['geoip'], | |
0e77de07 PD |
686 | ) |
687 | ||
8af54cc6 AR |
688 | godbc_mssql_credentials = {"username": "sa", "password": "SAsa12%%"} |
689 | ||
7d862cb3 | 690 | godbc_config = f''' |
8af54cc6 AR |
691 | [pdns-mssql-docker] |
692 | Driver=FreeTDS | |
693 | Trace=No | |
7d862cb3 | 694 | Server={auth_backend_ip_addr} |
8af54cc6 AR |
695 | Port=1433 |
696 | Database=pdns | |
697 | TDS_Version=7.1 | |
698 | ||
699 | [pdns-mssql-docker-nodb] | |
700 | Driver=FreeTDS | |
701 | Trace=No | |
7d862cb3 | 702 | Server={auth_backend_ip_addr} |
8af54cc6 AR |
703 | Port=1433 |
704 | TDS_Version=7.1 | |
705 | ||
706 | [pdns-sqlite3-1] | |
707 | Driver = SQLite3 | |
708 | Database = pdns.sqlite3 | |
709 | ||
710 | [pdns-sqlite3-2] | |
711 | Driver = SQLite3 | |
712 | Database = pdns.sqlite32 | |
713 | ''' | |
714 | ||
715 | def setup_godbc_mssql(c): | |
716 | with open(os.path.expanduser("~/.odbc.ini"), "a") as f: | |
717 | f.write(godbc_config) | |
718 | c.sudo('sh -c \'echo "Threading=1" | cat /usr/share/tdsodbc/odbcinst.ini - | tee -a /etc/odbcinst.ini\'') | |
719 | c.sudo('sed -i "s/libtdsodbc.so/\/usr\/lib\/x86_64-linux-gnu\/odbc\/libtdsodbc.so/g" /etc/odbcinst.ini') | |
720 | c.run(f'echo "create database pdns" | isql -v pdns-mssql-docker-nodb {godbc_mssql_credentials["username"]} {godbc_mssql_credentials["password"]}') | |
721 | # FIXME: Skip 8bit-txt-unescaped test | |
722 | c.run('touch ${PWD}/regression-tests/tests/8bit-txt-unescaped/skip') | |
723 | ||
724 | def setup_godbc_sqlite3(c): | |
725 | with open(os.path.expanduser("~/.odbc.ini"), "a") as f: | |
726 | f.write(godbc_config) | |
727 | c.sudo('sed -i "s/libsqlite3odbc.so/\/usr\/lib\/x86_64-linux-gnu\/odbc\/libsqlite3odbc.so/g" /etc/odbcinst.ini') | |
728 | ||
c4a7e1df | 729 | def setup_ldap_client(c): |
699d088a | 730 | c.sudo('DEBIAN_FRONTEND=noninteractive apt-get install -y ldap-utils') |
7d862cb3 | 731 | c.sudo(f'sh -c \'echo "{auth_backend_ip_addr} ldapserver" | tee -a /etc/hosts\'') |
c4a7e1df | 732 | |
0e77de07 PD |
733 | @task |
734 | def test_auth_backend(c, backend): | |
7d862cb3 | 735 | pdns_auth_env_vars = f'PDNS=/opt/pdns-auth/sbin/pdns_server PDNS2=/opt/pdns-auth/sbin/pdns_server SDIG=/opt/pdns-auth/bin/sdig NOTIFY=/opt/pdns-auth/bin/pdns_notify NSEC3DIG=/opt/pdns-auth/bin/nsec3dig SAXFR=/opt/pdns-auth/bin/saxfr ZONE2SQL=/opt/pdns-auth/bin/zone2sql ZONE2LDAP=/opt/pdns-auth/bin/zone2ldap ZONE2JSON=/opt/pdns-auth/bin/zone2json PDNSUTIL=/opt/pdns-auth/bin/pdnsutil PDNSCONTROL=/opt/pdns-auth/bin/pdns_control PDNSSERVER=/opt/pdns-auth/sbin/pdns_server SDIG=/opt/pdns-auth/bin/sdig GMYSQLHOST={auth_backend_ip_addr} GMYSQL2HOST={auth_backend_ip_addr} MYSQL_HOST={auth_backend_ip_addr} PGHOST={auth_backend_ip_addr} PGPORT=5432' |
8af54cc6 | 736 | |
0e77de07 | 737 | if backend == 'remote': |
e55d3a4b | 738 | ci_auth_install_remotebackend_test_deps(c) |
0e77de07 | 739 | |
222d17e2 | 740 | if backend == 'authpy': |
7d862cb3 | 741 | c.sudo(f'sh -c \'echo "{auth_backend_ip_addr} kerberos-server" | tee -a /etc/hosts\'') |
222d17e2 | 742 | with c.cd('regression-tests.auth-py'): |
8af54cc6 AR |
743 | c.run(f'{pdns_auth_env_vars} WITHKERBEROS=YES ./runtests') |
744 | return | |
745 | ||
746 | if backend == 'godbc_sqlite3': | |
747 | setup_godbc_sqlite3(c) | |
748 | with c.cd('regression-tests'): | |
749 | for variant in backend_regress_tests[backend]: | |
750 | c.run(f'{pdns_auth_env_vars} GODBC_SQLITE3_DSN=pdns-sqlite3-1 ./start-test-stop 5300 {variant}') | |
751 | return | |
752 | ||
753 | if backend == 'godbc_mssql': | |
754 | setup_godbc_mssql(c) | |
755 | with c.cd('regression-tests'): | |
756 | for variant in backend_regress_tests[backend]: | |
757 | c.run(f'{pdns_auth_env_vars} GODBC_MSSQL_PASSWORD={godbc_mssql_credentials["password"]} GODBC_MSSQL_USERNAME={godbc_mssql_credentials["username"]} GODBC_MSSQL_DSN=pdns-mssql-docker GODBC_MSSQL2_PASSWORD={godbc_mssql_credentials["password"]} GODBC_MSSQL2_USERNAME={godbc_mssql_credentials["username"]} GODBC_MSSQL2_DSN=pdns-mssql-docker ./start-test-stop 5300 {variant}') | |
222d17e2 PD |
758 | return |
759 | ||
c4a7e1df AR |
760 | if backend == 'ldap': |
761 | setup_ldap_client(c) | |
762 | ||
763 | if backend == 'geoip_mmdb': | |
764 | with c.cd('regression-tests'): | |
765 | for variant in backend_regress_tests[backend]: | |
766 | c.run(f'{pdns_auth_env_vars} geoipdatabase=../modules/geoipbackend/regression-tests/GeoLiteCity.mmdb ./start-test-stop 5300 {variant}') | |
767 | return | |
768 | ||
0e77de07 | 769 | with c.cd('regression-tests'): |
b33a88da PD |
770 | if backend == 'lua2': |
771 | c.run('touch trustedkeys') # avoid silly error during cleanup | |
772 | for variant in backend_regress_tests[backend]: | |
8af54cc6 | 773 | c.run(f'{pdns_auth_env_vars} ./start-test-stop 5300 {variant}') |
222d17e2 PD |
774 | |
775 | if backend == 'gsqlite3': | |
7d862cb3 AR |
776 | if os.getenv('SKIP_IPV6_TESTS'): |
777 | pdns_auth_env_vars += ' context=noipv6' | |
222d17e2 | 778 | with c.cd('regression-tests.nobackend'): |
8af54cc6 | 779 | c.run(f'{pdns_auth_env_vars} ./runtests') |
222d17e2 PD |
780 | c.run('/opt/pdns-auth/bin/pdnsutil test-algorithms') |
781 | return | |
b33a88da PD |
782 | |
783 | @task | |
784 | def test_ixfrdist(c): | |
785 | with c.cd('regression-tests.ixfrdist'): | |
786 | c.run('IXFRDISTBIN=/opt/pdns-auth/bin/ixfrdist ./runtests') | |
0e77de07 | 787 | |
99bb3530 PD |
788 | @task |
789 | def test_dnsdist(c): | |
790 | c.run('chmod +x /opt/dnsdist/bin/*') | |
791 | c.run('ls -ald /var /var/agentx /var/agentx/master') | |
792 | c.run('ls -al /var/agentx/master') | |
793 | with c.cd('regression-tests.dnsdist'): | |
794 | c.run('DNSDISTBIN=/opt/dnsdist/bin/dnsdist ./runtests') | |
d3cb00f9 | 795 | |
6b45d67b O |
796 | @task |
797 | def test_regression_recursor(c): | |
798 | c.run('/opt/pdns-recursor/sbin/pdns_recursor --version') | |
7d862cb3 | 799 | c.run('PDNSRECURSOR=/opt/pdns-recursor/sbin/pdns_recursor RECCONTROL=/opt/pdns-recursor/bin/rec_control ./build-scripts/test-recursor') |
6b45d67b O |
800 | |
801 | @task | |
375c8fd6 O |
802 | def test_bulk_recursor(c, threads, mthreads, shards): |
803 | # We run an extremely small version of the bulk test, as GH does not seem to be able to handle the UDP load | |
6b45d67b O |
804 | with c.cd('regression-tests'): |
805 | c.run('curl -LO http://s3-us-west-1.amazonaws.com/umbrella-static/top-1m.csv.zip') | |
806 | c.run('unzip top-1m.csv.zip -d .') | |
807 | c.run('chmod +x /opt/pdns-recursor/bin/* /opt/pdns-recursor/sbin/*') | |
375c8fd6 | 808 | c.run(f'DNSBULKTEST=/usr/bin/dnsbulktest RECURSOR=/opt/pdns-recursor/sbin/pdns_recursor RECCONTROL=/opt/pdns-recursor/bin/rec_control THRESHOLD=95 TRACE=no ./timestamp ./recursor-test 5300 100 {threads} {mthreads} {shards}') |
6b45d67b | 809 | |
dab788a9 PD |
810 | @task |
811 | def install_swagger_tools(c): | |
812 | c.run('npm install -g api-spec-converter') | |
813 | ||
814 | @task | |
815 | def swagger_syntax_check(c): | |
816 | c.run('api-spec-converter docs/http-api/swagger/authoritative-api-swagger.yaml -f swagger_2 -t openapi_3 -s json -c') | |
817 | ||
66c07369 | 818 | @task |
df23d4bf RG |
819 | def install_coverity_tools(c, project): |
820 | token = os.getenv('COVERITY_TOKEN') | |
821 | c.run(f'curl -s https://scan.coverity.com/download/linux64 --data "token={token}&project={project}" | gunzip | sudo tar xvf /dev/stdin --strip-components=1 --no-same-owner -C /usr/local', hide=True) | |
66c07369 RG |
822 | |
823 | @task | |
824 | def coverity_clang_configure(c): | |
7d862cb3 | 825 | c.sudo(f'/usr/local/bin/cov-configure --template --comptype clangcc --compiler clang++-{clang_version}') |
66c07369 RG |
826 | |
827 | @task | |
828 | def coverity_make(c): | |
829 | c.run('/usr/local/bin/cov-build --dir cov-int make -j8 -k') | |
830 | ||
831 | @task | |
832 | def coverity_tarball(c, tarball): | |
833 | c.run(f'tar caf {tarball} cov-int') | |
834 | ||
835 | @task | |
df23d4bf RG |
836 | def coverity_upload(c, email, project, tarball): |
837 | token = os.getenv('COVERITY_TOKEN') | |
66c07369 RG |
838 | c.run(f'curl --form token={token} \ |
839 | --form email="{email}" \ | |
840 | --form file=@{tarball} \ | |
841 | --form version="$(./builder-support/gen-version)" \ | |
842 | --form description="master build" \ | |
df23d4bf | 843 | https://scan.coverity.com/builds?project={project}', hide=True) |
66c07369 | 844 | |
d3cb00f9 PD |
845 | # this is run always |
846 | def setup(): | |
847 | if '/usr/lib/ccache' not in os.environ['PATH']: | |
848 | os.environ['PATH']='/usr/lib/ccache:'+os.environ['PATH'] | |
849 | ||
850 | setup() |