create 'journal' system test

tests that version 1 journal files containing version 1 transaction
headers are rolled forward correctly on server startup, then updated
into version 2 journals. also checks journal file consistency and
'max-journal-size' behavior.

(cherry picked from commit a0aefa1de6459000ddfd1876be886e3b12cf7692)

diff --git a/bin/tests/system/conf.sh.common b/bin/tests/system/conf.sh.common
index 65fce6267e6a3684c161772c3269acc8b2fde501..405f88bb9f6c659496ce383fb6340e9e5de935c9 100644 (file)
--- a/bin/tests/system/conf.sh.common
+++ b/bin/tests/system/conf.sh.common
@@ -88,6 +88,7 @@ idna \
 inline \
 integrity \
 ixfr \
+journal \
 kasp \
 keepalive \
 legacy \

diff --git a/bin/tests/system/journal/clean.sh b/bin/tests/system/journal/clean.sh
new file mode 100644 (file)
index 0000000..1b86495
--- /dev/null
+++ b/bin/tests/system/journal/clean.sh
@@ -0,0 +1,14 @@
+# Copyright (C) Internet Systems Consortium, Inc. ("ISC")
+#
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, you can obtain one at https://mozilla.org/MPL/2.0/.
+#
+# See the COPYRIGHT file distributed with this work for additional
+# information regarding copyright ownership.
+
+rm -f */named.memstats
+rm -f */named.run
+rm -f */named.conf
+rm -f */*.db */*.jnl
+rm -f dig.out*

diff --git a/bin/tests/system/journal/ns1/changed.ver1.jnl.saved b/bin/tests/system/journal/ns1/changed.ver1.jnl.saved
new file mode 100644 (file)
index 0000000..b449a7d
Binary files /dev/null and b/bin/tests/system/journal/ns1/changed.ver1.jnl.saved differ

diff --git a/bin/tests/system/journal/ns1/changed.ver2.jnl.saved b/bin/tests/system/journal/ns1/changed.ver2.jnl.saved
new file mode 100644 (file)
index 0000000..d2fa199
Binary files /dev/null and b/bin/tests/system/journal/ns1/changed.ver2.jnl.saved differ

diff --git a/bin/tests/system/journal/ns1/d1212.jnl.saved b/bin/tests/system/journal/ns1/d1212.jnl.saved
new file mode 100644 (file)
index 0000000..57c1497
Binary files /dev/null and b/bin/tests/system/journal/ns1/d1212.jnl.saved differ

diff --git a/bin/tests/system/journal/ns1/d2121.jnl.saved b/bin/tests/system/journal/ns1/d2121.jnl.saved
new file mode 100644 (file)
index 0000000..ec21372
Binary files /dev/null and b/bin/tests/system/journal/ns1/d2121.jnl.saved differ

diff --git a/bin/tests/system/journal/ns1/generic.db.in b/bin/tests/system/journal/ns1/generic.db.in
new file mode 100644 (file)
index 0000000..bb75d3c
--- /dev/null
+++ b/bin/tests/system/journal/ns1/generic.db.in
@@ -0,0 +1,16 @@
+; Copyright (C) Internet Systems Consortium, Inc. ("ISC")
+;
+; This Source Code Form is subject to the terms of the Mozilla Public
+; License, v. 2.0. If a copy of the MPL was not distributed with this
+; file, You can obtain one at http://mozilla.org/MPL/2.0/.
+;
+; See the COPYRIGHT file distributed with this work for additional
+; information regarding copyright ownership.
+
+
+$TTL 600
+@              SOA     ns hostmaster 2012010901 3600 1200 604800 1200
+                NS      ns
+ns             A       192.0.2.1
+
+addr1  A      10.53.0.1

diff --git a/bin/tests/system/journal/ns1/ixfr.db.in b/bin/tests/system/journal/ns1/ixfr.db.in
new file mode 100644 (file)
index 0000000..cee52ca
--- /dev/null
+++ b/bin/tests/system/journal/ns1/ixfr.db.in
@@ -0,0 +1,17 @@
+; Copyright (C) Internet Systems Consortium, Inc. ("ISC")
+;
+; This Source Code Form is subject to the terms of the Mozilla Public
+; License, v. 2.0. If a copy of the MPL was not distributed with this
+; file, You can obtain one at http://mozilla.org/MPL/2.0/.
+;
+; See the COPYRIGHT file distributed with this work for additional
+; information regarding copyright ownership.
+
+
+$TTL 600
+@              SOA     ns hostmaster 2012010902 3600 1200 604800 1200
+                NS      ns
+ns             A       192.0.2.1
+
+addr1  A      10.53.0.1
+addr2  A      10.53.0.2

diff --git a/bin/tests/system/journal/ns1/ixfr.ver1.jnl.saved b/bin/tests/system/journal/ns1/ixfr.ver1.jnl.saved
new file mode 100644 (file)
index 0000000..10b5116
Binary files /dev/null and b/bin/tests/system/journal/ns1/ixfr.ver1.jnl.saved differ

diff --git a/bin/tests/system/journal/ns1/maxjournal.jnl.saved b/bin/tests/system/journal/ns1/maxjournal.jnl.saved
new file mode 100644 (file)
index 0000000..7c79e5c
Binary files /dev/null and b/bin/tests/system/journal/ns1/maxjournal.jnl.saved differ

diff --git a/bin/tests/system/journal/ns1/maxjournal2.jnl.saved b/bin/tests/system/journal/ns1/maxjournal2.jnl.saved
new file mode 100644 (file)
index 0000000..e200905
Binary files /dev/null and b/bin/tests/system/journal/ns1/maxjournal2.jnl.saved differ

diff --git a/bin/tests/system/journal/ns1/named.conf.in b/bin/tests/system/journal/ns1/named.conf.in
new file mode 100644 (file)
index 0000000..3fdd96b
--- /dev/null
+++ b/bin/tests/system/journal/ns1/named.conf.in
@@ -0,0 +1,90 @@
+/*
+ * Copyright (C) Internet Systems Consortium, Inc. ("ISC")
+ *
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/.
+ *
+ * See the COPYRIGHT file distributed with this work for additional
+ * information regarding copyright ownership.
+ */
+
+options {
+       query-source address 10.53.0.1;
+       notify-source 10.53.0.1;
+       transfer-source 10.53.0.1;
+       port @PORT@;
+       session-keyfile "session.key";
+       pid-file "named.pid";
+       listen-on { 10.53.0.1; };
+       listen-on-v6 { none; };
+       dnssec-validation yes;
+       minimal-responses no;
+       recursion no;
+       notify yes;
+};
+
+key rndc_key {
+       secret "1234abcd8765";
+       algorithm hmac-sha256;
+};
+
+controls {
+       inet 10.53.0.1 port @CONTROLPORT@ allow { any; } keys { rndc_key; };
+};
+
+zone changed {
+       type primary;
+       update-policy local;
+       file "changed.db";
+};
+
+zone unchanged {
+       type primary;
+       update-policy local;
+       file "unchanged.db";
+};
+
+zone changed2 {
+       type primary;
+       update-policy local;
+       file "changed2.db";
+};
+
+zone unchanged2 {
+       type primary;
+       update-policy local;
+       file "unchanged2.db";
+};
+
+zone hdr1d1d2d1d2 {
+       type primary;
+       update-policy local;
+       file "d1212.db";
+};
+
+zone hdr1d2d1d2d1 {
+       type primary;
+       update-policy local;
+       file "d2121.db";
+};
+
+zone ixfr {
+       type primary;
+       ixfr-from-differences yes;
+       file "ixfr.db";
+};
+
+zone maxjournal {
+       type primary;
+       max-journal-size 1k;
+       update-policy local;
+       file "maxjournal.db";
+};
+
+zone maxjournal2 {
+       type primary;
+       max-journal-size 1k;
+       update-policy local;
+       file "maxjournal2.db";
+};

diff --git a/bin/tests/system/journal/ns1/unchanged.ver1.jnl.saved b/bin/tests/system/journal/ns1/unchanged.ver1.jnl.saved
new file mode 100644 (file)
index 0000000..f7885d9
Binary files /dev/null and b/bin/tests/system/journal/ns1/unchanged.ver1.jnl.saved differ

diff --git a/bin/tests/system/journal/ns1/unchanged.ver2.jnl.saved b/bin/tests/system/journal/ns1/unchanged.ver2.jnl.saved
new file mode 100644 (file)
index 0000000..d974be4
Binary files /dev/null and b/bin/tests/system/journal/ns1/unchanged.ver2.jnl.saved differ

diff --git a/bin/tests/system/journal/setup.sh b/bin/tests/system/journal/setup.sh
new file mode 100644 (file)
index 0000000..23c9639
--- /dev/null
+++ b/bin/tests/system/journal/setup.sh
@@ -0,0 +1,42 @@
+#!/bin/sh
+#
+# Copyright (C) Internet Systems Consortium, Inc. ("ISC")
+#
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, you can obtain one at https://mozilla.org/MPL/2.0/.
+#
+# See the COPYRIGHT file distributed with this work for additional
+# information regarding copyright ownership.
+
+. ../conf.sh
+
+$SHELL clean.sh
+
+copy_setports ns1/named.conf.in ns1/named.conf
+cp ns1/generic.db.in ns1/changed.db
+cp ns1/changed.ver1.jnl.saved ns1/changed.db.jnl
+
+cp ns1/generic.db.in ns1/unchanged.db
+cp ns1/unchanged.ver1.jnl.saved ns1/unchanged.db.jnl
+
+cp ns1/generic.db.in ns1/changed2.db
+cp ns1/changed.ver2.jnl.saved ns1/changed2.db.jnl
+
+cp ns1/generic.db.in ns1/unchanged2.db
+cp ns1/unchanged.ver2.jnl.saved ns1/unchanged2.db.jnl
+
+cp ns1/ixfr.db.in ns1/ixfr.db
+cp ns1/ixfr.ver1.jnl.saved ns1/ixfr.db.jnl
+
+cp ns1/generic.db.in ns1/d1212.db
+cp ns1/d1212.jnl.saved ns1/d1212.db.jnl
+
+cp ns1/generic.db.in ns1/d2121.db
+cp ns1/d2121.jnl.saved ns1/d2121.db.jnl
+
+cp ns1/generic.db.in ns1/maxjournal.db
+cp ns1/maxjournal.jnl.saved ns1/maxjournal.db.jnl
+
+cp ns1/generic.db.in ns1/maxjournal2.db
+cp ns1/maxjournal2.jnl.saved ns1/maxjournal2.db.jnl

diff --git a/bin/tests/system/journal/tests.sh b/bin/tests/system/journal/tests.sh
new file mode 100644 (file)
index 0000000..7fd12dc
--- /dev/null
+++ b/bin/tests/system/journal/tests.sh
@@ -0,0 +1,201 @@
+#!/bin/sh
+#
+# Copyright (C) Internet Systems Consortium, Inc. ("ISC")
+#
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, you can obtain one at https://mozilla.org/MPL/2.0/.
+#
+# See the COPYRIGHT file distributed with this work for additional
+# information regarding copyright ownership.
+
+. ../conf.sh
+
+dig_with_opts() {
+    "$DIG" @10.53.0.1 -p "$PORT" +tcp "$@"
+}
+
+rndc_with_opts() {
+    "$RNDC" -c ../common/rndc.conf -p "$CONTROLPORT" -s "$@"
+}
+
+status=0
+n=0
+
+n=`expr $n + 1`
+echo_i "check outdated journal rolled forward (dynamic) ($n)"
+ret=0
+dig_with_opts changed soa > dig.out.test$n
+grep 'status: NOERROR' dig.out.test$n > /dev/null || ret=1
+grep '2012010902' dig.out.test$n > /dev/null || ret=1
+grep 'zone changed/IN: retried using old journal format' ns1/named.run > /dev/null || ret=1
+[ $ret -eq 0 ] || echo_i "failed"
+status=`expr $status + $ret`
+
+n=`expr $n + 1`
+echo_i "check outdated empty journal did not cause an error (dynamic) ($n)"
+ret=0
+dig_with_opts unchanged soa > dig.out.test$n
+grep 'status: NOERROR' dig.out.test$n > /dev/null || ret=1
+grep '2012010901' dig.out.test$n > /dev/null || ret=1
+[ $ret -eq 0 ] || echo_i "failed"
+status=`expr $status + $ret`
+
+n=`expr $n + 1`
+echo_i "check outdated journals were updated or removed (dynamic) ($n)"
+ret=0
+cat -v ns1/changed.db.jnl | grep "BIND LOG V9.2" > /dev/null || ret=1
+[ -f ns1/unchanged.db.jnl ] && ret=1
+[ $ret -eq 0 ] || echo_i "failed"
+status=`expr $status + $ret`
+
+n=`expr $n + 1`
+echo_i "check updated journal has correct RR count (dynamic) ($n)"
+ret=0
+$JOURNALPRINT -x ns1/changed.db.jnl | grep "rrcount 3 " > /dev/null || ret=1
+[ $ret -eq 0 ] || echo_i "failed"
+status=`expr $status + $ret`
+
+n=`expr $n + 1`
+echo_i "check new-format journal rolled forward (dynamic) ($n)"
+ret=0
+dig_with_opts changed2 soa > dig.out.test$n
+grep 'status: NOERROR' dig.out.test$n > /dev/null || ret=1
+grep '2012010902' dig.out.test$n > /dev/null || ret=1
+grep 'zone changed2/IN: retried using old journal format' ns1/named.run > /dev/null && ret=1
+[ $ret -eq 0 ] || echo_i "failed"
+status=`expr $status + $ret`
+
+n=`expr $n + 1`
+echo_i "check new-format empty journal did not cause error (dynamic) ($n)"
+ret=0
+dig_with_opts unchanged2 soa > dig.out.test$n
+grep 'status: NOERROR' dig.out.test$n > /dev/null || ret=1
+grep '2012010901' dig.out.test$n > /dev/null || ret=1
+grep 'zone unchanged2/IN: retried using old journal format' ns1/named.run > /dev/null && ret=1
+[ $ret -eq 0 ] || echo_i "failed"
+status=`expr $status + $ret`
+
+n=`expr $n + 1`
+echo_i "check new-format journals were updated or removed (dynamic) ($n)"
+ret=0
+cat -v ns1/changed2.db.jnl | grep "BIND LOG V9.2" > /dev/null || ret=1
+[ -f ns1/unchanged2.db.jnl ] && ret=1
+[ $ret -eq 0 ] || echo_i "failed"
+status=`expr $status + $ret`
+
+n=`expr $n + 1`
+echo_i "check outdated up-to-date journal succeeded (ixfr-from-differences) ($n)"
+ret=0
+dig_with_opts -t soa ixfr > dig.out.test$n
+grep 'status: NOERROR' dig.out.test$n > /dev/null || ret=1
+grep '2012010902' dig.out.test$n > /dev/null || ret=1
+grep 'zone ixfr/IN: journal rollforward completed successfully: recoverable' ns1/named.run > /dev/null || ret=1
+[ $ret -eq 0 ] || echo_i "failed"
+status=`expr $status + $ret`
+
+n=`expr $n + 1`
+echo_i "check outdated journal was updated (ixfr-from-differences) ($n)"
+ret=0
+cat -v ns1/ixfr.db.jnl | grep "BIND LOG V9.2" > /dev/null || ret=1
+[ $ret -eq 0 ] || echo_i "failed"
+status=`expr $status + $ret`
+
+n=`expr $n + 1`
+echo_i "check journal with mixed headers succeeded (version 1,2,1,2) ($n)"
+ret=0
+dig_with_opts -t soa hdr1d1d2d1d2 > dig.out.test$n
+grep 'status: NOERROR' dig.out.test$n > /dev/null || ret=1
+grep '2012010905' dig.out.test$n > /dev/null || ret=1
+grep 'zone hdr1d1d2d1d2/IN: journal rollforward completed successfully: recoverable' ns1/named.run > /dev/null || ret=1
+[ $ret -eq 0 ] || echo_i "failed"
+status=`expr $status + $ret`
+
+n=`expr $n + 1`
+echo_i "check journal with mixed headers was updated (version 1,2,1,2) ($n)"
+ret=0
+[ $($JOURNALPRINT -x ns1/d1212.jnl.saved | grep -c "version 1") -eq 2 ] || ret=1
+[ $($JOURNALPRINT -x ns1/d1212.jnl.saved | grep -c "version 2") -eq 2 ] || ret=1
+[ $($JOURNALPRINT -x ns1/d1212.db.jnl | grep -c "version 1") -eq 0 ] || ret=1
+[ $($JOURNALPRINT -x ns1/d1212.db.jnl | grep -c "version 2") -eq 4 ] || ret=1
+[ $ret -eq 0 ] || echo_i "failed"
+status=`expr $status + $ret`
+
+n=`expr $n + 1`
+echo_i "check journal with mixed headers succeeded (version 2,1,2,1) ($n)"
+ret=0
+dig_with_opts -t soa hdr1d2d1d2d1 > dig.out.test$n
+grep 'status: NOERROR' dig.out.test$n > /dev/null || ret=1
+grep '2012010905' dig.out.test$n > /dev/null || ret=1
+grep 'zone hdr1d2d1d2d1/IN: journal rollforward completed successfully: recoverable' ns1/named.run > /dev/null || ret=1
+[ $ret -eq 0 ] || echo_i "failed"
+status=`expr $status + $ret`
+
+n=`expr $n + 1`
+echo_i "check journal with mixed headers was updated (version 2,1,2,1) ($n)"
+ret=0
+[ $($JOURNALPRINT -x ns1/d2121.jnl.saved | grep -c "version 1") -eq 2 ] || ret=1
+[ $($JOURNALPRINT -x ns1/d2121.jnl.saved | grep -c "version 2") -eq 2 ] || ret=1
+[ $($JOURNALPRINT -x ns1/d2121.db.jnl | grep -c "version 1") -eq 0 ] || ret=1
+[ $($JOURNALPRINT -x ns1/d2121.db.jnl | grep -c "version 2") -eq 4 ] || ret=1
+[ $ret -eq 0 ] || echo_i "failed"
+status=`expr $status + $ret`
+
+n=`expr $n + 1`
+echo_i "check there are no journals left un-updated ($n)"
+ret=0
+c1=$(cat -v ns1/*.jnl | grep -c "BIND LOG V9")
+c2=$(cat -v ns1/*.jnl | grep -c "BIND LOG V9.2")
+[ ${c1} -eq ${c2} ] || ret=1
+[ $ret -eq 0 ] || echo_i "failed"
+status=`expr $status + $ret`
+
+n=`expr $n + 1`
+echo_i "check journal downgrade/upgrade ($n)"
+ret=0
+cp ns1/changed.db.jnl ns1/temp.jnl
+$JOURNALPRINT -d ns1/temp.jnl
+[ $($JOURNALPRINT -x ns1/temp.jnl | grep -c "version 1") -eq 1 ] || ret=1
+$JOURNALPRINT -x ns1/temp.jnl | grep -q "Header version = 1" || ret=1
+$JOURNALPRINT -u ns1/temp.jnl
+$JOURNALPRINT -x ns1/temp.jnl | grep -q "Header version = 2" || ret=1
+[ $($JOURNALPRINT -x ns1/temp.jnl | grep -c "version 2") -eq 1 ] || ret=1
+[ $ret -eq 0 ] || echo_i "failed"
+status=`expr $status + $ret`
+
+n=`expr $n + 1`
+echo_i "check max-journal-size works after journal update ($n)"
+ret=0
+# a dump should have been triggered by repairing the journal,
+# which would have resulted in the journal already being
+# compacted.
+[ $(wc -c < ns1/maxjournal.db.jnl) -lt 4000 ] || ret=1
+[ $ret -eq 0 ] || echo_i "failed"
+status=`expr $status + $ret`
+
+n=`expr $n + 1`
+echo_i "check max-journal-size works with non-updated journals ($n)"
+ret=0
+# journal was not repaired, so it should still be big
+[ $(wc -c < ns1/maxjournal2.db.jnl) -gt 12000 ] || ret=1
+# the zone hasn't been dumped yet, so 'rndc sync' should work without
+# needing a zone update first.
+rndc_with_opts 10.53.0.1 sync maxjournal2
+check_size() (
+    [ $(wc -c < ns1/maxjournal2.db.jnl) -lt 4000 ]
+)
+retry_quiet 10 check_size || ret=1
+[ $ret -eq 0 ] || echo_i "failed"
+status=`expr $status + $ret`
+
+n=`expr $n + 1`
+echo_i "check journal index consistency ($n)"
+ret=0
+for jnl in ns1/*.jnl; do
+    $JOURNALPRINT -x $jnl 2>&1 | grep -q "Offset mismatch" && ret=1
+done
+[ $ret -eq 0 ] || echo_i "failed"
+status=`expr $status + $ret`
+
+echo_i "exit status: $status"
+[ $status -eq 0 ] || exit 1

diff --git a/util/copyrights b/util/copyrights
index 9a9d53e44cc5c8a5386f273502d9d66f1360417e..1e4b6c779c481efa2feb0dd8c02c84fdff80252e 100644 (file)
--- a/util/copyrights
+++ b/util/copyrights
@@ -638,6 +638,18 @@
 ./bin/tests/system/ixfr/prereq.sh              SH      2001,2004,2007,2012,2014,2016,2018,2019,2020,2021
 ./bin/tests/system/ixfr/setup.sh               SH      2001,2004,2007,2011,2012,2013,2014,2016,2018,2019,2020,2021
 ./bin/tests/system/ixfr/tests.sh               SH      2001,2004,2007,2011,2012,2014,2016,2018,2019,2020,2021
+./bin/tests/system/journal/clean.sh            SH      2021
+./bin/tests/system/journal/ns1/changed.ver1.jnl.saved  X       2021
+./bin/tests/system/journal/ns1/changed.ver2.jnl.saved  X       2021
+./bin/tests/system/journal/ns1/d1212.jnl.saved X       2021
+./bin/tests/system/journal/ns1/d2121.jnl.saved X       2021
+./bin/tests/system/journal/ns1/ixfr.ver1.jnl.saved     X       2021
+./bin/tests/system/journal/ns1/maxjournal.jnl.saved    X       2021
+./bin/tests/system/journal/ns1/maxjournal2.jnl.saved   X       2021
+./bin/tests/system/journal/ns1/unchanged.ver1.jnl.saved        X       2021
+./bin/tests/system/journal/ns1/unchanged.ver2.jnl.saved        X       2021
+./bin/tests/system/journal/setup.sh            SH      2021
+./bin/tests/system/journal/tests.sh            SH      2021
 ./bin/tests/system/kasp/README                 TXT.BRIEF       2019,2020,2021
 ./bin/tests/system/kasp/clean.sh               SH      2019,2020,2021
 ./bin/tests/system/kasp/ns2/setup.sh           SH      2019,2020,2021