From: Thomas Markwalder Date: Fri, 24 Aug 2018 19:38:20 +0000 (-0400) Subject: [#15,!11] Schema upgrades for Kea 1.5, now update subnet ids in existing host reserva... X-Git-Tag: gitlab116_base~18^2~4 X-Git-Url: http://git.ipfire.org/gitweb/gitweb.cgi?a=commitdiff_plain;h=8d04ad0ae6b999bfacedb9e827046fba52674f47;p=thirdparty%2Fkea.git [#15,!11] Schema upgrades for Kea 1.5, now update subnet ids in existing host reservations src/bin/admin/tests/pgsql_tests.sh.in src/bin/admin/tests/cql_tests.sh.in src/bin/admin/tests/mysql_tests.sh.in Added functions that verify subnet id values are updated src/share/database/scripts/cql/upgrade_2.0_to_3.0.sh.in src/share/database/scripts/mysql/upgrade_6.0_to_7.0.sh.in src/share/database/scripts/pgsql/upgrade_4.0_to_5.0.sh.in Added logic to update subnet id values in existing reservations and options --- diff --git a/src/bin/admin/tests/cql_tests.sh.in b/src/bin/admin/tests/cql_tests.sh.in index 082101ad9f..3b233e778b 100644 --- a/src/bin/admin/tests/cql_tests.sh.in +++ b/src/bin/admin/tests/cql_tests.sh.in @@ -22,12 +22,17 @@ db_name="keatest" # Set location of the kea-admin. keaadmin=@abs_top_builddir@/src/bin/admin/kea-admin +cql_wipe() { + # Wipe the database. + cql_execute_script $db_scripts_dir/cql/dhcpdb_drop.cql + assert_eq 0 $? "drop table query failed, exit code %d, expected %d" +} + cql_lease_init_test() { test_start "cql.init" # Wipe the database. - cql_execute_script $db_scripts_dir/cql/dhcpdb_drop.cql - assert_eq 0 $? "drop table query failed, exit code %d, expected %d" + cql_wipe # Create the database $keaadmin lease-init cql -u $db_user -p $db_password -n $db_name -d $db_scripts_dir @@ -63,8 +68,7 @@ cql_lease_init_test() { assert_eq 2 $? "kea-admin failed to deny lease-init, expected exit code: %d, actual: %d" # Wipe the database. - cql_execute_script $db_scripts_dir/cql/dhcpdb_drop.cql - assert_eq 0 $? "drop table query failed, exit code %d, expected %d" + cql_wipe test_finish 0 } @@ -73,8 +77,7 @@ cql_lease_version_test() { test_start "cql.version" # Wipe the database. - cql_execute_script $db_scripts_dir/cql/dhcpdb_drop.cql - assert_eq 0 $? "drop table query failed, exit code %d, expected %d" + cql_wipe # Create the database. $keaadmin lease-init cql -u $db_user -p $db_password -n $db_name -d $db_scripts_dir @@ -85,8 +88,7 @@ cql_lease_version_test() { assert_str_eq "3.0" $version "Expected kea-admin to return %s, returned value was %s" # Wipe the database. - cql_execute_script $db_scripts_dir/cql/dhcpdb_drop.cql - assert_eq 0 $? "drop table query failed, exit code %d, expected %d" + cql_wipe test_finish 0 } @@ -95,8 +97,7 @@ cql_upgrade_test() { test_start "cql.upgrade" # Wipe the database. - cql_execute_script $db_scripts_dir/cql/dhcpdb_drop.cql - assert_eq 0 $? "drop table query failed, exit code %d, expected %d" + cql_wipe # Initialize database to schema 1.0. cql_execute_script @abs_top_srcdir@/src/bin/admin/tests/dhcpdb_create_1.0.cql @@ -106,8 +107,7 @@ cql_upgrade_test() { assert_eq 0 $? "lease-upgrade failed, expected exit code: %d, actual: %d" # Wipe the database. - cql_execute_script $db_scripts_dir/cql/dhcpdb_drop.cql - assert_eq 0 $? "drop table query failed, exit code %d, expected %d" + cql_wipe test_finish 0 } @@ -136,8 +136,7 @@ cql_lease4_dump_test() { fi # Wipe the database. - cql_execute_script $db_scripts_dir/cql/dhcpdb_drop.cql - assert_eq 0 $? "drop table query failed, exit code %d, expected %d" + cql_wipe # Create the database $keaadmin lease-init cql -u $db_user -p $db_password -n $db_name -d $db_scripts_dir @@ -178,8 +177,7 @@ INSERT INTO lease4(address, hwaddr, client_id, valid_lifetime, expire, subnet_id rm $output_file # Wipe the database. - cql_execute_script $db_scripts_dir/cql/dhcpdb_drop.cql - assert_eq 0 $? "drop table query failed, exit code %d, expected %d" + cql_wipe test_finish 0 } @@ -208,8 +206,7 @@ cql_lease6_dump_test() { fi # Wipe the database. - cql_execute_script $db_scripts_dir/cql/dhcpdb_drop.cql - assert_eq 0 $? "drop table query failed, exit code %d, expected %d" + cql_wipe # Create the database. $keaadmin lease-init cql -u $db_user -p $db_password -n $db_name -d $db_scripts_dir @@ -251,15 +248,117 @@ INSERT INTO lease6(address, duid, valid_lifetime, expire, subnet_id,\ rm $output_file # Wipe the database. - cql_execute_script $db_scripts_dir/cql/dhcpdb_drop.cql - assert_eq 0 $? "drop table query failed, exit code %d, expected %d" + cql_wipe test_finish 0 } +# Upgrades an existing schema to a target newer version +# param target_version - desired schema version as "major.minor" +cql_upgrade_schema_to_version() { + target_version=$1 + + # Check if the scripts directory exists at all. + if [ ! -d ${db_scripts_dir}/cql ]; then + log_error "Invalid scripts directory: ${db_scripts_dir}/mysql" + exit 1 + fi + + # Check if there are any files in it + num_files=$(find ${db_scripts_dir}/cql/upgrade*.sh -type f | wc -l) + if [ $num_files -eq 0 ]; then + log_error "No scripts in ${db_scripts_dir}/cql?" + exit 1 + fi + + version=`cql_version` + for script in ${db_scripts_dir}/cql/upgrade*.sh + do + if [ $version = "$target_version" ] + then + break; + fi + + echo "Processing $script file..." + sh ${script} -u ${db_user} -p ${db_password} -k ${db_name} + version=`cql_version` + done + + echo "Schema upgraded to $version" +} + +# Verifies that you can upgrade from an earlier version and +# that unused subnet ID values in hosts and options tables are +# converted to -1 +cql_unused_subnet_id_test() { + test_start "cql.unused_subnet_id_test" + + # Let's wipe the whole database + cql_wipe + + # We need to create an older database with lease data so we can + # verify the upgrade mechanisms which convert subnet id values + # + # Initialize database to scheme 1.0. + cql_execute_script @abs_top_srcdir@/src/bin/admin/tests/dhcpdb_create_1.0.cql + + # Now upgrade to schema 2.0, the version just before global HRs + cql_upgrade_schema_to_version 2.0 + + # Now we need insert some hosts to "migrate" for both v4 and v6 + qry=\ +"insert into host_reservations (id, host_identifier_type, host_identifier, host_ipv4_subnet_id, host_ipv6_subnet_id, option_subnet_id)\ + values (1, 0, textAsBlob('0123456'), 0, 0, 0);\ + insert into host_reservations (id, host_identifier_type, host_identifier, host_ipv4_subnet_id, host_ipv6_subnet_id, option_subnet_id)\ + values (2, 0, textAsBlob('1123456'), 4, 0, 4);\ + insert into host_reservations (id, host_identifier_type, host_identifier, host_ipv4_subnet_id, host_ipv6_subnet_id, option_subnet_id)\ + values (3, 0, textAsBlob('2123456'), 0, 6, 6);\ + insert into host_reservations (id, host_identifier_type, host_identifier, host_ipv4_subnet_id, host_ipv6_subnet_id, option_subnet_id)\ + values (4, 0, textAsBlob('3123456'), 4, 6, 0);\ + insert into host_reservations (id, host_identifier_type, host_identifier, host_ipv4_subnet_id, host_ipv6_subnet_id, option_subnet_id)\ + values (5, 0, textAsBlob('3123456'), -1, 6, 6);" + + cql_execute "$qry" + assert_eq 0 $? "insert hosts failed, expected exit code: %d, actual: %d" + + # Ok, we have a 2.0 database with hosts and options. Let's upgrade it. + ${keaadmin} lease-upgrade cql -u $db_user -p $db_password -n $db_name -d $db_scripts_dir + ERRCODE=$? + + # Upgrade should succeed + assert_eq 0 $ERRCODE "upgrade failed" + + test_dir="@abs_top_srcdir@/src/bin/admin/tests" + ref_file="$test_dir/data/cql.subnet_id_test.reference.csv" + + output_dir="@abs_top_builddir@/src/bin/admin/tests" + export_file="$output_dir/data/cql.subnet_id_test.csv" + + # Fetch host_reservation data for comparison + echo "Exporting host_reservation data to $export_file ..." + + qry=\ +"select id, host_ipv4_subnet_id, host_ipv6_subnet_id, option_subnet_id from\ + host_reservations where id in(1,2,3,4,5);" + + cql_execute "$qry" >$export_file + assert_eq 0 $? "insert hosts failed, expected exit code: %d, actual: %d" + # Compare the dump output to reference file, they should be identical. + cmp -s $export_file $ref_file + assert_eq 0 $? "export file does not match reference file, expected exit code %d, actual %d" + + # remove the output file. + rm $export_file + + # Wipe the database. + cql_wipe +} + + # Run tests. cql_lease_init_test cql_lease_version_test cql_upgrade_test cql_lease4_dump_test cql_lease6_dump_test +cql_unused_subnet_id_test diff --git a/src/bin/admin/tests/data/Makefile.am b/src/bin/admin/tests/data/Makefile.am index f95d694ebb..2d3812f697 100644 --- a/src/bin/admin/tests/data/Makefile.am +++ b/src/bin/admin/tests/data/Makefile.am @@ -3,4 +3,5 @@ EXTRA_DIST = mysql.lease4_dump_test.reference.csv \ pgsql.lease4_dump_test.reference.csv \ pgsql.lease6_dump_test.reference.csv \ cql.lease4_dump_test.reference.csv \ - cql.lease6_dump_test.reference.csv + cql.lease6_dump_test.reference.csv \ + cql.subnet_id_test.reference.csv diff --git a/src/bin/admin/tests/data/cql.subnet_id_test.reference.csv b/src/bin/admin/tests/data/cql.subnet_id_test.reference.csv new file mode 100644 index 0000000000..2d474bde9e --- /dev/null +++ b/src/bin/admin/tests/data/cql.subnet_id_test.reference.csv @@ -0,0 +1,10 @@ + + id | host_ipv4_subnet_id | host_ipv6_subnet_id | option_subnet_id +----+---------------------+---------------------+------------------ + 1 | -1 | -1 | -1 + 2 | 4 | -1 | 4 + 3 | -1 | 6 | 6 + 4 | 4 | 6 | -1 + 5 | -1 | 6 | 6 + +(5 rows) diff --git a/src/bin/admin/tests/mysql_tests.sh.in b/src/bin/admin/tests/mysql_tests.sh.in index a8dcf2b7cd..d0ca957f7d 100644 --- a/src/bin/admin/tests/mysql_tests.sh.in +++ b/src/bin/admin/tests/mysql_tests.sh.in @@ -869,6 +869,89 @@ mysql_lease_stat_upgrade_test() { mysql_wipe } +# Verifies that you can upgrade from an earlier version and +# that unused subnet ID values in hosts and options tables are +# converted to NULL. +mysql_unused_subnet_id_test() { + test_start "mysql.unused_subnet_id_test" + + # Let's wipe the whole database + mysql_wipe + + # We need to create an older database with lease data so we can + # verify the upgrade mechanisms which convert subnet id values + # + # Initialize database to scheme 1.0. + mysql -u$db_user -p$db_password $db_name < @abs_top_srcdir@/src/bin/admin/tests/dhcpdb_create_1.0.mysql + + # Now upgrade to schema 6.0, this has lease_state in it + mysql_upgrade_schema_to_version 6.0 + + # Now we need insert some hosts to "migrate" for both v4 and v6 + qry=\ +"insert into hosts (dhcp_identifier_type, dhcp_identifier, dhcp4_subnet_id, dhcp6_subnet_id, hostname)\ + values (0, '0123456', 0, 0, 'both'); \ + insert into hosts (dhcp_identifier_type, dhcp_identifier, dhcp4_subnet_id, dhcp6_subnet_id, hostname)\ + values (0, '1123456', 4, 0, 'v4only'); + insert into hosts (dhcp_identifier_type, dhcp_identifier, dhcp4_subnet_id, dhcp6_subnet_id, hostname)\ + values (0, '2123456', 0, 6, 'v6only');\ + insert into hosts (dhcp_identifier_type, dhcp_identifier, dhcp4_subnet_id, dhcp6_subnet_id, hostname) \ + values (0, '3123456', 4, 6, 'neither');" + + run_statement "insert hosts" "$qry" + + # Now we need insert some options to "migrate" for both v4 and v6 + qry=\ +"insert into dhcp4_options (code, dhcp4_subnet_id, scope_id) values (1, 4, 0);\ + insert into dhcp4_options (code, dhcp4_subnet_id, scope_id) values (2, 0, 0);\ + insert into dhcp6_options (code, dhcp6_subnet_id, scope_id) values (1, 6, 0);\ + insert into dhcp6_options (code, dhcp6_subnet_id, scope_id) values (2, 0, 0);" + + run_statement "insert options" "$qry" + + # Ok, we have a 4.0 database with hosts and options. Let's upgrade it to 5.0 + ${keaadmin} lease-upgrade mysql -u $db_user -p $db_password -n $db_name -d $db_scripts_dir + ERRCODE=$? + + # Upgrade should succeed + assert_eq 0 $ERRCODE "upgrade failed" + + # Two hosts should have null v4 subnet ids + qry="select count(host_id) from hosts where dhcp4_subnet_id is null;" + run_statement "#hosts.1" "$qry" 2 + + # Two hosts should have v4 subnet ids = 4 + qry="select count(host_id) from hosts where dhcp4_subnet_id = 4;" + run_statement "#hosts.2" "$qry" 2 + + # Two hosts should have null v6 subnet ids + qry="select count(host_id) from hosts where dhcp6_subnet_id is null;" + run_statement "#hosts.3" "$qry" 2 + + # Two hosts should should have v6 subnet ids = 6 + qry="select count(host_id) from hosts where dhcp6_subnet_id = 6;" + run_statement "#hosts.4" "$qry" 2 + + # One option should have null v4 subnet id + qry="select count(option_id) from dhcp4_options where dhcp4_subnet_id is null;" + run_statement "#options.1" "$qry" 1 + + # One option should have v4 subnet id = 4 + qry="select count(option_id) from dhcp4_options where dhcp4_subnet_id = 4;" + run_statement "#options.2" "$qry" 1 + + # One option should have null v6 subnet id + qry="select count(option_id) from dhcp6_options where dhcp6_subnet_id is null;" + run_statement "#options.3" "$qry" 1 + + # One option should have v4 subnet id = 6 + qry="select count(option_id) from dhcp6_options where dhcp6_subnet_id = 6;" + run_statement "#options.4" "$qry" 1 + + # Let's wipe the whole database + mysql_wipe +} + mysql_lease_init_test mysql_host_reservation_init_test mysql_lease_version_test @@ -878,3 +961,4 @@ mysql_lease6_dump_test mysql_lease4_stat_test mysql_lease6_stat_test mysql_lease_stat_upgrade_test +mysql_unused_subnet_id_test diff --git a/src/bin/admin/tests/pgsql_tests.sh.in b/src/bin/admin/tests/pgsql_tests.sh.in index 0584ff2012..43c01c5e14 100644 --- a/src/bin/admin/tests/pgsql_tests.sh.in +++ b/src/bin/admin/tests/pgsql_tests.sh.in @@ -791,6 +791,92 @@ pgsql_lease_stat_upgrade_test() { pgsql_wipe } +# Verifies that you can upgrade from earlier version and +# that unused subnet ID values in hosts and options tables are +# converted to NULL. +pgsql_unused_subnet_id_test() { + test_start "pgsql.unused_subnet_id_test" + + # Let's wipe the whole database + pgsql_wipe + + # We need to create an older database with lease data so we can + # verify the upgrade mechanisms which prepopulate the lease stat + # tables. + # + # Initialize database to scheme 1.0. + pgsql_execute_script @abs_top_srcdir@/src/bin/admin/tests/dhcpdb_create_1.0.pgsql + assert_eq 0 $? "cannot initialize 1.0 database, expected exit code: %d, actual: %d" + + # Now upgrade to schema 4.0 + pgsql_upgrade_schema_to_version 4.0 + + # Now we need insert some hosts to "migrate" for both v4 and v6 + qry=\ +"insert into hosts (dhcp_identifier_type, dhcp_identifier, dhcp4_subnet_id, dhcp6_subnet_id, hostname)\ + values (0, '0123456', 0, 0, 'both'); \ + insert into hosts (dhcp_identifier_type, dhcp_identifier, dhcp4_subnet_id, dhcp6_subnet_id, hostname)\ + values (0, '1123456', 4, 0, 'v4only'); + insert into hosts (dhcp_identifier_type, dhcp_identifier, dhcp4_subnet_id, dhcp6_subnet_id, hostname)\ + values (0, '2123456', 0, 6, 'v6only');\ + insert into hosts (dhcp_identifier_type, dhcp_identifier, dhcp4_subnet_id, dhcp6_subnet_id, hostname) \ + values (0, '3123456', 4, 6, 'neither');" + + run_statement "insert hosts" "$qry" + + # Now we need insert some options to "migrate" for both v4 and v6 + qry=\ +"insert into dhcp4_options (code, dhcp4_subnet_id, scope_id) values (1, 4, 0);\ + insert into dhcp4_options (code, dhcp4_subnet_id, scope_id) values (2, 0, 0);\ + insert into dhcp6_options (code, dhcp6_subnet_id, scope_id) values (1, 6, 0);\ + insert into dhcp6_options (code, dhcp6_subnet_id, scope_id) values (2, 0, 0);" + + run_statement "insert options" "$qry" + + # Ok, we have a 4.0 database with hosts and options. Let's upgrade it to 5.0 + ${keaadmin} lease-upgrade pgsql -u $db_user -p $db_password -n $db_name -d $db_scripts_dir + ERRCODE=$? + + # Upgrade should succeed + assert_eq 0 $ERRCODE "upgrade failed" + + # Two hosts should have null v4 subnet ids + qry="select count(host_id) from hosts where dhcp4_subnet_id is null;" + run_statement "#hosts.1" "$qry" 2 + + # Two hosts should have v4 subnet ids = 4 + qry="select count(host_id) from hosts where dhcp4_subnet_id = 4;" + run_statement "#hosts.2" "$qry" 2 + + # Two hosts should have null v6 subnet ids + qry="select count(host_id) from hosts where dhcp6_subnet_id is null;" + run_statement "#hosts.3" "$qry" 2 + + # Two hosts should should have v6 subnet ids = 6 + qry="select count(host_id) from hosts where dhcp6_subnet_id = 6;" + run_statement "#hosts.4" "$qry" 2 + + # One option should have null v4 subnet id + qry="select count(option_id) from dhcp4_options where dhcp4_subnet_id is null;" + run_statement "#options.1" "$qry" 1 + + # One option should have v4 subnet id = 4 + qry="select count(option_id) from dhcp4_options where dhcp4_subnet_id = 4;" + run_statement "#options.2" "$qry" 1 + + # One option should have null v6 subnet id + qry="select count(option_id) from dhcp6_options where dhcp6_subnet_id is null;" + run_statement "#options.3" "$qry" 1 + + # One option should have v4 subnet id = 6 + qry="select count(option_id) from dhcp6_options where dhcp6_subnet_id = 6;" + run_statement "#options.4" "$qry" 1 + + # Let's wipe the whole database + pgsql_wipe +} + + pgsql_lease_init_test pgsql_lease_version_test pgsql_upgrade_test @@ -799,3 +885,4 @@ pgsql_lease6_dump_test pgsql_lease4_stat_test pgsql_lease6_stat_test pgsql_lease_stat_upgrade_test +pgsql_unused_subnet_id_test diff --git a/src/share/database/scripts/cql/upgrade_2.0_to_3.0.sh.in b/src/share/database/scripts/cql/upgrade_2.0_to_3.0.sh.in index 0deb78f595..35805be8d9 100644 --- a/src/share/database/scripts/cql/upgrade_2.0_to_3.0.sh.in +++ b/src/share/database/scripts/cql/upgrade_2.0_to_3.0.sh.in @@ -9,14 +9,28 @@ else . @abs_top_builddir@/src/bin/admin/admin-utils.sh fi -version=$(cql_version "$@") - -if [ "${version}" != "2.0" ]; then - printf "This script upgrades 2.0 to 3.0. Reported version is %s. Skipping upgrade.\n" "${version}" - exit 0 +# Need a path for temporary files created during upgrade data migration +# Use the state directory in the install path directory if it exists, otherwise +# use the build tree +if [ -e @localstatedir@/@PACKAGE_NAME@ ]; then + temp_file_dir="@localstatedir@/@PACKAGE_NAME@" +else + temp_file_dir="@abs_top_builddir@/src/share/database/scripts/cql" fi -cqlsh "$@" <> $update_file + let update_cnt++ + fi + + IFS="$xIFS" + done < $export_file + + # If we didn't record any updates, then hey, we're good to go! + if [ $update_cnt == 0 ] + then + exit_now 0 "Completedly successfully: No updates were needed" + fi + + # We have at least one update in the update file, so submit it # to cqlsh. + echo "$update_cnt update statements written to $update_file" + echo "Running the updates..." + cqlsh $cqlargs -f "$update_file" + if [ $? -ne 0 ] + then + exit_now -1 "Cassandra updates failed" + exit -1 + fi + + exit_now 0 "Updated $update_cnt of $line_cnt records" +} + +check_version +update_schema +migrate_host_data diff --git a/src/share/database/scripts/mysql/upgrade_6.0_to_7.0.sh.in b/src/share/database/scripts/mysql/upgrade_6.0_to_7.0.sh.in index 99f56f9dd7..370320cfe3 100644 --- a/src/share/database/scripts/mysql/upgrade_6.0_to_7.0.sh.in +++ b/src/share/database/scripts/mysql/upgrade_6.0_to_7.0.sh.in @@ -106,6 +106,12 @@ CREATE INDEX timestamp_index ON logs (timestamp); ALTER TABLE hosts ADD COLUMN auth_key VARCHAR(16) NULL; +# Convert subnet-id values of 0 to NULL +UPDATE hosts SET dhcp4_subnet_id = NULL WHERE dhcp4_subnet_id = 0; +UPDATE dhcp4_options SET dhcp4_subnet_id = NULL WHERE dhcp4_subnet_id = 0; +UPDATE hosts SET dhcp6_subnet_id = NULL WHERE dhcp6_subnet_id = 0; +UPDATE dhcp6_options SET dhcp6_subnet_id = NULL WHERE dhcp6_subnet_id = 0; + # Update the schema version number UPDATE schema_version SET version = '7', minor = '0'; diff --git a/src/share/database/scripts/pgsql/upgrade_4.0_to_5.0.sh.in b/src/share/database/scripts/pgsql/upgrade_4.0_to_5.0.sh.in index 0fca541770..243b94692a 100644 --- a/src/share/database/scripts/pgsql/upgrade_4.0_to_5.0.sh.in +++ b/src/share/database/scripts/pgsql/upgrade_4.0_to_5.0.sh.in @@ -136,6 +136,12 @@ CREATE INDEX address_id ON logs (address); -- Create auth_key in hosts table for storing keys for DHCPv6 reconfigure. ALTER TABLE hosts ADD COLUMN auth_key VARCHAR(16) DEFAULT NULL; +-- Convert subnet-id values of 0 to NULL +UPDATE hosts SET dhcp4_subnet_id = NULL WHERE dhcp4_subnet_id = 0; +UPDATE dhcp4_options SET dhcp4_subnet_id = NULL WHERE dhcp4_subnet_id = 0; +UPDATE hosts SET dhcp6_subnet_id = NULL WHERE dhcp6_subnet_id = 0; +UPDATE dhcp6_options SET dhcp6_subnet_id = NULL WHERE dhcp6_subnet_id = 0; + -- Set 5.0 schema version. UPDATE schema_version SET version = '5', minor = '0';