From: Thomas Markwalder Date: Tue, 12 Mar 2019 14:47:57 +0000 (-0400) Subject: [#526,!269] Added env var control and CQL symmetry X-Git-Tag: Kea-1.6.0-beta~391^2~3 X-Git-Url: http://git.ipfire.org/cgi-bin/gitweb.cgi?a=commitdiff_plain;h=878ce0b44fd0185a92048419c0b2902792b9fefb;p=thirdparty%2Fkea.git [#526,!269] Added env var control and CQL symmetry Data wiping is on by default, for both MySQL and CQL (not yet implemented for Postgresql) and but can be overridden by KEA_TEST_DB_WIPE_DATA_ONLY. src/lib/mysql/testutils/mysql_schema.* destroyMySQLSchema() createMySQLSchema() - added check of softWipeEnabled() to see if env var has turned it OFF. Changed CQL to work the same way as MySQL: src/share/database/scripts/cql wipe_data.sh.in - new file, replaces soft_wipe.cql src/lib/cql/testutils/cql_schema.* destroyCqlSchema() - now destroys the schema if data wipe fails or is disabled createCqlSchema() - now drops/creates the schema if data wipe fails or is disabled. runCqlScript(() - added request-timeout to prevent timeout errors in slow environments wipeCqlData(bool show_err) - new function that attempts to run the wipe shell script src/lib/dhcpsrv/tests/cql_lease_mgr_unittest.cc Removed now superflous destroyCqlSchema calls --- diff --git a/configure.ac b/configure.ac index 69fde01b2b..113c618413 100644 --- a/configure.ac +++ b/configure.ac @@ -1685,6 +1685,7 @@ AC_CONFIG_FILES([Makefile src/share/database/scripts/cql/Makefile src/share/database/scripts/cql/upgrade_1.0_to_2.0.sh src/share/database/scripts/cql/upgrade_2.0_to_3.0.sh + src/share/database/scripts/cql/wipe_data.sh src/share/database/scripts/mysql/Makefile src/share/database/scripts/mysql/upgrade_1.0_to_2.0.sh src/share/database/scripts/mysql/upgrade_2.0_to_3.0.sh diff --git a/src/lib/cql/testutils/cql_schema.cc b/src/lib/cql/testutils/cql_schema.cc index 2cfa28a5e0..87a3b64cdc 100644 --- a/src/lib/cql/testutils/cql_schema.cc +++ b/src/lib/cql/testutils/cql_schema.cc @@ -32,19 +32,18 @@ validCqlConnectionString() { } void -destroyCqlSchema(bool force_wipe, bool show_err) { - if (force_wipe || !softWipeEnabled()) { - // Do full wipe +destroyCqlSchema(bool force, bool show_err) { + // If force is true or wipeCqlData() fails, destory the schema. + if (force || (!softWipeEnabled()) || wipeCqlData(show_err)) { runCqlScript(DATABASE_SCRIPTS_DIR, "cql/dhcpdb_drop.cql", show_err); - } else { - // do soft wipe (just remove the data, not the structures) - runCqlScript(DATABASE_SCRIPTS_DIR, "cql/soft_wipe.cql", show_err); } } void -createCqlSchema(bool force_wipe, bool show_err) { - if (force_wipe || !softWipeEnabled()) { +createCqlSchema(bool force, bool show_err) { + // If force is true or wipeCqlData() fails, recreate the schema. + if (force || (!softWipeEnabled()) || wipeCqlData(show_err)) { + destroyCqlSchema(show_err, true); runCqlScript(DATABASE_SCRIPTS_DIR, "cql/dhcpdb_create.cql", show_err); } } @@ -54,7 +53,7 @@ runCqlScript(const std::string& path, const std::string& script_name, bool show_err) { std::ostringstream cmd; - cmd << "cqlsh -u keatest -p keatest -k keatest"; + cmd << "cqlsh -u keatest -p keatest -k keatest --request-timeout=6000"; if (!show_err) { cmd << " 2>/dev/null "; } @@ -74,6 +73,28 @@ runCqlScript(const std::string& path, } } +bool wipeCqlData(bool show_err) { + std::ostringstream cmd; + cmd << "sh " << DATABASE_SCRIPTS_DIR << "/"; + + std::ostringstream version; + version << CQL_SCHEMA_VERSION_MAJOR << "." << CQL_SCHEMA_VERSION_MINOR; + + cmd << "cql/wipe_data.sh" << " " << version.str() + << " -u keatest -p keatest -k keatest --request-timeout=6000"; + + if (!show_err) { + cmd << " 2>/dev/null "; + } + + int retval = ::system(cmd.str().c_str()); + if (retval) { + std::cerr << "wipeCqlData failed:[" << cmd.str() << "]" << std::endl; + } + + return(retval); +} + } // namespace test } // namespace dhcp } // namespace isc diff --git a/src/lib/cql/testutils/cql_schema.h b/src/lib/cql/testutils/cql_schema.h index 0d3049d241..995c1fc92f 100644 --- a/src/lib/cql/testutils/cql_schema.h +++ b/src/lib/cql/testutils/cql_schema.h @@ -22,35 +22,52 @@ extern const char* CQL_VALID_TYPE; /// @return valid CQL connection string. std::string validCqlConnectionString(); -/// @brief Clear everything from the database +/// @brief Clear the unit test database /// -/// Submits the current schema drop script: +/// In order to reduce test execution time, this function +/// defaults to first attempting to delete transient data +/// from the database by calling @c wipeCqlData. If that +/// function fails it will then attempt to destroy the database +/// schema by running the SQL script: /// /// /cql/dhcpdb_drop.cql /// -/// to the unit test CQL database. If the script fails, the invoking test -/// will fail. The output of stderr is suppressed unless the parameter, -/// show_err is true. +/// The default behavior of wiping the data only may be overridden +/// in one of two ways: /// -/// @param force_wipe forces wipe of the database, even if -/// KEA_TEST_CASSANDRA_WIPE is set. +/// -# Setting the force parameter to true +/// -# Defining the environment variable: +/// KEA_TEST_DB_WIPE_DATA_ONLY="false" +/// +/// @param force if true, the function will skip deleting the data and /// @param show_err flag which governs whether or not stderr is suppressed. -void destroyCqlSchema(bool force_wipe, bool show_err = false); +/// destroy the schema. +void destroyCqlSchema(bool force, bool show_err = false); -/// @brief Create the CQL Schema +/// @brief Create the unit test Cql Schema /// -/// Submits the current schema creation script: +/// Ensures the unit test database is a empty and version-correct. +/// Unless, the force parameter is true, it will first attempt +/// to wipe the data from the database by calling @c wipeCqlData. +/// If this call succeeds the function returns, otherwise it will +/// will call @c destroyCqlSchema to forcibly remove the +/// existing schema and then submits the SQL script: /// /// /cql/dhcpdb_create.cql /// -/// to the unit test CQL database. If the script fails, the invoking test -/// will fail. The output of stderr is suppressed unless the parameter, -/// show_err is true. +/// to the unit test Cql database. +/// +/// The default behavior of wiping the data only may be overridden +/// in one of two ways: +/// +/// -# Setting the force parameter to true +/// -# Defining the environment variable: +/// KEA_TEST_DB_WIPE_DATA_ONLY="false" /// -/// @param force_wipe forces wipe of the database, even if -/// KEA_TEST_CASSANDRA_WIPE is set. +/// @param force flag when true, the function will recreate the database +/// schema. /// @param show_err flag which governs whether or not stderr is suppressed. -void createCqlSchema(bool force_wipe, bool show_err = false); +void createCqlSchema(bool force, bool show_err = false); /// @brief Run a CQL script against the CQL unit test database /// @@ -65,6 +82,22 @@ void createCqlSchema(bool force_wipe, bool show_err = false); /// @throw Unexpected when the script returns an error. void runCqlScript(const std::string& path, const std::string& script_name, bool show_err); + +/// @brief Attempts to wipe data from the Cql unit test database +/// +/// Runs the shell script +/// +/// /cql/wipe_data.sh +/// +/// This will fail if there is no schema, if the existing schema +/// version is incorrect (i.e. does not match CQL_SCHEMA_VERSION_MAJOR +/// and CQL_SCHEMA_VERSION_MINOR), or a SQL error occurs. Otherwise, +/// the script is should delete all transient data, leaving intact +/// reference tables. +/// +/// @param show_err flag which governs whether or not stderr is suppressed. +bool wipeCqlData(bool show_err); + }; }; }; diff --git a/src/lib/dhcpsrv/tests/cql_lease_mgr_unittest.cc b/src/lib/dhcpsrv/tests/cql_lease_mgr_unittest.cc index 21441fd4f2..dccf207736 100644 --- a/src/lib/dhcpsrv/tests/cql_lease_mgr_unittest.cc +++ b/src/lib/dhcpsrv/tests/cql_lease_mgr_unittest.cc @@ -56,7 +56,6 @@ public: /// @brief Clears the database and opens connection to it. void initializeTest() { // Ensure schema is the correct one. - destroyCqlSchema(false, true); createCqlSchema(false, true); // Connect to the database @@ -341,7 +340,6 @@ public: TEST(CqlOpenTest, OpenDatabase) { // Schema needs to be created for the test to work. - destroyCqlSchema(false, true); createCqlSchema(false, true); // Check that lease manager open the database opens correctly and tidy up. diff --git a/src/lib/mysql/testutils/mysql_schema.cc b/src/lib/mysql/testutils/mysql_schema.cc index f82196a9d4..ef72313abc 100644 --- a/src/lib/mysql/testutils/mysql_schema.cc +++ b/src/lib/mysql/testutils/mysql_schema.cc @@ -31,21 +31,21 @@ validMySQLConnectionString() { } void destroyMySQLSchema(bool show_err, bool force) { - // If force is true or wipeData() fails, destory the schema. - if (force || (!softWipeEnabled()) || wipeData(show_err)) { + // If force is true or wipeMySQLData() fails, destory the schema. + if (force || (!softWipeEnabled()) || wipeMySQLData(show_err)) { runMySQLScript(DATABASE_SCRIPTS_DIR, "mysql/dhcpdb_drop.mysql", show_err); } } void createMySQLSchema(bool show_err, bool force) { - // If force is true or wipeData() fails, recreate the schema. - if (force || (!softWipeEnabled()) || wipeData(show_err)) { + // If force is true or wipeMySQLData() fails, recreate the schema. + if (force || (!softWipeEnabled()) || wipeMySQLData(show_err)) { destroyMySQLSchema(show_err, true); runMySQLScript(DATABASE_SCRIPTS_DIR, "mysql/dhcpdb_create.mysql", show_err); } } -bool wipeData(bool show_err) { +bool wipeMySQLData(bool show_err) { std::ostringstream cmd; cmd << "sh " << DATABASE_SCRIPTS_DIR << "/"; @@ -60,7 +60,7 @@ bool wipeData(bool show_err) { int retval = ::system(cmd.str().c_str()); if (retval) { - std::cerr << "wipeData failed:[" << cmd.str() << "]" << std::endl; + std::cerr << "wipeMySQLData failed:[" << cmd.str() << "]" << std::endl; } return(retval); diff --git a/src/lib/mysql/testutils/mysql_schema.h b/src/lib/mysql/testutils/mysql_schema.h index d4f19269a6..f3e5d11341 100644 --- a/src/lib/mysql/testutils/mysql_schema.h +++ b/src/lib/mysql/testutils/mysql_schema.h @@ -24,28 +24,31 @@ std::string validMySQLConnectionString(); /// @brief Clear the unit test database /// -/// Either calls @c wipeData to wipe the data from the -/// database /or destroys the database itself by submitting the -/// SQL script: +/// In order to reduce test execution time, this function +/// defaults to first attempting to delete transient data +/// from the database by calling @c wipeMySQLData. If that +/// function fails it will then attempt to destroy the database +/// schema by running the SQL script: /// /// /mysql/dhcpdb_drop.mysql /// -/// If wipeData() is called and fails, it will destroy -/// the schema. If the schema destruction fails, the -/// invoking test should fail. +/// The default behavior of wiping the data only may be overridden +/// in one of two ways: /// -/// The output stderr is suppressed unless the parameter, -/// show_err is true. +/// -# Setting the force parameter to true +/// -# Defining the environment variable: +/// KEA_TEST_DB_WIPE_DATA_ONLY="false" /// /// @param show_err flag which governs whether or not stderr is suppressed. -/// @param force if true, the function will simply destroy the schema. +/// @param force if true, the function will skip deleting the data and +/// destroy the schema. void destroyMySQLSchema(bool show_err = false, bool force = false); /// @brief Create the unit test MySQL Schema /// /// Ensures the unit test database is a empty and version-correct. /// Unless, the force parameter is true, it will first attempt -/// to wipe the data from the database by calling @c wipeData. +/// to wipe the data from the database by calling @c wipeMySQLData. /// If this call succeeds the function returns, otherwise it will /// will call @c destroyMySQLSchema to forcibly remove the /// existing schema and then submits the SQL script: @@ -54,6 +57,13 @@ void destroyMySQLSchema(bool show_err = false, bool force = false); /// /// to the unit test MySQL database. /// +/// The default behavior of wiping the data only may be overridden +/// in one of two ways: +/// +/// -# Setting the force parameter to true +/// -# Defining the environment variable: +/// KEA_TEST_DB_WIPE_DATA_ONLY="false" +/// /// @param show_err flag which governs whether or not stderr is suppressed. /// @param force flag when true, the function will recreate the database /// schema. @@ -73,7 +83,7 @@ void createMySQLSchema(bool show_err = false, bool force = false); /// reference tables. /// /// @param show_err flag which governs whether or not stderr is suppressed. -bool wipeData(bool show_err = false); +bool wipeMySQLData(bool show_err = false); /// @brief Run a MySQL SQL script against the MySQL unit test database /// diff --git a/src/share/database/scripts/cql/.gitignore b/src/share/database/scripts/cql/.gitignore index 7f3704b0cc..f8469c2087 100644 --- a/src/share/database/scripts/cql/.gitignore +++ b/src/share/database/scripts/cql/.gitignore @@ -1,3 +1,3 @@ /upgrade_1.0_to_2.0.sh /upgrade_2.0_to_3.0.sh - +/wipe_data.sh diff --git a/src/share/database/scripts/cql/Makefile.am b/src/share/database/scripts/cql/Makefile.am index c5def18567..c7c2388664 100644 --- a/src/share/database/scripts/cql/Makefile.am +++ b/src/share/database/scripts/cql/Makefile.am @@ -5,6 +5,6 @@ sqlscripts_DATA = dhcpdb_create.cql sqlscripts_DATA += dhcpdb_drop.cql sqlscripts_DATA += upgrade_1.0_to_2.0.sh sqlscripts_DATA += upgrade_2.0_to_3.0.sh -sqlscripts_DATA += soft_wipe.cql +sqlscripts_DATA += wipe_data.sh EXTRA_DIST = ${sqlscripts_DATA} diff --git a/src/share/database/scripts/cql/soft_wipe.cql b/src/share/database/scripts/cql/soft_wipe.cql deleted file mode 100644 index 722de01d79..0000000000 --- a/src/share/database/scripts/cql/soft_wipe.cql +++ /dev/null @@ -1,27 +0,0 @@ --- Copyright (C) 2016-2018 Internet Systems Consortium. --- --- This Source Code Form is subject to the terms of the Mozilla Public --- License, v. 2.0. If a copy of the MPL was not distributed with this --- file, You can obtain one at http://mozilla.org/MPL/2.0/. - --- This is the soft wipe script for CQL (Cassandra). This capability may be --- used when running Cassandra unit-tests. --- --- In some deployments (e.g. in case of Tomek's dev system) Cassandra tests take --- a very long time to execute. This was traced back to slow table/indexes --- creation/deletion. With full wipe and recreation of all structures, it --- took over 60 seconds for each test to execute. To avoid this problem, a --- feature called soft-wipe has been implemented. If enabled, it does not --- remove the structures, just the data from essential tables. To enable --- it set KEA_TEST_CASSANDRA_WIPE environment variable to 'soft'. Make sure --- that the database schema is set up properly before running in soft-wipe --- mode. - -TRUNCATE TABLE lease4; -TRUNCATE TABLE lease6; -TRUNCATE TABLE lease6_types; -TRUNCATE TABLE lease_hwaddr_source; -TRUNCATE TABLE lease_state; -TRUNCATE TABLE schema_version; -TRUNCATE TABLE host_reservations; -TRUNCATE TABLE logs; diff --git a/src/share/database/scripts/cql/wipe_data.sh.in b/src/share/database/scripts/cql/wipe_data.sh.in new file mode 100644 index 0000000000..df1a5ce547 --- /dev/null +++ b/src/share/database/scripts/cql/wipe_data.sh.in @@ -0,0 +1,54 @@ +# Copyright (C) 2019 Internet Systems Consortium, Inc. ("ISC") +# +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +#!/bin/sh + +# This script is primarily used for CQL unit tests, which need to +# ensure an empty, but schema correct database for each test. It +# deletes ALL transient data from an existing Kea CQL schema, +# including leases, reservations, etc... Use at your own peril. +# Reference tables will be left in-tact. + +# Include utilities. Use installed version if available and +# use build version if it isn't. +if [ -e @datarootdir@/@PACKAGE_NAME@/scripts/admin-utils.sh ]; then + . @datarootdir@/@PACKAGE_NAME@/scripts/admin-utils.sh +else + . @abs_top_builddir@/src/bin/admin/admin-utils.sh +fi + +# First argument is must be the expected schema version . +exp_version="$1" +shift; + +# Remaining arguments are used as cql command line arguments + +# If the existing schema doesn't match, the fail +VERSION=`cql_version "$@"` +if [ "$VERSION" = "" ]; then + printf "Cannot wipe data, schema version could not be detected.\n" + exit 1 +fi + +if [ "$VERSION" != "$exp_version" ]; then + printf "Cannot wipe data, wrong schema version. Expected $exp_version, found version $VERSION.\n" + exit 1 +fi + +# Delete transient data from tables. +cqlsh "$@" <