From: Andrei Pavel Date: Wed, 22 Dec 2021 12:09:17 +0000 (+0200) Subject: [#2039] kea-admin lease-upload mysql X-Git-Tag: Kea-2.1.2~63 X-Git-Url: http://git.ipfire.org/cgi-bin/gitweb.cgi?a=commitdiff_plain;h=d274a702d71c580fbcd7354f38ede354b1a03063;p=thirdparty%2Fkea.git [#2039] kea-admin lease-upload mysql --- diff --git a/src/bin/admin/kea-admin.in b/src/bin/admin/kea-admin.in index 740a94d226..30dfc795bf 100644 --- a/src/bin/admin/kea-admin.in +++ b/src/bin/admin/kea-admin.in @@ -13,6 +13,7 @@ # - database version check # - database version upgrade # - lease database dump +# - lease upload to the database # - lease database recount # shellcheck disable=SC1091 @@ -33,7 +34,7 @@ scripts_dir="${SCRIPTS_DIR_DEFAULT}" VERSION="@PACKAGE_VERSION@" # lease dump parameters -dump_type=0 +dhcp_version=0 dump_file="" dump_qry="" @@ -63,6 +64,7 @@ COMMAND: Currently supported operations are: - for checking database version when preparing for an upgrade. - db-upgrade: Upgrades your database schema. - lease-dump: Dumps current leases to a memfile-ready CSV file. + - lease-upload: Uploads leases from a CSV file to the database. - stats-recount: Recounts lease statistics. BACKEND - one of the supported backends: memfile|mysql|pgsql|cql @@ -79,14 +81,14 @@ PARAMETERS: Parameters are optional in general, but may be required -d or --directory - path to upgrade scripts (default: %s) -v or --version - print kea-admin version and quit. - Parameters specific to lease-dump: + Parameters specific to lease-dump, lease-upload: -4 to dump IPv4 leases to file -6 to dump IPv6 leases to file - -o or --output - name of file to which leases will be dumped + -i or --input to specify the name of file from which leases will be uploaded + -o or --output to specify the name of file to which leases will be dumped ' "${VERSION}" "${0}" "${SCRIPTS_DIR_DEFAULT}" } - ### Logging functions ### # Logs message at the error level. @@ -130,7 +132,6 @@ is_in_list() { done } - ### Functions that implement database initialization commands memfile_init() { @@ -484,7 +485,7 @@ get_dump_query() { ;; esac - dump_qry="${invoke} lease${dump_type}DumpHeader();${invoke} lease${dump_type}DumpData();"; + dump_qry="${invoke} lease${dhcp_version}DumpHeader();${invoke} lease${dhcp_version}DumpData();"; } memfile_dump() { @@ -495,7 +496,7 @@ memfile_dump() { mysql_dump() { # Check the lease type was given - if [ $dump_type -eq 0 ]; then + if [ ${dhcp_version} -eq 0 ]; then log_error "lease-dump: lease type ( -4 or -6 ) needs to be specified" usage exit 1 @@ -548,14 +549,14 @@ mysql_dump() { # delete the tmp file on success rm $tmp_file - echo lease$dump_type successfully dumped to "$dump_file" + echo lease${dhcp_version} successfully dumped to "${dump_file}" exit 0 } ### Functions used for dump pgsql_dump() { # Check the lease type was given - if [ $dump_type -eq 0 ]; then + if [ ${dhcp_version} -eq 0 ]; then log_error "lease-dump: lease type ( -4 or -6 ) needs to be specified" usage exit 1 @@ -589,16 +590,16 @@ pgsql_dump() { exit 1 fi - echo lease$dump_type successfully dumped to "$dump_file" + echo lease${dhcp_version} successfully dumped to "${dump_file}" exit 0 } cql_dump() { # Get the query appropriate to lease version. Explicitly specify all columns # so that they are returned in expected order. - if [ $dump_type -eq 4 ]; then + if [ ${dhcp_version} -eq 4 ]; then dump_query="SELECT address,hwaddr,client_id,valid_lifetime,expire,subnet_id,fqdn_fwd,fqdn_rev,hostname,state,user_context FROM lease4" - elif [ $dump_type -eq 6 ]; then + elif [ ${dhcp_version} -eq 6 ]; then dump_query="SELECT address,duid,valid_lifetime,expire,subnet_id,pref_lifetime,lease_type,iaid,prefix_len,fqdn_fwd,fqdn_rev,hostname,hwaddr,hwtype,hwaddr_source,state,user_context FROM lease6" else log_error "lease-dump: lease type ( -4 or -6 ) needs to be specified" @@ -631,10 +632,150 @@ cql_dump() { # by address. awk script replaces head -n -2 which is not portable. echo "${OUTPUT}" | tail -n +4 | awk 'n>=2 { print a[n%2] } { a[n%2]=$0; n=n+1 }' | sed -e 's/[[:space:]]*//g' | sed -e 's/|/,/g' | sort -r >> "$dump_file" - echo "lease$dump_type successfully dumped to $dump_file" + echo "lease${dhcp_version} successfully dumped to ${dump_file}" exit 0 } +######################## functions used in lease-upload ######################## + +# Finds the position of a column by name, starting with 1. +get_column_position() { + local column_name="${1}"; shift + + # Look only in the header, count the number of commas up to the column. + position=$(head -n 1 "${input_file}" | grep -Eo ",|${column_name}" | uniq -c | \ + head -n 1 | grep ',' | tr -s ' ' | cut -d ' ' -f 2) + + if test -z "${position}"; then + # If no commas, that means position 1. + printf '1' + else + # Else increment, so that it starts at 1. + printf '%s' "$((position + 1))" + fi +} + +# Checks that an input file was provided and that it has at least one lease. +lease_upload_checks() { + # Check that an input file was specified. + if test -z "${input_file-}"; then + log_error 'you must specify an input file with -i or --input for lease-upload' + usage + exit 1 + fi + + # Check that the input file has at least one row of values. + if test "$(wc -l < "${input_file}")" -le 1; then + log_error 'CSV file has no leases' + exit 1 + fi +} + +# Adds quotes around values at given positions starting with 1 in a CSV line. +stringify_positions_in_line() { + local positions="${1}"; shift + local line="${1}"; shift + + i=1 + output= + for p in ${positions}; do + # Get everything up to position p. + if test "${i}" -lt "${p}"; then + up_until_p=$(printf '%s' "${line}" | cut -d ',' -f "${i}-$((p - 1))") + else + up_until_p='' + fi + + # Get value at position p. + p_word=$(printf '%s' "${line}" | cut -d ',' -f "${p}") + + # Add comma unless we're doing the first append. + if test "${i}" != 1; then + output="${output}," + fi + + # Append everything up to position p. + output="${output}${up_until_p}" + + # Add comma if we're not stringifying position 1 and if there is + # anything up to position p. In the second case, the comma was already + # added at a previous step. + if test "${p}" != 1 && test -n "${up_until_p}"; then + output="${output}," + fi + + # Add value at position p. + output="${output}'${p_word}'" + + # Skip position p when getting the values between positions next time. + i=$((p + 1)) + done + + # The last position might be somewhere in the middle, so add everything + # until the end. + the_rest=$(printf '%s' "${line}" | cut -d ',' -f ${i}-) + + # Add comma unless we're adding the whole line or nothing. + if test "${i}" != 1 && test -n "${the_rest}"; then + output="${output}," + fi + + # Append. + output="${output}${the_rest}" + + # Print back to the caller. + printf '%s' "${output}" +} + +# Entry point for the lease-upload mysql command. +mysql_lease_upload() { + # Do checks. + lease_upload_checks + + # Determine the columns whose values need to be stringified to avoid syntax + # errors in the MySQL client. These are columns which are VARCHARs or need + # to be further processed by a procedure. + if test "${dhcp_version}" = '4'; then + string_columns='address hwaddr client_id hostname user_context' + elif test "${dhcp_version}" = '6'; then + string_columns='address duid hostname hwaddr user_context' + else + log_error "lease-upload: lease type -4 or -6 needs to be specified" + usage + exit 1 + fi + + # Get positions of string columns. + string_positions= + for i in ${string_columns}; do + string_positions="${string_positions} $(get_column_position "${i}")" + done + + # Construct the SQL insert statements. + header_parsed=false + sql_statement='START TRANSACTION;\n' + while read -r line; do + if "${header_parsed}"; then + line=$(stringify_positions_in_line "${string_positions}" "${line}") + sql_statement="${sql_statement}CALL lease${dhcp_version}Upload(${line});\\n" + else + header_parsed=true + fi + done < "${input_file}" + sql_statement="${sql_statement}COMMIT;\n" + + # Execute the SQL insert statements. + output="$(mysql_execute "${sql_statement}")" + + # Print a confirmation message. + printf 'lease%s successfully updated.\n' "${dhcp_version}" +} + +# Entry point for the lease-upload pgsql command. +pgsql_lease_upload() { + log_error 'not implemented' +} + ### Functions used for recounting statistics mysql_recount() { printf "Recount lease statistics from database\n" @@ -692,7 +833,7 @@ if test "${command}" = "-v" || test "${command}" = "--version" ; then exit 0 fi -is_in_list "${command}" "db-init db-version db-upgrade lease-dump stats-recount" +is_in_list "${command}" "db-init db-version db-upgrade lease-dump lease-upload stats-recount" if [ "${_inlist}" -eq 0 ]; then log_error "invalid command: ${command}" usage @@ -789,21 +930,31 @@ do ;; # specify DHCPv4 lease type -4) - if [ $dump_type -eq 6 ]; then + if [ ${dhcp_version} -eq 6 ]; then log_error "you may not specify both -4 and -6" usage exit 1 fi - dump_type=4 + dhcp_version=4 ;; # specify DHCPv6 lease type -6) - if [ $dump_type -eq 4 ]; then + if [ ${dhcp_version} -eq 4 ]; then log_error "you may not specify both -4 and -6" usage exit 1 fi - dump_type=6 + dhcp_version=6 + ;; + # specify input file, used by lease-upload + -i|--input) + shift + input_file=${1-} + if [ -z "${input_file}" ]; then + log_error '-i or --input requires a parameter' + usage + exit 1 + fi ;; # specify output file, currently only used by lease dump -o|--output) @@ -898,6 +1049,24 @@ case ${command} in ;; esac ;; + lease-upload) + case ${backend} in + memfile) + log_error 'uploading from memfile to memfile' + exit 1 + ;; + mysql) + mysql_lease_upload + ;; + pgsql) + pgsql_lease_upload + ;; + cql) + log_error 'lease-export cql is deprecated' + exit 1 + ;; + esac + ;; stats-recount) case ${backend} in memfile) diff --git a/src/share/database/scripts/mysql/dhcpdb_create.mysql b/src/share/database/scripts/mysql/dhcpdb_create.mysql index bbf03c1080..c63c6b438c 100644 --- a/src/share/database/scripts/mysql/dhcpdb_create.mysql +++ b/src/share/database/scripts/mysql/dhcpdb_create.mysql @@ -4174,6 +4174,112 @@ BEGIN END $$ DELIMITER ; +-- Create a procedure that inserts a v4 lease from memfile data. +DELIMITER $$ +CREATE PROCEDURE lease4Upload( + IN address VARCHAR(15), + IN hwaddr VARCHAR(20), + IN client_id VARCHAR(128), + IN valid_lifetime INT UNSIGNED, + IN expire BIGINT UNSIGNED, + IN subnet_id INT UNSIGNED, + IN fqdn_fwd TINYINT, + IN fqdn_rev TINYINT, + IN hostname VARCHAR(255), + IN state INT UNSIGNED, + IN user_context TEXT +) +BEGIN + INSERT INTO lease4 ( + address, + hwaddr, + client_id, + valid_lifetime, + expire, + subnet_id, + fqdn_fwd, + fqdn_rev, + hostname, + state, + user_context + ) VALUES ( + INET_ATON(address), + UNHEX(REPLACE(hwaddr, ':', '')), + UNHEX(REPLACE(client_id, ':', '')), + valid_lifetime, + FROM_UNIXTIME(expire), + subnet_id, + fqdn_fwd, + fqdn_rev, + REPLACE(hostname, ',', ','), + state, + REPLACE(user_context, ',', ',') + ); +END $$ +DELIMITER ; + +-- Create a procedure that inserts a v6 lease from memfile data. +DELIMITER $$ +CREATE PROCEDURE lease6Upload( + IN address VARCHAR(39), + IN duid VARCHAR(128), + IN valid_lifetime INT UNSIGNED, + IN expire BIGINT UNSIGNED, + IN subnet_id INT UNSIGNED, + IN pref_lifetime INT UNSIGNED, + IN lease_type TINYINT, + IN iaid INT UNSIGNED, + IN prefix_len TINYINT UNSIGNED, + IN fqdn_fwd TINYINT, + IN fqdn_rev TINYINT, + IN hostname VARCHAR(255), + IN hwaddr VARCHAR(64), + IN state INT UNSIGNED, + IN user_context TEXT, + IN hwtype SMALLINT, + IN hwaddr_source INT UNSIGNED +) +BEGIN + INSERT INTO lease6 ( + address, + duid, + valid_lifetime, + expire, + subnet_id, + pref_lifetime, + lease_type, + iaid, + prefix_len, + fqdn_fwd, + fqdn_rev, + hostname, + hwaddr, + state, + user_context, + hwtype, + hwaddr_source + ) VALUES ( + address, + UNHEX(REPLACE(duid, ':', '')), + valid_lifetime, + FROM_UNIXTIME(expire), + subnet_id, + pref_lifetime, + lease_type, + iaid, + prefix_len, + fqdn_fwd, + fqdn_rev, + REPLACE(hostname, ',', ','), + UNHEX(REPLACE(hwaddr, ':', '')), + state, + REPLACE(user_context, ',', ','), + hwtype, + hwaddr_source + ); +END $$ +DELIMITER ; + -- Update the schema version number. UPDATE schema_version SET version = '13', minor = '0'; diff --git a/src/share/database/scripts/mysql/dhcpdb_drop.mysql b/src/share/database/scripts/mysql/dhcpdb_drop.mysql index 35faadb8fa..6ea8fd4a20 100644 --- a/src/share/database/scripts/mysql/dhcpdb_drop.mysql +++ b/src/share/database/scripts/mysql/dhcpdb_drop.mysql @@ -123,3 +123,5 @@ DROP TRIGGER IF EXISTS dhcp6_client_class_AUPD; DROP TRIGGER IF EXISTS dhcp6_client_class_ADEL; DROP TRIGGER IF EXISTS dhcp6_client_class_dependency_BINS; DROP TRIGGER IF EXISTS dhcp6_client_class_dependency_AINS; +DROP PROCEDURE IF EXISTS lease4Upload; +DROP PROCEDURE IF EXISTS lease6Upload; diff --git a/src/share/database/scripts/mysql/upgrade_012_to_013.sh.in b/src/share/database/scripts/mysql/upgrade_012_to_013.sh.in index a8e09c7b5a..ee133d5928 100644 --- a/src/share/database/scripts/mysql/upgrade_012_to_013.sh.in +++ b/src/share/database/scripts/mysql/upgrade_012_to_013.sh.in @@ -52,6 +52,7 @@ then fi mysql "$@" <