return (0);
}
+/// @brief ha-maintenance-cancel command handler implementation.
+int maintenance_cancel_command(CalloutHandle& handle) {
+ try {
+ impl->maintenanceCancelHandler(handle);
+
+ } catch (const std::exception& ex) {
+ LOG_ERROR(ha_logger, HA_MAINTENANCE_CANCEL_HANDLER_FAILED)
+ .arg(ex.what());
+ }
+
+ return (0);
+}
+
/// @brief This function is called when the library is loaded.
///
/// @param handle library handle
handle.registerCommandCallout("ha-continue", continue_command);
handle.registerCommandCallout("ha-maintenance-notify", maintenance_notify_command);
handle.registerCommandCallout("ha-maintenance-start", maintenance_start_command);
+ handle.registerCommandCallout("ha-maintenance-cancel", maintenance_cancel_command);
} catch (const std::exception& ex) {
LOG_ERROR(ha_logger, HA_CONFIGURATION_FAILED)
callout_handle.setArgument("response", response);
}
+void
+HAImpl::maintenanceCancelHandler(hooks::CalloutHandle& callout_handle) {
+ ConstElementPtr response = service_->processMaintenanceCancel();
+ callout_handle.setArgument("response", response);
+}
+
} // end of namespace isc::ha
} // end of namespace isc
/// @param callout_handle Callout handle provided to the callout.
void maintenanceStartHandler(hooks::CalloutHandle& callout_handle);
+ /// @brief Implements handler for the ha-maintenance-cancel command.
+ ///
+ /// @param callout_handle Callout handle provided to the callout.
+ void maintenanceCancelHandler(hooks::CalloutHandle& callout_handle);
+
protected:
/// @brief Holds parsed configuration.
-// File created from ../../../../src/hooks/dhcp/high_availability/ha_messages.mes on Wed Jan 15 2020 11:05
+// File created from ../../../../src/hooks/dhcp/high_availability/ha_messages.mes on Thu Jan 16 2020 09:28
#include <cstddef>
#include <log/message_types.h>
extern const isc::log::MessageID HA_LOAD_BALANCING_IDENTIFIER_MISSING = "HA_LOAD_BALANCING_IDENTIFIER_MISSING";
extern const isc::log::MessageID HA_LOCAL_DHCP_DISABLE = "HA_LOCAL_DHCP_DISABLE";
extern const isc::log::MessageID HA_LOCAL_DHCP_ENABLE = "HA_LOCAL_DHCP_ENABLE";
+extern const isc::log::MessageID HA_MAINTENANCE_CANCEL_HANDLER_FAILED = "HA_MAINTENANCE_CANCEL_HANDLER_FAILED";
+extern const isc::log::MessageID HA_MAINTENANCE_NOTIFY_CANCEL_COMMUNICATIONS_FAILED = "HA_MAINTENANCE_NOTIFY_CANCEL_COMMUNICATIONS_FAILED";
+extern const isc::log::MessageID HA_MAINTENANCE_NOTIFY_CANCEL_FAILED = "HA_MAINTENANCE_NOTIFY_CANCEL_FAILED";
extern const isc::log::MessageID HA_MAINTENANCE_NOTIFY_COMMUNICATIONS_FAILED = "HA_MAINTENANCE_NOTIFY_COMMUNICATIONS_FAILED";
extern const isc::log::MessageID HA_MAINTENANCE_NOTIFY_FAILED = "HA_MAINTENANCE_NOTIFY_FAILED";
extern const isc::log::MessageID HA_MAINTENANCE_NOTIFY_HANDLER_FAILED = "HA_MAINTENANCE_NOTIFY_HANDLER_FAILED";
"HA_LOAD_BALANCING_IDENTIFIER_MISSING", "load balancing failed for the DHCPv4 message (transaction id: %1) because HW address and client identifier are missing",
"HA_LOCAL_DHCP_DISABLE", "local DHCP service is disabled while the %1 is in the %2 state",
"HA_LOCAL_DHCP_ENABLE", "local DHCP service is enabled while the %1 is in the %2 state",
+ "HA_MAINTENANCE_CANCEL_HANDLER_FAILED", "ha-maintenance-cancel command failed: %1",
+ "HA_MAINTENANCE_NOTIFY_CANCEL_COMMUNICATIONS_FAILED", "failed to send ha-maintenance-notify to %1 in attempt to cancel its maintenance: %2",
+ "HA_MAINTENANCE_NOTIFY_CANCEL_FAILED", "error returned while processing ha-maintenance-notify by %1 in attempt to cancel its maintenance: %2",
"HA_MAINTENANCE_NOTIFY_COMMUNICATIONS_FAILED", "failed to send ha-maintenance-notify to %1: %2",
"HA_MAINTENANCE_NOTIFY_FAILED", "error returned while processing ha-maintenance-notify by %1: %2",
"HA_MAINTENANCE_NOTIFY_HANDLER_FAILED", "ha-maintenance-notify command failed: %1",
-// File created from ../../../../src/hooks/dhcp/high_availability/ha_messages.mes on Wed Jan 15 2020 11:05
+// File created from ../../../../src/hooks/dhcp/high_availability/ha_messages.mes on Thu Jan 16 2020 09:28
#ifndef HA_MESSAGES_H
#define HA_MESSAGES_H
extern const isc::log::MessageID HA_LOAD_BALANCING_IDENTIFIER_MISSING;
extern const isc::log::MessageID HA_LOCAL_DHCP_DISABLE;
extern const isc::log::MessageID HA_LOCAL_DHCP_ENABLE;
+extern const isc::log::MessageID HA_MAINTENANCE_CANCEL_HANDLER_FAILED;
+extern const isc::log::MessageID HA_MAINTENANCE_NOTIFY_CANCEL_COMMUNICATIONS_FAILED;
+extern const isc::log::MessageID HA_MAINTENANCE_NOTIFY_CANCEL_FAILED;
extern const isc::log::MessageID HA_MAINTENANCE_NOTIFY_COMMUNICATIONS_FAILED;
extern const isc::log::MessageID HA_MAINTENANCE_NOTIFY_FAILED;
extern const isc::log::MessageID HA_MAINTENANCE_NOTIFY_HANDLER_FAILED;
respond to the DHCP clients. The first argument specifies server name.
The second argument specifies server's state.
+% HA_MAINTENANCE_CANCEL_HANDLER_FAILED ha-maintenance-cancel command failed: %1
+This error message is issued to indicate that the ha-maintenance-cancel command
+handler failed while processing the command. The argument provides the reason for
+failure.
+
+% HA_MAINTENANCE_NOTIFY_CANCEL_COMMUNICATIONS_FAILED failed to send ha-maintenance-notify to %1 in attempt to cancel its maintenance: %2
+This warning message indicates that there was a problem in communication with a
+HA peer while sending the ha-maintenance-notify command with the cancel flag
+set to true. The first argument provides the remote server's name. The second
+argument provides a reason for failure.
+
+% HA_MAINTENANCE_NOTIFY_CANCEL_FAILED error returned while processing ha-maintenance-notify by %1 in attempt to cancel its maintenance: %2
+This warning message indicates that a peer returned an error status code
+in response to a ha-maintenance-notify command with the cancel flag set to
+true. The first argument provides the remote server's name. The second
+argument provides a reason for failure.
+
% HA_MAINTENANCE_NOTIFY_COMMUNICATIONS_FAILED failed to send ha-maintenance-notify to %1: %2
This warning message indicates that there was a problem in communication with a
HA peer while sending the ha-maintenance-notify command. The first argument provides the
verboseTransition(getPrevState());
runModel(NOP_EVT);
- return (createAnswer(CONTROL_RESULT_SUCCESS, "Server maintenance cancelled."));
+ return (createAnswer(CONTROL_RESULT_SUCCESS, "Server maintenance canceled."));
}
switch (getCurrState()) {
HAConfig::PeerConfigPtr remote_config = config_->getFailoverPeerConfig();
- // Create HTTP/1.1 request including our command.
+ // Create HTTP/1.1 request including ha-maintenance-notify command
+ // with the cancel flag set to false.
PostHttpRequestJsonPtr request = boost::make_shared<PostHttpRequestJson>
(HttpRequest::Method::HTTP_POST, "/", HttpVersion::HTTP_11(),
HostHttpHeader(remote_config->getUrl().getHostname()));
" can be now safely shut down."));
}
+ConstElementPtr
+HAService::processMaintenanceCancel() {
+ if (getCurrState() != HA_PARTNER_MAINTAINED_ST) {
+ return (createAnswer(CONTROL_RESULT_ERROR, "Unable to cancel maintenance"
+ " request because the server is not in the"
+ " partner-maintained state."));
+ }
+
+ HAConfig::PeerConfigPtr remote_config = config_->getFailoverPeerConfig();
+
+ // Create HTTP/1.1 request including ha-maintenance-notify command
+ // with the cancel flag set to true.
+ PostHttpRequestJsonPtr request = boost::make_shared<PostHttpRequestJson>
+ (HttpRequest::Method::HTTP_POST, "/", HttpVersion::HTTP_11(),
+ HostHttpHeader(remote_config->getUrl().getHostname()));
+ request->setBodyAsJson(CommandCreator::createMaintenanceNotify(true, server_type_));
+ request->finalize();
+
+ // Response object should also be created because the HTTP client needs
+ // to know the type of the expected response.
+ HttpResponseJsonPtr response = boost::make_shared<HttpResponseJson>();
+
+ IOService io_service;
+ HttpClient client(io_service);
+
+ std::string error_message;
+
+ // Schedule asynchronous HTTP request.
+ client.asyncSendRequest(remote_config->getUrl(), request, response,
+ [this, remote_config, &io_service, &error_message]
+ (const boost::system::error_code& ec,
+ const HttpResponsePtr& response,
+ const std::string& error_str) {
+
+ io_service.stop();
+
+ // Handle first two groups of errors.
+ if (ec || !error_str.empty()) {
+ error_message = (ec ? ec.message() : error_str);
+ LOG_ERROR(ha_logger, HA_MAINTENANCE_NOTIFY_CANCEL_COMMUNICATIONS_FAILED)
+ .arg(remote_config->getLogLabel())
+ .arg(error_message);
+
+ } else {
+
+ // Handle third group of errors.
+ try {
+ int rcode = 0;
+ static_cast<void>(verifyAsyncResponse(response, rcode));
+
+ } catch (const std::exception& ex) {
+ error_message = ex.what();
+ LOG_ERROR(ha_logger, HA_MAINTENANCE_NOTIFY_CANCEL_FAILED)
+ .arg(remote_config->getLogLabel())
+ .arg(error_message);
+ }
+ }
+
+ // If there was an error communicating with the partner, mark the
+ // partner as unavailable.
+ if (!error_message.empty()) {
+ communication_state_->setPartnerState("unavailable");
+ }
+ },
+ HttpClient::RequestTimeout(TIMEOUT_DEFAULT_HTTP_CLIENT_REQUEST),
+ boost::bind(&HAService::clientConnectHandler, this, _1, _2),
+ boost::bind(&HAService::clientCloseHandler, this, _1)
+ );
+
+ // Run the IO service until it is stopped by any of the callbacks. This
+ // makes it synchronous.
+ io_service.run();
+
+ // There was an error in communication with the partner or the
+ // partner was unable to revert its state.
+ if (!error_message.empty()) {
+ return (createAnswer(CONTROL_RESULT_ERROR,
+ "Unable to cancel maintenance. The partner server responded"
+ " with the following message to the ha-maintenance-notify"
+ " commmand: " + error_message + "."));
+ }
+
+ // Successfully reverted partner's state. Let's also revert our state to the
+ // previous one.
+ verboseTransition(getPrevState());
+ runModel(NOP_EVT);
+
+ return (createAnswer(CONTROL_RESULT_SUCCESS,
+ "Server maintenance successfully canceled."));
+}
+
ConstElementPtr
HAService::verifyAsyncResponse(const HttpResponsePtr& response, int& rcode) {
// The response must cast to JSON type.
/// maintained state.
///
/// @param cancel boolean value indicating if the maintenance is being
- /// cancelled with this operation. If it is set to false the maintenance
+ /// canceled with this operation. If it is set to false the maintenance
/// is being started.
///
/// @return Pointer to the reponse to the ha-maintenance-notify.
/// @return Pointer to the response to the ha-maintenance-start.
data::ConstElementPtr processMaintenanceStart();
+ /// @brief Processes ha-maintenance-cancel command and returns a response.
+ ///
+ /// The server receiving this command will try to revert the partner's
+ /// state from maintained to the previous state, and also it will try to
+ /// revert its own state from partner-maintained to the previous state.
+ /// It effectively means canceling the request for maintenance signaled
+ /// with the ha-maintenance-start command.
+ ///
+ /// In some cases canceling the maintenace is no longer possible, e.g.
+ /// if the server has already got into the partner-down state. Generally,
+ /// canceling the maintenance is only possible if this server is in the
+ /// partner-maintained state and the partner is in the maintained state.
+ ///
+ /// @return Pointer to the response to the ha-maintenance-cancel.
+ data::ConstElementPtr processMaintenanceCancel();
+
protected:
/// @brief Checks if the response is valid or contains an error.
-// Copyright (C) 2018-2019 Internet Systems Consortium, Inc. ("ISC")
+// Copyright (C) 2018-2020 Internet Systems Consortium, Inc. ("ISC")
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this
ASSERT_TRUE(response);
checkAnswer(response, CONTROL_RESULT_SUCCESS, "Server is in maintained state.");
-
}
}
// The server should have responded.
ASSERT_TRUE(rsp);
- checkAnswer(rsp, CONTROL_RESULT_SUCCESS, "Server maintenance cancelled.");
+ checkAnswer(rsp, CONTROL_RESULT_SUCCESS, "Server maintenance canceled.");
// The state machine should have been transitioned to the state it was in
// prior to transitioning to the maintained state.
EXPECT_EQ(HA_WAITING_ST, service.getCurrState());
}
+// This test verifies the case when the server receiving the ha-maintenance-cancel
+// command successfully transitions out of the partner-maintained state.
+TEST_F(HAServiceTest, processMaintenanceCancelSuccess) {
+ // Create HA configuration for 3 servers. This server is
+ // server 1.
+ HAConfigPtr config_storage = createValidConfiguration();
+
+ // Start the servers.
+ ASSERT_NO_THROW({
+ listener_->start();
+ listener2_->start();
+ });
+
+ TestHAService service(io_service_, network_state_, config_storage);
+
+ ASSERT_NO_THROW(service.verboseTransition(HA_PARTNER_MAINTAINED_ST));
+
+ // The tested function is synchronous, so we need to run server side IO service
+ // in background to not block the main thread.
+ auto thread = runIOServiceInThread();
+
+ // Process ha-maintenance-cancel command.
+ ConstElementPtr rsp;
+ ASSERT_NO_THROW(rsp = service.processMaintenanceCancel());
+
+ // Stop the IO service. This should cause the thread to terminate.
+ io_service_->stop();
+ thread->join();
+ io_service_->get_io_service().reset();
+ io_service_->poll();
+
+ // The partner of our server is online and should have responded with
+ // the success status. Therefore, this server should have transitioned
+ // to the partner-maintained state.
+ ASSERT_TRUE(rsp);
+ checkAnswer(rsp, CONTROL_RESULT_SUCCESS, "Server maintenance successfully canceled.");
+
+ EXPECT_EQ(HA_WAITING_ST, service.getCurrState());
+}
+
+// This test verifies that the maintenance is not canceled in case the
+// partner returns an error.
+TEST_F(HAServiceTest, processMaintenanceCancelPartnerError) {
+ // Create HA configuration for 3 servers. This server is
+ // server 1.
+ HAConfigPtr config_storage = createValidConfiguration();
+
+ // Simulate an error returned by the partner.
+ factory2_->getResponseCreator()->setControlResult(CONTROL_RESULT_ERROR);
+
+ // Start the servers.
+ ASSERT_NO_THROW({
+ listener_->start();
+ listener2_->start();
+ });
+
+ TestHAService service(io_service_, network_state_, config_storage);
+
+ ASSERT_NO_THROW(service.verboseTransition(HA_PARTNER_MAINTAINED_ST));
+
+ // The tested function is synchronous, so we need to run server side IO service
+ // in background to not block the main thread.
+ auto thread = runIOServiceInThread();
+
+ // Process ha-maintenance-cancel command.
+ ConstElementPtr rsp;
+ ASSERT_NO_THROW(rsp = service.processMaintenanceCancel());
+
+ // Stop the IO service. This should cause the thread to terminate.
+ io_service_->stop();
+ thread->join();
+ io_service_->get_io_service().reset();
+ io_service_->poll();
+
+ // The partner should have responded with an error.
+ ASSERT_TRUE(rsp);
+ checkAnswer(rsp, CONTROL_RESULT_ERROR,
+ "Unable to cancel maintenance. The partner server responded"
+ " with the following message to the ha-maintenance-notify"
+ " commmand: response returned, error code 1.");
+
+ // The state of this server should not change.
+ EXPECT_EQ(HA_PARTNER_MAINTAINED_ST, service.getCurrState());
+}
+
+
/// @brief HA partner to the server under test.
///
/// This is a wrapper class around @c HttpListener which simulates a
projects / thirdparty / kea.git / commitdiff
