From: Miroslav Lichvar Date: Thu, 13 Nov 2025 15:02:17 +0000 (+0100) Subject: ntp: add alternative method of retrieving transmitted messages X-Git-Url: http://git.ipfire.org/cgi-bin/gitweb.cgi?a=commitdiff_plain;h=e5abf3ad2b46fd6237ad5f1226ba2f96bf15e983;p=thirdparty%2Fchrony.git ntp: add alternative method of retrieving transmitted messages When chronyd gets a kernel or hardware transmit timestamp after sending an NTP message to a server, peer, or client (using interleaved mode), it needs the address and content of the message to be able to correctly assign the timestamp to the server, peer, or client. The timestamps are processed asynchronously. The kernel provides with each timestamp the data-link frame that was timestamped, but chronyd can extract the necessary data only from plain IPv4 and IPv6 packets in Ethernet frames, possibly including VLAN tags. If the NTP packets are transmitted by a non-Ethernet device, or they are encapsulated in another layer (e.g. a WireGuard tunnel), chronyd is not able to extract the data and use the kernel or hardware transmit timestamps, having to fall back to less accurate daemon timestamps. Add an alternative method using transmit IDs assigned to each message (supported since Linux 6.13), which are provided by the kernel with the timestamp in the error queue, and map them to messages, addresses and ports saved in a ring buffer, whose size can be configured by the new maxtxbuffers directive. Fow now, set the default maxtxbuffers to 0 (disabled). If set to a non-zero value, allocate the ring buffer to the maximum size on start. As a future improvement, it could be allocated only when the extraction of the UDP payload fails, or the extracted message is not the expected NTP message. The size could grow dynamically when a transmit ID is missed. --- diff --git a/conf.c b/conf.c index d081759f..d7a22f69 100644 --- a/conf.c +++ b/conf.c @@ -300,6 +300,9 @@ static ARR_Instance hwts_interfaces; /* Timeout for resuming reading from sockets waiting for HW TX timestamp */ static double hwts_timeout = 0.001; +/* Maximum number of saved messages for TX timestamp identification */ +static int max_tx_buffers = 0; + /* PTP event port (disabled by default) */ static int ptp_port = 0; /* PTP domain number of NTP-over-PTP messages */ @@ -708,6 +711,8 @@ CNF_ParseLine(const char *filename, int number, char *line) parse_int(p, &max_stratum, 0, INT_MAX); } else if (!strcasecmp(command, "maxupdateskew")) { parse_double(p, &max_update_skew); + } else if (!strcasecmp(command, "maxtxbuffers")) { + parse_int(p, &max_tx_buffers, 0, 1048576); } else if (!strcasecmp(command, "minsamples")) { parse_int(p, &min_samples, 0, INT_MAX); } else if (!strcasecmp(command, "minsources")) { @@ -2778,6 +2783,14 @@ CNF_GetHwTsTimeout(void) /* ================================================== */ +int +CNF_GetMaxTxBuffers(void) +{ + return max_tx_buffers; +} + +/* ================================================== */ + int CNF_GetPtpPort(void) { diff --git a/conf.h b/conf.h index a5ff1640..4b2ad211 100644 --- a/conf.h +++ b/conf.h @@ -162,6 +162,7 @@ typedef struct { extern int CNF_GetHwTsInterface(unsigned int index, CNF_HwTsInterface **iface); extern double CNF_GetHwTsTimeout(void); +extern int CNF_GetMaxTxBuffers(void); extern int CNF_GetPtpPort(void); extern int CNF_GetPtpDomain(void); diff --git a/configure b/configure index ca64475d..ab0cfcf9 100755 --- a/configure +++ b/configure @@ -748,9 +748,10 @@ then struct scm_ts_pktinfo pktinfo; pktinfo.if_index = pktinfo.pkt_length = 0; return pktinfo.if_index + pktinfo.pkt_length + HWTSTAMP_FILTER_NTP_ALL + - SCM_TIMESTAMPING_PKTINFO + + SCM_TIMESTAMPING_PKTINFO + SOF_TIMESTAMPING_OPT_ID + SOF_TIMESTAMPING_OPT_PKTINFO + SOF_TIMESTAMPING_OPT_TX_SWHW;'; then add_def HAVE_LINUX_TIMESTAMPING_RXFILTER_NTP 1 + add_def HAVE_LINUX_TIMESTAMPING_OPT_ID 1 add_def HAVE_LINUX_TIMESTAMPING_OPT_PKTINFO 1 add_def HAVE_LINUX_TIMESTAMPING_OPT_TX_SWHW 1 fi diff --git a/doc/chrony.conf.adoc b/doc/chrony.conf.adoc index 3e57da81..2097632b 100644 --- a/doc/chrony.conf.adoc +++ b/doc/chrony.conf.adoc @@ -2881,6 +2881,38 @@ same interface (which would compete with timestamping of the server's own requests), increasing the timeout to 0.01 or possibly even longer might help. Note that the maximum timeout is limited by the NTP polling interval. +[[maxtxbuffers]]*maxtxbuffers* _messages_:: +When *chronyd* gets a kernel or hardware transmit timestamp after sending an +NTP message to a server, peer, or client (using interleaved mode), it needs the +address and content of the message to be able to correctly assign the +timestamp to the server, peer, or client. The timestamps are processed +asynchronously. The kernel provides with each timestamp the data-link frame +that was timestamped, but *chronyd* can extract the necessary data only from +plain IPv4 and IPv6 packets in Ethernet frames, possibly including VLAN tags. +If the NTP packets are transmitted by a non-Ethernet device, or they are +encapsulated in another layer (e.g. a WireGuard tunnel), *chronyd* is not +able to extract the data and use the kernel or hardware transmit timestamps, +having to fall back to less accurate daemon timestamps. ++ +On Linux 6.13 and newer, *chronyd* supports an alternative method of processing +kernel and hardware transmit timestamps, which works with any device that can +provide kernel and/or hardware transmit timestamps. *chronyd* can save its +transmitted messages with their addresses and ports in a ring buffer, each +assigned an ID that the kernel will provide back with the timestamp. This +transmit ID enables *chronyd* to assign the timestamp to the correct server, +peer, or client. ++ +The directive specifies the maximum number of messages that *chronyd* is +allowed to save in the ring buffer. The minimum value is 0 and the maximum +value is 1048576. It needs about 1.5 KB of allocated memory per message. +*chronyd* will log a warning if the number of messages is insufficient for the +actual rate of transmit timestamping. The default value is 0 (disabled). ++ +Example: +---- +maxtxbuffers 128 +---- + [[keyfile]]*keyfile* _file_:: This directive is used to specify the location of the file containing symmetric keys, which are shared between NTP servers and clients, or peers, in order to diff --git a/ntp_io.c b/ntp_io.c index 7a034f72..71ccfab9 100644 --- a/ntp_io.c +++ b/ntp_io.c @@ -626,7 +626,7 @@ NIO_SendPacket(NTP_Packet *packet, NTP_Remote_Address *remote_addr, #ifdef HAVE_LINUX_TIMESTAMPING if (process_tx) - NIO_Linux_RequestTxTimestamp(&message, local_addr->sock_fd); + NIO_Linux_RequestTxTimestamp(&message, local_addr->sock_fd, remote_addr); #endif if (!SCK_SendMessage(local_addr->sock_fd, &message, 0)) diff --git a/ntp_io_linux.c b/ntp_io_linux.c index 808a008d..e917a785 100644 --- a/ntp_io_linux.c +++ b/ntp_io_linux.c @@ -44,6 +44,7 @@ #include "ntp_io.h" #include "ntp_io_linux.h" #include "ntp_sources.h" +#include "ptp.h" #include "sched.h" #include "socket.h" #include "sys_linux.h" @@ -94,6 +95,24 @@ static int permanent_ts_options; enabling the timestamping and sending a request */ static int dummy_rxts_socket; +struct SavedTxMessage { + uint32_t tx_id; + int length; + NTP_Remote_Address remote_addr; + union { + NTP_Packet ntp_msg; + PTP_NtpMessage ptp_msg; + } message; +}; + +/* Ring buffer of transmitted messages to provide missing data needed for + processing of transmit timestamps */ +static ARR_Instance saved_tx_messages; +static uint32_t last_saved_tx_id; + +/* Transmit IDs start at a non-zero value to detect missing kernel support */ +#define MIN_SAVED_TX_ID 0x1000 + #define INVALID_SOCK_FD -3 /* ================================================== */ @@ -367,8 +386,8 @@ void NIO_Linux_Initialise(void) { CNF_HwTsInterface *conf_iface; + int hwts, tx_buffers; unsigned int i; - int hwts; interfaces = ARR_CreateInstance(sizeof (struct Interface)); @@ -407,6 +426,32 @@ NIO_Linux_Initialise(void) #endif } + saved_tx_messages = NULL; + last_saved_tx_id = 0; + tx_buffers = CNF_GetMaxTxBuffers(); + + if (tx_buffers > 0) { +#if defined(HAVE_LINUX_TIMESTAMPING_OPT_ID) && defined(SCM_TS_OPT_ID) + /* Enable identification of packets looped back to the error queue using + a 32-bit integer and mapping of the IDs to saved messages to be able to + process TX timestamps of packets going out over tunnels or non-Ethernet + interfaces, where extract_udp_data() fails, or does not extract the + NTP message. If the SCM_TS_OPT_ID control message (setting the ID for + given packet) is not supported by the kernel, a zero ID will be received + and this functionality disabled in get_saved_tx_message(). */ + + if (check_timestamping_option(SOF_TIMESTAMPING_OPT_ID)) { + ts_tx_flags |= SOF_TIMESTAMPING_OPT_ID; + + saved_tx_messages = ARR_CreateInstance(sizeof (struct SavedTxMessage)); + ARR_SetSize(saved_tx_messages, tx_buffers); + memset(ARR_GetElements(saved_tx_messages), 0, + tx_buffers * sizeof (struct SavedTxMessage)); + } else +#endif + LOG(LOGS_WARN, "Transmit ID not supported"); + } + /* Enable IP_PKTINFO in messages looped back to the error queue */ ts_flags |= SOF_TIMESTAMPING_OPT_CMSG; @@ -424,6 +469,9 @@ NIO_Linux_Finalise(void) struct Interface *iface; unsigned int i; + if (saved_tx_messages) + ARR_DestroyInstance(saved_tx_messages); + if (dummy_rxts_socket != INVALID_SOCK_FD) SCK_CloseSocket(dummy_rxts_socket); @@ -629,6 +677,73 @@ process_sw_timestamp(struct timespec *sw_ts, NTP_Local_Timestamp *local_ts) local_ts->source = NTP_TS_KERNEL; } +/* ================================================== */ + +static void +save_tx_message(SCK_Message *message, NTP_Remote_Address *remote_addr) +{ + struct SavedTxMessage *saved_msg; + + if (!saved_tx_messages || ARR_GetSize(saved_tx_messages) == 0 || + message->length > sizeof (saved_msg->message)) + return; + + last_saved_tx_id++; + if (last_saved_tx_id < MIN_SAVED_TX_ID) + last_saved_tx_id = MIN_SAVED_TX_ID; + + saved_msg = ARR_GetElement(saved_tx_messages, + last_saved_tx_id % ARR_GetSize(saved_tx_messages)); + saved_msg->tx_id = last_saved_tx_id; + saved_msg->length = message->length; + saved_msg->remote_addr = *remote_addr; + memcpy(&saved_msg->message, message->data, message->length); + + message->timestamp.tx_id = saved_msg->tx_id; +} + +/* ================================================== */ + +static int +get_saved_tx_message(SCK_Message *message) +{ +#ifdef HAVE_LINUX_TIMESTAMPING_OPT_ID + struct SavedTxMessage *saved_msg; + + if (!saved_tx_messages) + return 0; + + if (message->timestamp.tx_id < MIN_SAVED_TX_ID) { + LOG(LOGS_WARN, "Transmit ID not supported"); + + ARR_DestroyInstance(saved_tx_messages); + saved_tx_messages = NULL; + ts_tx_flags &= ~SOF_TIMESTAMPING_OPT_ID; + return 0; + } + + saved_msg = ARR_GetElement(saved_tx_messages, + message->timestamp.tx_id % ARR_GetSize(saved_tx_messages)); + + if (message->timestamp.tx_id != saved_msg->tx_id) { + static int warned = 0; + if (!warned) { + LOG(LOGS_WARN, "maxtxbuffers too small"); + warned = 1; + } + return 0; + } + + message->data = &saved_msg->message; + message->length = saved_msg->length; + message->remote_addr.ip = saved_msg->remote_addr; + + return 1; +#else + return 0; +#endif +} + /* ================================================== */ /* Extract UDP data from a layer 2 message. Supported is Ethernet with optional VLAN tags. */ @@ -771,7 +886,11 @@ NIO_Linux_ProcessMessage(SCK_Message *message, NTP_Local_Address *local_addr, currently doesn't seem to be a better way to get them both. */ l2_length = message->length; - if (extract_udp_data(message)) { + if (get_saved_tx_message(message)) { + DEBUG_LOG("Found saved message for %s fd=%d len=%d", + UTI_IPSockAddrToString(&message->remote_addr.ip), + local_addr->sock_fd, message->length); + } else if (extract_udp_data(message)) { DEBUG_LOG("Extracted message for %s fd=%d len=%d", UTI_IPSockAddrToString(&message->remote_addr.ip), local_addr->sock_fd, message->length); @@ -808,11 +927,14 @@ NIO_Linux_ProcessMessage(SCK_Message *message, NTP_Local_Address *local_addr, /* ================================================== */ void -NIO_Linux_RequestTxTimestamp(SCK_Message *message, int sock_fd) +NIO_Linux_RequestTxTimestamp(SCK_Message *message, int sock_fd, + NTP_Remote_Address *remote_addr) { if (!ts_flags) return; + save_tx_message(message, remote_addr); + /* Check if TX timestamping is disabled on this socket */ if (permanent_ts_options || !NIO_IsServerSocket(sock_fd)) return; diff --git a/ntp_io_linux.h b/ntp_io_linux.h index 79788895..75c863ce 100644 --- a/ntp_io_linux.h +++ b/ntp_io_linux.h @@ -40,6 +40,7 @@ extern int NIO_Linux_SetTimestampSocketOptions(int sock_fd, int client_only, int extern int NIO_Linux_ProcessMessage(SCK_Message *message, NTP_Local_Address *local_addr, NTP_Local_Timestamp *local_ts, int event); -extern void NIO_Linux_RequestTxTimestamp(SCK_Message *message, int sock_fd); +extern void NIO_Linux_RequestTxTimestamp(SCK_Message *message, int sock_fd, + NTP_Remote_Address *remote_addr); #endif