From: Michael Vogt Date: Tue, 16 Jun 2026 14:20:10 +0000 (+0200) Subject: journal: expose last 10 high priority logs as metrics X-Git-Url: http://git.ipfire.org/cgi-bin/gitweb.cgi?a=commitdiff_plain;h=b436be5938c75885bfcb18fea9072aa46510db52;p=thirdparty%2Fsystemd.git journal: expose last 10 high priority logs as metrics This commit exposes the last 10 high priority logs as metrics so that the systemd-report reports them. The entries are reported as `io.systemd.Journal.HighPriorityMessage` and include all field as the new METRIC_FAMILY_TYPE_OBJECT. Individual fields from a journal entry that are unprintable (invalid utf-8) are skipped. This is archived via a new socket-activated unit listens on /run/systemd/report/io.systemd.Journal --- diff --git a/presets/90-systemd.preset b/presets/90-systemd.preset index 005280cd94d..85108dcdd53 100644 --- a/presets/90-systemd.preset +++ b/presets/90-systemd.preset @@ -24,6 +24,7 @@ enable systemd-boot-update.service enable systemd-confext.service enable systemd-homed.service enable systemd-homed-activate.service +enable systemd-journalctl-metrics.socket enable systemd-journald-audit.socket enable systemd-mountfsd.socket enable systemd-network-generator.service diff --git a/src/journal/journalctl-metrics.c b/src/journal/journalctl-metrics.c new file mode 100644 index 00000000000..bcba2ef00e3 --- /dev/null +++ b/src/journal/journalctl-metrics.c @@ -0,0 +1,93 @@ +/* SPDX-License-Identifier: LGPL-2.1-or-later */ + +#include "sd-journal.h" +#include "sd-json.h" +#include "sd-varlink.h" + +#include "journalctl.h" +#include "journalctl-filter.h" +#include "journalctl-metrics.h" +#include "log.h" +#include "logs-show.h" +#include "metrics.h" +#include "output-mode.h" + +/* Fallback cap so we never stream unbounded entries when --lines= is "all" or unset. */ +#define N_RECENT_HIGH_PRIORITY 10 + +/* Set a maximum scan factor to avoid unbound iterating through the journal */ +#define RECENT_HIGH_PRIORITY_SCAN_FACTOR 50 + +static int recent_high_priority_generate(const MetricFamily *mf, sd_varlink *link, void *userdata) { + _cleanup_(sd_journal_closep) sd_journal *j = NULL; + int r; + + assert(mf && mf->name); + assert(link); + + r = sd_journal_open(&j, SD_JOURNAL_LOCAL_ONLY | SD_JOURNAL_SYSTEM | SD_JOURNAL_ASSUME_IMMUTABLE); + if (r < 0) + return log_debug_errno(r, "Failed to open journal, ignoring: %m"); + + /* Get filters (--priority=, units, matches, ...) from the command-line. */ + r = add_filters(j, /* matches= */ NULL); + if (r < 0) + return log_debug_errno(r, "Failed to add journal filters: %m"); + + r = sd_journal_seek_tail(j); + if (r < 0) + return log_debug_errno(r, "Failed to seek to journal tail: %m"); + + /* The --lines=+N argument does not make much sense for a metrics provider so we ignore it here. */ + if (arg_lines_oldest) + log_warning("--lines=+N is not supported when serving the metrics interface, ignoring."); + + uint64_t max_lines = arg_lines_needs_seek_end() ? (uint64_t) arg_lines : N_RECENT_HIGH_PRIORITY; + uint64_t max_scan = max_lines * RECENT_HIGH_PRIORITY_SCAN_FACTOR; + + for (uint64_t found = 0, scanned = 0; found < max_lines && scanned < max_scan; scanned++) { + _cleanup_(sd_json_variant_unrefp) sd_json_variant *entry = NULL; + + r = sd_journal_previous(j); + if (r < 0) + return log_debug_errno(r, "Failed to iterate to previous journal entry: %m"); + if (r == 0) + break; + + r = journal_entry_to_json(j, OUTPUT_SHOW_ALL | OUTPUT_SKIP_UNPRINTABLE, /* output_fields= */ NULL, &entry); + if (r < 0) { + log_debug_errno(r, "Failed to convert journal entry to JSON, skipping entry: %m"); + continue; + } + if (r == 0) + continue; + + const char *ident = sd_json_variant_string(sd_json_variant_by_key(entry, "SYSLOG_IDENTIFIER")); + + r = metric_build_send_object(mf, link, /* object= */ ident, entry, /* fields= */ NULL); + if (r < 0) + return log_debug_errno(r, "Failed to send journal metric: %m"); + + found++; + } + + return 0; +} + +static const MetricFamily journal_metric_family_table[] = { + { + .name = METRIC_IO_SYSTEMD_JOURNAL_PREFIX "HighPriorityMessage", + .description = "The most recent high-priority journal messages", + .type = METRIC_FAMILY_TYPE_OBJECT, + .generate = recent_high_priority_generate, + }, + {} +}; + +int vl_method_describe_metrics(sd_varlink *link, sd_json_variant *parameters, sd_varlink_method_flags_t flags, void *userdata) { + return metrics_method_describe(journal_metric_family_table, link, parameters, flags, userdata); +} + +int vl_method_list_metrics(sd_varlink *link, sd_json_variant *parameters, sd_varlink_method_flags_t flags, void *userdata) { + return metrics_method_list(journal_metric_family_table, link, parameters, flags, userdata); +} diff --git a/src/journal/journalctl-metrics.h b/src/journal/journalctl-metrics.h new file mode 100644 index 00000000000..8250b43855e --- /dev/null +++ b/src/journal/journalctl-metrics.h @@ -0,0 +1,9 @@ +/* SPDX-License-Identifier: LGPL-2.1-or-later */ +#pragma once + +#include "shared-forward.h" + +#define METRIC_IO_SYSTEMD_JOURNAL_PREFIX "io.systemd.Journal." + +int vl_method_list_metrics(sd_varlink *link, sd_json_variant *parameters, sd_varlink_method_flags_t flags, void *userdata); +int vl_method_describe_metrics(sd_varlink *link, sd_json_variant *parameters, sd_varlink_method_flags_t flags, void *userdata); diff --git a/src/journal/journalctl.c b/src/journal/journalctl.c index 5d0b8e22857..b5c4ec7aeef 100644 --- a/src/journal/journalctl.c +++ b/src/journal/journalctl.c @@ -16,6 +16,7 @@ #include "journalctl.h" #include "journalctl-authenticate.h" #include "journalctl-catalog.h" +#include "journalctl-metrics.h" #include "journalctl-misc.h" #include "journalctl-show.h" #include "journalctl-varlink.h" @@ -40,6 +41,7 @@ #include "syslog-util.h" #include "time-util.h" #include "varlink-io.systemd.JournalAccess.h" +#include "varlink-io.systemd.Metrics.h" #include "varlink-util.h" #define DEFAULT_FSS_INTERVAL_USEC (15*USEC_PER_MINUTE) @@ -214,6 +216,43 @@ default_noarg: return 0; } +static int parse_priorities(const char *arg) { + assert(arg); + + const char *dots = strstr(arg, ".."); + if (dots) { + /* a range */ + _cleanup_free_ char *a = strndup(arg, dots - arg); + if (!a) + return log_oom(); + + int from = log_level_from_string(a), + to = log_level_from_string(dots + 2); + + if (from < 0 || to < 0) + return log_error_errno(from < 0 ? from : to, + "Failed to parse log level range %s", arg); + + arg_priorities = 0; + if (from < to) + for (int i = from; i <= to; i++) + arg_priorities |= 1 << i; + else + for (int i = to; i <= from; i++) + arg_priorities |= 1 << i; + } else { + int p = log_level_from_string(arg); + if (p < 0) + return log_error_errno(p, "Unknown log level %s", arg); + + arg_priorities = 0; + for (int i = 0; i <= p; i++) + arg_priorities |= 1 << i; + } + + return 0; +} + static int help_facilities(void) { if (!arg_quiet) puts("Available facilities:"); @@ -277,13 +316,23 @@ static int vl_server(void) { if (r < 0) return log_error_errno(r, "Failed to allocate Varlink server: %m"); - r = sd_varlink_server_add_interface(varlink_server, &vl_interface_io_systemd_JournalAccess); + /* Serve both interfaces regardless of which socket activated us: both activating sockets share the + * same access controls (systemd-journal group), so anyone reaching either is already entitled to + * both. */ + r = sd_varlink_server_add_interface_many( + varlink_server, + &vl_interface_io_systemd_JournalAccess, + &vl_interface_io_systemd_Metrics); if (r < 0) - return log_error_errno(r, "Failed to add Varlink interface: %m"); + return log_error_errno(r, "Failed to add Varlink interfaces: %m"); - r = sd_varlink_server_bind_method(varlink_server, "io.systemd.JournalAccess.GetEntries", vl_method_get_entries); + r = sd_varlink_server_bind_method_many( + varlink_server, + "io.systemd.JournalAccess.GetEntries", vl_method_get_entries, + "io.systemd.Metrics.List", vl_method_list_metrics, + "io.systemd.Metrics.Describe", vl_method_describe_metrics); if (r < 0) - return log_error_errno(r, "Failed to bind Varlink method: %m"); + return log_error_errno(r, "Failed to bind Varlink methods: %m"); r = sd_varlink_server_loop_auto(varlink_server); if (r < 0) @@ -319,13 +368,35 @@ static int parse_argv(int argc, char *argv[], char ***remaining_args) { OPTION_COMMON_USER: arg_varlink_runtime_scope = RUNTIME_SCOPE_USER; break; - } + + OPTION('p', "priority", "RANGE", "Show entries within the specified priority range"): + r = parse_priorities(opts.arg); + if (r < 0) + return r; + break; + + OPTION_FULL(OPTION_OPTIONAL_ARG, 'n', "lines", "[+]INTEGER", + "Number of journal entries to show"): { + const char *p = opts.arg ?: option_parser_peek_next_arg(&opts); + + r = parse_lines(p, /* graceful= */ !opts.arg); + if (r < 0) + return r; + if (r > 0 && !opts.arg) + (void) option_parser_consume_next_arg(&opts); + + break; + }} if (arg_varlink_runtime_scope < 0) return log_error_errno(arg_varlink_runtime_scope, "Cannot run in Varlink mode with no runtime scope specified."); if (option_parser_get_n_args(&opts) > 0) - return log_error_errno(SYNTHETIC_ERRNO(EINVAL), "No arguments expected in Varlink mode."); + return log_error_errno(SYNTHETIC_ERRNO(EINVAL), "No positional arguments expected in Varlink mode."); + + /* We return early, skipping the tristate resolution the regular path does below; resolve it + * here so add_filters() doesn't trip over the unresolved -1. */ + arg_boot = false; *remaining_args = NULL; return 1; @@ -526,42 +597,11 @@ static int parse_argv(int argc, char *argv[], char ***remaining_args) { return log_oom(); break; - OPTION('p', "priority", "RANGE", "Show entries within the specified priority range"): { - - const char *dots = strstr(opts.arg, ".."); - if (dots) { - /* a range */ - _cleanup_free_ char *a = strndup(opts.arg, dots - opts.arg); - if (!a) - return log_oom(); - - int from = log_level_from_string(a), - to = log_level_from_string(dots + 2); - - if (from < 0 || to < 0) - return log_error_errno(from < 0 ? from : to, - "Failed to parse log level range %s", opts.arg); - - arg_priorities = 0; - if (from < to) - for (int i = from; i <= to; i++) - arg_priorities |= 1 << i; - else - for (int i = to; i <= from; i++) - arg_priorities |= 1 << i; - - } else { - int p = log_level_from_string(opts.arg); - if (p < 0) - return log_error_errno(p, "Unknown log level %s", opts.arg); - - arg_priorities = 0; - for (int i = 0; i <= p; i++) - arg_priorities |= 1 << i; - } - + OPTION('p', "priority", "RANGE", "Show entries within the specified priority range"): + r = parse_priorities(opts.arg); + if (r < 0) + return r; break; - } OPTION_LONG("facility", "FACILITY…", "Show entries with the specified facilities"): for (const char *p = opts.arg;;) { diff --git a/src/journal/meson.build b/src/journal/meson.build index 75cd7b7a30a..38f2e7bb7ed 100644 --- a/src/journal/meson.build +++ b/src/journal/meson.build @@ -34,6 +34,7 @@ journalctl_sources = files( 'journalctl-authenticate.c', 'journalctl-catalog.c', 'journalctl-filter.c', + 'journalctl-metrics.c', 'journalctl-misc.c', 'journalctl-show.c', 'journalctl-util.c', diff --git a/test/units/TEST-04-JOURNAL.journalctl-metrics.sh b/test/units/TEST-04-JOURNAL.journalctl-metrics.sh new file mode 100755 index 00000000000..88aa91dd16b --- /dev/null +++ b/test/units/TEST-04-JOURNAL.journalctl-metrics.sh @@ -0,0 +1,53 @@ +#!/usr/bin/env bash +# SPDX-License-Identifier: LGPL-2.1-or-later +set -eux +set -o pipefail + +# journalctl's Varlink server also exposes the io.systemd.Metrics provider that reports the most recent +# high-priority journal messages, socket-activated under /run/systemd/report/. +SOCKET="/run/systemd/report/io.systemd.Journal" + +# Ensure the socket is running, as some distros don't enable it by default. +systemctl start systemd-journalctl-metrics.socket +test -S "$SOCKET" + +# Both activating sockets share the same systemd-journal group access controls, so the server exposes both +# io.systemd.Metrics and io.systemd.JournalAccess regardless of which socket was connected to. +varlinkctl list-interfaces "$SOCKET" | grep io.systemd.Metrics >/dev/null +varlinkctl list-interfaces "$SOCKET" | grep io.systemd.JournalAccess >/dev/null + +# Describe must advertise our metric family. +varlinkctl --more call "$SOCKET" io.systemd.Metrics.Describe '{}' | + grep "io.systemd.Journal.HighPriorityMessage" >/dev/null + +# Seed a unique high-priority (crit) message right before listing, so it is guaranteed to be among the 10 +# most recent matches. The service runs with --priority=crit, so the level must be crit or higher. We run +# as root (_UID=0), so it lands in the system journal that the provider reads. +TAG="$(systemd-id128 new)" +systemd-cat -t "high-$TAG" -p crit echo "metrics-hiprio-$TAG" +# A low-priority (info) message that must NOT be reported. +systemd-cat -t "info-$TAG" -p info echo "metrics-info-$TAG" +journalctl --sync + +LIST="$(varlinkctl --more call "$SOCKET" io.systemd.Metrics.List '{}')" + +# Output is valid application/json-seq. +jq --seq . >/dev/null <<<"$LIST" + +# The crit message is reported as the whole journal entry object in .value, with every field under its raw +# journal name (journal_entry_to_json()). systemd-cat -t sets SYSLOG_IDENTIFIER, which maps to "object". +[ "$(jq --seq -r --arg o "high-$TAG" 'select(.object == $o) | .value.MESSAGE' <<<"$LIST")" = "metrics-hiprio-$TAG" ] +[ "$(jq --seq -r --arg o "high-$TAG" 'select(.object == $o) | .value.PRIORITY' <<<"$LIST")" = "2" ] + +# The info message is filtered out (priority below crit). +[ -z "$(jq --seq -r --arg o "info-$TAG" 'select(.object == $o) | .value.MESSAGE' <<<"$LIST")" ] + +# The list is capped at 10. Seed more than that and confirm exactly 10 are returned. (Count raw .name +# lines; jq --seq frames non-string output like `length` with an RS byte, but -r string output is clean.) +for ((i = 0; i < 15; i++)); do + systemd-cat -t "cap-$TAG" -p crit echo "metrics-cap-$TAG-$i" +done +journalctl --sync +LIST="$(varlinkctl --more call "$SOCKET" io.systemd.Metrics.List '{}')" +COUNT="$(jq --seq -r '.name' <<<"$LIST" | wc -l)" +test "$COUNT" -eq 10 diff --git a/units/meson.build b/units/meson.build index 16fa36cf8d5..f261fded6ff 100644 --- a/units/meson.build +++ b/units/meson.build @@ -473,6 +473,8 @@ units = [ 'file' : 'systemd-journalctl.socket', 'symlinks' : ['sockets.target.wants/'], }, + { 'file' : 'systemd-journalctl-metrics@.service' }, + { 'file' : 'systemd-journalctl-metrics.socket' }, { 'file' : 'systemd-journald-audit.socket' }, { 'file' : 'systemd-journald-dev-log.socket', diff --git a/units/systemd-journalctl-metrics.socket b/units/systemd-journalctl-metrics.socket new file mode 100644 index 00000000000..93e126d6ba2 --- /dev/null +++ b/units/systemd-journalctl-metrics.socket @@ -0,0 +1,28 @@ +# SPDX-License-Identifier: LGPL-2.1-or-later +# +# This file is part of systemd. +# +# systemd is free software; you can redistribute it and/or modify it +# under the terms of the GNU Lesser General Public License as published by +# the Free Software Foundation; either version 2.1 of the License, or +# (at your option) any later version. + +[Unit] +Description=Journal Metrics Report Socket +Documentation=man:journalctl(1) +DefaultDependencies=no +Before=sockets.target +After=systemd-sysusers.service + +[Socket] +ListenStream=/run/systemd/report/io.systemd.Journal +FileDescriptorName=varlink +# Same permissions as systemd-journalctl.socket for io.systemd.JournalAccess +SocketGroup=systemd-journal +SocketMode=0660 +Accept=yes +MaxConnectionsPerSource=16 +RemoveOnStop=yes + +[Install] +WantedBy=sockets.target diff --git a/units/systemd-journalctl-metrics@.service b/units/systemd-journalctl-metrics@.service new file mode 100644 index 00000000000..92d62960912 --- /dev/null +++ b/units/systemd-journalctl-metrics@.service @@ -0,0 +1,44 @@ +# SPDX-License-Identifier: LGPL-2.1-or-later +# +# This file is part of systemd. +# +# systemd is free software; you can redistribute it and/or modify it +# under the terms of the GNU Lesser General Public License as published by +# the Free Software Foundation; either version 2.1 of the License, or +# (at your option) any later version. + +[Unit] +Description=Journal Metrics Report Service +Documentation=man:journalctl(1) +DefaultDependencies=no +Conflicts=shutdown.target +Before=shutdown.target +RequiresMountsFor=/var/log/journal + +[Service] +ExecStart=journalctl --system --lines=10 --priority=crit +DynamicUser=yes +User=systemd-journal-access +SupplementaryGroups=systemd-journal +CapabilityBoundingSet= +DeviceAllow= +LockPersonality=yes +MemoryDenyWriteExecute=yes +PrivateDevices=yes +PrivateIPC=yes +PrivateNetwork=yes +PrivateTmp=disconnected +ProtectControlGroups=yes +ProtectHome=yes +ProtectHostname=yes +ProtectKernelLogs=yes +ProtectKernelModules=yes +ProtectKernelTunables=yes +ProtectSystem=strict +RestrictAddressFamilies=AF_UNIX +RestrictNamespaces=yes +RestrictRealtime=yes +RuntimeMaxSec=1min +SystemCallArchitectures=native +SystemCallErrorNumber=EPERM +SystemCallFilter=@system-service