From: Pavel Filipenský Date: Wed, 28 Jun 2023 12:59:29 +0000 (+0200) Subject: s3:script: Replace --merge by --merge-by-timestamp in samba-log-parser X-Git-Tag: ldb-2.8.0~86 X-Git-Url: http://git.ipfire.org/gitweb.cgi?a=commitdiff_plain;h=ca5cc05b22bf92dd7236d2c0d4a424340a0048f3;p=thirdparty%2Fsamba.git s3:script: Replace --merge by --merge-by-timestamp in samba-log-parser For --merge-by-timestamp the traces do not need to contain the traceid header field. Signed-off-by: Pavel Filipenský Reviewed-by: Douglas Bagnall Autobuild-User(master): Pavel Filipensky Autobuild-Date(master): Thu Jul 20 19:14:05 UTC 2023 on atb-devel-224 --- diff --git a/docs-xml/manpages/samba-log-parser.1.xml b/docs-xml/manpages/samba-log-parser.1.xml index a6a1c27ac88..7a62cd10a47 100644 --- a/docs-xml/manpages/samba-log-parser.1.xml +++ b/docs-xml/manpages/samba-log-parser.1.xml @@ -20,7 +20,7 @@ options: --traceid ID specify the traceid of the trace records --pid PID specify the pid of winbind client --breakdown breakdown the traces into per traceid files - --merge merge logs by timestamp + --merge-by-timestamp merge logs by timestamp --flow show the request/sub-request flow traces --flow-compact show the request/sub-request flow traces without dcerpc details @@ -31,7 +31,7 @@ options: --pid=PID --traceid=ID --breakdown - --merge + --merge-by-timestamp --flow --flow-compact -h|--help @@ -79,8 +79,9 @@ options: - --merge + --merge-by-timestamp Sort the trace lines according to the timestamp. + Works with log files without traceid header field as well. @@ -118,7 +119,7 @@ options: - # samba-log-parser --pid 999999 --merge /var/log/samba + # samba-log-parser --pid 999999 --merge-by-timestamp /var/log/samba Break down the traces into separate files according to traceid sorted @@ -126,7 +127,15 @@ options: - # samba-log-parser --breakdown --merge /var/log/samba + # samba-log-parser --breakdown --merge-by-timestamp /var/log/samba + + + Sort traces using the timestamp for log files found in the samba log + directory. Traces do not need to contain the traceid header field. + + + + # samba-log-parser --merge-by-timestamp /var/log/samba diff --git a/source3/script/samba-log-parser b/source3/script/samba-log-parser index 1c1d2ced33d..a07dfdb4a6c 100755 --- a/source3/script/samba-log-parser +++ b/source3/script/samba-log-parser @@ -5,11 +5,12 @@ # A script to parse samba (especially winbind) logfiles. # Trace files should be in a non-syslog format (debug syslog format = no). # -# --traceid ... Specify the traceid of the request to parse -# --pid ... Specify the pid -# --merge ... Merge logs by timestamp -# --flow ... Show the request/sub-request call flow -# --flow-compact ... Show the request/sub-request call flow without dcerpc +# --traceid ... Specify the traceid of the request to parse +# --pid ... Specify the pid +# --breakdown ... Break to separate files per each traceid +# --merge-by-timestamp ... Merge logs by timestamp +# --flow ... Show the request/sub-request call flow +# --flow-compact ... Show the request/sub-request call flow without dcerpc # # # Copyright (c) 2023 Andreas Schneider @@ -39,6 +40,16 @@ from argparse import ArgumentParser from collections import defaultdict # Trace record consists of a trace header followed by one or more text lines. +# +# This tool expects trace header format based on these smb.conf parameters: +# +# debug syslog format = no +# debug hires timestamp = yes +# winbind debug traceid = yes +# +# If 'winbind debug traceid = no' is set, then the option --merge-by-timestamp +# still can be used. +# # Each trace header contains a traceid, which is the main identifier for this # tool. A single traceid is either provided via command line option --traceid # or a list of traceids is derived from the PID specified via option --pid. @@ -56,6 +67,26 @@ from collections import defaultdict # memory if files are big. +def process_file_no_traceid(record_list, fname): + with open(fname, "r") as infile: + data = infile.readlines() + date = "" + record_lines = [] + + RE_HEADER_NO_TRACEID = re.compile( + r"^\[(\d{4}/\d{2}/\d{2} \d{2}:\d{2}:\d{2}\.\d{6}).*") + for line in data: + header = RE_HEADER_NO_TRACEID.search(line) + if header: + # Append all previous trace lines of a record + if record_lines: + record_list.append((date, None, record_lines, fname)) + record_lines = [] + # Remember the new date + date = header.group(1) + record_lines.append(line) + + def process_file(record_list, traceid_set, fname, opid, otraceid): with open(fname, "r") as infile: data = infile.readlines() @@ -204,7 +235,7 @@ def setup_parser(): help="breakdown the traces into per traceid files" ) parser.add_argument( - "--merge", + "--merge-by-timestamp", action="store_true", dest="merge", default=False, @@ -235,11 +266,16 @@ def main(): # noqa parser = setup_parser() options = parser.parse_args() - if not options.traceid and not options.pid and not options.breakdown: - print("One of --traceid or --pid is needed or --breakdown.") + if (not options.traceid and not options.pid and not options.breakdown + and not options.merge): + print("One of --traceid or --pid is needed" + " or --breakdown or --merge-by-timestamp.") sys.exit(1) elif options.traceid and options.pid: - print("Only one of --traceid or --pid or --breakdown is allowed.") + print("Only one of --traceid or --pid is allowed.") + sys.exit(1) + elif options.breakdown and (options.traceid or options.pid): + print("--breakdown cannot be combined with --traceid and --pid.") sys.exit(1) if options.flow and not options.traceid: @@ -259,29 +295,50 @@ def main(): # noqa print("Path to logfile or directory with logs is needed.") sys.exit(1) + merge_with_no_traceid = (not options.traceid and not options.pid + and not options.breakdown) and options.merge + path = options.path if os.path.isdir(path): for root, dirs, files in os.walk(path): for name in files: - process_file( - record_list, - traceid_set, - os.path.join(root, name), - options.pid, - options.traceid, - ) + if merge_with_no_traceid: + process_file_no_traceid( + record_list, + os.path.join(root, name) + ) + else: + process_file( + record_list, + traceid_set, + os.path.join(root, name), + options.pid, + options.traceid, + ) elif os.path.isfile(path): - process_file( - record_list, - traceid_set, - path, - options.pid, - options.traceid - ) + if merge_with_no_traceid: + process_file_no_traceid( + record_list, + path + ) + else: + process_file( + record_list, + traceid_set, + path, + options.pid, + options.traceid + ) else: print(path, "Path is neither file or directory.") sys.exit(1) + # Sort only using timestamps, no use of traceid + if merge_with_no_traceid: + record_list.sort() + print_record_list(record_list, sys.stdout) + sys.exit(0) + # Keep only records with matching traceids if not options.breakdown: record_list = filter_traceids(record_list, traceid_set)