Parser: Adjust HTTP parser to be compatible with newer log format.

[people/stevee/guardian.git] / guardian.in

#!/usr/bin/perl
###############################################################################
#                                                                             #
# IPFire.org - A linux based firewall                                         #
# Copyright (C) 2015-2016  IPFire Development Team                            #
#                                                                             #
# This program is free software: you can redistribute it and/or modify        #
# it under the terms of the GNU General Public License as published by        #
# the Free Software Foundation, either version 3 of the License, or           #
# (at your option) any later version.                                         #
#                                                                             #
# This program is distributed in the hope that it will be useful,             #
# but WITHOUT ANY WARRANTY; without even the implied warranty of              #
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the               #
# GNU General Public License for more details.                                #
#                                                                             #
# You should have received a copy of the GNU General Public License           #
# along with this program.  If not, see <http://www.gnu.org/licenses/>.       #
#                                                                             #
###############################################################################

use strict;
use threads;
use threads::shared;
use Getopt::Long;
use Thread::Queue;
use Linux::Inotify2;
use Time::HiRes qw[ time sleep ];

require Guardian::Base;
require Guardian::Config;
require Guardian::Daemon;
require Guardian::Events;
require Guardian::Logger;
require Guardian::Parser;
require Guardian::Socket;

use warnings;

# Disable warnings of unjoined threads when stopping guardian.
no warnings 'threads';

# Define version.
my $version ="@PACKAGE_VERSION@";

# Get and store the given command line arguments in a hash.
my %cmdargs = ();

&GetOptions (\%cmdargs,
	'foreground|f',
	'config|c=s',
	'help|h',
	'version|v',
);

# Show help / version information.
if (defined($cmdargs{"help"})) {
	print "Guardian $version \n";
	print "Usage: guardian <optional arguments>\n";
	print " -c, --config\t\tspecifiy a configuration file other than the default (/etc/guardian/guardian.conf)\n";
	print " -f, --foreground\turn in the foreground (doesn't fork, any output goes to STDOUT)\n";
	print " -h, --help\t\tshows this help\n";
	print " -v, --version\t\tdisplay programm version and exit.\n";
	exit;
} elsif (defined($cmdargs{"version"})) {
	print "Guardian $version \n";
	exit;
}

# Check if another instance of guardian is allready running.
if (&Guardian::Daemon::IsRunning()) {
	die "Another instance of Guardian is already running...\n";
}

# Read-in the configuration file and store the settings.
# Push the may be given config file argument.
my %mainsettings = &Guardian::Config::UseConfig($cmdargs{"config"});

# Initialize Logger.
my $logger = Guardian::Logger->Init(%mainsettings);
$logger->Log("debug", "Logger successfully initialized...");

# Add the logger object to the mainsettings for passing
# it to the modules.
$mainsettings{Logger} = $logger;

# Redirect perls "die" messages to the logger before exiting.
$SIG{__DIE__} = sub { $logger->Log("err", "@_"); };

# Initialize the parser module.
my $parser = Guardian::Parser->Init(%mainsettings);

# Initialize the event handler.
my $events = Guardian::Events->Init(%mainsettings);

# Hash to store the currently monitored files and their configured
# parsers.
my %monitored_files = ();

# Shared hash between the main process and all threads. It will store the
# monitored files and their current file position.
my %file_positions :shared = ();

# Create the main queue. It is used to store and process all events which are
# reported and enqueued by the worker threads.
my $queue :shared = new Thread::Queue or die "Could not create new, empty queue. $!";;

# Hash to store all currently running worker objects and their corresponding files.
# (Does not include the socket thread)
my %running_workers;

# Variable to store if the workers should pause or compute.
my $workers_pause :shared = 0;

# Check if guardian should be daemonized or keep in the foreground.
unless (defined($cmdargs{"foreground"})) {
	# Fork into background.
	&Guardian::Daemon::Daemonize();
} else {
	# Write PID (process-id).
	&Guardian::Daemon::WritePID();
}

# Call Init function to initzialize guardian.
&Init();

# Log successfully started process.
$logger->Log("info", "Guardian $version successfully started...");

# Infinite main loop, which processes all queued events.
while(1) {
	# Get the amount of elements in our queue.
	# "undef" will be returned if it is empty.
	my $current_events = $queue->pending();

	# If there is at least one element enqued
	if($current_events > 0) {
		# Grab the data of the top enqueued event.
		my $event = $queue->peek();

		# Log processed event.
		$logger->Log("debug", "QUEUE - Processed event: $event");

		# Send event data to the events parser to determine
		# if any action is required.
		$events->CheckAction($event);

		# Drop processed event from queue.
		$queue->dequeue();
	}

	# Call RemoveBlocks routine from the Events module to check
	# if items from the block list can be dropped.
	$events->RemoveBlocks();

	# Sleep 50ms to reduce the load of the main process.
	sleep(0.05);
}

#
## Init function.
#
## This function contains code which has to be executed while guardian
## is starting.
#
sub Init () {
	# Setup signal handler.
	&SignalHandler();

	# Setup IPC mechanism via Socket in an own thread.
	threads->create(\&Socket);

	# Generate hash of monitored files.
	%monitored_files = &Guardian::Base::GenerateMonitoredFiles(\%mainsettings, \%monitored_files);

	# Start worker threads.
	&StartWorkers();
}

#
## Worker function.
#
## This function is responsible for monitoring modifications of the given logfile,
## read them and pass them to the message parser.
#
## To get file modifications the inotify subsystem of the linux kernel is used.
#
## In order to prevent from permanently read and keep files opened, or dealing
## with huge logfiles, at initialization time of the worker process, the file will
## be opened once and the cursor position of the end of file (EOF) get stored. When
## reading any newly added lines from the file, we directly can jump to the last
## known position and get these lines. Afterwards, we store the current cursor position
## again, so we can do it in this way over and over again.
#
## All read lines get stored in an array, which will be passed to the Parser.
#
## If any response (action) from the parser is received, it will be put into the
## shared event queue.
#
sub Worker ($) {
	my $file = $_[0];

	# Obtain the parser name which should be used to parse any
	# messages of this file.
	my $use_parser = $monitored_files{$file};

	# Signal handler to kill worker.
	$SIG{'KILL'} = sub { threads->exit(); };

	# Create inotify watcher.
	my $watcher = new Linux::Inotify2 or die "Could not use inotify. $!";

	# Monitor the specified file.
	$watcher->watch("$file", IN_MODIFY) or die "Could not monitor $file. $!";

	# Switch watcher into non-blocking mode.
	$watcher->blocking(0);

	# Log successfully spawned worker.
	$logger->Log("debug", "Spawned worker thread for: $file");

	# Infinite loop.
	while(1) {
		# Check if the workers should pause or perform their desired work.
		if ($workers_pause) {
			# Wait 1 second until the next check.
			sleep(1);
		} else {
			# Check for any events and perform them, if there
			# is a least one.
			if ($watcher->read) {
				my @message = ();

				# Obtain fileposition from hash.
				my $fileposition = $file_positions{$file};

				# Open the file.
				open (FILE, $file) or die "Could not open $file. $!";

				# Seek to the last known position.
				seek (FILE, $fileposition, 0);

				# Get the log message.
				while (my $line = <FILE>) {
					# Remove any newlines.
					chomp $line;

					# Add all lines to the message array.
					push (@message, $line);
				}

				{
					# Lock shared hash.
					lock(%file_positions);

					# Update fileposition.
					$file_positions{$file} = tell(FILE);
				}

				# Close file.
				close(FILE);

				# Send filename and message to the parser,
				# which will return if any actions have to be performed.
				my @actions = $parser->Parser("$use_parser", @message);

				# Send the action to the main process and put it into
				# the queue.
				if (@actions) {
					# Lock the queue.
					lock($queue);

					# Loop through the actions array, and perform
					# every single action.
					foreach my $action (@actions) {
						# Prevent from enqueuing empty actions.
						if (defined($action)) {
							# Put the required action into the queue.
							$queue->enqueue($action);
						}
					}
				}
			} else {
				# Sleep for 100ms until the next round of the loop will start.
				sleep(0.1);
			}
		}
	}
}

#
## Socket function.
#
## This function uses the Socket module to create and listen to an UNIX socket.
## It automatically accepts all incoming connections and pass the received
## data messages to the the Message_Parser function which is also part of the
## socket module.
#
## If a valid command has been sent through the socket, the corresponding event
## will be enqueued into the shared event queue.
#
sub Socket () {
	# Create the Server socket by calling the responsible function.
	my $server = &Guardian::Socket::Server($mainsettings{SocketOwner});

	# Log successfull creation of socket.
	$logger->Log("debug", "Listening to Socket...");

        # Accept incoming connections from the socket.
        while (my $connection = $server->accept()) {
		# Autoflush the socket after the data
		# has been received.
		$connection->autoflush(1);

		# Gather all data from the connection.
		while (my $message = <$connection>) {
			# Remove any newlines.
			chomp($message);

			# Log received socket command.
			$logger->Log("debug", "Socket - Recieved message: $message");

			# Send the received data message to the
			# socket parser.
			my $action = &Guardian::Socket::Message_Parser($message);

			# If the parser returns to perform an action,
			# add it to the main event queue.
			if ($action) {
				# Lock the queue.
				lock($queue);

				# Enqueue the returned action.
				$queue->enqueue($action);
			}
		}
	}
}

#
## Function for capturing process signals.
#
## This function captures any sent process signals and will call various
## actions, based on the captured signal.
#
sub SignalHandler {
	$SIG{INT} = \&Shutdown;
	$SIG{TERM} = \&Shutdown;
	$SIG{QUIT} = \&Shutdown;
	$SIG{HUP} = \&Reload;
	$SIG{USR1} = \&ReloadIgnoreList;
}

#
## Function to start the workers (threads) for all monitored files.
#
## This function will loop through the hash of monitored files and will
## spawn an own thread based worker for each file. Every created worker will
## be added to the array of running workers.
#
sub StartWorkers () {
	# Init/Update hash which contains the cursor position of EOF.
	%file_positions = &Guardian::Base::FilePositions(\%monitored_files, \%file_positions);

	# Loop through the hash which contains the monitored files and start
	# a worker thread for each single one.
	foreach my $file (keys %monitored_files) {
		# Check if an worker is already running for this file.
		# If not, start the worker.
		unless (exists($running_workers{$file})) {
			$logger->Log("debug", "Starting worker thread for $file");

			# Create worker thread for the file.
			$running_workers{$file} = threads->create(\&Worker,$file);
		}
	}
}

#
## Function to stop all running workers.
#
## This function is used to stop all currently running workers and will be
## called when reloading or shutting down guardian.
#
sub StopWorkers () {
	# Loop through all running workers.
	foreach my $worker (keys %running_workers) {
		# Determine if the worker should be stopped.
		# This happens if the file should not be longer monitored.
		unless(exists($monitored_files{$worker})) {
			$logger->Log("debug", "Stopping worker thread for $worker");

			# Send a "KILL" signal to the worker.
			$running_workers{$worker}->kill('KILL');

			# Remove worker from hash of running workers.
			delete($running_workers{$worker});
		}
	}

	# Get amount of currently running worker threads.
	if (! keys(%running_workers)) {
		$logger->Log("debug", "All workers have been stopped...");
	}

	# Return nothing.
	return;
}

#
## Function to pause all running workers.
#
## This function is used to pause all currently running workers.
#
sub PauseWorkers() {
	# Set workers_pause variable to "1".
	# All workers will sleep until the variable has been set to "0".
	$workers_pause = 1;

	# Log paused workers.
	$logger->Log("debug", "All workers have been paused...");

	# Return nothing.
	return;
}

#
## Function to continue all running workers.
#
## This function is used to continue all paused workers.
#
sub ResumeWorkers() {
	# Set workers_suspend variable to "0" - they will continue their work
	# again.
	$workers_pause = 0;

	# Log continued workers.
	$logger->Log("debug", "All workers are working again...");

	# Return nothing.
	return;
}

#
## Reload function.
#
## This function will get called if the signal handler receives a "SIGHUP" signal,
## or the reload command will be sent via socket connection. It is responsible for
## reloading all configure options and stopping/starting the worker threads.
#
sub Reload () {
	# Log reload.
	$logger->Log("info", "Reload configuration...");

	# Pause all running workers.
	&PauseWorkers();

	# Re-read configuration file.
	%mainsettings = &Guardian::Config::UseConfig($cmdargs{"config"});

	# Update Logger settings.
	$logger = Guardian::Logger->Init(%mainsettings);

	# Update logger object in mainsettings hash.
	$mainsettings{Logger} = $logger;

	# Update Parser handler.
	$parser->Update(\%mainsettings);

	# Update Event handler.
	$events->Update(\%mainsettings);

	# Update ignore list.
	&ReloadIgnoreList();

	# Re-generate hash of monitored files.
	%monitored_files = &Guardian::Base::GenerateMonitoredFiles(\%mainsettings, \%monitored_files);

	# Stop workers if they are not needed anymore.
	&StopWorkers();

	# Start new worker threads if required.
	&StartWorkers();

	# Resume workers.
	&ResumeWorkers();

	# Return nothing.
	return;
}

#
## ReloadIgnoreList function.
#
## This function will be called if the signal handler receives a "SIGUSR1" signal,
## or the reload-ignore-list command will be sent via the socket connection. It just
## performs a simple check if an ignore file has been configured and calls the responsible
## function on the events module.
#
sub ReloadIgnoreList () {
	# Update ignore list, if an ignorefile has been specified.
	if (exists($mainsettings{IgnoreFile})) {
		# Log reload of the ignore list.
		$logger->Log("info", "Reloading ignore list...");

		# Call responsible function from the events module.
		&Guardian::Events::GenerateIgnoreList($mainsettings{IgnoreFile});
	}
}

#
## Logrotate function.
#
## This function only get called when the logrotate command will be sent via
## the socket connection. It is responsible for validating and altering the current
## cursor positions of the monitored files and stopping/starting the worker threads.
#
sub Logrotate () {
	# Stop all running workers.
	&PauseWorkers();

	{
	# Lock shared hash.
	lock(%file_positions);

	# Loop through the hash which contains the current
	# file positions.
	foreach my $file (keys(%file_positions)) {
		# Obtain stored value from hash.
		my $stored_position = $file_positions{$file};

		# Call function to get the current position.
		my $current_position = &Guardian::Base::GetFileposition($file);

		# Compare if the current position still matches
		# the stored one.
		if ($current_position ne $stored_position) {
			# Update to the new position, because
			# they has changed during file rotation.
			$file_positions{$file} = $current_position;

			# Log notice about rotated file.
			$logger->Log("debug", "$file have been rotated - Using new file position.");
		}
	}

	# After this bracket, the lock of the hash will be released.
	}

	# Restart all worker threads.
	&ResumeWorkers();

	# Return nothing.
	return;
}

#
## Shutdown function.
#
## This function is used to do a clean shutdown of guardian. It will be called
## by the signal handler when receiving INT (2), QUIT (3) and TERM (15) signals.
#
sub Shutdown () {
	# Log shutdown.
	$logger->Log("info", "Shutting down...");

	# Reset hash of monitored files.
	%monitored_files = ();

	# Stop all workers.
	&StopWorkers();

	# Unblock all blocked hosts.
	&Guardian::Events::CallFlush();

	# Remove socket file on exit.
	&Guardian::Socket::RemoveSocketFile();

	# Remove pid file on exit.
	&Guardian::Daemon::RemovePIDFile();

	# Sleep for one second to give perl some
	# time to proper clean up everything before
	# exiting.
	sleep(1);

	# Log good bye message.
	$logger->Log("debug", "Good Bye!");

	# Exit guardian.
	exit;
}