[ipfire-2.x.git] / src / squid-accounting / acct.pl

#!/usr/bin/perl
###############################################################################
#                                                                             #
# IPFire.org - A linux based firewall                                         #
# Copyright (C) 2014  IPFire Team  <alexander.marx@ipfire.org>                #
#                                                                             #
# This program is free software: you can redistribute it and/or modify        #
# it under the terms of the GNU General Public License as published by        #
# the Free Software Foundation, either version 3 of the License, or           #
# (at your option) any later version.                                         #
#                                                                             #
# This program is distributed in the hope that it will be useful,             #
# but WITHOUT ANY WARRANTY; without even the implied warranty of              #
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the               #
# GNU General Public License for more details.                                #
#                                                                             #
# You should have received a copy of the GNU General Public License           #
# along with this program.  If not, see <http://www.gnu.org/licenses/>.       #
#                                                                             #
###############################################################################


###########
# Modules #
###########

use Time::Local;
use File::ReadBackwards;
use strict;
#use warnings;

require '/var/ipfire/general-functions.pl';
require "${General::swroot}/accounting/acct-lib.pl";
require "${General::swroot}/lang.pl";

#############
# Variables #
#############

my $count = 0;
my $dbh;
my $logfile = "/var/log/squid/access.log";
my $line = '';
my $checktime = 3600;  #1 hour = 3600 sec
my $starttime = time;
my ($time,$elapsed,$ip,$state,$bytes,$method,$url,$user,$peerstate,$type); #split logfileline into variables
my $name;
my $name1;
my $settingsfile = "${General::swroot}/accounting/settings.conf";
my $proxyenabled = "${General::swroot}/proxy/enable";
my %counter = ();
my %counterip = ();
my %settings = ();
my %toplist = ();
my ($sec,$min,$hour,$mday,$mon,$year,$wday,$ydat,$isdst)=localtime();
my $skipurlcount=0;
my $skipurlsum=0;
&General::readhash("$settingsfile", \%settings);
my $skipurl=$settings{'SKIPURLS'};
$skipurl="'".$skipurl."'";
my ($mini,$max)=&ACCT::getminmax;
my $now = localtime;
my $proxylog;
my $proxysrv;

########
# Main #
########

&checkproxy;

#If we have a disabled file and the proxy is off, we don't need to check anything, exit!
if((! -f $proxyenabled || $proxylog eq $Lang::tr{'stopped'}) && -f "${General::swroot}/accounting/disabled"){
	&ACCT::logger($settings{'LOG'}," Proxy or proxylogging disabled - exiting with no data collection\n");
	exit 0;
}
#If proxy was turned off within last hour, we need to check missing minutes and write a disabled file
if ((! -f $proxyenabled || $proxylog eq $Lang::tr{'stopped'}) && ! -f "${General::swroot}/accounting/disabled"){
	$checktime = (time-$max);
	open (FH,">${General::swroot}/accounting/disabled");
	close (FH);
	&ACCT::logger($settings{'LOG'}," Proxy or proxylogging was disabled during last hour - just checking meantime and disabling data collection\n");
}

#If proxy is on, we are doing a normal run. maybe we had a disabled file, so delete it here
if (-f $proxyenabled && $proxylog eq $Lang::tr{'running'}){
	#check if we are running again after the was shutdown and reenabled
	if (-f "${General::swroot}/accounting/disabled"){
		unlink("${General::swroot}/accounting/disabled");
	}
	#Find out if the month changed
	$dbh=&ACCT::connectdb;
	my $m=sprintf("%d",(localtime((time-3600)))[4]+1);
	&ACCT::logger($settings{'LOG'},"month before one hour $m, now is ".($mon+1)."\n");
	if ($m < ($mon+1) || $m == '12' && ($mon+1) == '1'){
		#Logrotate
		my $year1=$year+1900;
		system ("tar", "cfz", "/var/log/accounting-$m-$year1.tar.gz", "/var/log/accounting.log");
		unlink ("/var/log/accounting.log");
		open (FH,">/var/log/accounting.log");
		close (FH);
		chmod 0755, "/var/log/accounting.log";
		#move all db entries older than this month to second table and cumulate them daily
		&ACCT::movedbdata;
		&ACCT::logger($settings{'LOG'},"New Month. Old trafficvalues moved to ACCT_HIST Table\n");
		if ($settings{'USEMAIL'} eq 'on'){
			&ACCT::logger($settings{'LOG'},"Mailserver is activated - Now sending bills via mail...\n");
			my $res=&ACCT::getbillgroups;
			foreach my $line (@$res){
				my ($grp) = @$line;
				&sendbill($grp,$settings{'MWST'},$settings{'CURRENCY'});
			}
		}else{
			&ACCT::logger($settings{'LOG'},"Mailserver is deactivated - We are NOT sending bills via mail...\n");
		}
	}

	&ACCT::logger($settings{'LOG'},"Start reading last hour of access.log\n");
	&readlog;
	&fill_db;
	&ACCT::closedb;
	$skipurlsum=sprintf("%.2f",$skipurlsum/(1024*1024));
	&ACCT::logger($settings{'LOG'},"skipped: $skipurlcount Adressen\n");
	&ACCT::logger($settings{'LOG'},"skipped: $skipurlsum MB\n") if ($skipurl);
}
#############
# functions #
#############

sub checkproxy{
	if(-f "${General::swroot}/proxy/enable"){
		$proxysrv=$Lang::tr{'running'};
	}else{
		$proxysrv=$Lang::tr{'stopped'};
	}
	my $srce = "${General::swroot}/proxy/squid.conf";
	my $string1 = 'access\.log';
	open(FH, $srce);
	while(my $line = <FH>) {
		if($line =~ m/$string1/) {
			$proxylog=$Lang::tr{'running'};
		}
	}
	close FH;
	return;
}

sub readlog{
	my $url1;
	my $user1;
	$count = 0;
	my $urlcnt=0;
	&ACCT::logger($settings{'LOG'},"Start: $now. Reading data back till: ".localtime(($starttime-$checktime)).".\n");
	#Open Logfile and begin to read the file backwards
	my $bw = File::ReadBackwards->new( $logfile ) or die "can't read $logfile $!" ;
	while( defined( $line = $bw->readline ) ) {
		undef $url1;
		chomp $line;
		#Divide $line into single variables to get timestamp and check if we are within hte desired timerange
		($time,$elapsed,$ip,$state,$bytes,$method,$url,$user,$peerstate,$type)=split(m/\s+/, $line);
		$count += $bytes;
		$time = substr($time, 0, -4);
		if (($time > ($starttime-$checktime))){
			#Skip DENIED stated lines (can be reactivated later)
			next if ($state =~ m/DENIED/);

			#extract site name
			if ($url =~ m/([a-z]+:\/\/)??([a-z0-9\-]+\.){1}(([a-z0-9\-]+\.){0,})([a-z0-9\-]+){1}(:[0-9]+)?\/(.*)/o) {
			   $url=$2.$3.$5;
			} else {
			   my ($a,$b)=split(":",$url);
			   $url=$a;
			}

			#Skip special URLs like intranet and webservers from local network
			if ($url =~ m/$skipurl/o) {
			  $skipurlcount++;
			  $skipurlsum+=$bytes;
			  next;
			};

			#Increase urlcounter
			$urlcnt++;

			#Get Data for accounting
			$counter{$user}{'bytes'} += $bytes if ($user ne '-');
			$counter{$ip}{'bytes'} += $bytes;
		}else{
			#If we are out of timewindow, break
			last;
		}
	}
	$count=sprintf("%.2f",$count/(1024*1024));
	&ACCT::logger($settings{'LOG'},"got $count MB from $urlcnt URLs this run.\n");
	$bw->close;
}
sub fill_db{
	my $tim=time();
	#Fill ACCT table with accounting information
	foreach my $name (sort keys %counter){
		foreach my $bytes (keys %{ $counter{$name} }) {
			$dbh->do("insert into ACCT (TIME_RUN,NAME,BYTES) values ('$tim','$name','$counter{$name}{$bytes}');");
		}
	}
}
sub sendbill {
	my $rggrp=$_[0];
	my $mwst=$_[1];
	my $cur = $_[2];
	my @now = localtime(time);
	$now[5] = $now[5] + 1900;
	my $actmonth = $now[4];
	my $month;
	$month = '0'.$actmonth if $actmonth < 10;
	$month = '12' if $actmonth == 0;
	my $actyear  = $now[5];
	my ($from,$till)=&ACCT::getmonth($actmonth,$actyear);					#FIXME month and year as variables!
	my @billar = &ACCT::GetTaValues($from,$till,$rggrp);
	my $address_cust = &ACCT::getTaAddress($rggrp,'CUST');
	my $address_host = &ACCT::getTaAddress($rggrp,'HOST');
	my $billpos		= &ACCT::getextrabillpos($rggrp);
	my $no			= &ACCT::getBillNr;
	my $back = &ACCT::pdf2(\@billar,$actmonth,$actyear,$mwst,$address_cust,$address_host,$billpos,$rggrp,$cur);
	my ($company_cust,$type_cust,$name1_cust,$str_cust,$str_nr_cust,$plz_cust,$city_cust,$bank,$iban,$bic,$blz,$kto,$email,$internet,$hrb,$stnr,$tel_host,$fax_host,$ccmail,$billgrp,$text,$host,$cust,$cent);

	foreach my $addrline_cust (@$address_cust){
		($company_cust,$type_cust,$name1_cust,$str_cust,$str_nr_cust,$plz_cust,$city_cust,$bank,$iban,$bic,$blz,$kto,$email,$internet,$hrb,$stnr,$tel_host,$fax_host,$ccmail,$billgrp,$text,$host,$cust,$cent)=@$addrline_cust;
	}

	if ($back eq '0'){
		&ACCT::logger($settings{'LOG'},"Bill for $company_cust successfully created.\n");
		my $file="'/var/ipfire/accounting/bill/$rggrp/$month-$actyear-$no.pdf'";
		$settings{'MAILTXT'} =~ tr/\|/\r\n/ ;
		my $cmd = "/usr/local/bin/sendEmail ";
		$cmd .= " -f $settings{'MAILSENDER'}";							#Sender
		$cmd .= " -t $email";											#Recipient
		if ($ccmail){
			$cmd .= " -cc $ccmail";
		}
		#Send Mail via TLS?
		if ($settings{'TLS'} eq 'on'){
			$cmd .= " -o tls=yes";										#TLS
		}
		$cmd .= " -u '$settings{'MAILSUB'}'";							#Subject
		$cmd .= " -m '$settings{'MAILTXT'}'";							#Mailtext
		$cmd .= " -s $settings{'MAILSRV'}:$settings{'MAILPORT'}";		#Mailserver:port
		$cmd .= " -a $file";
		my $res=system ($cmd);
		if ($res == 0){
			&ACCT::logger($settings{'LOG'},"Bill for $company_cust successfully sent.\n");
		}elsif ($res > 0){
			&ACCT::logger($settings{'LOG'},"ERROR: Bill for $company_cust NOT sent.\n");
		}
		return 0;
		
	}else{
		&ACCT::logger($settings{'LOG'},"ERROR Bill for $company_cust could not be created.\n");
		my $cmd = "/usr/local/bin/sendEmail ";
		$cmd .= " -f $settings{'MAILSENDER'}";
		$cmd .= " -t $settings{'MAILSENDER'}";
		$cmd .= " -u Fehler Squid Accounting";
		$cmd .= " -m 'Die Rechnung konnte nicht erzeugt und per Mail versendet werden' $company_cust";
		$cmd .= " -s $settings{'MAILSRV'}:$settings{'MAILPORT'}";
		my $res=system ($cmd);
		return 0;
	}
}
Commit	Line	Data
db8a01e0 AM	1	#!/usr/bin/perl
	2	###############################################################################
	3	# #
	4	# IPFire.org - A linux based firewall #
	5	# Copyright (C) 2014 IPFire Team <alexander.marx@ipfire.org> #
	6	# #
	7	# This program is free software: you can redistribute it and/or modify #
	8	# it under the terms of the GNU General Public License as published by #
	9	# the Free Software Foundation, either version 3 of the License, or #
	10	# (at your option) any later version. #
	11	# #
	12	# This program is distributed in the hope that it will be useful, #
	13	# but WITHOUT ANY WARRANTY; without even the implied warranty of #
	14	# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the #
	15	# GNU General Public License for more details. #
	16	# #
	17	# You should have received a copy of the GNU General Public License #
	18	# along with this program. If not, see <http://www.gnu.org/licenses/>. #
	19	# #
	20	###############################################################################
	21
	22
	23	###########
	24	# Modules #
	25	###########
	26
	27	use Time::Local;
	28	use File::ReadBackwards;
	29	use strict;
	30	#use warnings;
	31
	32	require '/var/ipfire/general-functions.pl';
	33	require "${General::swroot}/accounting/acct-lib.pl";
	34	require "${General::swroot}/lang.pl";
	35
	36	#############
	37	# Variables #
	38	#############
	39
	40	my $count = 0;
	41	my $dbh;
	42	my $logfile = "/var/log/squid/access.log";
	43	my $line = '';
	44	my $checktime = 3600; #1 hour = 3600 sec
	45	my $starttime = time;
	46	my ($time,$elapsed,$ip,$state,$bytes,$method,$url,$user,$peerstate,$type); #split logfileline into variables
	47	my $name;
	48	my $name1;
	49	my $settingsfile = "${General::swroot}/accounting/settings.conf";
	50	my $proxyenabled = "${General::swroot}/proxy/enable";
	51	my %counter = ();
	52	my %counterip = ();
	53	my %settings = ();
	54	my %toplist = ();
	55	my ($sec,$min,$hour,$mday,$mon,$year,$wday,$ydat,$isdst)=localtime();
	56	my $skipurlcount=0;
	57	my $skipurlsum=0;
	58	&General::readhash("$settingsfile", \%settings);
	59	my $skipurl=$settings{'SKIPURLS'};
	60	$skipurl="'".$skipurl."'";
	61	my ($mini,$max)=&ACCT::getminmax;
	62	my $now = localtime;
	63	my $proxylog;
	64	my $proxysrv;
65
66	########
67	# Main #
68	########
69
70	&checkproxy;
71
72	#If we have a disabled file and the proxy is off, we don't need to check anything, exit!
73	if((! -f $proxyenabled \|\| $proxylog eq $Lang::tr{'stopped'}) && -f "${General::swroot}/accounting/disabled"){
74	&ACCT::logger($settings{'LOG'}," Proxy or proxylogging disabled - exiting with no data collection\n");
75	exit 0;
76	}
77	#If proxy was turned off within last hour, we need to check missing minutes and write a disabled file
78	if ((! -f $proxyenabled \|\| $proxylog eq $Lang::tr{'stopped'}) && ! -f "${General::swroot}/accounting/disabled"){
79	$checktime = (time-$max);
80	open (FH,">${General::swroot}/accounting/disabled");
81	close (FH);
82	&ACCT::logger($settings{'LOG'}," Proxy or proxylogging was disabled during last hour - just checking meantime and disabling data collection\n");
83	}
84
85	#If proxy is on, we are doing a normal run. maybe we had a disabled file, so delete it here
86	if (-f $proxyenabled && $proxylog eq $Lang::tr{'running'}){
87	#check if we are running again after the was shutdown and reenabled
88	if (-f "${General::swroot}/accounting/disabled"){
89	unlink("${General::swroot}/accounting/disabled");
90	}
91	#Find out if the month changed
92	$dbh=&ACCT::connectdb;
93	my $m=sprintf("%d",(localtime((time-3600)))[4]+1);
94	&ACCT::logger($settings{'LOG'},"month before one hour $m, now is ".($mon+1)."\n");
95	if ($m < ($mon+1) \|\| $m == '12' && ($mon+1) == '1'){
96	#Logrotate
97	my $year1=$year+1900;
98	system ("tar", "cfz", "/var/log/accounting-$m-$year1.tar.gz", "/var/log/accounting.log");
99	unlink ("/var/log/accounting.log");
100	open (FH,">/var/log/accounting.log");
101	close (FH);
102	chmod 0755, "/var/log/accounting.log";
0fa8d28e	103	#move all db entries older than this month to second table and cumulate them daily
db8a01e0 AM	104	&ACCT::movedbdata;
	105	&ACCT::logger($settings{'LOG'},"New Month. Old trafficvalues moved to ACCT_HIST Table\n");
	106	if ($settings{'USEMAIL'} eq 'on'){
	107	&ACCT::logger($settings{'LOG'},"Mailserver is activated - Now sending bills via mail...\n");
	108	my $res=&ACCT::getbillgroups;
	109	foreach my $line (@$res){
	110	my ($grp) = @$line;
	111	&sendbill($grp,$settings{'MWST'},$settings{'CURRENCY'});
	112	}
	113	}else{
	114	&ACCT::logger($settings{'LOG'},"Mailserver is deactivated - We are NOT sending bills via mail...\n");
	115	}
	116	}
	117
	118	&ACCT::logger($settings{'LOG'},"Start reading last hour of access.log\n");
	119	&readlog;
	120	&fill_db;
	121	&ACCT::closedb;
	122	$skipurlsum=sprintf("%.2f",$skipurlsum/(1024*1024));
	123	&ACCT::logger($settings{'LOG'},"skipped: $skipurlcount Adressen\n");
	124	&ACCT::logger($settings{'LOG'},"skipped: $skipurlsum MB\n") if ($skipurl);
	125	}
	126	#############
	127	# functions #
	128	#############
	129
	130	sub checkproxy{
	131	if(-f "${General::swroot}/proxy/enable"){
	132	$proxysrv=$Lang::tr{'running'};
	133	}else{
	134	$proxysrv=$Lang::tr{'stopped'};
	135	}
	136	my $srce = "${General::swroot}/proxy/squid.conf";
	137	my $string1 = 'access\.log';
	138	open(FH, $srce);
	139	while(my $line = <FH>) {
	140	if($line =~ m/$string1/) {
	141	$proxylog=$Lang::tr{'running'};
	142	}
	143	}
	144	close FH;
	145	return;
	146	}
	147
	148	sub readlog{
	149	my $url1;
	150	my $user1;
	151	$count = 0;
	152	my $urlcnt=0;
	153	&ACCT::logger($settings{'LOG'},"Start: $now. Reading data back till: ".localtime(($starttime-$checktime)).".\n");
	154	#Open Logfile and begin to read the file backwards
	155	my $bw = File::ReadBackwards->new( $logfile ) or die "can't read $logfile $!" ;
	156	while( defined( $line = $bw->readline ) ) {
	157	undef $url1;
	158	chomp $line;
	159	#Divide $line into single variables to get timestamp and check if we are within hte desired timerange
	160	($time,$elapsed,$ip,$state,$bytes,$method,$url,$user,$peerstate,$type)=split(m/\s+/, $line);
	161	$count += $bytes;
	162	$time = substr($time, 0, -4);
	163	if (($time > ($starttime-$checktime))){
	164	#Skip DENIED stated lines (can be reactivated later)
	165	next if ($state =~ m/DENIED/);
	166
	167	#extract site name
168	if ($url =~ m/([a-z]+:\/\/)??([a-z0-9\-]+\.){1}(([a-z0-9\-]+\.){0,})([a-z0-9\-]+){1}(:[0-9]+)?\/(.*)/o) {
169	$url=$2.$3.$5;
170	} else {
171	my ($a,$b)=split(":",$url);
172	$url=$a;
173	}
174
175	#Skip special URLs like intranet and webservers from local network
176	if ($url =~ m/$skipurl/o) {
177	$skipurlcount++;
178	$skipurlsum+=$bytes;
179	next;
180	};
181
182	#Increase urlcounter
183	$urlcnt++;
184
185	#Get Data for accounting
186	$counter{$user}{'bytes'} += $bytes if ($user ne '-');
187	$counter{$ip}{'bytes'} += $bytes;
188	}else{
189	#If we are out of timewindow, break
190	last;
191	}
192	}
193	$count=sprintf("%.2f",$count/(1024*1024));
194	&ACCT::logger($settings{'LOG'},"got $count MB from $urlcnt URLs this run.\n");
195	$bw->close;
196	}
197	sub fill_db{
198	my $tim=time();
199	#Fill ACCT table with accounting information
200	foreach my $name (sort keys %counter){
201	foreach my $bytes (keys %{ $counter{$name} }) {
202	$dbh->do("insert into ACCT (TIME_RUN,NAME,BYTES) values ('$tim','$name','$counter{$name}{$bytes}');");
203	}
204	}
205	}
206	sub sendbill {
207	my $rggrp=$_[0];
208	my $mwst=$_[1];
209	my $cur = $_[2];
210	my @now = localtime(time);
211	$now[5] = $now[5] + 1900;
d6413caa	212	my $actmonth = $now[4];
218d2d75 AM	213	my $month;
	214	$month = '0'.$actmonth if $actmonth < 10;
	215	$month = '12' if $actmonth == 0;
db8a01e0 AM	216	my $actyear = $now[5];
	217	my ($from,$till)=&ACCT::getmonth($actmonth,$actyear); #FIXME month and year as variables!
	218	my @billar = &ACCT::GetTaValues($from,$till,$rggrp);
	219	my $address_cust = &ACCT::getTaAddress($rggrp,'CUST');
	220	my $address_host = &ACCT::getTaAddress($rggrp,'HOST');
	221	my $billpos = &ACCT::getextrabillpos($rggrp);
	222	my $no = &ACCT::getBillNr;
	223	my $back = &ACCT::pdf2(\@billar,$actmonth,$actyear,$mwst,$address_cust,$address_host,$billpos,$rggrp,$cur);
	224	my ($company_cust,$type_cust,$name1_cust,$str_cust,$str_nr_cust,$plz_cust,$city_cust,$bank,$iban,$bic,$blz,$kto,$email,$internet,$hrb,$stnr,$tel_host,$fax_host,$ccmail,$billgrp,$text,$host,$cust,$cent);
	225
	226	foreach my $addrline_cust (@$address_cust){
	227	($company_cust,$type_cust,$name1_cust,$str_cust,$str_nr_cust,$plz_cust,$city_cust,$bank,$iban,$bic,$blz,$kto,$email,$internet,$hrb,$stnr,$tel_host,$fax_host,$ccmail,$billgrp,$text,$host,$cust,$cent)=@$addrline_cust;
	228	}
	229
	230	if ($back eq '0'){
	231	&ACCT::logger($settings{'LOG'},"Bill for $company_cust successfully created.\n");
	232	my $file="'/var/ipfire/accounting/bill/$rggrp/$month-$actyear-$no.pdf'";
	233	$settings{'MAILTXT'} =~ tr/\\|/\r\n/ ;
	234	my $cmd = "/usr/local/bin/sendEmail ";
	235	$cmd .= " -f $settings{'MAILSENDER'}"; #Sender
	236	$cmd .= " -t $email"; #Recipient
	237	if ($ccmail){
	238	$cmd .= " -cc $ccmail";
	239	}
	240	#Send Mail via TLS?
	241	if ($settings{'TLS'} eq 'on'){
	242	$cmd .= " -o tls=yes"; #TLS
	243	}
	244	$cmd .= " -u '$settings{'MAILSUB'}'"; #Subject
	245	$cmd .= " -m '$settings{'MAILTXT'}'"; #Mailtext
	246	$cmd .= " -s $settings{'MAILSRV'}:$settings{'MAILPORT'}"; #Mailserver:port
	247	$cmd .= " -a $file";
	248	my $res=system ($cmd);
	249	if ($res == 0){
	250	&ACCT::logger($settings{'LOG'},"Bill for $company_cust successfully sent.\n");
	251	}elsif ($res > 0){
	252	&ACCT::logger($settings{'LOG'},"ERROR: Bill for $company_cust NOT sent.\n");
	253	}
	254	return 0;
	255
	256	}else{
	257	&ACCT::logger($settings{'LOG'},"ERROR Bill for $company_cust could not be created.\n");
	258	my $cmd = "/usr/local/bin/sendEmail ";
	259	$cmd .= " -f $settings{'MAILSENDER'}";
	260	$cmd .= " -t $settings{'MAILSENDER'}";
	261	$cmd .= " -u Fehler Squid Accounting";
	262	$cmd .= " -m 'Die Rechnung konnte nicht erzeugt und per Mail versendet werden' $company_cust";
	263	$cmd .= " -s $settings{'MAILSRV'}:$settings{'MAILPORT'}";
	264	my $res=system ($cmd);
	265	return 0;
	266	}
	267	}