From: eldy <> Date: Mon, 14 Oct 2002 17:45:56 +0000 (+0000) Subject: Remove common2combined.pl tool. X-Git-Tag: AWSTATS_5_1_BETA~27 X-Git-Url: http://git.ipfire.org/cgi-bin/gitweb.cgi?a=commitdiff_plain;h=62a9b1ff32bfdd4ea719dbdda6c6e24fdfd0f0ab;p=thirdparty%2FAWStats.git Remove common2combined.pl tool. --- diff --git a/tools/common2combined.pl b/tools/common2combined.pl deleted file mode 100644 index a008d0cd..00000000 --- a/tools/common2combined.pl +++ /dev/null @@ -1,263 +0,0 @@ -#!/usr/bin/perl -# With some other Unix Os, first line may be -#!/usr/local/bin/perl -# With Apache for Windows and ActiverPerl, first line may be -#!C:/Program Files/ActiveState/bin/perl -#-Description------------------------------------------- -# Convert a common log file into a combined. -# This tool is part of AWStats log analyzer but can be use -# alone for any other log analyzer. -# See COPYING.TXT file about AWStats GNU General Public License. -#------------------------------------------------------- -use strict; no strict "refs"; -#use diagnostics; - - -#------------------------------------------------------- -# Defines -#------------------------------------------------------- -# Last change $Revision$ - $Author$ - $Date$ -my $REVISION='$Revision$'; $REVISION =~ /\s(.*)\s/; $REVISION=$1; -my $VERSION="1.1 (build $REVISION)"; - -# ---------- Init variables -------- -my $Debug=0; -my $ShowSteps=0; -my $DIR; -my $PROG; -my $Extension; -my $DNSLookup=0; -my $DirCgi=""; -my $DirData=""; -my $NbOfLinesForBenchmark=5000; -my $NewReferer="-"; # $NewReferer="http://www.referersite.com/refererpage.html"; -my $NewUserAgent="Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.0)"; -my $nowtime = my $nowweekofmonth = my $nowdaymod = my $nowsmallyear = 0; -my $nowsec = my $nowmin = my $nowhour = my $nowday = my $nowmonth = my $nowyear = my $nowwday = 0; -# ---------- Init hash arrays -------- -my %ParamFile=(); -my %linerecord=(); -my %timeconnexion=(); -my %corrupted=(); - - - -#------------------------------------------------------- -# Functions -#------------------------------------------------------- -sub error { - print "Error: $_[0].\n"; - exit 1; -} - -sub debug { - my $level = $_[1] || 1; - if ($Debug >= $level) { - my $debugstring = $_[0]; - if ($ENV{"GATEWAY_INTERFACE"}) { $debugstring =~ s/^ /   /; $debugstring .= "
"; } - print "DEBUG $level - ".time." : $debugstring\n"; - } - 0; -} - - - -#------------------------------------------------------- -# MAIN -#------------------------------------------------------- -my $QueryString=""; for (0..@ARGV-1) { $QueryString .= "$ARGV[$_] "; } -if ($QueryString =~ /debug=/i) { $Debug=$QueryString; $Debug =~ s/.*debug=//; $Debug =~ s/&.*//; $Debug =~ s/ .*//; } -if ($QueryString =~ /dnslookup/i) { $DNSLookup=1; } -if ($QueryString =~ /showsteps/i) { $ShowSteps=1; } -($DIR=$0) =~ s/([^\/\\]*)$//; ($PROG=$1) =~ s/\.([^\.]*)$//; $Extension=$1; - -my $cpt=1; -for (0..@ARGV-1) { - if ($ARGV[$_] =~ /^-/) { last; } - $ParamFile{$cpt}=$ARGV[$_]; - $cpt++; -} -if (scalar keys %ParamFile == 0) { - print "----- $PROG $VERSION (c) Laurent Destailleur -----\n"; - print "$PROG converts any Apache 'common' log file into a 'combined' file.\n"; - print "$PROG comes with ABSOLUTELY NO WARRANTY. It's a free software\n"; - print "distributed with a GNU General Public License (See COPYING.txt file).\n"; - print "$PROG is part of AWStats but can be used alone for any need.\n"; - print "\n"; - print "Value used for the 2 new fields added in conversion are :\n"; - print "New referer : \"$NewReferer\"\n"; - print "New user Agent : \"$NewUserAgent\"\n"; - print "\n"; - print "Usage:\n"; - print " $PROG.$Extension oldfile.log > newfile.log\n"; - print "\n"; - print "Options:\n"; - print " -showsteps to add benchmark informations every $NbOfLinesForBenchmark lines processed\n"; - print "\n"; - print "New versions and FAQ at http://awstats.sourceforge.net\n"; - exit 0; -} - -# Get current time -my $nowtime=time; -my ($nowsec,$nowmin,$nowhour,$nowday,$nowmonth,$nowyear) = localtime($nowtime); -if ($nowyear < 100) { $nowyear+=2000; } else { $nowyear+=1900; } -my $nowsmallyear=$nowyear;$nowsmallyear =~ s/^..//; -if (++$nowmonth < 10) { $nowmonth = "0$nowmonth"; } -if ($nowday < 10) { $nowday = "0$nowday"; } -if ($nowhour < 10) { $nowhour = "0$nowhour"; } -if ($nowmin < 10) { $nowmin = "0$nowmin"; } -if ($nowsec < 10) { $nowsec = "0$nowsec"; } -# Get tomorrow time (will be used to discard some record with corrupted date (future date)) -my ($tomorrowsec,$tomorrowmin,$tomorrowhour,$tomorrowday,$tomorrowmonth,$tomorrowyear) = localtime($nowtime+86400); -if ($tomorrowyear < 100) { $tomorrowyear+=2000; } else { $tomorrowyear+=1900; } -my $tomorrowsmallyear=$tomorrowyear;$tomorrowsmallyear =~ s/^..//; -if (++$tomorrowmonth < 10) { $tomorrowmonth = "0$tomorrowmonth"; } -if ($tomorrowday < 10) { $tomorrowday = "0$tomorrowday"; } -if ($tomorrowhour < 10) { $tomorrowhour = "0$tomorrowhour"; } -if ($tomorrowmin < 10) { $tomorrowmin = "0$tomorrowmin"; } -if ($tomorrowsec < 10) { $tomorrowsec = "0$tomorrowsec"; } -my $timetomorrow=$tomorrowyear.$tomorrowmonth.$tomorrowday.$tomorrowhour.$tomorrowmin.$tomorrowsec; - -# Init other parameters -if ($ENV{"GATEWAY_INTERFACE"}) { $DirCgi=""; } -if ($DirCgi && !($DirCgi =~ /\/$/) && !($DirCgi =~ /\\$/)) { $DirCgi .= "/"; } -if (! $DirData || $DirData eq ".") { $DirData=$DIR; } # If not defined or choosed to "." value then DirData is current dir -if (! $DirData) { $DirData="."; } # If current dir not defined then we put it to "." -$DirData =~ s/\/$//; -if ($DNSLookup) { use Socket; } -#my %monthlib = ( "01","$message[60]","02","$message[61]","03","$message[62]","04","$message[63]","05","$message[64]","06","$message[65]","07","$message[66]","08","$message[67]","09","$message[68]","10","$message[69]","11","$message[70]","12","$message[71]" ); -# monthnum must be in english because it's used to translate log date in apache log files which are always in english -my %monthnum = ( "Jan","01","jan","01","Feb","02","feb","02","Mar","03","mar","03","Apr","04","apr","04","May","05","may","05","Jun","06","jun","06","Jul","07","jul","07","Aug","08","aug","08","Sep","09","sep","09","Oct","10","oct","10","Nov","11","nov","11","Dec","12","dec","12" ); - -#------------------------------------------ -# PROCESSING CURRENT LOG(s) -#------------------------------------------ -my %LogFileToDo=(); my %NowNewLinePhase=(); my %NbOfLinesRead=(); my %NbOfLinesCorrupted=(); -my $NbOfNewLinesProcessed=0; -my $NbOfNewLinesCorrupted=0; -my $logfilechosen=0; -my $starttime=time(); - -# Define the LogFileToDo list -$cpt=1; -foreach my $key (keys %ParamFile) { - if ($ParamFile{$key} !~ /\*/ && $ParamFile{$key} !~ /\?/) { - &debug("Log file $ParamFile{$key} is added to LogFileToDo."); - $LogFileToDo{$cpt}=$ParamFile{$key}; - $cpt++; - } - else { - my $DirFile=$ParamFile{$key}; $DirFile =~ s/([^\/\\]*)$//; - $ParamFile{$key} = $1; - if ($DirFile eq "") { $DirFile = "."; } - $ParamFile{$key} =~ s/\./\\\./g; - $ParamFile{$key} =~ s/\*/\.\*/g; - $ParamFile{$key} =~ s/\?/\./g; - &debug("Search for file \"$ParamFile{$key}\" into \"$DirFile\""); - opendir(DIR,"$DirFile"); - my @filearray = sort readdir DIR; - close DIR; - foreach my $i (0..$#filearray) { - if ("$filearray[$i]" =~ /^$ParamFile{$key}$/ && "$filearray[$i]" ne "." && "$filearray[$i]" ne "..") { - &debug("Log file $filearray[$i] is added to LogFileToDo."); - $LogFileToDo{$cpt}="$DirFile/$filearray[$i]"; - $cpt++; - } - } - } -} - -# If no files to process -if (scalar keys %LogFileToDo == 0) { - error("No input log file found"); -} - -# Open all log files -&debug("Start of processing ".(scalar keys %LogFileToDo)." log file(s)"); -foreach my $logfilenb (keys %LogFileToDo) { - &debug("Open log file number $logfilenb: \"$LogFileToDo{$logfilenb}\""); - open("LOG$logfilenb","$LogFileToDo{$logfilenb}") || error("Couldn't open log file \"$LogFileToDo{$logfilenb}\" : $!"); -} - -while (1 == 1) -{ - # BEGIN Read new record (for each log file or only for log file with record just processed) - #------------------------------------------------------------------------------------------ - foreach my $logfilenb (keys %LogFileToDo) { - if (($logfilechosen == 0) || ($logfilechosen == $logfilenb)) { - &debug("Search next record in file number $logfilenb",3); - # Read chosen log file until we found a record with good date or reaching end of file - while (1 == 1) { - my $LOG="LOG$logfilenb"; $_=<$LOG>; # Read new line - if (! $_) { # No more records in log file number $logfilenb - &debug(" No more records in file number $logfilenb",2); - delete $LogFileToDo{$logfilenb}; - last; } # We have all the new records for each other files, we stop here - - chomp $_; s/\r//; - - if (/^#/) { next; } # Ignore comment lines (ISS writes such comments) - if (/^!!/) { next; } # Ignore comment lines (Webstar writes such comments) - if (/^$/) { next; } # Ignore blank lines (With ISS: happens sometimes, with Apache: possible when editing log file) - - $NbOfLinesRead{$logfilenb}++; - - # Check filters - #---------------------------------------------------------------------- - # Split DD/Month/YYYY:HH:MM:SS or YYYY-MM-DD HH:MM:SS or MM/DD/YY\tHH:MM:SS - $linerecord{$logfilenb}=$_; - my $year=0; my $month=0; my $day=0; my $hour=0; my $minute=0; my $second=0; - if ($_ =~ /(\d\d\d\d)-(\d\d)-(\d\d) (\d\d):(\d\d):(\d\d)/) { $year=$1; $month=$2; $day=$3; $hour=$4; $minute=$5; $second=$6; } - if ($_ =~ /\[(\d\d)\/(.*)\/(\d\d\d\d):(\d\d):(\d\d):(\d\d) /) { $year=$3; $month=$2; $day=$1; $hour=$4; $minute=$5; $second=$6; } - if ($monthnum{$month}) { $month=$monthnum{$month}; } # Change lib month in num month if necessary - - # Create $timeconnexion like YYYYMMDDHHMMSS - $timeconnexion{$logfilenb}=int("$year$month$day$hour$minute$second"); - if ($timeconnexion{$logfilenb}<10000000000000) { - &debug(" This record is corrupted (no date found)",3); - $corrupted{$logfilenb}++; - next; - } - &debug(" This is next record for file $logfilenb : timeconnexion=$timeconnexion{$logfilenb}",3); - last; - } - } - } - # END Read new lines for each log file. After this, following var are filled - # $timeconnexion{$logfilenb} - - # We choose wich record of wich log file to process - &debug("Choose of wich record of which log file to process",3); - $logfilechosen=-1; - my $timeref="99999999999999"; - foreach my $logfilenb (keys %LogFileToDo) { - &debug(" timeconnexion for file $logfilenb is $timeconnexion{$logfilenb}",4); - if ($timeconnexion{$logfilenb} < $timeref) { $logfilechosen=$logfilenb; $timeref=$timeconnexion{$logfilenb} } - } - if ($logfilechosen <= 0) { last; } # No more record to process - # Record is chosen - &debug(" We choosed to analyze record of file number $logfilechosen",3); - &debug(" Record is $linerecord{$logfilechosen}",3); - - # Record is approved. We found a new line to process in file number $logfilechosen - #---------------------------------------------------------------------------------- - $NbOfNewLinesProcessed++; - if (($ShowSteps) && ($NbOfNewLinesProcessed % $NbOfLinesForBenchmark == 0)) { print STDERR "$NbOfNewLinesProcessed lines processed (".(time()-$starttime)." seconds, ".($NbOfNewLinesProcessed/(time()-$starttime))." lines/seconds)\n"; } - - # Print record if ready - print "$linerecord{$logfilechosen} \"$NewReferer\" \"$NewUserAgent\"\n"; - - # End of processing all new records. -} -&debug("End of processing log file(s)"); - - -# Close all log files -foreach my $logfilenb (keys %LogFileToDo) { - &debug("Close log file number $logfilenb"); - close("LOG$logfilenb"); -} - - -0; # Do not remove this line