Added option -dnscache= in logresolvemerge.pl to use static file.

diff --git a/docs/awstats_changelog.txt b/docs/awstats_changelog.txt
index bd6076f2452388536d03dfb1a4d2c6e097f58a5b..a9e3cee19cd8ce518ad8be95c7cac85c39b5f7e3 100644 (file)
--- a/docs/awstats_changelog.txt
+++ b/docs/awstats_changelog.txt
@@ -32,6 +32,7 @@ New features/improvements:
   session.
 - Better management of plugin load failure.
 - Added LogType parameter.
+- Added option -dnscache= in logresolvemerge.pl to use dns static file.
 
 Other/Documentation:
 - The "Popup WhoIs link" code is now handled by new 'hostinfo' plugin.

diff --git a/tools/logresolvemerge.pl b/tools/logresolvemerge.pl
index 283cf9afe68da404d46e3c3428fbbda655ed4585..d13dc61c5b9ffd4db2e3570e1dbca88a38cf5547 100644 (file)
--- a/tools/logresolvemerge.pl
+++ b/tools/logresolvemerge.pl
@@ -41,8 +41,9 @@ my $DIR;
 my $PROG;
 my $Extension;
 my $DNSLookup=0;
-my $DirCgi="";
-my $DirData="";
+my $DNSCache='';
+my $DirCgi='';
+my $DirData='';
 my $DNSLookupAlreadyDone=0;
 my $NbOfLinesShowsteps=0;
 
@@ -56,10 +57,7 @@ my %corrupted=();
 my %TmpDNSLookup =();
 my %QueueHostsToResolve=();
 my %QueueRecords=();
-
-# These table is used to make fast reverse DNS lookup for particular IP adresses.
-# You can add your own IP adresses resolutions.
-my %MyDNSTable = ();
+my %MyDNSTable=();
 
 
 
@@ -127,18 +125,22 @@ sub MakeDNSLookup {
 #-----------------------------------------------------------------------------
 # MAIN
 #-----------------------------------------------------------------------------
-my $QueryString=''; for (0..@ARGV-1) { $QueryString .= "$ARGV[$_] "; }
-if ($QueryString =~ /debug=(\d)/i) { $Debug=$1; }
-if ($QueryString =~ /dnslookup/i)  { $DNSLookup=1; }
-if ($QueryString =~ /showsteps/i)  { $ShowSteps=1; }
-if ($QueryString =~ /addfilenum/i) { $AddFileNum=1; }
 ($DIR=$0) =~ s/([^\/\\]*)$//; ($PROG=$1) =~ s/\.([^\.]*)$//; $Extension=$1;
 
 my $cpt=1;
 for (0..@ARGV-1) {
-       if ($ARGV[$_] =~ /^-/) { next; }
-       $ParamFile{$cpt}=$ARGV[$_];
-       $cpt++;
+       if ($ARGV[$_] =~ /^-/) {\r
+               if ($ARGV[$_] =~ /debug=(\d)/i) { $Debug=$1; }\r
+               elsif ($ARGV[$_] =~ /dnscache=/i) { $DNSLookup||=2; $DNSCache=$ARGV[$_]; $DNSCache =~ s/-dnscache=//; }\r
+               elsif ($ARGV[$_] =~ /dnslookup/i) { $DNSLookup=1; }\r
+               elsif ($ARGV[$_] =~ /showsteps/i) { $ShowSteps=1; }\r
+               elsif ($ARGV[$_] =~ /addfilenum/i) { $AddFileNum=1; }\r
+               else { print "Unknown argument $ARGV[$_] ignored\n"; }\r
+       }\r
+       else {\r
+               $ParamFile{$cpt}=$ARGV[$_];\r
+               $cpt++;\r
+       }\r
 }
 if (scalar keys %ParamFile == 0) {
        print "----- $PROG $VERSION (c) Laurent Destailleur -----\n";
@@ -155,11 +157,12 @@ if (scalar keys %ParamFile == 0) {
        print "  $PROG.$Extension [options] file1 ... filen\n";
        print "  $PROG.$Extension [options] *.*\n";
        print "Options:\n";
-       print "  -dnslookup    make a reverse DNS lookup on IP adresses (not done by default)\n";
-#      print "  -dnslookup:n  same with a n parallel threads instead of $QueuePoolSize by default\n";
-       print "  -showsteps    print on stderr benchmark information every $NBOFLINESFORBENCHMARK lines\n";
-       print "  -addfilenum   if used with several files, file number can be added in first\n";
-       print "                field of output file.\n";
+       print "  -dnslookup     make a reverse DNS lookup on IP adresses\n";
+#      print "  -dnslookup:n   same with a n parallel threads instead of $QueuePoolSize by default\n";
+       print "  -dnscache=file make DNS lookup from cache file first before network lookup\n";
+       print "  -showsteps     print on stderr benchmark information every $NBOFLINESFORBENCHMARK lines\n";
+       print "  -addfilenum    if used with several files, file number can be added in first\n";
+       print "                 field of output file.\n";
        print "\n";
        print "This runs $PROG in command line to open one or several web\n";
        print "server log files to merge them (sorted on date) and/or to make a reverse\n";
@@ -218,6 +221,20 @@ if ($DNSLookup) { use Socket; }
 # monthnum must be in english because it's used to translate log date in apache log files which are always in english
 my %monthnum =  ( "Jan","01","jan","01","Feb","02","feb","02","Mar","03","mar","03","Apr","04","apr","04","May","05","may","05","Jun","06","jun","06","Jul","07","jul","07","Aug","08","aug","08","Sep","09","sep","09","Oct","10","oct","10","Nov","11","nov","11","Dec","12","dec","12" );
 
+&debug("DNSLookup=$DNSLookup");
+&debug("DNSCache=$DNSCache");
+
+if ($DNSCache) {\r
+       &debug("Load DNS Cache file $DNSCache",2);
+       open(CACHE, "<$DNSCache") or error("Can't open cache file $DNSCache");\r
+       while (<CACHE>) {\r
+               my ($time, $ip, $name) = split;\r
+               $name='ip' if $name eq '*';\r
+               $MyDNSTable{$ip}=$name;\r
+       }\r
+       close CACHE;\r
+}\r
+
 #-----------------------------------------------------------------------------
 # PROCESSING CURRENT LOG(s)
 #-----------------------------------------------------------------------------
@@ -356,7 +373,7 @@ while (1 == 1)
                }
        }
 
-       # Analyze: IP-address
+       # Do DNS lookup
        #--------------------
        my $Host='';
        my $HostResolved='';
@@ -368,7 +385,7 @@ while (1 == 1)
                        # Check in static DNS cache file
                        $HostResolved=$MyDNSTable{$Host};
                        if ($HostResolved) {
-                               if ($Debug) { debug("  DNS lookup asked for $Host and found in DNS cache file: $HostResolved",4); }
+                               if ($Debug) { debug("  DNS lookup asked for $Host and found in static DNS cache file: $HostResolved",4); }
                        }
                        elsif ($DNSLookup==1) {
                                # Check in session cache (dynamic DNS cache file + session DNS cache)
@@ -406,7 +423,7 @@ while (1 == 1)
                        }
                        else {
                                $HostResolved='*';
-                               if ($Debug) { debug("  DNS lookup by file asked for $Host but not found in DNS cache file.",4); }
+                               if ($Debug) { debug("  DNS lookup by static DNS cache file asked for $Host but not found.",4); }
                        }
                }
                else {
@@ -415,8 +432,8 @@ while (1 == 1)
                }
        }
        else {
-               if ($Host =~ /^\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}$/) { $HostResolved='*'; $ip=4; }      # IPv4
-               elsif ($Host =~ /^[0-9A-F]*:/i) { $HostResolved='*'; $ip=6; }                                           # IPv6
+               if ($linerecord{$logfilechosen} =~ /(\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3})/) { $HostResolved='*'; $ip=4; $Host=$1; }      # IPv4
+               elsif ($linerecord{$logfilechosen} =~ /([0-9A-F]*:)/i) { $HostResolved='*'; $ip=6; $Host=$1; }                                          # IPv6
                if ($Debug) { debug("  No DNS lookup asked.",4); }
        }
 
@@ -424,19 +441,38 @@ while (1 == 1)
        debug("Add record $NbOfLinesParsed in queue (with host value ".($Host?$Host:'*').")",4);
        $QueueRecords{$NbOfLinesParsed}=$linerecord{$logfilechosen};
        # If there is a host to resolve, we add line to queue with value of HostResolved
-       $QueueHostsToResolve{$NbOfLinesParsed}=$Host?$Host:'*';
+       # $Host is '' (no ip found) or is ip
+       if ($DNSLookup==0) {
+               $QueueHostsToResolve{$NbOfLinesParsed}='*';
+       }
+       if ($DNSLookup==1) { 
+               $QueueHostsToResolve{$NbOfLinesParsed}=$Host?$Host:'*';
+       }
+       if ($DNSLookup==2) { 
+               # If ($HostResolved ne '*' and $Host) then MyDNSTable is defined for $Host
+               $QueueHostsToResolve{$NbOfLinesParsed}=($Host && $HostResolved ne '*')?$Host:'*';
+       }
 
        # Print all records in head of queue that are ready
        debug("Check head of queue to write records ready to flush (QueueCursor=$QueueCursor, QueueSize=".(scalar keys %QueueRecords).")",4);
-       while ( $QueueHostsToResolve{$QueueCursor} && ( ($QueueHostsToResolve{$QueueCursor} eq '*') || ($TmpDNSLookup{$QueueHostsToResolve{$QueueCursor}}) ) ) {
+       while ( $QueueHostsToResolve{$QueueCursor} && ( ($QueueHostsToResolve{$QueueCursor} eq '*') || ($MyDNSTable{$QueueHostsToResolve{$QueueCursor}}) || ($TmpDNSLookup{$QueueHostsToResolve{$QueueCursor}}) ) ) {
+               # $QueueCursor point to a ready record
                if ($QueueHostsToResolve{$QueueCursor} eq '*') {
-                       debug(" First elem in queue has a lookup result ending with 'no result'. We pull it.",4);
+                       debug(" First elem in queue is ready. No change on it. We pull it.",4);
                }
                else {
-                       if ($TmpDNSLookup {$QueueHostsToResolve{$QueueCursor}} ne '*') {
-                               $QueueRecords{$QueueCursor}=~s/$QueueHostsToResolve{$QueueCursor}/$TmpDNSLookup{$QueueHostsToResolve{$QueueCursor}}/;
+                       if ($MyDNSTable{$QueueHostsToResolve{$QueueCursor}}) {
+                               if ($MyDNSTable{$QueueHostsToResolve{$QueueCursor}} ne '*') {
+                                       $QueueRecords{$QueueCursor}=~s/$QueueHostsToResolve{$QueueCursor}/$MyDNSTable{$QueueHostsToResolve{$QueueCursor}}/;
+                                       debug(" First elem in queue has been resolved ($MyDNSTable{$QueueHostsToResolve{$QueueCursor}}). We pull it.",4);
+                               }
+                       }
+                       elsif ($TmpDNSLookup{$QueueHostsToResolve{$QueueCursor}}) {
+                               if ($TmpDNSLookup{$QueueHostsToResolve{$QueueCursor}} ne '*') {
+                                       $QueueRecords{$QueueCursor}=~s/$QueueHostsToResolve{$QueueCursor}/$TmpDNSLookup{$QueueHostsToResolve{$QueueCursor}}/;
+                                       debug(" First elem in queue has been resolved ($TmpDNSLookup{$QueueHostsToResolve{$QueueCursor}}). We pull it.",4);
+                               }
                        }
-                       debug(" First elem in queue has been resolved ($TmpDNSLookup{$QueueHostsToResolve{$QueueCursor}}). We pull it.",4);
                }
                # Record is ready, we output it.
                if ($AddFileNum) { print "$logfilechosen $QueueRecords{$QueueCursor}\n"; }
@@ -462,6 +498,17 @@ foreach my $logfilenb (keys %LogFileToDo) {
 
 
 # DNSLookup warning
-if ($DNSLookup && $DNSLookupAlreadyDone) { warning("Warning: $PROG has detected that some host names were already resolved in your logfile $DNSLookupAlreadyDone.\nIf DNS lookup was already made by the logger (web server) in ALL your log files, you should not use -dnslookup option to increase $PROG speed."); }
+if ($DNSLookup==1 && $DNSLookupAlreadyDone) {
+       warning("Warning: $PROG has detected that some host names were already resolved in your logfile $DNSLookupAlreadyDone.\nIf DNS lookup was already made by the logger (web server) in ALL your log files, you should not use -dnslookup option to increase $PROG speed.");
+}
+
+#if ($DNSCache) {
+#      open(CACHE, ">$DNSCache") or die;
+#      foreach (keys %TmpDNSLookup) {
+#              $TmpDNSLookup{$_}="*" if $TmpDNSLookup{$_} eq "ip";
+#              print CACHE "0\t$_\t$TmpDNSLookup{$_}\n";
+#      }
+#      close CACHE;
+#}
 
 0;     # Do not remove this line