From: Neil Gierman Date: Tue, 17 Apr 2018 21:10:53 +0000 (-0500) Subject: GeoIP2 Support Plugin X-Git-Tag: AWSTATS_7_8~22^2~8 X-Git-Url: http://git.ipfire.org/cgi-bin/gitweb.cgi?a=commitdiff_plain;h=9185afdd75c33cd3527b8a04cf9b3ddd777f9658;p=thirdparty%2FAWStats.git GeoIP2 Support Plugin Initial implementation. Looksup only Country code for IPv4 and IPv6 --- diff --git a/wwwroot/cgi-bin/awstats.model.conf b/wwwroot/cgi-bin/awstats.model.conf index 6b0f430f..2112cd8c 100644 --- a/wwwroot/cgi-bin/awstats.model.conf +++ b/wwwroot/cgi-bin/awstats.model.conf @@ -180,8 +180,8 @@ HostAliases="localhost 127.0.0.1 REGEX[myserver\.com$]" # dramatically AWStats update process speed. Do not use on large web sites. # Note: Reverse DNS lookup is done on IPv4 only (Enable ipv6 plugin for IPv6). # Note: Result of DNS Lookup can be used to build the Country report. However -# it is highly recommanded to enable the plugin 'geoip' or 'geoipfree' to -# have an accurate Country report with no need of DNS Lookup. +# it is highly recommanded to enable the plugin 'geoip', 'geoipfree', or 'geoip2' +# to have an accurate Country report with no need of DNS Lookup. # Possible values: # 0 - No DNS Lookup # 1 - DNS Lookup is fully enabled @@ -1459,6 +1459,16 @@ color_x="C1B2E2" # Background color for number of exit pages (Default = "C1B2 # #LoadPlugin="geoip GEOIP_STANDARD /pathto/GeoIP.dat" +# PLUGIN: GeoIP2 +# REQUIRED MODULES: GeoIP2::Database::Reader (from Maxmind) +# PARAMETERS: [/pathto/GeoLite2-Country.mmdb[+/pathto/override.txt]] +# DESCRIPTION: Builds a country chart and adds an entry to the hosts +# table with country name. This uses the new schema of GeoIP2 replacing +# the now expired Legacy schema. +# Replace spaces in the path of geoip data file with string "%20". +# +#LoadPlugin="geoip2 /pathto/GeoLite2-Country.mmdb" + # PLUGIN: GeoIP6 # REQUIRED MODULES: Geo::IP or Geo::IP::PurePerl (from Maxmind, version >= 1.40) # PARAMETERS: [GEOIP_STANDARD | GEOIP_MEMORY_CACHE] [/pathto/geoipv6.dat[+/pathto/override.txt]] diff --git a/wwwroot/cgi-bin/awstats.pl b/wwwroot/cgi-bin/awstats.pl index b04ed6f8..dc08a48b 100755 --- a/wwwroot/cgi-bin/awstats.pl +++ b/wwwroot/cgi-bin/awstats.pl @@ -3169,6 +3169,7 @@ sub Read_Plugins { 'geoipfree' => 'u', 'geoip' => 'ou', 'geoip6' => 'ou', + 'geoip2' => 'ou', 'geoip_region_maxmind' => 'mou', 'geoip_city_maxmind' => 'mou', 'geoip_isp_maxmind' => 'mou', @@ -3345,7 +3346,8 @@ sub Read_Plugins { # In output mode, geo ip plugins are not loaded, so message changes are done here (can't be done in plugin init function) if ( $PluginsLoaded{'init'}{'geoip'} || $PluginsLoaded{'init'}{'geoip6'} - || $PluginsLoaded{'init'}{'geoipfree'} ) + || $PluginsLoaded{'init'}{'geoipfree'} + || $PluginsLoaded{'init'}{'geoip2'}) { $Message[17] = $Message[25] = $Message[148]; } @@ -19404,6 +19406,9 @@ if ( $UpdateStats && $FrameName ne 'index' && $FrameName ne 'mainleft' ) elsif ( $PluginsLoaded{'GetCountryCodeByAddr'}{'geoipfree'} ) { $Domain = GetCountryCodeByAddr_geoipfree($HostResolved); } + elsif ( $PluginsLoaded{'GetCountryCodeByAddr'}{'geoip2'} ) { + $Domain = GetCountryCodeByAddr_geoip2($HostResolved); + } if ($AtLeastOneSectionPlugin) { foreach my $pluginname ( keys %{ $PluginsLoaded{'SectionProcessIp'} } ) @@ -19438,6 +19443,11 @@ if ( $UpdateStats && $FrameName ne 'index' && $FrameName ne 'mainleft' ) { $Domain = GetCountryCodeByAddr_geoipfree($Host); } + elsif ( + $PluginsLoaded{'GetCountryCodeByAddr'}{'geoip2'} ) + { + $Domain = GetCountryCodeByAddr_geoip2($Host); + } elsif ( $HostResolved =~ /\.(\w+)$/ ) { $Domain = $1; } if ($AtLeastOneSectionPlugin) { foreach my $pluginname ( @@ -19467,6 +19477,11 @@ if ( $UpdateStats && $FrameName ne 'index' && $FrameName ne 'mainleft' ) { $Domain = GetCountryCodeByName_geoipfree($HostResolved); } + elsif ( + $PluginsLoaded{'GetCountryCodeByName'}{'geoip2'} ) + { + $Domain = GetCountryCodeByName_geoip2($HostResolved); + } elsif ( $HostResolved =~ /\.(\w+)$/ ) { $Domain = $1; } if ($AtLeastOneSectionPlugin) { foreach my $pluginname ( diff --git a/wwwroot/cgi-bin/plugins/geoip2.pm b/wwwroot/cgi-bin/plugins/geoip2.pm new file mode 100644 index 00000000..6e644a32 --- /dev/null +++ b/wwwroot/cgi-bin/plugins/geoip2.pm @@ -0,0 +1,272 @@ +#!/usr/bin/perl +#----------------------------------------------------------------------------- +# GeoIp2 Maxmind AWStats plugin +# This plugin allow you to get country report with countries detected +# from a Geographical database (GeoIP2 internal database) instead of domain +# hostname suffix. +# Need the country database from Maxmind (free). +#----------------------------------------------------------------------------- +# Perl Required Module: GeoIP2::Database::Reader +#----------------------------------------------------------------------------- + + +# <----- +# ENTER HERE THE USE COMMAND FOR ALL REQUIRED PERL MODULES +use vars qw/ $type /; +$type='geoip2'; +if (!eval ('require "GeoIP2/Database/Reader.pm";')) { + $error=$@; + $ret=($error)?"Error:\n$error":""; + $ret.="Error: Need Perl module GeoIP2::Database::Reader"; + return $ret; +} +# GeoIP2 Perl API doesn't have a ByName lookup so we need to do the resolution ourselves +if (!eval ('require "Socket.pm";')) { + $error=$@; + $ret=($error)?"Error:\n$error":""; + $ret.="Error: Need Perl module Socket"; + return $ret; +} +# -----> +#use strict; +no strict "refs"; + + + +#----------------------------------------------------------------------------- +# PLUGIN VARIABLES +#----------------------------------------------------------------------------- +# <----- +# ENTER HERE THE MINIMUM AWSTATS VERSION REQUIRED BY YOUR PLUGIN +# AND THE NAME OF ALL FUNCTIONS THE PLUGIN MANAGE. +my $PluginNeedAWStatsVersion="5.4"; +my $PluginHooksFunctions="GetCountryCodeByAddr GetCountryCodeByName ShowInfoHost"; +my $PluginName = "geoip2"; +my $LoadedOverride=0; +my $OverrideFile=""; +my %TmpDomainLookup; +# -----> + +# <----- +# IF YOUR PLUGIN NEED GLOBAL VARIABLES, THEY MUST BE DECLARED HERE. +use vars qw/ +$reader +/; +# -----> + + +#----------------------------------------------------------------------------- +# PLUGIN FUNCTION: Init_pluginname +#----------------------------------------------------------------------------- +sub Init_geoip2 { + my $InitParams=shift; + my $checkversion=&Check_Plugin_Version($PluginNeedAWStatsVersion); + + # <----- + # ENTER HERE CODE TO DO INIT PLUGIN ACTIONS + debug(" Plugin $PluginName: InitParams=$InitParams",1); + my $datafile=$InitParams; + if (! $datafile) { $datafile="$PluginName.dat"; } + else { $datafile =~ s/%20/ /g; } + # if ($type eq 'geoippureperl') { + # if ($mode eq '' || $mode eq 'GEOIP_MEMORY_CACHE') { $mode=Geo::IP::PurePerl::GEOIP_MEMORY_CACHE(); } + # else { $mode=Geo::IP::PurePerl::GEOIP_STANDARD(); } + # } else { + # if ($mode eq '' || $mode eq 'GEOIP_MEMORY_CACHE') { $mode=Geo::IP::GEOIP_MEMORY_CACHE(); } + # else { $mode=Geo::IP::GEOIP_STANDARD(); } + # } + if ($override){$OverrideFile=$override;} + %TmpDomainLookup=(); + debug(" Plugin $PluginName: GeoIP2 try to initialize type=$type mode=$mode override=$override datafile=$datafile",1); + $reader = GeoIP2::Database::Reader->new( + file => $datafile, + locales => [ 'en', 'de', ] + ); + + # Fails on some GeoIP version + # debug(" Plugin $PluginName: GeoIP initialized database_info=".$reader->database_info()); + if ($reader) { debug(" Plugin $PluginName: GeoIP2 plugin and reader object initialized",1); } + else { return "Error: Failed to create reader object for datafile=".$datafile; } + # -----> + + return ($checkversion?$checkversion:"$PluginHooksFunctions"); +} + + +#----------------------------------------------------------------------------- +# PLUGIN FUNCTION: GetCountryCodeByAddr_pluginname +# UNIQUE: YES (Only one plugin using this function can be loaded) +# GetCountryCodeByAddr is called to translate an ip into a country code in lower case. +#----------------------------------------------------------------------------- +sub GetCountryCodeByAddr_geoip2 { + my $param="$_[0]"; + # <----- + if (! $param) { return ''; } + my $res= TmpLookup_geoip2($param); + if (! $res) { + $res=lc($reader->country( ip => $param)->country()->iso_code()) || 'unknown'; + $TmpDomainLookup{$param}=$res; + if ($Debug) { debug(" Plugin $PluginName: GetCountryCodeByAddr for $param: [$res]",5); } + } + elsif ($Debug) { debug(" Plugin $PluginName: GetCountryCodeByAddr for $param: Already resolved to [$res]",5); } + # -----> + return $res; +} + + +#----------------------------------------------------------------------------- +# PLUGIN FUNCTION: GetCountryCodeByName_pluginname +# UNIQUE: YES (Only one plugin using this function can be loaded) +# GetCountryCodeByName is called to translate a host name into a country code in lower case. +#----------------------------------------------------------------------------- +sub GetCountryCodeByName_geoip2 { + my $param="$_[0]"; + # <----- + if (! $param) { return ''; } + my $res = TmpLookup_geoip($param); + if (! $res) { + # First resolve the name to an IP + $address = inet_ntoa(inet_aton($param)); + # Now do the same lookup from the IP + $res=lc($reader->country($address)->country()->iso_code()) || 'unknown'; + $TmpDomainLookup{$param}=$res; + if ($Debug) { debug(" Plugin $PluginName: GetCountryCodeByName for $param: [$res]",5); } + } + elsif ($Debug) { debug(" Plugin $PluginName: GetCountryCodeByName for $param: Already resolved to [$res]",5); } + # -----> + return $res; +} + + +#----------------------------------------------------------------------------- +# PLUGIN FUNCTION: ShowInfoHost_pluginname +# UNIQUE: NO (Several plugins using this function can be loaded) +# Function called to add additionnal columns to the Hosts report. +# This function is called when building rows of the report (One call for each +# row). So it allows you to add a column in report, for example with code : +# print "This is a new cell for $param"; +# Parameters: Host name or ip +#----------------------------------------------------------------------------- +sub ShowInfoHost_geoip2 { + my $param="$_[0]"; + # <----- + if ($param eq '__title__') { + my $NewLinkParams=${QueryString}; + $NewLinkParams =~ s/(^|&)update(=\w*|$)//i; + $NewLinkParams =~ s/(^|&)output(=\w*|$)//i; + $NewLinkParams =~ s/(^|&)staticlinks(=\w*|$)//i; + $NewLinkParams =~ s/(^|&)framename=[^&]*//i; + my $NewLinkTarget=''; + if ($DetailedReportsOnNewWindows) { $NewLinkTarget=" target=\"awstatsbis\""; } + if (($FrameName eq 'mainleft' || $FrameName eq 'mainright') && $DetailedReportsOnNewWindows < 2) { + $NewLinkParams.="&framename=mainright"; + $NewLinkTarget=" target=\"mainright\""; + } + $NewLinkParams =~ tr/&/&/s; $NewLinkParams =~ s/^&//; $NewLinkParams =~ s/&$//; + if ($NewLinkParams) { $NewLinkParams="${NewLinkParams}&"; } + + print ""; + print "GeoIP2
Country"; + print ""; + } + elsif ($param) { + my $ip=0; + my $key; + if ($param =~ /^\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}$/) { # IPv4 address + $ip=4; + $key=$param; + } + elsif ($param =~ /^[0-9A-F]*:/i) { # IPv6 address + $ip=6; + $key=$param; + } + print ""; + if ($key && $ip==4) { + my $res = TmpLookup_geoip2($param); + if (!$res){$res=lc($reader->country($param)->country()->iso_code()) if $reader;} + if ($Debug) { debug(" Plugin $PluginName: GetCountryByIp for $param: [$res]",5); } + if ($res) { print $DomainsHashIDLib{$res}?$DomainsHashIDLib{$res}:"$Message[0]"; } + else { print "$Message[0]"; } + } + if ($key && $ip==6) { # GeoIP2 supports both IPv4 and IPv6 + my $res = TmpLookup_geoip2($param); + if (!$res){$res=lc($reader->country($param)->country()->iso_code()) if $reader;} + if ($Debug) { debug(" Plugin $PluginName: GetCountryByIp for $param: [$res]",5); } + if ($res) { print $DomainsHashIDLib{$res}?$DomainsHashIDLib{$res}:"$Message[0]"; } + else { print "$Message[0]"; } + } + if (! $key) { + my $res = TmpLookup_geoip2($param); + if (!$res){$res=lc($reader->country($param)->country()->iso_code()) if $reader;} + if ($Debug) { debug(" Plugin $PluginName: GetCountryByHostname for $param: [$res]",5); } + if ($res) { print $DomainsHashIDLib{$res}?$DomainsHashIDLib{$res}:"$Message[0]"; } + else { print "$Message[0]"; } + } + print ""; + } + else { + print " "; + } + return 1; + # -----> +} + +#----------------------------------------------------------------------------- +# PLUGIN FUNCTION: LoadOverrideFile +# Attempts to load a comma delimited file that will override the GeoIP database +# Useful for Intranet records +# CSV format: IP,2-char Country code +#----------------------------------------------------------------------------- +sub LoadOverrideFile_geoip2{ + my $filetoload=""; + if ($OverrideFile){ + if (!open(GEOIPFILE, $OverrideFile)){ + debug("Plugin $PluginName: Unable to open override file: $OverrideFile"); + $LoadedOverride = 1; + return; + } + }else{ + my $conf = (exists(&Get_Config_Name) ? Get_Config_Name() : $SiteConfig); + if ($conf && open(GEOIPFILE,"$DirData/$PluginName.$conf.txt")) { $filetoload="$DirData/$PluginName.$conf.txt"; } + elsif (open(GEOIPFILE,"$DirData/$PluginName.txt")) { $filetoload="$DirData/$PluginName.txt"; } + else { debug("No override file \"$DirData/$PluginName.txt\": $!"); } + } + if ($filetoload) + { + # This is the fastest way to load with regexp that I know + while (){ + chomp $_; + s/\r//; + my @record = split(",", $_); + # replace quotes if they were used in the file + foreach (@record){ $_ =~ s/"//g; } + # store in hash + $TmpDomainLookup{$record[0]} = $record[1]; + } + close GEOIPFILE; + debug(" Plugin $PluginName: Overload file loaded: ".(scalar keys %TmpDomainLookup)." entries found."); + } + $LoadedOverride = 1; + return; +} + +#----------------------------------------------------------------------------- +# PLUGIN FUNCTION: TmpLookup +# Searches the temporary hash for the parameter value and returns the corresponding +# GEOIP entry +#----------------------------------------------------------------------------- +sub TmpLookup_geoip2(){ + $param = shift; + if (!$LoadedOverride){&LoadOverrideFile_geoip2();} + #my $val; + #if ($reader && + #(($type eq 'geoip' && $reader->VERSION >= 1.30) || + # $type eq 'geoippureperl' && $reader->VERSION >= 1.17)){ + # $val = $TmpDomainLookup{$reader->get_ip_address($param)}; + #} + #else {$val = $TmpDomainLookup{$param};} + #return $val || ''; + return $TmpDomainLookup{$param}||''; +} + +1; # Do not remove this line