From 9dd9b70ccf89ca10abc5399eecdf6df389a30d64 Mon Sep 17 00:00:00 2001 From: eldy <> Date: Tue, 25 Nov 2003 21:21:59 +0000 Subject: [PATCH] Summary report shows also "unviewed" traffic. --- docs/awstats_changelog.txt | 3 + docs/awstats_faq.html | 18 +- wwwroot/cgi-bin/awstats.model.conf | 1 + wwwroot/cgi-bin/awstats.pl | 357 +++++++++++++++++----------- wwwroot/cgi-bin/lang/awstats-en.txt | 6 +- wwwroot/cgi-bin/lang/awstats-fr.txt | 4 + 6 files changed, 245 insertions(+), 144 deletions(-) diff --git a/docs/awstats_changelog.txt b/docs/awstats_changelog.txt index 3cdf37dd..40e67c44 100644 --- a/docs/awstats_changelog.txt +++ b/docs/awstats_changelog.txt @@ -24,6 +24,7 @@ Fixes: - Fixed bug in counting hits for miscellanous and clusters chart when a temporary flush was done on disk during a long update. - ExtraSections now works on all records whatever is the error code. +- Click on "Summary" now returns to top of page even with rawlog plugin. New features/improvements: - Increased speed by 10 to 20%. @@ -31,6 +32,7 @@ New features/improvements: understand and reduce unknown bugs. - Some changes to make AWStats to be XML compliant ready. Need to set the new parameter BuildReportFormat to 'xml' in config file. +- Summary report shows also "unviewed" traffic. - A new search engine database to allow several "match id" for same search engine. Example: All google ip referer id are recognised. - Can use UA and HOST fields to build personalized ExtraSection reports. @@ -63,6 +65,7 @@ Other/Documentation: asmx and aspx added in mime.pm file. Microsoft URL Control added in robot list. - Documentation seriously updated. +- FAQ updated. - Debian icon and some missing country flags added. - Prepare code for decodeUTFkeys plugin. - Added Hebrew and Galician language. diff --git a/docs/awstats_faq.html b/docs/awstats_faq.html index c89d0f52..1b830432 100644 --- a/docs/awstats_faq.html +++ b/docs/awstats_faq.html @@ -260,6 +260,11 @@ LogFile="/var/log/xferlog"
LogFormat="%time1 %host %logname %method %url %code %bytesd"
LogSeparator="\t"
NotPageList="" +
LevelForBrowsersDetection=0 +
LevelForOSDetection=0 +
LevelForRefererAnalyze=0 +
LevelForRobotsDetection=0 +
LevelForSearchEnginesDetection=0
ShowLinksOnUrl=0
ShowMenu=1
ShowMonthStats=UVHB @@ -269,7 +274,7 @@ LogFile="/var/log/xferlog"
ShowDomainsStats=HB
ShowHostsStats=HBL
ShowAuthenticatedUsers=HBL -
ShowRobotsStats=1 +
ShowRobotsStats=0
ShowEMailSenders=0
ShowEMailReceivers=0
ShowSessionsStats=1 @@ -320,6 +325,12 @@ Then change:
LogType=M
LogFormat="%time2 %email %email_r %host %host_r %method %url %code %bytesd" +
LevelForBrowsersDetection=0 +
LevelForOSDetection=0 +
LevelForRefererAnalyze=0 +
LevelForRobotsDetection=0 +
LevelForSearchEnginesDetection=0 +
LevelForFileTypesDetection=0
ShowMenu=1
ShowMonthStats=HB
ShowDaysOfMonthStats=HB @@ -386,7 +397,9 @@ Your log file will probably looks like this:
Copy config awstats.model.conf file to "awstats.mediaserver.conf". Modify this new config file:
-LogFile="/pathtomediaserver/mediaserverlog" + +LogFile="/pathtomediaserverlog/mediaserverlog" +
LogType=S
LogFormat="%host %other %logname %time1 %methodurl %code %bytesd %uabracket %other %other %other %other %other %other %other %other"
LogSeparator="\s+"
ShowMenu=1 @@ -482,6 +495,7 @@ Modify this new config file: \n":"   "); } # When $linetitle=&AtLeastOneNotNull($ShowMonthStats,$ShowDaysOfMonthStats,$ShowDaysOfWeekStats,$ShowHoursStats); if ($linetitle) { print "\n"; } if ($linetitle) { print ($frame?"\n":"\n":"   "); } + if ($ShowMonthStats) { print ($frame?"\n":"   "); } #if ($ShowMonthDayStats) { print ($frame?"\n":"   "); } if ($ShowDaysOfMonthStats) { print ($frame?"\n":"   "); } if ($ShowDaysOfWeekStats) { print ($frame?"\n":"   "); } @@ -7058,20 +7101,25 @@ if (scalar keys %HTMLOutput) { # FirstTime LastTime TotalVisits TotalUnique TotalPages TotalHits TotalBytes TotalHostsKnown TotalHostsUnknown my $FirstTime=0; my $LastTime=0; - $TotalUnique=$TotalVisits=$TotalPages=$TotalHits=$TotalBytes=$TotalHostsKnown=$TotalHostsUnknown=0; + $TotalUnique=$TotalVisits=$TotalPages=$TotalHits=$TotalBytes=0; + $TotalNotViewedPages=$TotalNotViewedHits=$TotalNotViewedBytes=0; + $TotalHostsKnown=$TotalHostsUnknown=0; my $beginmonth=$MonthRequired;my $endmonth=$MonthRequired; if ($MonthRequired eq 'all') { $beginmonth=1;$endmonth=12; } for (my $month=$beginmonth; $month<=$endmonth; $month++) { my $monthix=sprintf("%02s",$month); if ($FirstTime{$YearRequired.$monthix} && ($FirstTime == 0 || $FirstTime > $FirstTime{$YearRequired.$monthix})) { $FirstTime = $FirstTime{$YearRequired.$monthix}; } if ($LastTime < ($LastTime{$YearRequired.$monthix}||0)) { $LastTime = $LastTime{$YearRequired.$monthix}; } + $TotalHostsKnown+=$MonthHostsKnown{$YearRequired.$monthix}||0; # Wrong in year view + $TotalHostsUnknown+=$MonthHostsUnknown{$YearRequired.$monthix}||0; # Wrong in year view + $TotalUnique+=$MonthUnique{$YearRequired.$monthix}||0; # Wrong in year view $TotalVisits+=$MonthVisits{$YearRequired.$monthix}||0; $TotalPages+=$MonthPages{$YearRequired.$monthix}||0; $TotalHits+=$MonthHits{$YearRequired.$monthix}||0; $TotalBytes+=$MonthBytes{$YearRequired.$monthix}||0; - $TotalUnique+=$MonthUnique{$YearRequired.$monthix}||0; # Wrong in year view - $TotalHostsKnown+=$MonthHostsKnown{$YearRequired.$monthix}||0; # Wrong in year view - $TotalHostsUnknown+=$MonthHostsUnknown{$YearRequired.$monthix}||0; # Wrong in year view + $TotalNotViewedPages+=$MonthNotViewedPages{$YearRequired.$monthix}||0; + $TotalNotViewedHits+=$MonthNotViewedHits{$YearRequired.$monthix}||0; + $TotalNotViewedBytes+=$MonthNotViewedBytes{$YearRequired.$monthix}||0; } # TotalHitsErrors TotalBytesErrors my $TotalHitsErrors=0; my $TotalBytesErrors=0; @@ -8156,7 +8204,6 @@ if (scalar keys %HTMLOutput) { #--------------------------------------------------------------------- if ($ShowMonthStats) { if ($Debug) { debug("ShowMonthStats",2); } - print "$Center 
\n"; my $title="$Message[128]"; &tab_head("$title",0,0,'month'); @@ -8178,11 +8225,19 @@ if (scalar keys %HTMLOutput) { if ($TotalVisits > 0) { $RatioHits=int($TotalHits/$TotalVisits*100)/100; } if ($TotalVisits > 0) { $RatioBytes=int(($TotalBytes/1024)*100/($LogType eq 'M'?$TotalHits:$TotalVisits))/100; } - print "\n"; + my $colspan=5; + my $w='20'; + if ($LogType eq 'W') { $w='17'; $colspan=6; } + + print ""; + if ($LogType eq 'W') { print ""; } + print "\n"; print ""; print "\n"; - if ($FirstTime) { print ""; } - else { print ""; } + print ""; + if ($LogType eq 'W') { print ""; } + if ($FirstTime) { print ""; } + else { print ""; } print ""; @@ -8190,26 +8245,45 @@ if (scalar keys %HTMLOutput) { else { print "\n"; } # Show main indicators print ""; - if ($ShowMonthStats =~ /U/i) { print ""; } else { print ""; } - if ($ShowMonthStats =~ /V/i) { print ""; } else { print ""; } - if ($ShowMonthStats =~ /P/i) { print ""; } else { print ""; } - if ($ShowMonthStats =~ /H/i) { print ""; } else { print ""; } - if ($ShowMonthStats =~ /B/i) { print ""; } else { print ""; } + if ($LogType eq 'W') { print ""; } + if ($ShowMonthStats =~ /U/i) { print ""; } else { print ""; } + if ($ShowMonthStats =~ /V/i) { print ""; } else { print ""; } + if ($ShowMonthStats =~ /P/i) { print ""; } else { print ""; } + if ($ShowMonthStats =~ /H/i) { print ""; } else { print ""; } + if ($ShowMonthStats =~ /B/i) { print ""; } else { print ""; } print "\n"; print ""; + if ($LogType eq 'W') { $w='17'; print ""; } if ($ShowMonthStats =~ /U/i) { print ""; } else { print ""; } if ($ShowMonthStats =~ /V/i) { print ""; } else { print ""; } if ($ShowMonthStats =~ /P/i) { print ""; } else { print ""; } if ($ShowMonthStats =~ /H/i) { print ""); } else { print ""; } if ($ShowMonthStats =~ /B/i) { print ""; } else { print ""; } print "\n"; + print ""; + if ($LogType eq 'W') { + print ""; + print "\n"; + if ($ShowMonthStats =~ /P/i) { print ""; } else { print ""; } + if ($ShowMonthStats =~ /H/i) { print ""; } else { print ""; } + if ($ShowMonthStats =~ /B/i) { print ""; } else { print ""; } + } + print "\n"; + &tab_end($LogType eq 'W'?"* $Message[159]":""); - my $colspan=5; - print "\n"; &tab_end; } + } print "\n \n\n"; @@ -9773,13 +9848,13 @@ else { # So it's new line approved # If other month/year, create/update tmp file and purge data arrays with # &Read_History_With_TmpUpdate(lastprocessedyear,lastprocessedmonth,UPDATE,PURGE,"all",lastlinenumber,lastlineoffset,CheckSum($_)); +# Define a clean Url and Query (set urlwithnoquery, tokenquery and standalonequery and $field[$pos_url]) +# Define PageBool and extension # Analyze: Misc tracker --> complete %misc # Analyze: Add to favorites --> complete %_misc, next on loop # Analyze: Worms --> complete %_worms, countedtraffic=1 # If (!countedtraffic) Analyze: Status code --> complete %_error_, %_sider404, %_referrer404 --> countedtraffic=1 # If (!countedtraffic) Analyze: Robots --> complete %_robot, countedtraffic=1 -# Define a clean Url and Query (set urlwithnoquery, tokenquery and standalonequery and $field[$pos_url]) -# Define PageBool and extension # If (!countedtraffic) Analyze: File types - Compression # If (!countedtraffic) Analyze: Date - Hour - Pages - Hits - Kilo # If (!countedtraffic) Analyze: Login diff --git a/wwwroot/cgi-bin/lang/awstats-en.txt b/wwwroot/cgi-bin/lang/awstats-en.txt index f78e6812..53b63b3b 100644 --- a/wwwroot/cgi-bin/lang/awstats-en.txt +++ b/wwwroot/cgi-bin/lang/awstats-en.txt @@ -158,4 +158,8 @@ message154=Codes shown here gave hits or traffic "not viewed" by visitors, so th message155=Cluster message156=Robots shown here gave hits or traffic "not viewed" by visitors, so thay are not included in other charts. message157=Numbers after + are successful hits on "robots.txt" files. -message158=Worms shown here gave hits or traffic "not viewed" by visitors, so thay are not included in other charts. \ No newline at end of file +message158=Worms shown here gave hits or traffic "not viewed" by visitors, so thay are not included in other charts. +message159=Not viewed trafic is trafic generated by robots, worms or answers with special HTTP status code. +message160=Viewed traffic +message161=Not viewed traffic +message162=Monthly history diff --git a/wwwroot/cgi-bin/lang/awstats-fr.txt b/wwwroot/cgi-bin/lang/awstats-fr.txt index e24a87f7..c5103387 100644 --- a/wwwroot/cgi-bin/lang/awstats-fr.txt +++ b/wwwroot/cgi-bin/lang/awstats-fr.txt @@ -159,3 +159,7 @@ message155=Cluster message156=Les robots présentés ici sont à l'origine de hits ou de traffic "non vus" par les visiteurs donc non représentés dans les autres tableaux. message157=Les nombres après le + indiquent les hits avec succès sur les fichiers "robots.txt". message158=Les vers présentés ici sont à l'origine de hits ou de traffic "non vus" par les visiteurs donc non représentés dans les autres tableaux. +message159=Le trafic 'non vu' est le trafic généré par les robots, vers ou réponses avec un code retour HTTP spécial. +message160=Trafic 'vu' +message161=Trafic 'non vu' +message162=Historique mensuel -- 2.47.3
LogFile="/pathtomediaserver/mediaserverlog" +
LogType=S
LogFormat="c-ip date time cs-uri-stem c-starttime x-duration c-rate c-status c-playerid c-playerversion c-playerlanguage cs(User-Agent) cs(Referer) c-hostexe c-hostexever c-os c-osversion c-cpu filelength filesize avgbandwidth protocol transport audiocodec videocodec channelURL sc-bytes"
DecodeUA=1
ShowMenu=1 diff --git a/wwwroot/cgi-bin/awstats.model.conf b/wwwroot/cgi-bin/awstats.model.conf index c001bd80..9545872d 100644 --- a/wwwroot/cgi-bin/awstats.model.conf +++ b/wwwroot/cgi-bin/awstats.model.conf @@ -54,6 +54,7 @@ LogFile="/var/log/httpd/mylog.log" # Enter the log file type you want to analyze. # Possible values: # W - For a web log file +# S - For a streaming log file # M - For a mail log file # F - For a ftp log file # Example: W diff --git a/wwwroot/cgi-bin/awstats.pl b/wwwroot/cgi-bin/awstats.pl index 7e5f6571..c089b4d0 100644 --- a/wwwroot/cgi-bin/awstats.pl +++ b/wwwroot/cgi-bin/awstats.pl @@ -51,7 +51,9 @@ $LastLine $LastLineNumber $LastLineOffset $LastLineChecksum $LastUpdate $lowerval $PluginMode $TotalUnique $TotalVisits $TotalHostsKnown $TotalHostsUnknown -$TotalPages $TotalHits $TotalBytes $TotalEntries $TotalExits $TotalBytesPages $TotalDifferentPages +$TotalPages $TotalHits $TotalBytes +$TotalNotViewedPages $TotalNotViewedHits $TotalNotViewedBytes +$TotalEntries $TotalExits $TotalBytesPages $TotalDifferentPages $TotalKeyphrases $TotalKeywords $TotalDifferentKeyphrases $TotalDifferentKeywords $TotalSearchEnginesPages $TotalSearchEnginesHits $TotalRefererPages $TotalRefererHits $TotalDifferentSearchEngines $TotalDifferentReferer $FrameName $Center $FileConfig $FileSuffix $Host $DayRequired $MonthRequired $YearRequired @@ -69,7 +71,9 @@ $LastLine = $LastLineNumber = $LastLineOffset = $LastLineChecksum = $LastUpdate $lowerval = 0; $PluginMode = ''; $TotalUnique = $TotalVisits = $TotalHostsKnown = $TotalHostsUnknown = 0; -$TotalPages = $TotalHits = $TotalBytes = $TotalEntries = $TotalExits = $TotalBytesPages = $TotalDifferentPages = 0; +$TotalPages = $TotalHits = $TotalBytes = 0; +$TotalNotViewedPages = $TotalNotViewedHits = $TotalNotViewedBytes = 0; +$TotalEntries = $TotalExits = $TotalBytesPages = $TotalDifferentPages = 0; $TotalKeyphrases = $TotalKeywords = $TotalDifferentKeyphrases = $TotalDifferentKeywords = 0; $TotalSearchEnginesPages = $TotalSearchEnginesHits = $TotalRefererPages = $TotalRefererHits = $TotalDifferentSearchEngines = $TotalDifferentReferer = 0; ($FrameName, $Center, $FileConfig, $FileSuffix, $Host, $DayRequired, $MonthRequired, $YearRequired, @@ -210,11 +214,13 @@ use vars qw/ @BrowsersSearchIDOrder @OSSearchIDOrder @WordsToExtractSearchUrl @WordsToCleanSearchUrl @WormsSearchIDOrder @RobotsSearchIDOrder @SearchEnginesSearchIDOrder -@_from_p @_from_h @_time_p @_time_h @_time_k +@_from_p @_from_h +@_time_p @_time_h @_time_k @_time_nv_p @_time_nv_h @_time_nv_k @DOWIndex @fieldlib @keylist /; @RobotsSearchIDOrder = @SearchEnginesSearchIDOrder = (); -@_from_p = @_from_h = @_time_p = @_time_h = @_time_k = (); +@_from_p = @_from_h = (); +@_time_p = @_time_h = @_time_k = @_time_nv_p = @_time_nv_h = @_time_nv_k = (); @DOWIndex = @fieldlib = @keylist = (); use vars qw/ @MiscListOrder %MiscListCalc @@ -291,7 +297,10 @@ use vars qw/ %TmpDNSLookup = %TmpOS = %TmpRefererServer = %TmpRobot = %TmpBrowser = %MyDNSTable = (); use vars qw/ %FirstTime %LastTime -%MonthUnique %MonthVisits %MonthPages %MonthHits %MonthBytes %MonthHostsKnown %MonthHostsUnknown +%MonthHostsKnown %MonthHostsUnknown +%MonthUnique %MonthVisits +%MonthPages %MonthHits %MonthBytes +%MonthNotViewedPages %MonthNotViewedHits %MonthNotViewedBytes %_session %_browser_h %_domener_p %_domener_h %_domener_k %_errors_h %_errors_k %_filetypes_h %_filetypes_k %_filetypes_gz_in %_filetypes_gz_out @@ -477,7 +486,12 @@ use vars qw/ @Message /; 'Codes shown here gave hits or traffic "not viewed" by visitors, so they are not included in other charts.', 'Cluster', 'Robots shown here gave hits or traffic "not viewed" by visitors, so they are not included in other charts.', -'Numbers after + are successful hits on "robots.txt" files' +'Numbers after + are successful hits on "robots.txt" files', +'Worms shown here gave hits or traffic "not viewed" by visitors, so thay are not included in other charts.', +'Not viewed trafic is trafic generated by robots, worms or answers with special HTTP status code', +'Traffic viewed', +'Traffic not viewed', +'Monthly history' ); @@ -607,7 +621,7 @@ sub html_end { } print "\n"; if ($FrameName ne 'index') { - if ($BuildReportFormat eq 'html') { print "
\n"; } + if ($FrameName ne 'mainleft' && $BuildReportFormat eq 'html') { print "
\n"; } print "\n"; } print "\n"; @@ -2110,6 +2124,7 @@ sub Read_History_With_TmpUpdate { # BEGIN_TIME if ($field[0] eq 'BEGIN_TIME') { my $monthpages=0;my $monthhits=0;my $monthbytes=0; + my $monthnotviewedpages=0;my $monthnotviewedhits=0;my $monthnotviewedbytes=0; if ($Debug) { debug(" Begin of TIME section"); } $field[0]=''; my $count=0;my $countloaded=0; @@ -2122,10 +2137,16 @@ sub Read_History_With_TmpUpdate { if ($field[1]) { $_time_p[$field[0]]+=int($field[1]); } if ($field[2]) { $_time_h[$field[0]]+=int($field[2]); } if ($field[3]) { $_time_k[$field[0]]+=int($field[3]); } + if ($field[4]) { $_time_nv_p[$field[0]]+=int($field[4]); } + if ($field[5]) { $_time_nv_h[$field[0]]+=int($field[5]); } + if ($field[6]) { $_time_nv_k[$field[0]]+=int($field[6]); } } $monthpages+=int($field[1]); $monthhits+=int($field[2]); $monthbytes+=int($field[3]); + $monthnotviewedpages+=int($field[4]||0); + $monthnotviewedhits+=int($field[5]||0); + $monthnotviewedbytes+=int($field[6]||0); } } $_=; @@ -2138,10 +2159,13 @@ sub Read_History_With_TmpUpdate { $MonthPages{$year.$month}+=$monthpages; $MonthHits{$year.$month}+=$monthhits; $MonthBytes{$year.$month}+=$monthbytes; + $MonthNotViewedPages{$year.$month}+=$monthnotviewedpages; + $MonthNotViewedHits{$year.$month}+=$monthnotviewedhits; + $MonthNotViewedBytes{$year.$month}+=$monthnotviewedbytes; delete $SectionsToLoad{'time'}; if ($SectionsToSave{'time'}) { Save_History('time',$year,$month); delete $SectionsToSave{'time'}; - if ($withpurge) { @_time_p=(); @_time_h=(); @_time_k=(); } + if ($withpurge) { @_time_p=(); @_time_h=(); @_time_k=(); @_time_nv_p=(); @_time_nv_h=(); @_time_nv_k=(); } } if (! scalar %SectionsToLoad) { debug(" Stop reading history file. Got all we need."); last; } next; @@ -3259,10 +3283,10 @@ sub Save_History { # When if ($sectiontosave eq 'time') { print HISTORYTMP "\n"; - print HISTORYTMP "# Hour - Pages - Hits - Bandwidth\n"; + print HISTORYTMP "# Hour - Pages - Hits - Bandwidth - Not viewed Pages - Not viewed Hits - Not viewed Bandwidth\n"; $ValueInFile{$sectiontosave}=tell HISTORYTMP; print HISTORYTMP "BEGIN_TIME 24\n"; - for (my $ix=0; $ix<=23; $ix++) { print HISTORYTMP "$ix ".int($_time_p[$ix])." ".int($_time_h[$ix])." ".int($_time_k[$ix])."\n"; } + for (my $ix=0; $ix<=23; $ix++) { print HISTORYTMP "$ix ".int($_time_p[$ix])." ".int($_time_h[$ix])." ".int($_time_k[$ix])." ".int($_time_nv_p[$ix])." ".int($_time_nv_h[$ix])." ".int($_time_nv_k[$ix])."\n"; } print HISTORYTMP "END_TIME\n"; } if ($sectiontosave eq 'day') { # This section must be saved after VISITOR section is read @@ -3916,13 +3940,14 @@ sub Init_HashArray { if ($Debug) { debug("Call to Init_HashArray"); } # Reset global hash arrays %FirstTime = %LastTime = (); + %MonthHostsKnown = %MonthHostsUnknown = (); %MonthVisits = %MonthUnique = (); %MonthPages = %MonthHits = %MonthBytes = (); - %MonthHostsKnown = %MonthHostsUnknown = (); + %MonthNotViewedPages = %MonthNotViewedHits = %MonthNotViewedBytes = (); %DayPages = %DayHits = %DayBytes = %DayVisits = (); # Reset all arrays with name beginning by _ for (my $ix=0; $ix<6; $ix++) { $_from_p[$ix]=0; $_from_h[$ix]=0; } - for (my $ix=0; $ix<24; $ix++) { $_time_h[$ix]=0; $_time_k[$ix]=0; $_time_p[$ix]=0; } + for (my $ix=0; $ix<24; $ix++) { $_time_h[$ix]=0; $_time_k[$ix]=0; $_time_p[$ix]=0; $_time_nv_h[$ix]=0; $_time_nv_k[$ix]=0; $_time_nv_p[$ix]=0; } # Reset all hash arrays with name beginning by _ %_session = %_browser_h = (); %_domener_p = %_domener_h = %_domener_k = %_errors_h = %_errors_k = (); @@ -5710,10 +5735,11 @@ if ($UpdateStats && $FrameName ne 'index' && $FrameName ne 'mainleft') { # Updat # Now @dateparts is (DD,MM,YYYY,HH,MM,SS) and we're going to create $timerecord=YYYYMMDDHHMMSS # Plugin call : Convert a @datepart into another @datepart if ($PluginsLoaded{'ChangeTime'}{'timezone'}) { @dateparts=ChangeTime_timezone(\@dateparts); } - my $yearmonthdayrecord=sprintf("$dateparts[2]%02i%02i",$dateparts[1],$dateparts[0]); - my $timerecord=((int("$yearmonthdayrecord")*100+$dateparts[3])*100+$dateparts[4])*100+$dateparts[5]; my $yearrecord=int($dateparts[2]); my $monthrecord=int($dateparts[1]); + my $hourrecord=int($dateparts[3]); + my $yearmonthdayrecord=sprintf("$dateparts[2]%02i%02i",$dateparts[1],$dateparts[0]); + my $timerecord=((int("$yearmonthdayrecord")*100+$dateparts[3])*100+$dateparts[4])*100+$dateparts[5]; # Check date #----------------------- @@ -5799,13 +5825,89 @@ if ($UpdateStats && $FrameName ne 'index' && $FrameName ne 'mainleft') { # Updat # Convert $field[$pos_size] # if ($field[$pos_size] eq '-') { $field[$pos_size]=0; } - # Check misc tracker (must be before return code) - #------------------------------------------------ - if ($field[$pos_url] =~ /$MiscTrackerUrl/o) { + # Define a clean target URL and referrer URL + # We keep a clean $field[$pos_url] and + # we store original value for urlwithnoquery, tokenquery and standalonequery + #--------------------------------------------------------------------------- + if ($URLNotCaseSensitive) { $field[$pos_url]=lc($field[$pos_url]); } + # Possible URL syntax for $field[$pos_url]: /mydir/mypage.ext?param1=x¶m2=y#aaa, /mydir/mypage.ext#aaa, / + my $urlwithnoquery; my $tokenquery; my $standalonequery; my $anchor=''; + if ($field[$pos_url] =~ s/$regtruncanchor//o) { $anchor=$1; } # Remove and save anchor + if ($URLWithQuery) { + $urlwithnoquery=$field[$pos_url]; + my $foundparam=($urlwithnoquery =~ s/$regtruncurl//o); + $tokenquery=$1||''; + $standalonequery=$2||''; + # For IIS setup, if pos_query is enabled we need to combine the URL to query strings + if (! $foundparam && $pos_query >=0 && $field[$pos_query] && $field[$pos_query] ne '-') { + $foundparam=1; + $tokenquery='?'; + $standalonequery=$field[$pos_query]; + # Define query + $field[$pos_url] .= '?'.$field[$pos_query]; + } + if ($foundparam) { + # Keep only params that are defined in URLWithQueryWithOnlyFollowingParameters + my $newstandalonequery=''; + if (@URLWithQueryWithOnly) { + foreach (@URLWithQueryWithOnly) { + foreach my $p (split(/&/,$standalonequery)) { + if ($URLNotCaseSensitive) { if ($p =~ /^$_=/i) { $newstandalonequery.="$p&"; last; } } + else { if ($p =~ /^$_=/) { $newstandalonequery.="$p&"; last; } } + } + } + chop $newstandalonequery; + } + # Remove params that are marked to be ignored in URLWithQueryWithoutFollowingParameters + elsif (@URLWithQueryWithout) { + foreach my $p (split(/&/,$standalonequery)) { + my $found=0; + foreach (@URLWithQueryWithout) { + #debug(" Check if '$_=' is param '$p' to remove it from query",5); + if ($URLNotCaseSensitive) { if ($p =~ /^$_=/i) { $found=1; last; } } + else { if ($p =~ /^$_=/) { $found=1; last; } } + } + if (! $found) { $newstandalonequery.="$p&"; } + } + chop $newstandalonequery; + } + else { $newstandalonequery=$standalonequery; } + # Define query + $field[$pos_url]=$urlwithnoquery; + if ($newstandalonequery) { $field[$pos_url].="$tokenquery$newstandalonequery"; } + } + } + else { + # Trunc parameters of URL + $field[$pos_url] =~ s/$regtruncurl//o; + $urlwithnoquery=$field[$pos_url]; + $tokenquery=$1||''; + $standalonequery=$2||''; + } + if ($URLWithAnchor && $anchor) { $field[$pos_url].="#$anchor"; } # Restore anchor + # Here now urlwithnoquery is /mydir/mypage.ext, /mydir, /, /page#XXX + # Here now tokenquery is '' or '?' or ';' + # Here now standalonequery is '' or 'param1=x' + + # Define page and extension + #-------------------------- + my $PageBool=1; + # Extension + my $extension; + if ($urlwithnoquery =~ /$regext/o || ($urlwithnoquery =~ /[\\\/]$/ && $DefaultFile[0] =~ /$regext/o)) { + $extension=($LevelForFileTypesDetection>=2 || $MimeHashFamily{$1})?lc($1):'Unknown'; + if ($NotPageList{$extension}) { $PageBool=0; } + } + else { + $extension='Unknown'; + } + + # Analyze: misc tracker (must be before return code) + #--------------------------------------------------- + if ($urlwithnoquery =~ /$MiscTrackerUrl/o) { my $query=$field[$pos_url]; - if ($pos_query >=0 && $field[$pos_query]) { $query=$field[$pos_query]; } # For this fucking IIS in pos_query mode my $foundparam=0; - foreach (split(/&/,$query)) { + foreach (split(/&/,$standalonequery)) { if ($_ =~ /SCREEN=(\d+)x(\d+)/) { $foundparam++; $_screensize_h{"$1x$2"}++; next; } # if ($_ =~ /CDI=(\d+)/) { $foundparam++; $_screendepth_h{"$1"}++; next; } if ($_ =~ /JAVA=(\w+)/) { $foundparam++; if ($1 eq 'true') { $_misc_h{"JavaEnabled"}++; } next; } @@ -5818,10 +5920,11 @@ if ($UpdateStats && $FrameName ne 'index' && $FrameName ne 'mainleft') { # Updat } if ($foundparam) { $_misc_h{"TotalMisc"}++; } } - # Check favicon - #----------------------------------------------- - elsif ($field[$pos_url] =~ /\/favicon\.ico$/i) { - if (($field[$pos_code] != 404 || $field[$pos_url] !~ /\/.+\/favicon\.ico$/i) && ($field[$pos_agent] =~ /MSIE/)) { + + # Analyze: favicon + #----------------- + if ($urlwithnoquery =~ /\/favicon\.ico$/i) { + if (($field[$pos_code] != 404 || $urlwithnoquery !~ /\/.+\/favicon\.ico$/i) && ($field[$pos_agent] =~ /MSIE/)) { # We don't count one hit if (not on root and error) and MSIE # If error not on root, another hit will be made on root. If not MSIE, hit are made not only for "Adding". $_misc_h{'AddToFavourites'}++; # Hit on favicon on root or without error, we count it @@ -5834,18 +5937,22 @@ if ($UpdateStats && $FrameName ne 'index' && $FrameName ne 'mainleft') { # Updat if ($LevelForWormsDetection) { foreach (@WormsSearchIDOrder) { if ($field[$pos_url] =~ /$_/) { + # It's a worm my $worm=$WormsHashID{$_}; $_worm_h{$worm}++; $_worm_k{$worm}+=int($field[$pos_size]); $_worm_l{$worm}=$timerecord; $countedtraffic=1; + if ($PageBool) { $_time_nv_p[$hourrecord]++; } + $_time_nv_h[$hourrecord]++; + $_time_nv_k[$hourrecord]+=int($field[$pos_size]); last; } } } - # Check return status code - #------------------------- + # Analyze: Status code + #--------------------- if (! $countedtraffic) { if ($LogType eq 'W' || $LogType eq 'S') { # HTTP record or Stream record if ($ValidHTTPCodes{$field[$pos_code]}) { # Code is valid @@ -5868,13 +5975,20 @@ if ($UpdateStats && $FrameName ne 'index' && $FrameName ne 'mainleft') { # Updat } if ($Debug) { debug(" Record stored in the status code chart (status code=$field[$pos_code])",2); } $countedtraffic=1; + if ($PageBool) { $_time_nv_p[$hourrecord]++; } + $_time_nv_h[$hourrecord]++; + $_time_nv_k[$hourrecord]+=int($field[$pos_size]); } } elsif ($LogType eq 'M') { # Mail record if (! $ValidSMTPCodes{$field[$pos_code]}) { # Code is not valid $_errors_h{$field[$pos_code]}++; #$_errors_k{$field[$pos_code]}+=int($field[$pos_size]); # Useless since pos_size is often 0 or ? when error + if ($Debug) { debug(" Record stored in the status code chart (status code=$field[$pos_code])",2); } $countedtraffic=1; + if ($PageBool) { $_time_nv_p[$hourrecord]++; } + $_time_nv_h[$hourrecord]++; + $_time_nv_k[$hourrecord]+=int($field[$pos_size]); } } elsif ($LogType eq 'F') { # FTP record @@ -5909,99 +6023,28 @@ if ($UpdateStats && $FrameName ne 'index' && $FrameName ne 'mainleft') { # Updat $_robot_h{$uarobot}++; $_robot_k{$uarobot}+=int($field[$pos_size]); $_robot_l{$uarobot}=$timerecord; - if ($field[$pos_url] =~ /$regrobot/o) { $_robot_r{$uarobot}++; } + if ($urlwithnoquery =~ /$regrobot/o) { $_robot_r{$uarobot}++; } $countedtraffic=1; + if ($PageBool) { $_time_nv_p[$hourrecord]++; } + $_time_nv_h[$hourrecord]++; + $_time_nv_k[$hourrecord]+=int($field[$pos_size]); } } } # It's not a known robot or robot detection disabled. Check if hit on robots.txt file - if ($field[$pos_url] =~ /$regrobot/o) { + if ($urlwithnoquery =~ /$regrobot/o) { if ($Debug) { debug(" It's an unknown robot",2); } $_robot_h{'unknown'}++; $_robot_k{'unknown'}+=int($field[$pos_size]); $_robot_l{'unknown'}=$timerecord; $_robot_r{'unknown'}++; $countedtraffic=1; + if ($PageBool) { $_time_nv_p[$hourrecord]++; } + $_time_nv_h[$hourrecord]++; + $_time_nv_k[$hourrecord]+=int($field[$pos_size]); } } - # Define a clean target URL and referrer URL - # We keep a clean $field[$pos_url] and - # we store original value for urlwithnoquery, tokenquery and standalonequery - #--------------------------------------------------------------------------- - if ($URLNotCaseSensitive) { $field[$pos_url]=lc($field[$pos_url]); } - # Possible URL syntax for $field[$pos_url]: /mydir/mypage.ext?param1=x¶m2=y#aaa, /mydir/mypage.ext#aaa, / - my $urlwithnoquery; my $tokenquery; my $standalonequery; my $anchor=''; - if ($field[$pos_url] =~ s/$regtruncanchor//o) { $anchor=$1; } # Remove and save anchor - if ($URLWithQuery) { - $urlwithnoquery=$field[$pos_url]; - my $foundparam=($urlwithnoquery =~ s/$regtruncurl//o); - $tokenquery=$1||''; - $standalonequery=$2||''; - # For IIS setup, if pos_query is enabled we need to combine the URL to query strings - if (! $foundparam && $pos_query >=0 && $field[$pos_query] && $field[$pos_query] ne '-') { - $foundparam=1; - $tokenquery='?'; - $standalonequery=$field[$pos_query]; - # Define query - $field[$pos_url] .= '?'.$field[$pos_query]; - } - if ($foundparam) { - # Keep only params that are defined in URLWithQueryWithOnlyFollowingParameters - my $newstandalonequery=''; - if (@URLWithQueryWithOnly) { - foreach (@URLWithQueryWithOnly) { - foreach my $p (split(/&/,$standalonequery)) { - if ($URLNotCaseSensitive) { if ($p =~ /^$_=/i) { $newstandalonequery.="$p&"; last; } } - else { if ($p =~ /^$_=/) { $newstandalonequery.="$p&"; last; } } - } - } - chop $newstandalonequery; - } - # Remove params that are marked to be ignored in URLWithQueryWithoutFollowingParameters - elsif (@URLWithQueryWithout) { - foreach my $p (split(/&/,$standalonequery)) { - my $found=0; - foreach (@URLWithQueryWithout) { - #debug(" Check if '$_=' is param '$p' to remove it from query",5); - if ($URLNotCaseSensitive) { if ($p =~ /^$_=/i) { $found=1; last; } } - else { if ($p =~ /^$_=/) { $found=1; last; } } - } - if (! $found) { $newstandalonequery.="$p&"; } - } - chop $newstandalonequery; - } - else { $newstandalonequery=$standalonequery; } - # Define query - $field[$pos_url]=$urlwithnoquery; - if ($newstandalonequery) { $field[$pos_url].="$tokenquery$newstandalonequery"; } - } - } - else { - # Trunc parameters of URL - $field[$pos_url] =~ s/$regtruncurl//o; - $urlwithnoquery=$field[$pos_url]; - $tokenquery=$1||''; - $standalonequery=$2||''; - } - if ($URLWithAnchor && $anchor) { $field[$pos_url].="#$anchor"; } # Restore anchor - # Here now urlwithnoquery is /mydir/mypage.ext, /mydir, /, /page#XXX - # Here now tokenquery is '' or '?' or ';' - # Here now standalonequery is '' or 'param1=x' - - # Define page and extension - #-------------------------- - my $PageBool=1; - # Extension - my $extension; - if ($urlwithnoquery =~ /$regext/o || ($urlwithnoquery =~ /[\\\/]$/ && $DefaultFile[0] =~ /$regext/o)) { - $extension=($LevelForFileTypesDetection>=2 || $MimeHashFamily{$1})?lc($1):'Unknown'; - if ($NotPageList{$extension}) { $PageBool=0; } - } - else { - $extension='Unknown'; - } - # Analyze: File type - Compression #--------------------------------- if (! $countedtraffic) { @@ -6026,7 +6069,6 @@ if ($UpdateStats && $FrameName ne 'index' && $FrameName ne 'mainleft') { # Updat # Analyze: Date - Hour - Pages - Hits - Kilo #------------------------------------------- - my $hourrecord=int($dateparts[3]); if ($PageBool) { # Replace default page name with / only ('if' is to increase speed when only 1 value in @DefaultFile) if (@DefaultFile > 1) { foreach my $elem (@DefaultFile) { if ($field[$pos_url] =~ s/\/$elem$/\//o) { last; } } } @@ -6035,14 +6077,18 @@ if ($UpdateStats && $FrameName ne 'index' && $FrameName ne 'mainleft') { # Updat $FirstTime{$lastprocessedyearmonth}||=$timerecord; $LastTime{$lastprocessedyearmonth}=$timerecord; $DayPages{$yearmonthdayrecord}++; - $MonthPages{$lastprocessedyearmonth}++; - $_time_p[$hourrecord]++; #Count accesses for hour (page) +# $MonthPages{$lastprocessedyearmonth}++; $_url_p{$field[$pos_url]}++; #Count accesses for page (page) $_url_k{$field[$pos_url]}+=int($field[$pos_size]); + $_time_p[$hourrecord]++; #Count accesses for hour (page) } - $_time_h[$hourrecord]++; $MonthHits{$lastprocessedyearmonth}++; $DayHits{$yearmonthdayrecord}++; #Count accesses for hour (hit) - $_time_k[$hourrecord]+=int($field[$pos_size]); $MonthBytes{$lastprocessedyearmonth}+=int($field[$pos_size]); $DayBytes{$yearmonthdayrecord}+=int($field[$pos_size]); #Count accesses for hour (kb) - + $_time_h[$hourrecord]++; + $_time_k[$hourrecord]+=int($field[$pos_size]); + $DayHits{$yearmonthdayrecord}++; #Count accesses for hour (hit) + $DayBytes{$yearmonthdayrecord}+=int($field[$pos_size]); #Count accesses for hour (kb) +# $MonthHits{$lastprocessedyearmonth}++; +# $MonthBytes{$lastprocessedyearmonth}+=int($field[$pos_size]); + # Analyze: Login #--------------- if ($pos_logname>=0 && $field[$pos_logname] && $field[$pos_logname] ne '-') { @@ -6051,9 +6097,7 @@ if ($UpdateStats && $FrameName ne 'index' && $FrameName ne 'mainleft') { # Updat if ($AuthenticatedUsersNotCaseSensitive) { $field[$pos_logname]=lc($field[$pos_logname]); } # We found an authenticated user - if ($PageBool) { - $_login_p{$field[$pos_logname]}++; #Count accesses for page (page) - } + if ($PageBool) { $_login_p{$field[$pos_logname]}++; } #Count accesses for page (page) $_login_h{$field[$pos_logname]}++; #Count accesses for page (hit) $_login_k{$field[$pos_logname]}+=int($field[$pos_size]); #Count accesses for page (kb) $_login_l{$field[$pos_logname]}=$timerecord; @@ -6518,9 +6562,7 @@ if ($UpdateStats && $FrameName ne 'index' && $FrameName ne 'mainleft') { # Updat # Check cluster #-------------- if ($pos_cluster>=0) { - if ($PageBool) { - $_cluster_p{$field[$pos_cluster]}++; #Count accesses for page (page) - } + if ($PageBool) { $_cluster_p{$field[$pos_cluster]}++; } #Count accesses for page (page) $_cluster_h{$field[$pos_cluster]}++; #Count accesses for page (hit) $_cluster_k{$field[$pos_cluster]}+=int($field[$pos_size]); #Count accesses for page (kb) } @@ -6754,12 +6796,6 @@ if (scalar keys %HTMLOutput) { $NewLinkParams =~ tr/&/&/s; $NewLinkParams =~ s/^&//; $NewLinkParams =~ s/&$//; if ($NewLinkParams) { $NewLinkParams="${NewLinkParams}&"; } - # Call to plugins' function AddHTMLBodyHeader - foreach my $pluginname (keys %{$PluginsLoaded{'AddHTMLBodyHeader'}}) { - my $function="AddHTMLBodyHeader_$pluginname()"; - eval("$function"); - } - if ($FrameName ne 'mainleft') { # READING DATA @@ -6785,6 +6821,12 @@ if (scalar keys %HTMLOutput) { print "\n"; } + # Call to plugins' function AddHTMLBodyHeader + foreach my $pluginname (keys %{$PluginsLoaded{'AddHTMLBodyHeader'}}) { + my $function="AddHTMLBodyHeader_$pluginname()"; + eval("$function"); + } + # MENU #--------------------------------------------------------------------- if ($ShowMenu || $FrameName eq 'mainleft') { @@ -6930,11 +6972,12 @@ if (scalar keys %HTMLOutput) { my $menuicon=0; # Menu HTML print "\n"; + if ($ShowMonthStats) { print ($frame?"
":""); print "$Message[128]"; print ($frame?"
".($menuicon?" ":"")."$Message[93]:
"); } - if ($ShowMonthStats) { print ($frame?"
":""); print "$Message[128]"; print ($frame?"
":""); print "$Message[162]"; print ($frame?"
  \"...\" ":""); print "$Message[130]\n"; print ($frame?"
":""); print "$Message[138]"; print ($frame?"
":""); print "$Message[91]"; print ($frame?"
$Message[8]
 $Message[8]$Message[128]$Message[9]
".Format_Date($FirstTime,0)."
NA
 ".Format_Date($FirstTime,0)."NA"; print ($MonthRequired eq 'all'?"$Message[6] $YearRequired":"$Message[5] ".$MonthNumLib{$MonthRequired}." $YearRequired"); print "NA
$Message[11] $Message[10] $Message[56] $Message[57] $Message[75]  $Message[11] $Message[10] $Message[56] $Message[57] $Message[75] 
$Message[160] *".($MonthRequired eq 'all'?"<= $TotalUnique
$Message[129]":"$TotalUnique
 ")."
 $TotalVisits
($RatioVisits $Message[52])
 $TotalPages
($RatioPages ".lc($Message[56]."/".$Message[12]).")
 $TotalHits".($LogType eq 'M'?"":"
($RatioHits ".lc($Message[57]."/".$Message[12]).")
 ".Format_Bytes(int($TotalBytes))."
($RatioBytes $Message[108]/".lc($Message[($LogType eq 'M'?149:12)]).")
 
$Message[161] * 
 
$TotalNotViewedPages $TotalNotViewedHits ".Format_Bytes(int($TotalNotViewedBytes))." 
"; + # BY MONTH + #--------------------------------------------------------------------- + if ($ShowMonthStats) { + + if ($Debug) { debug("ShowMonthStats",2); } + print "$Center 
\n"; + my $title="$Message[162]"; + &tab_head("$title",0,0,'month'); + print "
\n"; + print "
\n"; $average_nb=$average_u=$average_v=$average_p=$average_h=$average_k=0; $total_u=$total_v=$total_p=$total_h=$total_k=0; - print "
\n"; $max_v=$max_p=$max_h=$max_k=1; # Define total and max @@ -8343,6 +8417,7 @@ if (scalar keys %HTMLOutput) { print "