2 * SARG Squid Analysis Report Generator http://sarg.sourceforge.net
6 * please look at http://sarg.sourceforge.net/donations.php
8 * http://sourceforge.net/projects/sarg/forums/forum/363374
9 * ---------------------------------------------------------------------
11 * This program is free software; you can redistribute it and/or modify
12 * it under the terms of the GNU General Public License as published by
13 * the Free Software Foundation; either version 2 of the License, or
14 * (at your option) any later version.
16 * This program is distributed in the hope that it will be useful,
17 * but WITHOUT ANY WARRANTY; without even the implied warranty of
18 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
19 * GNU General Public License for more details.
21 * You should have received a copy of the GNU General Public License
22 * along with this program; if not, write to the Free Software
23 * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111, USA.
27 #include "include/conf.h"
28 #include "include/defs.h"
29 #include "include/readlog.h"
30 #include "include/filelist.h"
32 #define REPORT_EVERY_X_LINES 5000
33 #define MAX_OPEN_USER_FILES 10
37 struct userfilestruct
*next
;
38 struct userinfostruct
*user
;
42 enum ExcludeReasonEnum
44 //! User name too long.
46 //! Squid logged an incomplete query received from the client.
48 //! Log file turned over.
50 //! Excluded by exclude_string from sarg.conf.
52 //! Unknown input log file format.
54 //! Line to be ignored from the input log file.
56 //! Entry not withing the requested date range.
62 //! User is not in the include_users list.
64 //! HTTP code excluded by exclude_code file.
66 //! Invalid character found in user name.
70 //! Not the IP address requested with -a.
72 //! URL excluded by -c or exclude_hosts.
74 //! Entry time outside of requested hour range.
76 //! Not the URL requested by -s.
80 //! Not the user requested by -u.
84 //! User ignored by exclude_users
87 ER_Last
//!< last entry of the list
90 numlist weekdays
= { { 0, 1, 2, 3, 4, 5, 6 }, 7 };
91 numlist hours
= { { 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23 }, 24 };
93 extern char *userfile
;
94 extern FileListObject AccessLog
;
96 extern const struct ReadLogProcessStruct ReadSquidLog
;
97 extern const struct ReadLogProcessStruct ReadCommonLog
;
98 extern const struct ReadLogProcessStruct ReadSargLog
;
99 extern const struct ReadLogProcessStruct ReadExtLog
;
101 //! The list of the supported log formats.
102 static const struct ReadLogProcessStruct
const *LogFormats
[]=
110 //! The path to the sarg log file.
111 static char SargLogFile
[4096]="";
112 //! Handle to the sarg log file. NULL if not created.
113 static FILE *fp_log
=NULL
;
114 //! The number of records read from the input logs.
115 static long int totregsl
=0;
116 //! The number of records kept.
117 static long int totregsg
=0;
118 //! The number of records excluded.
119 static long int totregsx
=0;
120 //! The beginning of a linked list of user's file.
121 static struct userfilestruct
*first_user_file
=NULL
;
122 //! Count the number of occurence of each input log format.
123 static unsigned long int format_count
[sizeof(LogFormats
)/sizeof(*LogFormats
)];
124 //! The minimum date found in the input logs.
125 static int mindate
=0;
126 static int maxdate
=0;
127 //! Count the number of excluded records.
128 static unsigned long int excluded_count
[ER_Last
];
129 //! Earliest date found in the log.
130 static int EarliestDate
=-1;
131 //! The earliest date in time format.
132 static struct tm EarliestDateTime
;
133 //! Latest date found in the log.
134 static int LatestDate
=-1;
135 //! The latest date in time format.
136 static struct tm LatestDateTime
;
139 * Initialize the memory structure needed by LogLine_Parse() to parse
142 * \param log_line The structure to initialize.
144 void LogLine_Init(struct LogLineStruct
*log_line
)
146 log_line
->current_format
=NULL
;
147 log_line
->current_format_idx
=-1;
148 log_line
->file_name
="";
149 log_line
->successive_errors
=0;
150 log_line
->total_errors
=0;
154 * Set the name of the log file being parsed.
156 * \param log_line Data structure to parse the log line.
157 * \param file_name The name of the log file being read.
159 void LogLine_File(struct LogLineStruct
*log_line
,const char *file_name
)
161 log_line
->file_name
=file_name
;
165 * Parse the next line from a log file.
167 * \param log_line A buffer to store the data about the current parsing.
168 * \param log_entry The variable to store the parsed data.
169 * \param linebuf The text line read from the log file.
173 enum ReadLogReturnCodeEnum
LogLine_Parse(struct LogLineStruct
*log_line
,struct ReadLogStruct
*log_entry
,char *linebuf
)
175 enum ReadLogReturnCodeEnum log_entry_status
=RLRC_Unknown
;
178 if (log_line
->current_format
)
180 memset(log_entry
,0,sizeof(*log_entry
));
181 log_entry_status
=log_line
->current_format
->ReadEntry(linebuf
,log_entry
);
184 // find out what line format to use
185 if (log_entry_status
==RLRC_Unknown
)
187 for (x
=0 ; x
<(int)(sizeof(LogFormats
)/sizeof(*LogFormats
)) ; x
++)
189 if (LogFormats
[x
]==log_line
->current_format
) continue;
190 memset(log_entry
,0,sizeof(*log_entry
));
191 log_entry_status
=LogFormats
[x
]->ReadEntry(linebuf
,log_entry
);
192 if (log_entry_status
!=RLRC_Unknown
)
194 log_line
->current_format
=LogFormats
[x
];
195 log_line
->current_format_idx
=x
;
196 if (debugz
>=LogLevel_Process
)
198 /* TRANSLATORS: The argument is the log format name as translated by you. */
199 debuga(_("Log format identified as \"%s\" for %s\n"),_(log_line
->current_format
->Name
),log_line
->file_name
);
204 if (x
>=(int)(sizeof(LogFormats
)/sizeof(*LogFormats
)))
206 if (++log_line
->successive_errors
>NumLogSuccessiveErrors
) {
207 debuga(ngettext("%d consecutive error found in the input log file %s\n",
208 "%d consecutive errors found in the input log file %s\n",log_line
->successive_errors
),log_line
->successive_errors
,log_line
->file_name
);
211 if (NumLogTotalErrors
>=0 && ++log_line
->total_errors
>NumLogTotalErrors
) {
212 debuga(ngettext("%d error found in the input log file (last in %s)\n",
213 "%d errors found in the input log file (last in %s)\n",log_line
->total_errors
),log_line
->total_errors
,log_line
->file_name
);
216 debuga(_("The following line read from %s could not be parsed and is ignored\n%s\n"),log_line
->file_name
,linebuf
);
219 log_line
->successive_errors
=0;
222 if (log_line
->current_format_idx
<0 || log_line
->current_format
==NULL
) {
223 debuga(_("Sarg failed to determine the format of the input log file %s\n"),log_line
->file_name
);
226 if (log_entry_status
==RLRC_InternalError
) {
227 debuga(_("Internal error encountered while processing %s\nSee previous message to know the reason for that error.\n"),log_line
->file_name
);
230 return(log_entry_status
);
234 Read a single log file.
236 \param arq The log file name to read.
238 static void ReadOneLogFile(struct ReadLogDataStruct
*Filter
,const char *arq
)
246 char tmp3
[MAXLEN
]="";
247 char download_url
[MAXLEN
];
248 char smartfilter
[MAXLEN
];
251 int OutputNonZero
= REPORT_EVERY_X_LINES
;
256 int maxopenfiles
=MAX_OPEN_USER_FILES
;
257 unsigned long int recs1
=0UL;
258 unsigned long int recs2
=0UL;
262 bool download_flag
=false;
264 enum ReadLogReturnCodeEnum log_entry_status
;
266 struct getwordstruct gwarea
;
267 struct userfilestruct
*prev_ufile
;
268 struct userinfostruct
*uinfo
;
269 struct userfilestruct
*ufile
;
270 struct userfilestruct
*ufile1
;
271 struct ReadLogStruct log_entry
;
272 struct LogLineStruct log_line
;
274 LogLine_Init(&log_line
);
275 LogLine_File(&log_line
,arq
);
276 for (x
=0 ; x
<sizeof(LogFormats
)/sizeof(*LogFormats
) ; x
++)
277 if (LogFormats
[x
]->NewFile
)
278 LogFormats
[x
]->NewFile(arq
);
280 if (arq
[0]=='-' && arq
[1]=='\0') {
282 debuga(_("Reading access log file: from stdin\n"));
286 if (Filter
->DateRange
[0]!='\0') {
287 if (stat(arq
,&logstat
)!=0) {
288 debuga(_("Cannot get the modification time of input log file %s (%s). Processing it anyway\n"),arq
,strerror(errno
));
290 struct tm
*logtime
=localtime(&logstat
.st_mtime
);
291 if ((logtime
->tm_year
+1900)*10000+(logtime
->tm_mon
+1)*100+logtime
->tm_mday
<dfrom
) {
292 debuga(_("Ignoring old log file %s\n"),arq
);
297 fp_in
=decomp(arq
,&from_pipe
);
299 debuga(_("Cannot open input log file \"%s\": %s\n"),arq
,strerror(errno
));
302 if(debug
) debuga(_("Reading access log file: %s\n"),arq
);
311 // pre-read the file only if we have to show stats
312 if (ShowReadStatistics
&& ShowReadPercent
&& !from_stdin
&& !from_pipe
) {
317 while ((nread
=fread(tmp4
,1,sizeof(tmp4
),fp_in
))>0) {
318 for (i
=0 ; i
<nread
; i
++)
320 if (tmp4
[i
]!='\n' && tmp4
[i
]!='\r') {
324 if (tmp4
[i
]=='\n' || tmp4
[i
]=='\r') {
331 printf(_("SARG: Records in file: %lu, reading: %3.2f%%"),recs1
,(float) 0);
336 if ((line
=longline_create())==NULL
) {
337 debuga(_("Not enough memory to read log file %s\n"),arq
);
341 while ((linebuf
=longline_read(fp_in
,line
))!=NULL
) {
345 if (ShowReadStatistics
&& --OutputNonZero
<=0) {
347 double perc
= recs2
* 100. / recs1
;
348 printf(_("SARG: Records in file: %lu, reading: %3.2lf%%"),recs2
,perc
);
350 printf(_("SARG: Records in file: %lu"),recs2
);
354 OutputNonZero
= REPORT_EVERY_X_LINES
;
358 The following checks are retained here as I don't know to
359 what format they apply. They date back to pre 2.4 versions.
361 //if(blen < 58) continue; //this test conflict with the reading of the sarg log header line
362 if(strstr(linebuf
,"HTTP/0.0") != 0) {//recorded by squid when encountering an incomplete query
363 excluded_count
[ER_IncompleteQuery
]++;
366 if(strstr(linebuf
,"logfile turned over") != 0) {//reported by newsyslog
367 excluded_count
[ER_LogfileTurnedOver
]++;
372 if(ExcludeString
[0] != '\0') {
374 getword_start(&gwarea
,ExcludeString
);
375 while(strchr(gwarea
.current
,':') != 0) {
376 if (getword_multisep(val1
,sizeof(val1
),&gwarea
,':')<0) {
377 debuga(_("Maybe you have a broken record or garbage in your exclusion string\n"));
380 if((str
=(char *) strstr(linebuf
,val1
)) != (char *) NULL
) {
385 if(!exstring
&& (str
=(char *) strstr(linebuf
,gwarea
.current
)) != (char *) NULL
)
388 excluded_count
[ER_ExcludeString
]++;
394 if (debugz
>=LogLevel_Data
)
395 printf("BUF=%s\n",linebuf
);
398 log_entry_status
=LogLine_Parse(&log_line
,&log_entry
,linebuf
);
399 if (log_entry_status
==RLRC_Unknown
)
401 excluded_count
[ER_UnknownFormat
]++;
404 if (log_entry_status
==RLRC_Ignore
) {
405 excluded_count
[ER_FormatData
]++;
408 format_count
[log_line
.current_format_idx
]++;
410 if (!fp_log
&& ParsedOutputLog
[0] && log_line
.current_format
!=&ReadSargLog
) {
411 if(access(ParsedOutputLog
,R_OK
) != 0) {
412 my_mkdir(ParsedOutputLog
);
414 if (snprintf(SargLogFile
,sizeof(SargLogFile
),"%s/sarg_temp.log",ParsedOutputLog
)>=sizeof(SargLogFile
)) {
415 debuga(_("Path too long: "));
416 debuga_more("%s/sarg_temp.log\n",ParsedOutputLog
);
419 if((fp_log
=MY_FOPEN(SargLogFile
,"w"))==NULL
) {
420 debuga(_("Cannot open file \"%s\": %s\n"),SargLogFile
,strerror(errno
));
423 fputs("*** SARG Log ***\n",fp_log
);
426 if (log_entry
.Ip
==NULL
) {
427 debuga(_("Unknown input log file format: no IP addresses\n"));
430 if (log_entry
.User
==NULL
) {
431 debuga(_("Unknown input log file format: no user\n"));
434 if (log_entry
.Url
==NULL
) {
435 debuga(_("Unknown input log file format: no URL\n"));
439 idata
=builddia(log_entry
.EntryTime
.tm_mday
,log_entry
.EntryTime
.tm_mon
+1,log_entry
.EntryTime
.tm_year
+1900);
440 if (debugz
>=LogLevel_Data
)
441 printf("DATE=%s IDATA=%d DFROM=%d DUNTIL=%d\n",Filter
->DateRange
,idata
,dfrom
,duntil
);
443 if (EarliestDate
<0 || idata
<EarliestDate
) {
445 memcpy(&EarliestDateTime
,&log_entry
.EntryTime
,sizeof(struct tm
));
447 if (LatestDate
<0 || idata
>LatestDate
) {
449 memcpy(&LatestDateTime
,&log_entry
.EntryTime
,sizeof(struct tm
));
451 if(Filter
->DateRange
[0] != '\0'){
452 if(idata
< dfrom
|| idata
> duntil
) {
453 excluded_count
[ER_OutOfDateRange
]++;
458 // Record only hours usage which is required
459 if( bsearch( &( log_entry
.EntryTime
.tm_wday
), weekdays
.list
, weekdays
.len
, sizeof( int ), compar
) == NULL
) {
460 excluded_count
[ER_OutOfWDayRange
]++;
464 if( bsearch( &( log_entry
.EntryTime
.tm_hour
), hours
.list
, hours
.len
, sizeof( int ), compar
) == NULL
) {
465 excluded_count
[ER_OutOfHourRange
]++;
470 if(strlen(log_entry
.User
) > MAX_USER_LEN
) {
471 if (debugz
>=LogLevel_Process
) debuga(_("User ID too long: %s\n"),log_entry
.User
);
472 excluded_count
[ER_UserNameTooLong
]++;
478 if(IncludeUsers
[0] != '\0') {
479 snprintf(val1
,sizeof(val1
),":%s:",log_entry
.User
);
480 if((str
=(char *) strstr(IncludeUsers
,val1
)) == (char *) NULL
) {
481 excluded_count
[ER_User
]++;
486 if(vercode(log_entry
.HttpCode
)) {
487 if (debugz
>=LogLevel_Process
) debuga(_("Excluded code: %s\n"),log_entry
.HttpCode
);
488 excluded_count
[ER_HttpCode
]++;
493 if(testvaliduserchar(log_entry
.User
)) {
494 excluded_count
[ER_InvalidUserChar
]++;
498 // replace any tab by a single space
499 for (str
=log_entry
.Url
; *str
; str
++)
500 if (*str
=='\t') *str
=' ';
501 for (str
=log_entry
.HttpCode
; *str
; str
++)
502 if (*str
=='\t') *str
=' ';
504 if (log_line
.current_format
!=&ReadSargLog
) {
506 The full URL is not saved in sarg log. There is no point in testing the URL to detect
509 download_flag
=is_download_suffix(log_entry
.Url
);
511 safe_strcpy(download_url
,log_entry
.Url
,sizeof(download_url
));
516 url
=process_url(log_entry
.Url
,LongUrl
);
517 if (!url
|| url
[0] == '\0') {
518 excluded_count
[ER_NoUrl
]++;
523 if(strcmp(addr
,log_entry
.Ip
)!=0) {
524 excluded_count
[ER_UntrackedIpAddr
]++;
528 if(Filter
->HostFilter
) {
529 if(!vhexclude(url
)) {
530 if (debugz
>=LogLevel_Data
) debuga(_("Excluded site: %s\n"),url
);
531 excluded_count
[ER_Url
]++;
537 if(Filter
->StartTime
>= 0 && Filter
->EndTime
>= 0) {
538 hmr
=log_entry
.EntryTime
.tm_hour
*100+log_entry
.EntryTime
.tm_min
;
539 if(hmr
< Filter
->StartTime
|| hmr
> Filter
->EndTime
) {
540 excluded_count
[ER_OutOfTimeRange
]++;
546 if(strstr(url
,site
)==0) {
547 excluded_count
[ER_UntrackedUrl
]++;
553 log_entry
.User
=log_entry
.Ip
;
557 if ((log_entry
.User
[0]=='\0') || (log_entry
.User
[1]=='\0' && (log_entry
.User
[0]=='-' || log_entry
.User
[0]==' '))) {
558 if(RecordsWithoutUser
== RECORDWITHOUTUSER_IP
) {
559 log_entry
.User
=log_entry
.Ip
;
562 if(RecordsWithoutUser
== RECORDWITHOUTUSER_IGNORE
) {
563 excluded_count
[ER_NoUser
]++;
566 if(RecordsWithoutUser
== RECORDWITHOUTUSER_EVERYBODY
)
567 log_entry
.User
="everybody";
569 if(NtlmUserFormat
== NTLMUSERFORMAT_USER
) {
570 if ((str
=strchr(log_entry
.User
,'+'))!=NULL
|| (str
=strchr(log_entry
.User
,'\\'))!=NULL
|| (str
=strchr(log_entry
.User
,'_'))!=NULL
) {
571 log_entry
.User
=str
+1;
578 if(strcmp(log_entry
.User
,us
)!=0) {
579 excluded_count
[ER_UntrackedUser
]++;
584 if(Filter
->SysUsers
) {
585 snprintf(wuser
,sizeof(wuser
),":%s:",log_entry
.User
);
586 if(strstr(userfile
, wuser
) == 0) {
587 excluded_count
[ER_SysUser
]++;
592 if(Filter
->UserFilter
) {
593 if(!vuexclude(log_entry
.User
)) {
594 if (debugz
>=LogLevel_Process
) debuga(_("Excluded user: %s\n"),log_entry
.User
);
595 excluded_count
[ER_IgnoredUser
]++;
601 user
=process_user(log_entry
.User
);
602 if (log_entry
.User
!=user
) {
606 if (log_entry
.User
[0]=='\0' || (log_entry
.User
[1]=='\0' && (log_entry
.User
[0]=='-' ||
607 log_entry
.User
[0]==' ' || log_entry
.User
[0]==':'))) {
608 excluded_count
[ER_NoUser
]++;
612 if (log_entry
.DataSize
<0) log_entry
.DataSize
=0;
614 if (log_entry
.ElapsedTime
<0) log_entry
.ElapsedTime
=0;
615 if (Filter
->max_elapsed
>0 && log_entry
.ElapsedTime
>Filter
->max_elapsed
) {
616 log_entry
.ElapsedTime
=0;
619 if((str
=(char *) strstr(linebuf
, "[SmartFilter:")) != (char *) NULL
) {
621 snprintf(smartfilter
,sizeof(smartfilter
),"\"%s\"",str
+1);
622 } else strcpy(smartfilter
,"\"\"");
626 for (ufile
=first_user_file
; ufile
&& strcmp(log_entry
.User
,ufile
->user
->id
)!=0 ; ufile
=ufile
->next
) {
628 if (ufile
->file
) nopen
++;
631 ufile
=malloc(sizeof(*ufile
));
633 debuga(_("Not enough memory to store the user %s\n"),log_entry
.User
);
636 memset(ufile
,0,sizeof(*ufile
));
637 ufile
->next
=first_user_file
;
638 first_user_file
=ufile
;
640 * This id_is_ip stuff is just to store the string only once if the user is
641 * identified by its IP address instead of a distinct ID and IP address.
643 uinfo
=userinfo_create(log_entry
.User
,(id_is_ip
) ? NULL
: log_entry
.Ip
);
648 prev_ufile
->next
=ufile
->next
;
649 ufile
->next
=first_user_file
;
650 first_user_file
=ufile
;
653 #ifdef ENABLE_DOUBLE_CHECK_DATA
654 if (strcmp(log_entry
.HttpCode
,"TCP_DENIED/407")!=0) {
655 ufile
->user
->nbytes
+=log_entry
.DataSize
;
656 ufile
->user
->elap
+=log_entry
.ElapsedTime
;
660 if (ufile
->file
==NULL
) {
661 if (nopen
>=maxopenfiles
) {
663 for (ufile1
=first_user_file
; ufile1
; ufile1
=ufile1
->next
) {
664 if (ufile1
->file
!=NULL
) {
665 if (x
>=maxopenfiles
) {
666 if (fclose(ufile1
->file
)==EOF
) {
667 debuga(_("Write error in the log file of user %s: %s\n"),ufile1
->user
->id
,strerror(errno
));
676 if (snprintf (tmp3
, sizeof(tmp3
), "%s/%s.user_unsort", tmp
, ufile
->user
->filename
)>=sizeof(tmp3
)) {
677 debuga(_("Temporary user file name too long: %s/%s.user_unsort\n"), tmp
, ufile
->user
->filename
);
680 if ((ufile
->file
= MY_FOPEN (tmp3
, "a")) == NULL
) {
681 debuga(_("(log) Cannot open temporary file %s: %s\n"), tmp3
, strerror(errno
));
686 strftime(dia
, sizeof(dia
), "%d/%m/%Y",&log_entry
.EntryTime
);
687 strftime(hora
,sizeof(hora
),"%H:%M:%S",&log_entry
.EntryTime
);
689 if (fprintf(ufile
->file
, "%s\t%s\t%s\t%s\t%"PRIu64
"\t%s\t%ld\t%s\n",dia
,hora
,
690 log_entry
.Ip
,url
,(uint64_t)log_entry
.DataSize
,
691 log_entry
.HttpCode
,log_entry
.ElapsedTime
,smartfilter
)<=0) {
692 debuga(_("Write error in the log file of user %s\n"),log_entry
.User
);
697 if (fp_log
&& log_line
.current_format
!=&ReadSargLog
) {
698 fprintf(fp_log
, "%s\t%s\t%s\t%s\t%s\t%"PRIu64
"\t%s\t%ld\t%s\n",dia
,hora
,
699 log_entry
.User
,log_entry
.Ip
,url
,(uint64_t)log_entry
.DataSize
,
700 log_entry
.HttpCode
,log_entry
.ElapsedTime
,smartfilter
);
705 denied_write(&log_entry
);
706 authfail_write(&log_entry
);
707 if (download_flag
) download_write(&log_entry
,download_url
);
709 if (log_line
.current_format
!=&ReadSargLog
) {
710 if (period
.start
.tm_year
==0 || idata
<mindate
|| compare_date(&period
.start
,&log_entry
.EntryTime
)>0){
712 memcpy(&period
.start
,&log_entry
.EntryTime
,sizeof(log_entry
.EntryTime
));
714 if (period
.end
.tm_year
==0 || idata
>maxdate
|| compare_date(&period
.end
,&log_entry
.EntryTime
)<0) {
716 memcpy(&period
.end
,&log_entry
.EntryTime
,sizeof(log_entry
.EntryTime
));
720 if (debugz
>=LogLevel_Data
){
721 printf("IP=\t%s\n",log_entry
.Ip
);
722 printf("USER=\t%s\n",log_entry
.User
);
723 printf("ELAP=\t%ld\n",log_entry
.ElapsedTime
);
724 printf("DATE=\t%s\n",dia
);
725 printf("TIME=\t%s\n",hora
);
726 //printf("FUNC=\t%s\n",fun);
727 printf("URL=\t%s\n",url
);
728 printf("CODE=\t%s\n",log_entry
.HttpCode
);
729 printf("LEN=\t%"PRIu64
"\n",(uint64_t)log_entry
.DataSize
);
732 longline_destroy(&line
);
739 if (ShowReadStatistics
) {
741 printf(_("SARG: Records in file: %lu, reading: %3.2f%%\n"),recs2
, (float) 100 );
743 printf(_("SARG: Records in file: %lu\n"),recs2
);
750 * Display a line with the excluded entries count.
752 * \param Explain A translated string explaining the exluded count.
753 * \param Reason The reason number.
755 static void DisplayExcludeCount(const char *Explain
,enum ExcludeReasonEnum Reason
)
757 if (excluded_count
[Reason
]>0) {
758 debuga(" %s: %lu\n",Explain
,excluded_count
[Reason
]);
765 \param Filter The filtering parameters for the file to load.
767 \retval 1 Records found.
768 \retval 0 No record found.
770 int ReadLogFile(struct ReadLogDataStruct
*Filter
)
774 struct userfilestruct
*ufile
;
775 struct userfilestruct
*ufile1
;
776 FileListIterator FIter
;
779 for (x
=0 ; x
<sizeof(format_count
)/sizeof(*format_count
) ; x
++) format_count
[x
]=0;
780 for (x
=0 ; x
<sizeof(excluded_count
)/sizeof(*excluded_count
) ; x
++) excluded_count
[x
]=0;
781 first_user_file
=NULL
;
789 FIter
=FileListIter_Open(AccessLog
);
790 while ((file
=FileListIter_Next(FIter
))!=NULL
)
791 ReadOneLogFile(Filter
,file
);
792 FileListIter_Close(FIter
);
796 char val4
[4096];//val4 must not be bigger than SargLogFile without fixing the strcpy below
798 if (fclose(fp_log
)==EOF
) {
799 debuga(_("Write error in \"%s\": %s\n"),SargLogFile
,strerror(errno
));
802 strftime(val2
,sizeof(val2
),"%d%m%Y_%H%M",&period
.start
);
803 strftime(val1
,sizeof(val1
),"%d%m%Y_%H%M",&period
.end
);
804 if (snprintf(val4
,sizeof(val4
),"%s/sarg-%s-%s.log",ParsedOutputLog
,val2
,val1
)>=sizeof(val4
)) {
805 debuga(_("Path too long: "));
806 debuga_more("%s/sarg-%s-%s.log\n",ParsedOutputLog
,val2
,val1
);
809 if (rename(SargLogFile
,val4
)) {
810 debuga(_("failed to rename %s to %s - %s\n"),SargLogFile
,val4
,strerror(errno
));
812 strcpy(SargLogFile
,val4
);
814 if(strcmp(ParsedOutputLogCompress
,"nocompress") != 0 && ParsedOutputLogCompress
[0] != '\0') {
816 No double quotes around ParsedOutputLogCompress because it may contain command line options. If double quotes are
817 necessary around the command name, put them in the configuration file.
819 if (snprintf(val1
,sizeof(val1
),"%s \"%s\"",ParsedOutputLogCompress
,SargLogFile
)>=sizeof(val1
)) {
820 debuga(_("Command too long: %s \"%s\"\n"),ParsedOutputLogCompress
,SargLogFile
);
823 cstatus
=system(val1
);
824 if (!WIFEXITED(cstatus
) || WEXITSTATUS(cstatus
)) {
825 debuga(_("command return status %d\n"),WEXITSTATUS(cstatus
));
826 debuga(_("command: %s\n"),val1
);
832 debuga(_("Sarg parsed log saved as %s\n"),SargLogFile
);
839 for (ufile
=first_user_file
; ufile
; ufile
=ufile1
) {
841 if (ufile
->file
!=NULL
&& fclose(ufile
->file
)==EOF
) {
842 debuga(_("Write error in the log file of user %s: %s\n"),ufile
->user
->id
,strerror(errno
));
849 unsigned long int totalcount
=0;
851 debuga(_(" Records read: %ld, written: %ld, excluded: %ld\n"),totregsl
,totregsg
,totregsx
);
853 for (x
=sizeof(excluded_count
)/sizeof(*excluded_count
)-1 ; x
>=0 && excluded_count
[x
]>0 ; x
--);
855 debuga(_("Reasons for excluded entries:\n"));
856 DisplayExcludeCount(_("User name too long"),ER_UserNameTooLong
);
857 DisplayExcludeCount(_("Squid logged an incomplete query received from the client"),ER_IncompleteQuery
);
858 DisplayExcludeCount(_("Log file turned over"),ER_LogfileTurnedOver
);
859 DisplayExcludeCount(_("Excluded by \"exclude_string\" in sarg.conf"),ER_ExcludeString
);
860 DisplayExcludeCount(_("Unknown input log file format"),ER_UnknownFormat
);
861 DisplayExcludeCount(_("Line ignored by the input log format"),ER_FormatData
);
862 DisplayExcludeCount(_("Time outside the requested date range (-d)"),ER_OutOfDateRange
);
863 DisplayExcludeCount(_("Ignored week day (\"weekdays\" parameter in sarg.conf)"),ER_OutOfWDayRange
);
864 DisplayExcludeCount(_("Ignored hour (\"hours\" parameter in sarg.conf)"),ER_OutOfHourRange
);
865 DisplayExcludeCount(_("User is not in the \"include_users\" list"),ER_User
);
866 DisplayExcludeCount(_("HTTP code excluded by \"exclude_code\" file"),ER_HttpCode
);
867 DisplayExcludeCount(_("Invalid character found in user name"),ER_InvalidUserChar
);
868 DisplayExcludeCount(_("No URL in entry"),ER_NoUrl
);
869 DisplayExcludeCount(_("Not the IP address requested with -a"),ER_UntrackedIpAddr
);
870 DisplayExcludeCount(_("URL excluded by -c or \"exclude_hosts\""),ER_Url
);
871 DisplayExcludeCount(_("Entry time outside of requested hour range (-t)"),ER_OutOfTimeRange
);
872 DisplayExcludeCount(_("Not the URL requested by -s"),ER_UntrackedUrl
);
873 DisplayExcludeCount(_("No user in entry"),ER_NoUser
);
874 DisplayExcludeCount(_("Not the user requested by -u"),ER_UntrackedUser
);
875 DisplayExcludeCount(_("System user as defined by \"password\" in sarg.conf"),ER_SysUser
);
876 DisplayExcludeCount(_("User ignored by \"exclude_users\""),ER_IgnoredUser
);
879 for (x
=0 ; x
<sizeof(LogFormats
)/sizeof(*LogFormats
) ; x
++) {
880 if (format_count
[x
]>0) {
881 /* TRANSLATORS: It displays the number of lines found in the input log files
882 * for each supported log format. The log format name is the %s and is a string
883 * you translate somewhere else. */
884 debuga(_("%s: %lu entries\n"),_(LogFormats
[x
]->Name
),format_count
[x
]);
885 totalcount
+=format_count
[x
];
889 if (totalcount
==0 && totregsg
)
890 debuga(_("Log with invalid format\n"));
893 return((totregsg
!=0) ? 1 : 0);
897 * Get the start and end date of the period covered by the log files.
899 void GetLogPeriod(struct tm
*Start
,struct tm
*End
)
901 if (EarliestDate
>=0) {
902 memcpy(Start
,&EarliestDateTime
,sizeof(struct tm
));
904 memset(Start
,0,sizeof(struct tm
));
907 memcpy(End
,&LatestDateTime
,sizeof(struct tm
));
909 memset(End
,0,sizeof(struct tm
));