2 * SARG Squid Analysis Report Generator http://sarg.sourceforge.net
6 * please look at http://sarg.sourceforge.net/donations.php
8 * http://sourceforge.net/projects/sarg/forums/forum/363374
9 * ---------------------------------------------------------------------
11 * This program is free software; you can redistribute it and/or modify
12 * it under the terms of the GNU General Public License as published by
13 * the Free Software Foundation; either version 2 of the License, or
14 * (at your option) any later version.
16 * This program is distributed in the hope that it will be useful,
17 * but WITHOUT ANY WARRANTY; without even the implied warranty of
18 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
19 * GNU General Public License for more details.
21 * You should have received a copy of the GNU General Public License
22 * along with this program; if not, write to the Free Software
23 * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111, USA.
27 #include "include/conf.h"
28 #include "include/defs.h"
29 #include "include/readlog.h"
30 #include "include/filelist.h"
32 #define REPORT_EVERY_X_LINES 5000
33 #define MAX_OPEN_USER_FILES 10
37 struct userfilestruct
*next
;
38 struct userinfostruct
*user
;
42 enum ExcludeReasonEnum
44 //! User name too long.
46 //! Squid logged an incomplete query received from the client.
48 //! Log file turned over.
50 //! Excluded by exclude_string from sarg.conf.
52 //! Unknown input log file format.
54 //! Line to be ignored from the input log file.
56 //! Entry not withing the requested date range.
62 //! User is not in the include_users list.
64 //! HTTP code excluded by exclude_code file.
66 //! Invalid character found in user name.
70 //! Not the IP address requested with -a.
72 //! URL excluded by -c or exclude_hosts.
74 //! Entry time outside of requested hour range.
76 //! Not the URL requested by -s.
80 //! Not the user requested by -u.
84 //! User ignored by exclude_users
87 ER_Last
//!< last entry of the list
90 numlist weekdays
= { { 0, 1, 2, 3, 4, 5, 6 }, 7 };
91 numlist hours
= { { 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23 }, 24 };
93 extern char *userfile
;
94 extern FileListObject AccessLog
;
96 extern const struct ReadLogProcessStruct ReadSquidLog
;
97 extern const struct ReadLogProcessStruct ReadCommonLog
;
98 extern const struct ReadLogProcessStruct ReadSargLog
;
99 extern const struct ReadLogProcessStruct ReadExtLog
;
101 //! The list of the supported log formats.
102 static const struct ReadLogProcessStruct
const *LogFormats
[]=
110 //! The path to the sarg log file.
111 static char SargLogFile
[4096]="";
112 //! Handle to the sarg log file. NULL if not created.
113 static FILE *fp_log
=NULL
;
114 //! The number of records read from the input logs.
115 static long int totregsl
=0;
116 //! The number of records kept.
117 static long int totregsg
=0;
118 //! The number of records excluded.
119 static long int totregsx
=0;
120 //! The beginning of a linked list of user's file.
121 static struct userfilestruct
*first_user_file
=NULL
;
122 //! Count the number of occurence of each input log format.
123 static unsigned long int format_count
[sizeof(LogFormats
)/sizeof(*LogFormats
)];
124 //! The minimum date found in the input logs.
125 static int mindate
=0;
126 static int maxdate
=0;
127 //! Count the number of excluded records.
128 static unsigned long int excluded_count
[ER_Last
];
129 //! Earliest date found in the log.
130 static int EarliestDate
=-1;
131 //! The earliest date in time format.
132 static struct tm EarliestDateTime
;
133 //! Latest date found in the log.
134 static int LatestDate
=-1;
135 //! The latest date in time format.
136 static struct tm LatestDateTime
;
139 * Read from standard input.
141 * \param Data The file object.
142 * \param Buffer The boffer to store the data read.
143 * \param Size How many bytes to read.
145 * \return The number of bytes read.
147 static int Stdin_Read(void *Data
,void *Buffer
,int Size
)
149 return(fread(Buffer
,1,Size
,(FILE *)Data
));
153 * Check if end of file is reached.
155 * \param Data The file object.
157 * \return \c True if end of file is reached.
159 static int Stdin_Eof(void *Data
)
161 return(feof((FILE *)Data
));
165 * Mimic a close of standard input but do nothing
167 * \param Data File to close.
169 * \return EOF on error.
171 static int Stdin_Close(void *Data
)
177 * Open a file object to read from standard input.
179 * \return The object to pass to other function in this module.
181 static FileObject
*Stdin_Open(void)
185 FileObject_SetLastOpenError(NULL
);
186 File
=calloc(1,sizeof(*File
));
189 FileObject_SetLastOpenError(_("Not enough memory"));
193 File
->Read
=Stdin_Read
;
196 File
->Close
=Stdin_Close
;
201 * Initialize the memory structure needed by LogLine_Parse() to parse
204 * \param log_line The structure to initialize.
206 void LogLine_Init(struct LogLineStruct
*log_line
)
208 log_line
->current_format
=NULL
;
209 log_line
->current_format_idx
=-1;
210 log_line
->file_name
="";
211 log_line
->successive_errors
=0;
212 log_line
->total_errors
=0;
216 * Set the name of the log file being parsed.
218 * \param log_line Data structure to parse the log line.
219 * \param file_name The name of the log file being read.
221 void LogLine_File(struct LogLineStruct
*log_line
,const char *file_name
)
223 log_line
->file_name
=file_name
;
227 * Parse the next line from a log file.
229 * \param log_line A buffer to store the data about the current parsing.
230 * \param log_entry The variable to store the parsed data.
231 * \param linebuf The text line read from the log file.
235 enum ReadLogReturnCodeEnum
LogLine_Parse(struct LogLineStruct
*log_line
,struct ReadLogStruct
*log_entry
,char *linebuf
)
237 enum ReadLogReturnCodeEnum log_entry_status
=RLRC_Unknown
;
240 if (log_line
->current_format
)
242 memset(log_entry
,0,sizeof(*log_entry
));
243 log_entry_status
=log_line
->current_format
->ReadEntry(linebuf
,log_entry
);
246 // find out what line format to use
247 if (log_entry_status
==RLRC_Unknown
)
249 for (x
=0 ; x
<(int)(sizeof(LogFormats
)/sizeof(*LogFormats
)) ; x
++)
251 if (LogFormats
[x
]==log_line
->current_format
) continue;
252 memset(log_entry
,0,sizeof(*log_entry
));
253 log_entry_status
=LogFormats
[x
]->ReadEntry(linebuf
,log_entry
);
254 if (log_entry_status
!=RLRC_Unknown
)
256 log_line
->current_format
=LogFormats
[x
];
257 log_line
->current_format_idx
=x
;
258 if (debugz
>=LogLevel_Process
)
260 /* TRANSLATORS: The argument is the log format name as translated by you. */
261 debuga(__FILE__
,__LINE__
,_("Log format identified as \"%s\" for %s\n"),_(log_line
->current_format
->Name
),log_line
->file_name
);
266 if (x
>=(int)(sizeof(LogFormats
)/sizeof(*LogFormats
)))
268 if (++log_line
->successive_errors
>NumLogSuccessiveErrors
) {
269 debuga(__FILE__
,__LINE__
,ngettext("%d consecutive error found in the input log file %s\n",
270 "%d consecutive errors found in the input log file %s\n",log_line
->successive_errors
),log_line
->successive_errors
,log_line
->file_name
);
273 if (NumLogTotalErrors
>=0 && ++log_line
->total_errors
>NumLogTotalErrors
) {
274 debuga(__FILE__
,__LINE__
,ngettext("%d error found in the input log file (last in %s)\n",
275 "%d errors found in the input log file (last in %s)\n",log_line
->total_errors
),log_line
->total_errors
,log_line
->file_name
);
278 debuga(__FILE__
,__LINE__
,_("The following line read from %s could not be parsed and is ignored\n%s\n"),log_line
->file_name
,linebuf
);
281 log_line
->successive_errors
=0;
284 if (log_line
->current_format_idx
<0 || log_line
->current_format
==NULL
) {
285 debuga(__FILE__
,__LINE__
,_("Sarg failed to determine the format of the input log file %s\n"),log_line
->file_name
);
288 if (log_entry_status
==RLRC_InternalError
) {
289 debuga(__FILE__
,__LINE__
,_("Internal error encountered while processing %s\nSee previous message to know the reason for that error.\n"),log_line
->file_name
);
292 return(log_entry_status
);
296 Read a single log file.
298 \param arq The log file name to read.
300 static void ReadOneLogFile(struct ReadLogDataStruct
*Filter
,const char *arq
)
308 char tmp3
[MAXLEN
]="";
309 char download_url
[MAXLEN
];
310 char smartfilter
[MAXLEN
];
313 int OutputNonZero
= REPORT_EVERY_X_LINES
;
318 int maxopenfiles
=MAX_OPEN_USER_FILES
;
319 unsigned long int recs1
=0UL;
320 unsigned long int recs2
=0UL;
321 FileObject
*fp_in
=NULL
;
322 bool download_flag
=false;
324 enum ReadLogReturnCodeEnum log_entry_status
;
326 struct getwordstruct gwarea
;
327 struct userfilestruct
*prev_ufile
;
328 struct userinfostruct
*uinfo
;
329 struct userfilestruct
*ufile
;
330 struct userfilestruct
*ufile1
;
331 struct ReadLogStruct log_entry
;
332 struct LogLineStruct log_line
;
334 LogLine_Init(&log_line
);
335 LogLine_File(&log_line
,arq
);
336 for (x
=0 ; x
<sizeof(LogFormats
)/sizeof(*LogFormats
) ; x
++)
337 if (LogFormats
[x
]->NewFile
)
338 LogFormats
[x
]->NewFile(arq
);
340 if (arq
[0]=='-' && arq
[1]=='\0') {
343 debuga(__FILE__
,__LINE__
,_("Reading access log file: from stdin\n"));
345 if (Filter
->DateRange
[0]!='\0') {
346 if (stat(arq
,&logstat
)!=0) {
347 debuga(__FILE__
,__LINE__
,_("Cannot get the modification time of input log file %s (%s). Processing it anyway\n"),arq
,strerror(errno
));
349 struct tm
*logtime
=localtime(&logstat
.st_mtime
);
350 if ((logtime
->tm_year
+1900)*10000+(logtime
->tm_mon
+1)*100+logtime
->tm_mday
<dfrom
) {
351 debuga(__FILE__
,__LINE__
,_("Ignoring old log file %s\n"),arq
);
358 debuga(__FILE__
,__LINE__
,_("Cannot open input log file \"%s\": %s\n"),arq
,FileObject_GetLastOpenError());
361 if (debug
) debuga(__FILE__
,__LINE__
,_("Reading access log file: %s\n"),arq
);
369 // pre-read the file only if we have to show stats
370 if (ShowReadStatistics
&& ShowReadPercent
&& fp_in
->Rewind
) {
375 while ((nread
=FileObject_Read(fp_in
,tmp4
,sizeof(tmp4
)))>0) {
376 for (i
=0 ; i
<nread
; i
++)
378 if (tmp4
[i
]!='\n' && tmp4
[i
]!='\r') {
382 if (tmp4
[i
]=='\n' || tmp4
[i
]=='\r') {
388 FileObject_Rewind(fp_in
);
389 printf(_("SARG: Records in file: %lu, reading: %3.2f%%"),recs1
,(float) 0);
394 if ((line
=longline_create())==NULL
) {
395 debuga(__FILE__
,__LINE__
,_("Not enough memory to read file \"%s\"\n"),arq
);
399 while ((linebuf
=longline_read(fp_in
,line
))!=NULL
) {
403 if (ShowReadStatistics
&& --OutputNonZero
<=0) {
405 double perc
= recs2
* 100. / recs1
;
406 printf(_("SARG: Records in file: %lu, reading: %3.2lf%%"),recs2
,perc
);
408 printf(_("SARG: Records in file: %lu"),recs2
);
412 OutputNonZero
= REPORT_EVERY_X_LINES
;
416 The following checks are retained here as I don't know to
417 what format they apply. They date back to pre 2.4 versions.
419 //if(blen < 58) continue; //this test conflict with the reading of the sarg log header line
420 if(strstr(linebuf
,"HTTP/0.0") != 0) {//recorded by squid when encountering an incomplete query
421 excluded_count
[ER_IncompleteQuery
]++;
424 if(strstr(linebuf
,"logfile turned over") != 0) {//reported by newsyslog
425 excluded_count
[ER_LogfileTurnedOver
]++;
430 if(ExcludeString
[0] != '\0') {
432 getword_start(&gwarea
,ExcludeString
);
433 while(strchr(gwarea
.current
,':') != 0) {
434 if (getword_multisep(val1
,sizeof(val1
),&gwarea
,':')<0) {
435 debuga(__FILE__
,__LINE__
,_("Invalid record in exclusion string\n"));
438 if((str
=(char *) strstr(linebuf
,val1
)) != (char *) NULL
) {
443 if(!exstring
&& (str
=(char *) strstr(linebuf
,gwarea
.current
)) != (char *) NULL
)
446 excluded_count
[ER_ExcludeString
]++;
452 if (debugz
>=LogLevel_Data
)
453 printf("BUF=%s\n",linebuf
);
456 log_entry_status
=LogLine_Parse(&log_line
,&log_entry
,linebuf
);
457 if (log_entry_status
==RLRC_Unknown
)
459 excluded_count
[ER_UnknownFormat
]++;
462 if (log_entry_status
==RLRC_Ignore
) {
463 excluded_count
[ER_FormatData
]++;
466 format_count
[log_line
.current_format_idx
]++;
468 if (!fp_log
&& ParsedOutputLog
[0] && log_line
.current_format
!=&ReadSargLog
) {
469 if(access(ParsedOutputLog
,R_OK
) != 0) {
470 my_mkdir(ParsedOutputLog
);
472 if (snprintf(SargLogFile
,sizeof(SargLogFile
),"%s/sarg_temp.log",ParsedOutputLog
)>=sizeof(SargLogFile
)) {
473 debuga(__FILE__
,__LINE__
,_("Path too long: "));
474 debuga_more("%s/sarg_temp.log\n",ParsedOutputLog
);
477 if((fp_log
=MY_FOPEN(SargLogFile
,"w"))==NULL
) {
478 debuga(__FILE__
,__LINE__
,_("Cannot open file \"%s\": %s\n"),SargLogFile
,strerror(errno
));
481 fputs("*** SARG Log ***\n",fp_log
);
484 if (log_entry
.Ip
==NULL
) {
485 debuga(__FILE__
,__LINE__
,_("Unknown input log file format: no IP addresses\n"));
488 if (log_entry
.User
==NULL
) {
489 debuga(__FILE__
,__LINE__
,_("Unknown input log file format: no user\n"));
492 if (log_entry
.Url
==NULL
) {
493 debuga(__FILE__
,__LINE__
,_("Unknown input log file format: no URL\n"));
497 idata
=builddia(log_entry
.EntryTime
.tm_mday
,log_entry
.EntryTime
.tm_mon
+1,log_entry
.EntryTime
.tm_year
+1900);
498 if (debugz
>=LogLevel_Data
)
499 printf("DATE=%s IDATA=%d DFROM=%d DUNTIL=%d\n",Filter
->DateRange
,idata
,dfrom
,duntil
);
501 if (EarliestDate
<0 || idata
<EarliestDate
) {
503 memcpy(&EarliestDateTime
,&log_entry
.EntryTime
,sizeof(struct tm
));
505 if (LatestDate
<0 || idata
>LatestDate
) {
507 memcpy(&LatestDateTime
,&log_entry
.EntryTime
,sizeof(struct tm
));
509 if(Filter
->DateRange
[0] != '\0'){
510 if(idata
< dfrom
|| idata
> duntil
) {
511 excluded_count
[ER_OutOfDateRange
]++;
516 // Record only hours usage which is required
517 if( bsearch( &( log_entry
.EntryTime
.tm_wday
), weekdays
.list
, weekdays
.len
, sizeof( int ), compar
) == NULL
) {
518 excluded_count
[ER_OutOfWDayRange
]++;
522 if( bsearch( &( log_entry
.EntryTime
.tm_hour
), hours
.list
, hours
.len
, sizeof( int ), compar
) == NULL
) {
523 excluded_count
[ER_OutOfHourRange
]++;
528 if(strlen(log_entry
.User
) > MAX_USER_LEN
) {
529 if (debugz
>=LogLevel_Process
) debuga(__FILE__
,__LINE__
,_("User ID too long: %s\n"),log_entry
.User
);
530 excluded_count
[ER_UserNameTooLong
]++;
536 if(IncludeUsers
[0] != '\0') {
537 snprintf(val1
,sizeof(val1
),":%s:",log_entry
.User
);
538 if((str
=(char *) strstr(IncludeUsers
,val1
)) == (char *) NULL
) {
539 excluded_count
[ER_User
]++;
544 if(vercode(log_entry
.HttpCode
)) {
545 if (debugz
>=LogLevel_Process
) debuga(__FILE__
,__LINE__
,_("Excluded code: %s\n"),log_entry
.HttpCode
);
546 excluded_count
[ER_HttpCode
]++;
551 if(testvaliduserchar(log_entry
.User
)) {
552 excluded_count
[ER_InvalidUserChar
]++;
556 // replace any tab by a single space
557 for (str
=log_entry
.Url
; *str
; str
++)
558 if (*str
=='\t') *str
=' ';
559 for (str
=log_entry
.HttpCode
; *str
; str
++)
560 if (*str
=='\t') *str
=' ';
562 if (log_line
.current_format
!=&ReadSargLog
) {
564 The full URL is not saved in sarg log. There is no point in testing the URL to detect
567 download_flag
=is_download_suffix(log_entry
.Url
);
569 safe_strcpy(download_url
,log_entry
.Url
,sizeof(download_url
));
574 url
=process_url(log_entry
.Url
,LongUrl
);
575 if (!url
|| url
[0] == '\0') {
576 excluded_count
[ER_NoUrl
]++;
581 if(strcmp(addr
,log_entry
.Ip
)!=0) {
582 excluded_count
[ER_UntrackedIpAddr
]++;
586 if(Filter
->HostFilter
) {
587 if(!vhexclude(url
)) {
588 if (debugz
>=LogLevel_Data
) debuga(__FILE__
,__LINE__
,_("Excluded site: %s\n"),url
);
589 excluded_count
[ER_Url
]++;
595 if(Filter
->StartTime
>= 0 && Filter
->EndTime
>= 0) {
596 hmr
=log_entry
.EntryTime
.tm_hour
*100+log_entry
.EntryTime
.tm_min
;
597 if(hmr
< Filter
->StartTime
|| hmr
> Filter
->EndTime
) {
598 excluded_count
[ER_OutOfTimeRange
]++;
604 if(strstr(url
,site
)==0) {
605 excluded_count
[ER_UntrackedUrl
]++;
611 log_entry
.User
=log_entry
.Ip
;
615 if ((log_entry
.User
[0]=='\0') || (log_entry
.User
[1]=='\0' && (log_entry
.User
[0]=='-' || log_entry
.User
[0]==' '))) {
616 if(RecordsWithoutUser
== RECORDWITHOUTUSER_IP
) {
617 log_entry
.User
=log_entry
.Ip
;
620 if(RecordsWithoutUser
== RECORDWITHOUTUSER_IGNORE
) {
621 excluded_count
[ER_NoUser
]++;
624 if(RecordsWithoutUser
== RECORDWITHOUTUSER_EVERYBODY
)
625 log_entry
.User
="everybody";
627 if(NtlmUserFormat
== NTLMUSERFORMAT_USER
) {
628 if ((str
=strchr(log_entry
.User
,'+'))!=NULL
|| (str
=strchr(log_entry
.User
,'\\'))!=NULL
|| (str
=strchr(log_entry
.User
,'_'))!=NULL
) {
629 log_entry
.User
=str
+1;
636 if(strcmp(log_entry
.User
,us
)!=0) {
637 excluded_count
[ER_UntrackedUser
]++;
642 if(Filter
->SysUsers
) {
643 snprintf(wuser
,sizeof(wuser
),":%s:",log_entry
.User
);
644 if(strstr(userfile
, wuser
) == 0) {
645 excluded_count
[ER_SysUser
]++;
650 if(Filter
->UserFilter
) {
651 if(!vuexclude(log_entry
.User
)) {
652 if (debugz
>=LogLevel_Process
) debuga(__FILE__
,__LINE__
,_("Excluded user: %s\n"),log_entry
.User
);
653 excluded_count
[ER_IgnoredUser
]++;
659 user
=process_user(log_entry
.User
);
660 if (log_entry
.User
!=user
) {
664 if (log_entry
.User
[0]=='\0' || (log_entry
.User
[1]=='\0' && (log_entry
.User
[0]=='-' ||
665 log_entry
.User
[0]==' ' || log_entry
.User
[0]==':'))) {
666 excluded_count
[ER_NoUser
]++;
670 if (log_entry
.DataSize
<0) log_entry
.DataSize
=0;
672 if (log_entry
.ElapsedTime
<0) log_entry
.ElapsedTime
=0;
673 if (Filter
->max_elapsed
>0 && log_entry
.ElapsedTime
>Filter
->max_elapsed
) {
674 log_entry
.ElapsedTime
=0;
677 if((str
=(char *) strstr(linebuf
, "[SmartFilter:")) != (char *) NULL
) {
679 snprintf(smartfilter
,sizeof(smartfilter
),"\"%s\"",str
+1);
680 } else strcpy(smartfilter
,"\"\"");
684 for (ufile
=first_user_file
; ufile
&& strcmp(log_entry
.User
,ufile
->user
->id
)!=0 ; ufile
=ufile
->next
) {
686 if (ufile
->file
) nopen
++;
689 ufile
=malloc(sizeof(*ufile
));
691 debuga(__FILE__
,__LINE__
,_("Not enough memory to store the user %s\n"),log_entry
.User
);
694 memset(ufile
,0,sizeof(*ufile
));
695 ufile
->next
=first_user_file
;
696 first_user_file
=ufile
;
698 * This id_is_ip stuff is just to store the string only once if the user is
699 * identified by its IP address instead of a distinct ID and IP address.
701 uinfo
=userinfo_create(log_entry
.User
,(id_is_ip
) ? NULL
: log_entry
.Ip
);
706 prev_ufile
->next
=ufile
->next
;
707 ufile
->next
=first_user_file
;
708 first_user_file
=ufile
;
711 #ifdef ENABLE_DOUBLE_CHECK_DATA
712 if (strcmp(log_entry
.HttpCode
,"TCP_DENIED/407")!=0) {
713 ufile
->user
->nbytes
+=log_entry
.DataSize
;
714 ufile
->user
->elap
+=log_entry
.ElapsedTime
;
718 if (ufile
->file
==NULL
) {
719 if (nopen
>=maxopenfiles
) {
721 for (ufile1
=first_user_file
; ufile1
; ufile1
=ufile1
->next
) {
722 if (ufile1
->file
!=NULL
) {
723 if (x
>=maxopenfiles
) {
724 if (fclose(ufile1
->file
)==EOF
) {
725 debuga(__FILE__
,__LINE__
,_("Write error in log file of user %s: %s\n"),ufile1
->user
->id
,strerror(errno
));
734 if (snprintf (tmp3
, sizeof(tmp3
), "%s/%s.user_unsort", tmp
, ufile
->user
->filename
)>=sizeof(tmp3
)) {
735 debuga(__FILE__
,__LINE__
,_("Temporary user file name too long: %s/%s.user_unsort\n"), tmp
, ufile
->user
->filename
);
738 if ((ufile
->file
= MY_FOPEN (tmp3
, "a")) == NULL
) {
739 debuga(__FILE__
,__LINE__
,_("(log) Cannot open temporary file %s: %s\n"), tmp3
, strerror(errno
));
744 strftime(dia
, sizeof(dia
), "%d/%m/%Y",&log_entry
.EntryTime
);
745 strftime(hora
,sizeof(hora
),"%H:%M:%S",&log_entry
.EntryTime
);
747 if (fprintf(ufile
->file
, "%s\t%s\t%s\t%s\t%"PRIu64
"\t%s\t%ld\t%s\n",dia
,hora
,
748 log_entry
.Ip
,url
,(uint64_t)log_entry
.DataSize
,
749 log_entry
.HttpCode
,log_entry
.ElapsedTime
,smartfilter
)<=0) {
750 debuga(__FILE__
,__LINE__
,_("Write error in the log file of user %s\n"),log_entry
.User
);
755 if (fp_log
&& log_line
.current_format
!=&ReadSargLog
) {
756 fprintf(fp_log
, "%s\t%s\t%s\t%s\t%s\t%"PRIu64
"\t%s\t%ld\t%s\n",dia
,hora
,
757 log_entry
.User
,log_entry
.Ip
,url
,(uint64_t)log_entry
.DataSize
,
758 log_entry
.HttpCode
,log_entry
.ElapsedTime
,smartfilter
);
763 denied_write(&log_entry
);
764 authfail_write(&log_entry
);
765 if (download_flag
) download_write(&log_entry
,download_url
);
767 if (log_line
.current_format
!=&ReadSargLog
) {
768 if (period
.start
.tm_year
==0 || idata
<mindate
|| compare_date(&period
.start
,&log_entry
.EntryTime
)>0){
770 memcpy(&period
.start
,&log_entry
.EntryTime
,sizeof(log_entry
.EntryTime
));
772 if (period
.end
.tm_year
==0 || idata
>maxdate
|| compare_date(&period
.end
,&log_entry
.EntryTime
)<0) {
774 memcpy(&period
.end
,&log_entry
.EntryTime
,sizeof(log_entry
.EntryTime
));
778 if (debugz
>=LogLevel_Data
){
779 printf("IP=\t%s\n",log_entry
.Ip
);
780 printf("USER=\t%s\n",log_entry
.User
);
781 printf("ELAP=\t%ld\n",log_entry
.ElapsedTime
);
782 printf("DATE=\t%s\n",dia
);
783 printf("TIME=\t%s\n",hora
);
784 //printf("FUNC=\t%s\n",fun);
785 printf("URL=\t%s\n",url
);
786 printf("CODE=\t%s\n",log_entry
.HttpCode
);
787 printf("LEN=\t%"PRIu64
"\n",(uint64_t)log_entry
.DataSize
);
790 longline_destroy(&line
);
792 if (FileObject_Close(fp_in
)) {
793 debuga(__FILE__
,__LINE__
,_("Read error in \"%s\": %s\n"),arq
,FileObject_GetLastCloseError());
796 if (ShowReadStatistics
) {
798 printf(_("SARG: Records in file: %lu, reading: %3.2f%%\n"),recs2
, (float) 100 );
800 printf(_("SARG: Records in file: %lu\n"),recs2
);
805 * Display a line with the excluded entries count.
807 * \param Explain A translated string explaining the exluded count.
808 * \param Reason The reason number.
810 static void DisplayExcludeCount(const char *Explain
,enum ExcludeReasonEnum Reason
)
812 if (excluded_count
[Reason
]>0) {
813 debuga(__FILE__
,__LINE__
," %s: %lu\n",Explain
,excluded_count
[Reason
]);
820 \param Filter The filtering parameters for the file to load.
822 \retval 1 Records found.
823 \retval 0 No record found.
825 int ReadLogFile(struct ReadLogDataStruct
*Filter
)
829 struct userfilestruct
*ufile
;
830 struct userfilestruct
*ufile1
;
831 FileListIterator FIter
;
834 for (x
=0 ; x
<sizeof(format_count
)/sizeof(*format_count
) ; x
++) format_count
[x
]=0;
835 for (x
=0 ; x
<sizeof(excluded_count
)/sizeof(*excluded_count
) ; x
++) excluded_count
[x
]=0;
836 first_user_file
=NULL
;
844 FIter
=FileListIter_Open(AccessLog
);
845 while ((file
=FileListIter_Next(FIter
))!=NULL
)
846 ReadOneLogFile(Filter
,file
);
847 FileListIter_Close(FIter
);
851 char val4
[4096];//val4 must not be bigger than SargLogFile without fixing the strcpy below
853 if (fclose(fp_log
)==EOF
) {
854 debuga(__FILE__
,__LINE__
,_("Write error in \"%s\": %s\n"),SargLogFile
,strerror(errno
));
857 strftime(val2
,sizeof(val2
),"%d%m%Y_%H%M",&period
.start
);
858 strftime(val1
,sizeof(val1
),"%d%m%Y_%H%M",&period
.end
);
859 if (snprintf(val4
,sizeof(val4
),"%s/sarg-%s-%s.log",ParsedOutputLog
,val2
,val1
)>=sizeof(val4
)) {
860 debuga(__FILE__
,__LINE__
,_("Path too long: "));
861 debuga_more("%s/sarg-%s-%s.log\n",ParsedOutputLog
,val2
,val1
);
864 if (rename(SargLogFile
,val4
)) {
865 debuga(__FILE__
,__LINE__
,_("failed to rename %s to %s - %s\n"),SargLogFile
,val4
,strerror(errno
));
867 strcpy(SargLogFile
,val4
);
869 if(strcmp(ParsedOutputLogCompress
,"nocompress") != 0 && ParsedOutputLogCompress
[0] != '\0') {
871 No double quotes around ParsedOutputLogCompress because it may contain command line options. If double quotes are
872 necessary around the command name, put them in the configuration file.
874 if (snprintf(val1
,sizeof(val1
),"%s \"%s\"",ParsedOutputLogCompress
,SargLogFile
)>=sizeof(val1
)) {
875 debuga(__FILE__
,__LINE__
,_("Command too long: %s \"%s\"\n"),ParsedOutputLogCompress
,SargLogFile
);
878 cstatus
=system(val1
);
879 if (!WIFEXITED(cstatus
) || WEXITSTATUS(cstatus
)) {
880 debuga(__FILE__
,__LINE__
,_("command return status %d\n"),WEXITSTATUS(cstatus
));
881 debuga(__FILE__
,__LINE__
,_("command: %s\n"),val1
);
887 debuga(__FILE__
,__LINE__
,_("Sarg parsed log saved as %s\n"),SargLogFile
);
894 for (ufile
=first_user_file
; ufile
; ufile
=ufile1
) {
896 if (ufile
->file
!=NULL
&& fclose(ufile
->file
)==EOF
) {
897 debuga(__FILE__
,__LINE__
,_("Write error in log file of user %s: %s\n"),ufile
->user
->id
,strerror(errno
));
904 unsigned long int totalcount
=0;
906 debuga(__FILE__
,__LINE__
,_(" Records read: %ld, written: %ld, excluded: %ld\n"),totregsl
,totregsg
,totregsx
);
908 for (x
=sizeof(excluded_count
)/sizeof(*excluded_count
)-1 ; x
>=0 && excluded_count
[x
]>0 ; x
--);
910 debuga(__FILE__
,__LINE__
,_("Reasons for excluded entries:\n"));
911 DisplayExcludeCount(_("User name too long"),ER_UserNameTooLong
);
912 DisplayExcludeCount(_("Squid logged an incomplete query received from the client"),ER_IncompleteQuery
);
913 DisplayExcludeCount(_("Log file turned over"),ER_LogfileTurnedOver
);
914 DisplayExcludeCount(_("Excluded by \"exclude_string\" in sarg.conf"),ER_ExcludeString
);
915 DisplayExcludeCount(_("Unknown input log file format"),ER_UnknownFormat
);
916 DisplayExcludeCount(_("Line ignored by the input log format"),ER_FormatData
);
917 DisplayExcludeCount(_("Time outside the requested date range (-d)"),ER_OutOfDateRange
);
918 DisplayExcludeCount(_("Ignored week day (\"weekdays\" parameter in sarg.conf)"),ER_OutOfWDayRange
);
919 DisplayExcludeCount(_("Ignored hour (\"hours\" parameter in sarg.conf)"),ER_OutOfHourRange
);
920 DisplayExcludeCount(_("User is not in the \"include_users\" list"),ER_User
);
921 DisplayExcludeCount(_("HTTP code excluded by \"exclude_code\" file"),ER_HttpCode
);
922 DisplayExcludeCount(_("Invalid character found in user name"),ER_InvalidUserChar
);
923 DisplayExcludeCount(_("No URL in entry"),ER_NoUrl
);
924 DisplayExcludeCount(_("Not the IP address requested with -a"),ER_UntrackedIpAddr
);
925 DisplayExcludeCount(_("URL excluded by -c or \"exclude_hosts\""),ER_Url
);
926 DisplayExcludeCount(_("Entry time outside of requested hour range (-t)"),ER_OutOfTimeRange
);
927 DisplayExcludeCount(_("Not the URL requested by -s"),ER_UntrackedUrl
);
928 DisplayExcludeCount(_("No user in entry"),ER_NoUser
);
929 DisplayExcludeCount(_("Not the user requested by -u"),ER_UntrackedUser
);
930 DisplayExcludeCount(_("System user as defined by \"password\" in sarg.conf"),ER_SysUser
);
931 DisplayExcludeCount(_("User ignored by \"exclude_users\""),ER_IgnoredUser
);
934 for (x
=0 ; x
<sizeof(LogFormats
)/sizeof(*LogFormats
) ; x
++) {
935 if (format_count
[x
]>0) {
936 /* TRANSLATORS: It displays the number of lines found in the input log files
937 * for each supported log format. The log format name is the %s and is a string
938 * you translate somewhere else. */
939 debuga(__FILE__
,__LINE__
,_("%s: %lu entries\n"),_(LogFormats
[x
]->Name
),format_count
[x
]);
940 totalcount
+=format_count
[x
];
944 if (totalcount
==0 && totregsg
)
945 debuga(__FILE__
,__LINE__
,_("Log with invalid format\n"));
948 return((totregsg
!=0) ? 1 : 0);
952 * Get the start and end date of the period covered by the log files.
954 bool GetLogPeriod(struct tm
*Start
,struct tm
*End
)
958 if (EarliestDate
>=0) {
959 memcpy(Start
,&EarliestDateTime
,sizeof(struct tm
));
962 memset(Start
,0,sizeof(struct tm
));
965 memcpy(End
,&LatestDateTime
,sizeof(struct tm
));
968 memset(End
,0,sizeof(struct tm
));