&ReadExtLog
};
-/*!
-Read the log files.
+//! The path to the sarg log file.
+static char SargLogFile[4096]="";
+//! Handle to the sarg log file. NULL if not created.
+static FILE *fp_log=NULL;
+//! The number of records read from the input logs.
+static long int totregsl=0;
+//! The number of records kept.
+static long int totregsg=0;
+//! The number of records excluded.
+static long int totregsx=0;
+//! The beginning of a linked list of user's file.
+static struct userfilestruct *first_user_file=NULL;
+//! Count the number of occurence of each input log format.
+static int format_count[sizeof(LogFormats)/sizeof(*LogFormats)];
+//! The minimum date found in the input logs.
+static int mindate=0;
+static int maxdate=0;
-\param Filter The filtering parameters for the file to load.
+/*!
+Read a single log file.
-\retval 1 Records found.
-\retval 0 No record found.
+\param arq The log file name to read.
*/
-int ReadLogFile(struct ReadLogDataStruct *Filter)
+static void ReadOneLogFile(struct ReadLogDataStruct *Filter,const char *arq)
{
longline line;
char *linebuf;
char *str;
- char arq_log[255];
char user[MAX_USER_LEN];
char hora[30];
- char tbuf2[128];
char dia[128]="";
char wuser[MAXLEN];
- char tmp3[MAXLEN];
- char start_hour[128];
+ char tmp3[MAXLEN]="";
char download_url[MAXLEN];
char smartfilter[MAXLEN];
- const char *arq;
const char *url;
- int iarq;
+ int current_format_idx;
int blen;
int OutputNonZero = REPORT_EVERY_X_LINES ;
int idata=0;
int hmr;
int nopen;
int maxopenfiles=MAX_OPEN_USER_FILES;
- int mindate=0;
- int maxdate=0;
- int cstatus;
- int current_format_idx;
int successive_errors=0;
int total_errors=0;
- int format_count[sizeof(LogFormats)/sizeof(*LogFormats)];
unsigned long int recs1=0UL;
unsigned long int recs2=0UL;
- long int totregsl=0;
- long int totregsg=0;
- long int totregsx=0;
FILE *fp_in=NULL;
- FILE *fp_log=NULL;
bool from_pipe;
bool from_stdin;
bool download_flag=false;
bool id_is_ip;
- bool totper=false;
enum ReadLogReturnCodeEnum log_entry_status;
struct stat logstat;
struct getwordstruct gwarea;
struct userfilestruct *prev_ufile;
struct userinfostruct *uinfo;
- struct userfilestruct *first_user_file=NULL;
struct userfilestruct *ufile;
struct userfilestruct *ufile1;
struct ReadLogStruct log_entry;
const struct ReadLogProcessStruct *current_format=NULL;
- for (x=0 ; x<sizeof(format_count)/sizeof(*format_count) ; x++) format_count[x]=0;
- tmp3[0]='\0';
- start_hour[0]='\0';
- first_user_file=NULL;
-
- if (!dataonly) {
- denied_open();
- authfail_open();
- download_open();
- }
+ current_format=NULL;
+ current_format_idx=-1;
+ for (x=0 ; x<sizeof(LogFormats)/sizeof(*LogFormats) ; x++)
+ if (LogFormats[x]->NewFile)
+ LogFormats[x]->NewFile(arq);
- if ((line=longline_create())==NULL) {
- debuga(_("Not enough memory to read a log file\n"));
- exit(EXIT_FAILURE);
- }
-
- for (iarq=0 ; iarq<NAccessLog ; iarq++) {
- arq=AccessLog[iarq];
-
- current_format=NULL;
- current_format_idx=-1;
- for (x=0 ; x<sizeof(LogFormats)/sizeof(*LogFormats) ; x++)
- if (LogFormats[x]->NewFile)
- LogFormats[x]->NewFile(arq);
-
- if (arq[0]=='-' && arq[1]=='\0') {
- if(debug)
- debuga(_("Reading access log file: from stdin\n"));
- fp_in=stdin;
- from_stdin=true;
- } else {
- if (Filter->DateRange[0]!='\0') {
- if (stat(arq,&logstat)!=0) {
- debuga(_("Cannot get the modification time of input log file %s (%s). Processing it anyway\n"),arq,strerror(errno));
- } else {
- struct tm *logtime=localtime(&logstat.st_mtime);
- if ((logtime->tm_year+1900)*10000+(logtime->tm_mon+1)*100+logtime->tm_mday<dfrom) {
- debuga(_("Ignoring old log file %s\n"),arq);
- continue;
- }
+ if (arq[0]=='-' && arq[1]=='\0') {
+ if(debug)
+ debuga(_("Reading access log file: from stdin\n"));
+ fp_in=stdin;
+ from_stdin=true;
+ } else {
+ if (Filter->DateRange[0]!='\0') {
+ if (stat(arq,&logstat)!=0) {
+ debuga(_("Cannot get the modification time of input log file %s (%s). Processing it anyway\n"),arq,strerror(errno));
+ } else {
+ struct tm *logtime=localtime(&logstat.st_mtime);
+ if ((logtime->tm_year+1900)*10000+(logtime->tm_mon+1)*100+logtime->tm_mday<dfrom) {
+ debuga(_("Ignoring old log file %s\n"),arq);
+ return;
}
}
- fp_in=decomp(arq,&from_pipe);
- if(fp_in==NULL) {
- debuga(_("(log) Cannot open log file: %s - %s\n"),arq,strerror(errno));
- exit(EXIT_FAILURE);
- }
- if(debug) debuga(_("Reading access log file: %s\n"),arq);
- from_stdin=false;
}
+ fp_in=decomp(arq,&from_pipe);
+ if(fp_in==NULL) {
+ debuga(_("(log) Cannot open log file: %s - %s\n"),arq,strerror(errno));
+ exit(EXIT_FAILURE);
+ }
+ if(debug) debuga(_("Reading access log file: %s\n"),arq);
+ from_stdin=false;
+ }
- download_flag=false;
+ download_flag=false;
- recs1=0UL;
- recs2=0UL;
+ recs1=0UL;
+ recs2=0UL;
- // pre-read the file only if we have to show stats
- if (ShowReadStatistics && ShowReadPercent && !from_stdin && !from_pipe) {
- size_t nread,i;
- bool skipcr=false;
- char tmp4[MAXLEN];
+ // pre-read the file only if we have to show stats
+ if (ShowReadStatistics && ShowReadPercent && !from_stdin && !from_pipe) {
+ size_t nread,i;
+ bool skipcr=false;
+ char tmp4[MAXLEN];
- while ((nread=fread(tmp4,1,sizeof(tmp4),fp_in))>0) {
- for (i=0 ; i<nread ; i++)
- if (skipcr) {
- if (tmp4[i]!='\n' && tmp4[i]!='\r') {
- skipcr=false;
- }
- } else {
- if (tmp4[i]=='\n' || tmp4[i]=='\r') {
- skipcr=true;
- recs1++;
- }
+ while ((nread=fread(tmp4,1,sizeof(tmp4),fp_in))>0) {
+ for (i=0 ; i<nread ; i++)
+ if (skipcr) {
+ if (tmp4[i]!='\n' && tmp4[i]!='\r') {
+ skipcr=false;
}
- }
- rewind(fp_in);
- printf(_("SARG: Records in file: %lu, reading: %3.2f%%"),recs1,(float) 0);
- putchar('\r');
- fflush( stdout ) ;
+ } else {
+ if (tmp4[i]=='\n' || tmp4[i]=='\r') {
+ skipcr=true;
+ recs1++;
+ }
+ }
}
+ rewind(fp_in);
+ printf(_("SARG: Records in file: %lu, reading: %3.2f%%"),recs1,(float) 0);
+ putchar('\r');
+ fflush( stdout ) ;
+ }
- longline_reset(line);
+ if ((line=longline_create())==NULL) {
+ debuga(_("Not enough memory to read log file %s\n"),arq);
+ exit(EXIT_FAILURE);
+ }
- while ((linebuf=longline_read(fp_in,line))!=NULL) {
- blen=strlen(linebuf);
- lines_read++;
+ while ((linebuf=longline_read(fp_in,line))!=NULL) {
+ blen=strlen(linebuf);
+ lines_read++;
- recs2++;
- if (ShowReadStatistics && --OutputNonZero<=0) {
- if (recs1>0) {
- double perc = recs2 * 100. / recs1 ;
- printf(_("SARG: Records in file: %lu, reading: %3.2lf%%"),recs2,perc);
- } else {
- printf(_("SARG: Records in file: %lu"),recs2);
- }
- putchar('\r');
- fflush (stdout);
- OutputNonZero = REPORT_EVERY_X_LINES ;
+ recs2++;
+ if (ShowReadStatistics && --OutputNonZero<=0) {
+ if (recs1>0) {
+ double perc = recs2 * 100. / recs1 ;
+ printf(_("SARG: Records in file: %lu, reading: %3.2lf%%"),recs2,perc);
+ } else {
+ printf(_("SARG: Records in file: %lu"),recs2);
}
+ putchar('\r');
+ fflush (stdout);
+ OutputNonZero = REPORT_EVERY_X_LINES ;
+ }
- /*
- The following checks are retained here as I don't know to
- what format they apply. They date back to pre 2.4 versions.
- */
- //if(blen < 58) continue; //this test conflict with the reading of the sarg log header line
- if(strstr(linebuf,"HTTP/0.0") != 0) continue;//recorded by squid when encountering an incomplete query
- if(strstr(linebuf,"logfile turned over") != 0) continue;//reported by newsyslog
- if(linebuf[0] == ' ') continue;
-
- // exclude_string
- if(ExcludeString[0] != '\0') {
- bool exstring=false;
- getword_start(&gwarea,ExcludeString);
- while(strchr(gwarea.current,':') != 0) {
- if (getword_multisep(val1,sizeof(val1),&gwarea,':')<0) {
- debuga(_("Maybe you have a broken record or garbage in your exclusion string\n"));
- exit(EXIT_FAILURE);
- }
- if((str=(char *) strstr(linebuf,val1)) != (char *) NULL ) {
- exstring=true;
- break;
- }
+ /*
+ The following checks are retained here as I don't know to
+ what format they apply. They date back to pre 2.4 versions.
+ */
+ //if(blen < 58) continue; //this test conflict with the reading of the sarg log header line
+ if(strstr(linebuf,"HTTP/0.0") != 0) continue;//recorded by squid when encountering an incomplete query
+ if(strstr(linebuf,"logfile turned over") != 0) continue;//reported by newsyslog
+ if(linebuf[0] == ' ') continue;
+
+ // exclude_string
+ if(ExcludeString[0] != '\0') {
+ bool exstring=false;
+ getword_start(&gwarea,ExcludeString);
+ while(strchr(gwarea.current,':') != 0) {
+ if (getword_multisep(val1,sizeof(val1),&gwarea,':')<0) {
+ debuga(_("Maybe you have a broken record or garbage in your exclusion string\n"));
+ exit(EXIT_FAILURE);
}
- if(!exstring && (str=(char *) strstr(linebuf,gwarea.current)) != (char *) NULL )
+ if((str=(char *) strstr(linebuf,val1)) != (char *) NULL ) {
exstring=true;
- if(exstring) continue;
+ break;
+ }
}
+ if(!exstring && (str=(char *) strstr(linebuf,gwarea.current)) != (char *) NULL )
+ exstring=true;
+ if(exstring) continue;
+ }
- totregsl++;
- if(debugm)
- printf("BUF=%s\n",linebuf);
+ totregsl++;
+ if(debugm)
+ printf("BUF=%s\n",linebuf);
- // process the line
- log_entry_status=RLRC_Unknown;
- memset(&log_entry,0,sizeof(log_entry));
- if (current_format) {
- log_entry_status=current_format->ReadEntry(linebuf,&log_entry);
- }
+ // process the line
+ log_entry_status=RLRC_Unknown;
+ memset(&log_entry,0,sizeof(log_entry));
+ if (current_format) {
+ log_entry_status=current_format->ReadEntry(linebuf,&log_entry);
+ }
- // find out what line format to use
- if (log_entry_status==RLRC_Unknown) {
- for (x=0 ; x<(int)(sizeof(LogFormats)/sizeof(*LogFormats)) ; x++) {
- if (LogFormats[x]==current_format) continue;
- memset(&log_entry,0,sizeof(log_entry));
- log_entry_status=LogFormats[x]->ReadEntry(linebuf,&log_entry);
- if (log_entry_status!=RLRC_Unknown) break;
- }
- if (x>=(int)(sizeof(LogFormats)/sizeof(*LogFormats))) {
- if (++successive_errors>NumLogSuccessiveErrors) {
- debuga(ngettext("%d consecutive error found in the input log file %s\n",
- "%d consecutive errors found in the input log file %s\n",successive_errors),successive_errors,arq);
- exit(EXIT_FAILURE);
- }
- if (NumLogTotalErrors>=0 && ++total_errors>NumLogTotalErrors) {
- debuga(ngettext("%d error found in the input log file (last in %s)\n",
- "%d errors found in the input log file (last in %s)\n",total_errors),total_errors,arq);
- exit(EXIT_FAILURE);
- }
- debuga(_("The following line read from %s could not be parsed and is ignored\n%s\n"),arq,linebuf);
- continue;
+ // find out what line format to use
+ if (log_entry_status==RLRC_Unknown) {
+ for (x=0 ; x<(int)(sizeof(LogFormats)/sizeof(*LogFormats)) ; x++) {
+ if (LogFormats[x]==current_format) continue;
+ memset(&log_entry,0,sizeof(log_entry));
+ log_entry_status=LogFormats[x]->ReadEntry(linebuf,&log_entry);
+ if (log_entry_status!=RLRC_Unknown) break;
+ }
+ if (x>=(int)(sizeof(LogFormats)/sizeof(*LogFormats))) {
+ if (++successive_errors>NumLogSuccessiveErrors) {
+ debuga(ngettext("%d consecutive error found in the input log file %s\n",
+ "%d consecutive errors found in the input log file %s\n",successive_errors),successive_errors,arq);
+ exit(EXIT_FAILURE);
}
- current_format=LogFormats[x];
- current_format_idx=x;
- if (debugz) {
- /* TRANSLATORS: The argument is the log format name as translated by you. */
- debuga(_("Log format identified as \"%s\" for %s\n"),_(current_format->Name),arq);
+ if (NumLogTotalErrors>=0 && ++total_errors>NumLogTotalErrors) {
+ debuga(ngettext("%d error found in the input log file (last in %s)\n",
+ "%d errors found in the input log file (last in %s)\n",total_errors),total_errors,arq);
+ exit(EXIT_FAILURE);
}
- successive_errors=0;
- }
- if (log_entry_status==RLRC_Ignore) {
+ debuga(_("The following line read from %s could not be parsed and is ignored\n%s\n"),arq,linebuf);
continue;
}
- if (current_format_idx<0 || current_format==NULL) {
- debuga(_("Sarg failed to determine the format of the input log file %s\n"),arq);
+ current_format=LogFormats[x];
+ current_format_idx=x;
+ if (debugz) {
+ /* TRANSLATORS: The argument is the log format name as translated by you. */
+ debuga(_("Log format identified as \"%s\" for %s\n"),_(current_format->Name),arq);
+ }
+ successive_errors=0;
+ }
+ if (log_entry_status==RLRC_Ignore) {
+ continue;
+ }
+ if (current_format_idx<0 || current_format==NULL) {
+ debuga(_("Sarg failed to determine the format of the input log file %s\n"),arq);
+ exit(EXIT_FAILURE);
+ }
+ if (log_entry_status==RLRC_InternalError) {
+ debuga(_("Internal error encountered while processing %s\nSee previous message to know the reason for that error.\n"),arq);
+ exit(EXIT_FAILURE);
+ }
+ format_count[current_format_idx]++;
+
+ if (!fp_log && ParsedOutputLog[0] && current_format!=&ReadSargLog) {
+ if(access(ParsedOutputLog,R_OK) != 0) {
+ my_mkdir(ParsedOutputLog);
+ }
+ if (snprintf(SargLogFile,sizeof(SargLogFile),"%s/sarg_temp.log",ParsedOutputLog)>=sizeof(SargLogFile)) {
+ debuga(_("File name too long: %s/sarg_temp.log\n"),ParsedOutputLog);
exit(EXIT_FAILURE);
}
- if (log_entry_status==RLRC_InternalError) {
- debuga(_("Internal error encountered while processing %s\nSee previous message to know the reason for that error.\n"),arq);
+ if((fp_log=MY_FOPEN(SargLogFile,"w"))==NULL) {
+ debuga(_("(log) Cannot open log file %s: %s\n"),SargLogFile,strerror(errno));
exit(EXIT_FAILURE);
}
- format_count[current_format_idx]++;
+ fputs("*** SARG Log ***\n",fp_log);
+ }
- if (!fp_log && ParsedOutputLog[0] && current_format!=&ReadSargLog) {
- if(access(ParsedOutputLog,R_OK) != 0) {
- my_mkdir(ParsedOutputLog);
- }
- if (snprintf(arq_log,sizeof(arq_log),"%s/sarg_temp.log",ParsedOutputLog)>=sizeof(arq_log)) {
- debuga(_("File name too long: %s/sarg_temp.log\n"),ParsedOutputLog);
- exit(EXIT_FAILURE);
- }
- if((fp_log=MY_FOPEN(arq_log,"w"))==NULL) {
- debuga(_("(log) Cannot open log file %s: %s\n"),arq_log,strerror(errno));
- exit(EXIT_FAILURE);
- }
- fputs("*** SARG Log ***\n",fp_log);
- }
+ if (log_entry.Ip==NULL) {
+ debuga(_("Unknown input log file format: no IP addresses\n"));
+ break;
+ }
+ if (log_entry.User==NULL) {
+ debuga(_("Unknown input log file format: no user\n"));
+ break;
+ }
+ if (log_entry.Url==NULL) {
+ debuga(_("Unknown input log file format: no URL\n"));
+ break;
+ }
- if (log_entry.Ip==NULL) {
- debuga(_("Unknown input log file format: no IP addresses\n"));
- break;
- }
- if (log_entry.User==NULL) {
- debuga(_("Unknown input log file format: no user\n"));
- break;
- }
- if (log_entry.Url==NULL) {
- debuga(_("Unknown input log file format: no URL\n"));
- break;
- }
+ idata=builddia(log_entry.EntryTime.tm_mday,log_entry.EntryTime.tm_mon+1,log_entry.EntryTime.tm_year+1900);
+ if(debugm)
+ printf("DATE=%s IDATA=%d DFROM=%d DUNTIL=%d\n",Filter->DateRange,idata,dfrom,duntil);
- idata=builddia(log_entry.EntryTime.tm_mday,log_entry.EntryTime.tm_mon+1,log_entry.EntryTime.tm_year+1900);
- if(debugm)
- printf("DATE=%s IDATA=%d DFROM=%d DUNTIL=%d\n",Filter->DateRange,idata,dfrom,duntil);
+ if(Filter->DateRange[0] != '\0'){
+ if(idata < dfrom || idata > duntil) continue;
+ }
- if(Filter->DateRange[0] != '\0'){
- if(idata < dfrom || idata > duntil) continue;
- }
+ // Record only hours usage which is required
+ if( bsearch( &( log_entry.EntryTime.tm_wday ), weekdays.list, weekdays.len, sizeof( int ), compar ) == NULL )
+ continue;
- // Record only hours usage which is required
- if( bsearch( &( log_entry.EntryTime.tm_wday ), weekdays.list, weekdays.len, sizeof( int ), compar ) == NULL )
- continue;
+ if( bsearch( &( log_entry.EntryTime.tm_hour ), hours.list, hours.len, sizeof( int ), compar ) == NULL )
+ continue;
- if( bsearch( &( log_entry.EntryTime.tm_hour ), hours.list, hours.len, sizeof( int ), compar ) == NULL )
- continue;
+ if(strlen(log_entry.User) > MAX_USER_LEN) {
+ if (debugm) printf(_("User ID too long: %s\n"),log_entry.User);
+ totregsx++;
+ continue;
+ }
- if(strlen(log_entry.User) > MAX_USER_LEN) {
- if (debugm) printf(_("User ID too long: %s\n"),log_entry.User);
- totregsx++;
+ // include_users
+ if(IncludeUsers[0] != '\0') {
+ snprintf(val1,sizeof(val1),":%s:",log_entry.User);
+ if((str=(char *) strstr(IncludeUsers,val1)) == (char *) NULL )
continue;
- }
+ }
- // include_users
- if(IncludeUsers[0] != '\0') {
- snprintf(val1,sizeof(val1),":%s:",log_entry.User);
- if((str=(char *) strstr(IncludeUsers,val1)) == (char *) NULL )
- continue;
+ if(vercode(log_entry.HttpCode)) {
+ if (debugm) printf(_("Excluded code: %s\n"),log_entry.HttpCode);
+ totregsx++;
+ continue;
+ }
+
+ if(testvaliduserchar(log_entry.User))
+ continue;
+
+ // replace any tab by a single space
+ for (str=log_entry.Url ; *str ; str++)
+ if (*str=='\t') *str=' ';
+ for (str=log_entry.HttpCode ; *str ; str++)
+ if (*str=='\t') *str=' ';
+
+ if (current_format!=&ReadSargLog) {
+ /*
+ The full URL is not saved in sarg log. There is no point in testing the URL to detect
+ a downloaded file.
+ */
+ download_flag=is_download_suffix(log_entry.Url);
+ if (download_flag) {
+ safe_strcpy(download_url,log_entry.Url,sizeof(download_url));
}
+ } else
+ download_flag=false;
- if(vercode(log_entry.HttpCode)) {
- if (debugm) printf(_("Excluded code: %s\n"),log_entry.HttpCode);
+ url=process_url(log_entry.Url,LongUrl);
+ if (!url || url[0] == '\0') continue;
+
+ if(addr[0] != '\0'){
+ if(strcmp(addr,log_entry.Ip)!=0) continue;
+ }
+ if(Filter->HostFilter) {
+ if(!vhexclude(url)) {
+ if (debugm) printf(_("Excluded site: %s\n"),url);
totregsx++;
continue;
}
+ }
- if(testvaliduserchar(log_entry.User))
- continue;
+ if(Filter->StartTime >= 0 && Filter->EndTime >= 0) {
+ hmr=log_entry.EntryTime.tm_hour*100+log_entry.EntryTime.tm_min;
+ if(hmr < Filter->StartTime || hmr > Filter->EndTime) continue;
+ }
- // replace any tab by a single space
- for (str=log_entry.Url ; *str ; str++)
- if (*str=='\t') *str=' ';
- for (str=log_entry.HttpCode ; *str ; str++)
- if (*str=='\t') *str=' ';
+ if(site[0] != '\0'){
+ if(strstr(url,site)==0) continue;
+ }
- if (current_format!=&ReadSargLog) {
- /*
- The full URL is not saved in sarg log. There is no point in testing the URL to detect
- a downloaded file.
- */
- download_flag=is_download_suffix(log_entry.Url);
- if (download_flag) {
- safe_strcpy(download_url,log_entry.Url,sizeof(download_url));
+ if(UserIp) {
+ log_entry.User=log_entry.Ip;
+ id_is_ip=true;
+ } else {
+ id_is_ip=false;
+ if ((log_entry.User[0]=='\0') || (log_entry.User[1]=='\0' && (log_entry.User[0]=='-' || log_entry.User[0]==' '))) {
+ if(RecordsWithoutUser == RECORDWITHOUTUSER_IP) {
+ log_entry.User=log_entry.Ip;
+ id_is_ip=true;
}
- } else
- download_flag=false;
-
- url=process_url(log_entry.Url,LongUrl);
- if (!url || url[0] == '\0') continue;
-
- if(addr[0] != '\0'){
- if(strcmp(addr,log_entry.Ip)!=0) continue;
- }
- if(Filter->HostFilter) {
- if(!vhexclude(url)) {
- if (debugm) printf(_("Excluded site: %s\n"),url);
- totregsx++;
+ if(RecordsWithoutUser == RECORDWITHOUTUSER_IGNORE)
continue;
- }
- }
-
- if(Filter->StartTime >= 0 && Filter->EndTime >= 0) {
- hmr=log_entry.EntryTime.tm_hour*100+log_entry.EntryTime.tm_min;
- if(hmr < Filter->StartTime || hmr > Filter->EndTime) continue;
- }
-
- if(site[0] != '\0'){
- if(strstr(url,site)==0) continue;
- }
-
- if(UserIp) {
- log_entry.User=log_entry.Ip;
- id_is_ip=true;
+ if(RecordsWithoutUser == RECORDWITHOUTUSER_EVERYBODY)
+ log_entry.User="everybody";
} else {
- id_is_ip=false;
- if ((log_entry.User[0]=='\0') || (log_entry.User[1]=='\0' && (log_entry.User[0]=='-' || log_entry.User[0]==' '))) {
- if(RecordsWithoutUser == RECORDWITHOUTUSER_IP) {
- log_entry.User=log_entry.Ip;
- id_is_ip=true;
- }
- if(RecordsWithoutUser == RECORDWITHOUTUSER_IGNORE)
- continue;
- if(RecordsWithoutUser == RECORDWITHOUTUSER_EVERYBODY)
- log_entry.User="everybody";
- } else {
- strlow(log_entry.User);
- if(NtlmUserFormat == NTLMUSERFORMAT_USER) {
- if ((str=strchr(user,'+'))!=NULL || (str=strchr(user,'\\'))!=NULL || (str=strchr(user,'_'))!=NULL) {
- log_entry.User=str+1;
- }
+ strlow(log_entry.User);
+ if(NtlmUserFormat == NTLMUSERFORMAT_USER) {
+ if ((str=strchr(user,'+'))!=NULL || (str=strchr(user,'\\'))!=NULL || (str=strchr(user,'_'))!=NULL) {
+ log_entry.User=str+1;
}
}
}
+ }
- if(us[0] != '\0'){
- if(strcmp(log_entry.User,us)!=0) continue;
- }
+ if(us[0] != '\0'){
+ if(strcmp(log_entry.User,us)!=0) continue;
+ }
- if(Filter->SysUsers) {
- snprintf(wuser,sizeof(wuser),":%s:",log_entry.User);
- if(strstr(userfile, wuser) == 0)
- continue;
- }
+ if(Filter->SysUsers) {
+ snprintf(wuser,sizeof(wuser),":%s:",log_entry.User);
+ if(strstr(userfile, wuser) == 0)
+ continue;
+ }
- if(Filter->UserFilter) {
- if(!vuexclude(log_entry.User)) {
- if (debugm) printf(_("Excluded user: %s\n"),log_entry.User);
- totregsx++;
- continue;
- }
+ if(Filter->UserFilter) {
+ if(!vuexclude(log_entry.User)) {
+ if (debugm) printf(_("Excluded user: %s\n"),log_entry.User);
+ totregsx++;
+ continue;
}
+ }
- if (log_entry.User[0]=='\0' || (log_entry.User[1]=='\0' && (log_entry.User[0]=='-' ||
- log_entry.User[0]==' ' || log_entry.User[0]==':')))
- continue;
+ if (log_entry.User[0]=='\0' || (log_entry.User[1]=='\0' && (log_entry.User[0]=='-' ||
+ log_entry.User[0]==' ' || log_entry.User[0]==':')))
+ continue;
- if (log_entry.DataSize<0) log_entry.DataSize=0;
+ if (log_entry.DataSize<0) log_entry.DataSize=0;
- if (log_entry.ElapsedTime<0) log_entry.ElapsedTime=0;
- if (Filter->max_elapsed>0 && log_entry.ElapsedTime>Filter->max_elapsed) {
- log_entry.ElapsedTime=0;
- }
+ if (log_entry.ElapsedTime<0) log_entry.ElapsedTime=0;
+ if (Filter->max_elapsed>0 && log_entry.ElapsedTime>Filter->max_elapsed) {
+ log_entry.ElapsedTime=0;
+ }
- if((str=(char *) strstr(linebuf, "[SmartFilter:")) != (char *) NULL ) {
- fixendofline(str);
- snprintf(smartfilter,sizeof(smartfilter),"\"%s\"",str+1);
- } else strcpy(smartfilter,"\"\"");
+ if((str=(char *) strstr(linebuf, "[SmartFilter:")) != (char *) NULL ) {
+ fixendofline(str);
+ snprintf(smartfilter,sizeof(smartfilter),"\"%s\"",str+1);
+ } else strcpy(smartfilter,"\"\"");
- nopen=0;
- prev_ufile=NULL;
- for (ufile=first_user_file ; ufile && strcmp(log_entry.User,ufile->user->id)!=0 ; ufile=ufile->next) {
- prev_ufile=ufile;
- if (ufile->file) nopen++;
- }
+ nopen=0;
+ prev_ufile=NULL;
+ for (ufile=first_user_file ; ufile && strcmp(log_entry.User,ufile->user->id)!=0 ; ufile=ufile->next) {
+ prev_ufile=ufile;
+ if (ufile->file) nopen++;
+ }
+ if (!ufile) {
+ ufile=malloc(sizeof(*ufile));
if (!ufile) {
- ufile=malloc(sizeof(*ufile));
- if (!ufile) {
- debuga(_("Not enough memory to store the user %s\n"),log_entry.User);
- exit(EXIT_FAILURE);
- }
- memset(ufile,0,sizeof(*ufile));
+ debuga(_("Not enough memory to store the user %s\n"),log_entry.User);
+ exit(EXIT_FAILURE);
+ }
+ memset(ufile,0,sizeof(*ufile));
+ ufile->next=first_user_file;
+ first_user_file=ufile;
+ uinfo=userinfo_create(log_entry.User);
+ ufile->user=uinfo;
+ uinfo->id_is_ip=id_is_ip;
+ nusers++;
+ } else {
+ if (prev_ufile) {
+ prev_ufile->next=ufile->next;
ufile->next=first_user_file;
first_user_file=ufile;
- uinfo=userinfo_create(log_entry.User);
- ufile->user=uinfo;
- uinfo->id_is_ip=id_is_ip;
- nusers++;
- } else {
- if (prev_ufile) {
- prev_ufile->next=ufile->next;
- ufile->next=first_user_file;
- first_user_file=ufile;
- }
}
+ }
#ifdef ENABLE_DOUBLE_CHECK_DATA
- ufile->user->nbytes+=log_entry.DataSize;
- ufile->user->elap+=log_entry.ElapsedTime;
+ ufile->user->nbytes+=log_entry.DataSize;
+ ufile->user->elap+=log_entry.ElapsedTime;
#endif
- if (ufile->file==NULL) {
- if (nopen>=maxopenfiles) {
- x=0;
- for (ufile1=first_user_file ; ufile1 ; ufile1=ufile1->next) {
- if (ufile1->file!=NULL) {
- if (x>=maxopenfiles) {
- if (fclose(ufile1->file)==EOF) {
- debuga(_("Write error in the log file of user %s: %s\n"),ufile1->user->id,strerror(errno));
- exit(EXIT_FAILURE);
- }
- ufile1->file=NULL;
+ if (ufile->file==NULL) {
+ if (nopen>=maxopenfiles) {
+ x=0;
+ for (ufile1=first_user_file ; ufile1 ; ufile1=ufile1->next) {
+ if (ufile1->file!=NULL) {
+ if (x>=maxopenfiles) {
+ if (fclose(ufile1->file)==EOF) {
+ debuga(_("Write error in the log file of user %s: %s\n"),ufile1->user->id,strerror(errno));
+ exit(EXIT_FAILURE);
}
- x++;
+ ufile1->file=NULL;
}
+ x++;
}
}
- if (snprintf (tmp3, sizeof(tmp3), "%s/%s.user_unsort", tmp, ufile->user->filename)>=sizeof(tmp3)) {
- debuga(_("Temporary user file name too long: %s/%s.user_unsort\n"), tmp, ufile->user->filename);
- exit(EXIT_FAILURE);
- }
- if ((ufile->file = MY_FOPEN (tmp3, "a")) == NULL) {
- debuga(_("(log) Cannot open temporary file %s: %s\n"), tmp3, strerror(errno));
- exit (1);
- }
}
-
- strftime(dia, sizeof(dia), "%d/%m/%Y",&log_entry.EntryTime);
- strftime(hora,sizeof(hora),"%H:%M:%S",&log_entry.EntryTime);
-
- if (fprintf(ufile->file, "%s\t%s\t%s\t%s\t%"PRIu64"\t%s\t%ld\t%s\n",dia,hora,
- log_entry.Ip,url,(uint64_t)log_entry.DataSize,
- log_entry.HttpCode,log_entry.ElapsedTime,smartfilter)<=0) {
- debuga(_("Write error in the log file of user %s\n"),log_entry.User);
+ if (snprintf (tmp3, sizeof(tmp3), "%s/%s.user_unsort", tmp, ufile->user->filename)>=sizeof(tmp3)) {
+ debuga(_("Temporary user file name too long: %s/%s.user_unsort\n"), tmp, ufile->user->filename);
exit(EXIT_FAILURE);
}
- records_kept++;
-
- if (fp_log && current_format!=&ReadSargLog) {
- fprintf(fp_log, "%s\t%s\t%s\t%s\t%s\t%"PRIu64"\t%s\t%ld\t%s\n",dia,hora,
- log_entry.User,log_entry.Ip,url,(uint64_t)log_entry.DataSize,
- log_entry.HttpCode,log_entry.ElapsedTime,smartfilter);
+ if ((ufile->file = MY_FOPEN (tmp3, "a")) == NULL) {
+ debuga(_("(log) Cannot open temporary file %s: %s\n"), tmp3, strerror(errno));
+ exit (1);
}
+ }
- totregsg++;
+ strftime(dia, sizeof(dia), "%d/%m/%Y",&log_entry.EntryTime);
+ strftime(hora,sizeof(hora),"%H:%M:%S",&log_entry.EntryTime);
- denied_write(&log_entry);
- authfail_write(&log_entry);
- download_write(&log_entry,download_url);
+ if (fprintf(ufile->file, "%s\t%s\t%s\t%s\t%"PRIu64"\t%s\t%ld\t%s\n",dia,hora,
+ log_entry.Ip,url,(uint64_t)log_entry.DataSize,
+ log_entry.HttpCode,log_entry.ElapsedTime,smartfilter)<=0) {
+ debuga(_("Write error in the log file of user %s\n"),log_entry.User);
+ exit(EXIT_FAILURE);
+ }
+ records_kept++;
- if (current_format!=&ReadSargLog) {
- if(!totper || idata<mindate){
- mindate=idata;
- memcpy(&period.start,&log_entry.EntryTime,sizeof(log_entry.EntryTime));
- strcpy(start_hour,tbuf2);
- }
- if (!totper || idata>maxdate) {
- maxdate=idata;
- memcpy(&period.end,&log_entry.EntryTime,sizeof(log_entry.EntryTime));
- }
- totper=true;
- }
+ if (fp_log && current_format!=&ReadSargLog) {
+ fprintf(fp_log, "%s\t%s\t%s\t%s\t%s\t%"PRIu64"\t%s\t%ld\t%s\n",dia,hora,
+ log_entry.User,log_entry.Ip,url,(uint64_t)log_entry.DataSize,
+ log_entry.HttpCode,log_entry.ElapsedTime,smartfilter);
+ }
+
+ totregsg++;
+
+ denied_write(&log_entry);
+ authfail_write(&log_entry);
+ download_write(&log_entry,download_url);
- if(debugm){
- printf("IP=\t%s\n",log_entry.Ip);
- printf("USER=\t%s\n",log_entry.User);
- printf("ELAP=\t%ld\n",log_entry.ElapsedTime);
- printf("DATE=\t%s\n",dia);
- printf("TIME=\t%s\n",hora);
- //printf("FUNC=\t%s\n",fun);
- printf("URL=\t%s\n",url);
- printf("CODE=\t%s\n",log_entry.HttpCode);
- printf("LEN=\t%"PRIu64"\n",(uint64_t)log_entry.DataSize);
+ if (current_format!=&ReadSargLog) {
+ if (period.start.tm_year==0 || idata<mindate || compare_date(&period.start,&log_entry.EntryTime)>0){
+ mindate=idata;
+ memcpy(&period.start,&log_entry.EntryTime,sizeof(log_entry.EntryTime));
+ }
+ if (period.end.tm_year==0 || idata>maxdate || compare_date(&period.end,&log_entry.EntryTime)<0) {
+ maxdate=idata;
+ memcpy(&period.end,&log_entry.EntryTime,sizeof(log_entry.EntryTime));
}
}
- if (!from_stdin) {
- if (from_pipe)
- pclose(fp_in);
- else {
- fclose(fp_in);
- if (ShowReadStatistics) {
- if (ShowReadPercent)
- printf(_("SARG: Records in file: %lu, reading: %3.2f%%\n"),recs2, (float) 100 );
- else
- printf(_("SARG: Records in file: %lu\n"),recs2);
- }
- }
+ if(debugm){
+ printf("IP=\t%s\n",log_entry.Ip);
+ printf("USER=\t%s\n",log_entry.User);
+ printf("ELAP=\t%ld\n",log_entry.ElapsedTime);
+ printf("DATE=\t%s\n",dia);
+ printf("TIME=\t%s\n",hora);
+ //printf("FUNC=\t%s\n",fun);
+ printf("URL=\t%s\n",url);
+ printf("CODE=\t%s\n",log_entry.HttpCode);
+ printf("LEN=\t%"PRIu64"\n",(uint64_t)log_entry.DataSize);
}
}
longline_destroy(&line);
+ if (!from_stdin) {
+ if (from_pipe)
+ pclose(fp_in);
+ else {
+ fclose(fp_in);
+ if (ShowReadStatistics) {
+ if (ShowReadPercent)
+ printf(_("SARG: Records in file: %lu, reading: %3.2f%%\n"),recs2, (float) 100 );
+ else
+ printf(_("SARG: Records in file: %lu\n"),recs2);
+ }
+ }
+ }
+}
+
+/*!
+Read the log files.
+
+\param Filter The filtering parameters for the file to load.
+
+\retval 1 Records found.
+\retval 0 No record found.
+*/
+int ReadLogFile(struct ReadLogDataStruct *Filter)
+{
+ int iarq;
+ int x;
+ int cstatus;
+ struct userfilestruct *ufile;
+ struct userfilestruct *ufile1;
+
+ for (x=0 ; x<sizeof(format_count)/sizeof(*format_count) ; x++) format_count[x]=0;
+ first_user_file=NULL;
+
+ if (!dataonly) {
+ denied_open();
+ authfail_open();
+ download_open();
+ }
+
+ for (iarq=0 ; iarq<NAccessLog ; iarq++) {
+ ReadOneLogFile(Filter,AccessLog[iarq]);
+ }
+
if(fp_log != NULL) {
- char end_hour[128];
char val2[40];
- char val4[255];//val4 must not be bigger than arq_log without fixing the strcpy below
+ char val4[4096];//val4 must not be bigger than SargLogFile without fixing the strcpy below
if (fclose(fp_log)==EOF) {
- debuga(_("Write error in %s: %s\n"),arq_log,strerror(errno));
+ debuga(_("Write error in %s: %s\n"),SargLogFile,strerror(errno));
exit(EXIT_FAILURE);
}
- safe_strcpy(end_hour,tbuf2,sizeof(end_hour));
- strftime(val2,sizeof(val2),"%d%m%Y",&period.start);
- strftime(val1,sizeof(val1),"%d%m%Y",&period.end);
- if (snprintf(val4,sizeof(val4),"%s/sarg-%s_%s-%s_%s.log",ParsedOutputLog,val2,start_hour,val1,end_hour)>=sizeof(val4)) {
- debuga(_("File name too long: %s/sarg-%s_%s-%s_%s.log\n"),ParsedOutputLog,val2,start_hour,val1,end_hour);
+ strftime(val2,sizeof(val2),"%d%m%Y_%H%M",&period.start);
+ strftime(val1,sizeof(val1),"%d%m%Y_%H%M",&period.end);
+ if (snprintf(val4,sizeof(val4),"%s/sarg-%s-%s.log",ParsedOutputLog,val2,val1)>=sizeof(val4)) {
+ debuga(_("File name too long: %s/sarg-%s-%s.log\n"),ParsedOutputLog,val2,val1);
exit(EXIT_FAILURE);
}
- if (rename(arq_log,val4)) {
- debuga(_("failed to rename %s to %s - %s\n"),arq_log,val4,strerror(errno));
+ if (rename(SargLogFile,val4)) {
+ debuga(_("failed to rename %s to %s - %s\n"),SargLogFile,val4,strerror(errno));
} else {
- strcpy(arq_log,val4);
+ strcpy(SargLogFile,val4);
if(strcmp(ParsedOutputLogCompress,"nocompress") != 0 && ParsedOutputLogCompress[0] != '\0') {
/*
No double quotes around ParsedOutputLogCompress because it may contain command line options. If double quotes are
necessary around the command name, put them in the configuration file.
*/
- if (snprintf(val1,sizeof(val1),"%s \"%s\"",ParsedOutputLogCompress,arq_log)>=sizeof(val1)) {
- debuga(_("Command too long: %s \"%s\"\n"),ParsedOutputLogCompress,arq_log);
+ if (snprintf(val1,sizeof(val1),"%s \"%s\"",ParsedOutputLogCompress,SargLogFile)>=sizeof(val1)) {
+ debuga(_("Command too long: %s \"%s\"\n"),ParsedOutputLogCompress,SargLogFile);
exit(EXIT_FAILURE);
}
cstatus=system(val1);
}
}
if(debug)
- debuga(_("Sarg parsed log saved as %s\n"),arq_log);
+ debuga(_("Sarg parsed log saved as %s\n"),SargLogFile);
}
denied_close();
debuga(_("Log with invalid format\n"));
}
- if(debugz) {
- debugaz(_("date=%s\n"),dia);
+ if (debugz)
debugaz(_("period=%s\n"),period.text);
- }
return((totregsg!=0) ? 1 : 0);
}
projects / thirdparty / sarg.git / commitdiff
