From: Frederic Marchal Date: Fri, 21 Aug 2015 17:24:55 +0000 (+0200) Subject: Rewrite the useragent log processing X-Git-Url: http://git.ipfire.org/?p=thirdparty%2Fsarg.git;a=commitdiff_plain;h=f009b9ad4c11cf2b90e06f6b184bd28942b0d6c5 Rewrite the useragent log processing Create more functions to distinguish the reading of the useragent log file from the report generation. The idea is to prepare the code to read the useragent data from other sources than the useragent_log option. --- diff --git a/include/defs.h b/include/defs.h index 19743a3..a6e3c24 100644 --- a/include/defs.h +++ b/include/defs.h @@ -336,7 +336,10 @@ void url_hostname(const char *url,char *hostname,int hostsize); void usage(const char *prog); // useragent.c -void useragent(void); +FILE *UserAgent_Open(void); +void UserAgent_Write(FILE *fp,const char *Ip,const char *User,const char *Agent); +void UserAgent_Readlog(void); +void UserAgent(void); // userinfo.c /*@shared@*/struct userinfostruct *userinfo_create(const char *userid, const char *ip); diff --git a/report.c b/report.c index dba3cd0..2d81566 100644 --- a/report.c +++ b/report.c @@ -112,7 +112,11 @@ void gerarel(void) debugaz(__FILE__,__LINE__,_("outdirname=%s\n"),outdirname); } - if (!FileList_IsEmpty(UserAgentLog) && email[0] == '\0') useragent(); + if (email[0] == '\0') { + if (!FileList_IsEmpty(UserAgentLog)) + UserAgent_Readlog(); + UserAgent(); + } snprintf(wdirname,sizeof(wdirname),"%s/sarg-general",outdirname); if((fp_gen=MY_FOPEN(wdirname,"w"))==NULL){ diff --git a/useragent.c b/useragent.c index 1e654d3..73ff399 100644 --- a/useragent.c +++ b/useragent.c @@ -30,60 +30,84 @@ FileListObject UserAgentLog=NULL; -void useragent(void) +//! Log file where the user agent data are written. +static char UserAgentTempLog[MAXLEN]=""; + +static struct tm UserAgentStartDate; +static struct tm UserAgentEndDate; + +/*! + * Open the temporary file to store the useragent entries to be + * reported. + * + * \return The file handle. It must be closed when the data have + * been written. + */ +FILE *UserAgent_Open(void) +{ + FILE *fp_ou; + + if (UserAgentTempLog[0]) { + debuga(__FILE__,__LINE__,_("Useragent log already opened\n")); + exit(EXIT_FAILURE); + } + snprintf(UserAgentTempLog,sizeof(UserAgentTempLog),"%s/squagent.int_unsort",tmp); + if ((fp_ou=fopen(UserAgentTempLog,"w"))==NULL) { + debuga(__FILE__,__LINE__,_("Cannot open file \"%s\": %s\n"),UserAgentTempLog,strerror(errno)); + exit(EXIT_FAILURE); + } + memset(&UserAgentStartDate,0,sizeof(UserAgentStartDate)); + memset(&UserAgentEndDate,0,sizeof(UserAgentEndDate)); + return(fp_ou); +} + +/*! + * Write a user agent entry into the temporary log. + * + * \param fp The file opened by UserAgent_Open(). + * \param Ip The IP address using this agent. + * \param User The user name. + * \param Agent The user agent string. + */ +void UserAgent_Write(FILE *fp,const char *Ip,const char *User,const char *Agent) +{ + fprintf(fp,"%s\t%s\t%s\n",Ip,Agent,User); + useragent_count++; +} + +/*! + * Read the user provided useragent file and create + * a temporary file with the data to report. + */ +void UserAgent_Readlog(void) { FileObject *fp_log; - FILE *fp_in = NULL, *fp_ou = NULL, *fp_ht = NULL; + FILE *fp_ou = NULL; char *ptr; - char buf[MAXLEN]; - char ip[MAXLEN], data[MAXLEN], agent[MAXLEN], user[MAXLEN]; - char ipbefore[MAXLEN], namebefore[MAXLEN]; - char tagent[MAXLEN]; - char user_old[MAXLEN]="$#%0a3bc6"; - char agent_old[MAXLEN]="$#%0a3bc6"; - char hfile[MAXLEN]; - char idate[MAXLEN], fdate[MAXLEN]; - char tmp2[MAXLEN]; - char tmp3[MAXLEN]; - char day[4],month[5],year[5], wdate[20]; - char csort[MAXLEN]; + char ip[80], data[50], agent[MAXLEN], user[MAXLEN]; + int day,month,year; + char monthname[5]; const char *FileName; - int agentot=0, agentot2=0, agentdif=0, cont=0, nagent; unsigned long totregsl=0; - int cstatus; int ndate; - double perc; struct getwordstruct gwarea, gwarea1; longline line; FileListIterator FIter; + struct tm logtime; - ip[0]='\0'; - data[0]='\0'; - agent[0]='\0'; - user[0]='\0'; - user_old[0]='\0'; - agent_old[0]='\0'; - ipbefore[0]='\0'; - namebefore[0]='\0'; - - snprintf(tmp3,sizeof(tmp3),"%s/squagent.int_unsort",tmp); - snprintf(tmp2,sizeof(tmp2),"%s/squagent.int_log",tmp); - - if((fp_ou=fopen(tmp3,"w"))==NULL) { - debuga(__FILE__,__LINE__,_("Cannot open file \"%s\": %s\n"),tmp3,strerror(errno)); - exit(EXIT_FAILURE); - } + fp_ou=UserAgent_Open(); if ((line=longline_create())==NULL) { debuga(__FILE__,__LINE__,_("Not enough memory to read useragent log\n")); exit(EXIT_FAILURE); } + memset(&logtime,0,sizeof(logtime)); FIter=FileListIter_Open(UserAgentLog); while ((FileName=FileListIter_Next(FIter))!=NULL) { longline_reset(line); - if((fp_log=decomp(FileName))==NULL) { + if ((fp_log=decomp(FileName))==NULL) { debuga(__FILE__,__LINE__,_("Cannot open file \"%s\": %s\n"),FileName,FileObject_GetLastOpenError()); exit(EXIT_FAILURE); } @@ -92,49 +116,56 @@ void useragent(void) debuga(__FILE__,__LINE__,_("Reading useragent log \"%s\"\n"),FileName); } - while((ptr=longline_read(fp_log,line))!=NULL) { + while ((ptr=longline_read(fp_log,line))!=NULL) { totregsl++; getword_start(&gwarea,ptr); - if (getword(ip,sizeof(ip),&gwarea,' ')<0 || getword_skip(MAXLEN,&gwarea,'[')<0 || + if (getword(ip,sizeof(ip),&gwarea,' ')<0 || getword_skip(10,&gwarea,'[')<0 || getword(data,sizeof(data),&gwarea,' ')<0) { debuga(__FILE__,__LINE__,_("Invalid record in file \"%s\"\n"),FileName); exit(EXIT_FAILURE); } getword_start(&gwarea1,data); - if (getword(day,sizeof(day),&gwarea1,'/')<0 || getword(month,sizeof(month),&gwarea1,'/')<0 || - getword(year,sizeof(year),&gwarea1,':')<0) { + if (getword_atoi(&day,&gwarea1,'/')<0 || getword(monthname,sizeof(monthname),&gwarea1,'/')<0 || + getword_atoi(&year,&gwarea1,':')<0) { debuga(__FILE__,__LINE__,_("Invalid date in file \"%s\"\n"),FileName); exit(EXIT_FAILURE); } + month=month2num(monthname)+1; + if (month>12) { + debuga(__FILE__,__LINE__,_("Invalid month name \"%s\" found in user agent file \"%s\""),monthname,FileName); + exit(EXIT_FAILURE); + } if (dfrom!=0 || duntil!=0){ - buildymd(day,month,year,wdate,sizeof(wdate)); - ndate=atoi(wdate); + ndate=year*10000+month*100+day; if (ndateduntil) break; } - if(totregsl == 1) - strcpy(idate,data); - strcpy(fdate,data); + logtime.tm_year=year-1900; + logtime.tm_mon=month-1; + logtime.tm_mday=day; + if (totregsl==1 || compare_date(&UserAgentStartDate,&logtime)>0) + memcpy(&UserAgentStartDate,&logtime,sizeof(logtime)); + if (compare_date(&UserAgentEndDate,&logtime)<0) + memcpy(&UserAgentEndDate,&logtime,sizeof(logtime)); if (getword_skip(MAXLEN,&gwarea,'"')<0 || getword(agent,sizeof(agent),&gwarea,'"')<0) { debuga(__FILE__,__LINE__,_("Invalid useragent in file \"%s\"\n"),FileName); exit(EXIT_FAILURE); } - if(gwarea.current[0]!='\0') { + if (gwarea.current[0]!='\0') { if (getword_skip(MAXLEN,&gwarea,' ')<0 || getword(user,sizeof(user),&gwarea,'\n')<0) { debuga(__FILE__,__LINE__,_("Invalid record in file \"%s\"\n"),FileName); exit(EXIT_FAILURE); } - if(user[0] == '-') + if (user[0] == '-') strcpy(user,ip); - if(user[0] == '\0') + if (user[0] == '\0') strcpy(user,ip); } else { strcpy(user,ip); } - fprintf(fp_ou,"%s\t%s\t%s\n",ip,agent,user); - useragent_count++; + UserAgent_Write(fp_ou,ip,user,agent); } if (FileObject_Close(fp_log)==EOF) { @@ -150,15 +181,40 @@ void useragent(void) } if (fclose(fp_ou)==EOF) { - debuga(__FILE__,__LINE__,_("Write error in \"%s\": %s\n"),tmp3,strerror(errno)); + debuga(__FILE__,__LINE__,_("Write error in \"%s\": %s\n"),UserAgentTempLog,strerror(errno)); exit(EXIT_FAILURE); } - if(debug) { +} + +void UserAgent(void) +{ + FILE *fp_in = NULL, *fp_ou = NULL, *fp_ht = NULL; + char buf[MAXLEN]; + char ip[80], agent[MAXLEN], user[MAXLEN]; + char ipbefore[MAXLEN]=""; + char namebefore[MAXLEN]=""; + char tagent[MAXLEN]; + char user_old[MAXLEN]=""; + char agent_old[MAXLEN]=""; + char hfile[MAXLEN]; + char idate[100], fdate[100]; + char tmp2[MAXLEN]; + char tmp3[MAXLEN]; + char csort[MAXLEN]; + int agentot=0, agentot2=0, agentdif=0, cont=0, nagent; + int cstatus; + double perc; + struct getwordstruct gwarea; + + if (!UserAgentTempLog[0]) return; + + snprintf(tmp2,sizeof(tmp2),"%s/squagent.int_log",tmp); + if (debug) { debuga(__FILE__,__LINE__,_("Sorting file \"%s\"\n"),tmp2); } - if (snprintf(csort,sizeof(csort),"sort -n -t \"\t\" -k 3,3 -k 2,2 -k 1,1 -o \"%s\" \"%s\"",tmp2,tmp3)>=sizeof(csort)) { - debuga(__FILE__,__LINE__,_("Sort command too long when sorting file \"%s\" to \"%s\"\n"),tmp2,tmp3); + if (snprintf(csort,sizeof(csort),"sort -n -t \"\t\" -k 3,3 -k 2,2 -k 1,1 -o \"%s\" \"%s\"",tmp2,UserAgentTempLog)>=sizeof(csort)) { + debuga(__FILE__,__LINE__,_("Sort command too long when sorting file \"%s\" to \"%s\"\n"),tmp2,UserAgentTempLog); exit(EXIT_FAILURE); } cstatus=system(csort); @@ -167,28 +223,30 @@ void useragent(void) debuga(__FILE__,__LINE__,_("sort command: %s\n"),csort); exit(EXIT_FAILURE); } - if((fp_in=fopen(tmp2,"r"))==NULL) { + if ((fp_in=fopen(tmp2,"r"))==NULL) { debuga(__FILE__,__LINE__,_("Cannot open file \"%s\": %s\n"),tmp2,strerror(errno)); debuga(__FILE__,__LINE__,_("sort command: %s\n"),csort); exit(EXIT_FAILURE); } - if (!KeepTempLog && unlink(tmp3)) { - debuga(__FILE__,__LINE__,_("Cannot delete \"%s\": %s\n"),tmp3,strerror(errno)); + if (!KeepTempLog && unlink(UserAgentTempLog)) { + debuga(__FILE__,__LINE__,_("Cannot delete \"%s\": %s\n"),UserAgentTempLog,strerror(errno)); exit(EXIT_FAILURE); } snprintf(hfile,sizeof(hfile),"%s/useragent.html", outdirname); - if((fp_ht=fopen(hfile,"w"))==NULL) { + if ((fp_ht=fopen(hfile,"w"))==NULL) { debuga(__FILE__,__LINE__,_("Cannot open file \"%s\": %s\n"),hfile,strerror(errno)); exit(EXIT_FAILURE); } - if(debug) + if (debug) debuga(__FILE__,__LINE__,_("Making Useragent report\n")); write_html_header(fp_ht,(IndexTree == INDEX_TREE_DATE) ? 3 : 1,_("Squid Useragent's Report"),HTML_JS_NONE); fprintf(fp_ht,"%s\n",_("Squid Useragent's Report")); + strftime(idate,sizeof(idate),"%x",&UserAgentStartDate); + strftime(fdate,sizeof(fdate),"%x",&UserAgentEndDate); fprintf(fp_ht,"%s: %s - %s\n",_("Period"),idate,fdate); close_html_header(fp_ht); @@ -199,15 +257,15 @@ void useragent(void) fprintf(fp_ht,"%s%s\n",_("USERID"),_("AGENT")); - while(fgets(buf,sizeof(buf),fp_in)!=NULL) { + while (fgets(buf,sizeof(buf),fp_in)!=NULL) { getword_start(&gwarea,buf); if (getword(ip,sizeof(ip),&gwarea,'\t')<0) { debuga(__FILE__,__LINE__,_("Invalid IP address in file \"%s\"\n"),tmp2); exit(EXIT_FAILURE); } - if(Ip2Name) { - if(strcmp(ip,ipbefore) != 0) { + if (Ip2Name) { + if (strcmp(ip,ipbefore) != 0) { strcpy(ipbefore,ip); ip2name(ip,sizeof(ip)); strcpy(namebefore,ip); @@ -223,13 +281,13 @@ void useragent(void) exit(EXIT_FAILURE); } - if(strcmp(user,user_old) != 0) { + if (strcmp(user,user_old) != 0) { fprintf(fp_ht,"%s",user); output_html_string(fp_ht,agent,250); fputs("\n",fp_ht); strcpy(user_old,user); strcpy(agent_old,agent); - } else if(strcmp(agent,agent_old) != 0) { + } else if (strcmp(agent,agent_old) != 0) { fputs("",fp_ht); output_html_string(fp_ht,agent,250); fputs("\n",fp_ht); @@ -243,6 +301,7 @@ void useragent(void) exit(EXIT_FAILURE); } + snprintf(tmp3,sizeof(tmp3),"%s/squagent2.int_log",tmp); if (snprintf(csort,sizeof(csort),"sort -t \"\t\" -k 2,2 -o \"%s\" \"%s\"",tmp3,tmp2)>=sizeof(csort)) { debuga(__FILE__,__LINE__,_("Sort command too long when sorting file \"%s\" to \"%s\"\n"),tmp2,tmp3); exit(EXIT_FAILURE); @@ -253,7 +312,7 @@ void useragent(void) debuga(__FILE__,__LINE__,_("sort command: %s\n"),csort); exit(EXIT_FAILURE); } - if((fp_in=fopen(tmp3,"r"))==NULL) { + if ((fp_in=fopen(tmp3,"r"))==NULL) { debuga(__FILE__,__LINE__,_("Cannot open file \"%s\": %s\n"),tmp3,strerror(errno)); debuga(__FILE__,__LINE__,_("sort command: %s\n"),csort); exit(EXIT_FAILURE); @@ -264,7 +323,7 @@ void useragent(void) exit(EXIT_FAILURE); } - if((fp_ou=fopen(tmp2,"w"))==NULL) { + if ((fp_ou=fopen(tmp2,"w"))==NULL) { debuga(__FILE__,__LINE__,_("Cannot open file \"%s\": %s\n"),tmp2,strerror(errno)); exit(EXIT_FAILURE); } @@ -272,7 +331,7 @@ void useragent(void) agent_old[0]='\0'; cont=0; - while(fgets(buf,sizeof(buf),fp_in)!=NULL) { + while (fgets(buf,sizeof(buf),fp_in)!=NULL) { getword_start(&gwarea,buf); if (getword(ip,sizeof(ip),&gwarea,'\t')<0) { debuga(__FILE__,__LINE__,_("Invalid IP address in file \"%s\"\n"),tmp3); @@ -283,12 +342,12 @@ void useragent(void) exit(EXIT_FAILURE); } - if(!cont) { + if (!cont) { cont++; strcpy(agent_old,agent); } - if(strcmp(agent,agent_old) != 0) { + if (strcmp(agent,agent_old) != 0) { agentdif++; fprintf(fp_ou,"%06d %s\n",agentot,agent_old); strcpy(agent_old,agent);