Rewrite the useragent log processing

Create more functions to distinguish the reading of the useragent log file
from the report generation.

The idea is to prepare the code to read the useragent data from other
sources than the useragent_log option.

diff --git a/include/defs.h b/include/defs.h
index 19743a31a10481e6b247a2a27f2c0df85ff4e9ee..a6e3c24c89c75a491961aa510bc87a13d10302ac 100644 (file)
--- a/include/defs.h
+++ b/include/defs.h
@@ -336,7 +336,10 @@ void url_hostname(const char *url,char *hostname,int hostsize);
 void usage(const char *prog);
 
 // useragent.c
-void useragent(void);
+FILE *UserAgent_Open(void);
+void UserAgent_Write(FILE *fp,const char *Ip,const char *User,const char *Agent);
+void UserAgent_Readlog(void);
+void UserAgent(void);
 
 // userinfo.c
 /*@shared@*/struct userinfostruct *userinfo_create(const char *userid, const char *ip);

diff --git a/report.c b/report.c
index dba3cd026bc0db93a069a3068a6ca9cadac8a00d..2d81566067d8421752a5db3de092f64002e1fd85 100644 (file)
--- a/report.c
+++ b/report.c
@@ -112,7 +112,11 @@ void gerarel(void)
                debugaz(__FILE__,__LINE__,_("outdirname=%s\n"),outdirname);
        }
 
-       if (!FileList_IsEmpty(UserAgentLog) && email[0] == '\0') useragent();
+       if (email[0] == '\0') {
+               if (!FileList_IsEmpty(UserAgentLog))
+                       UserAgent_Readlog();
+               UserAgent();
+       }
 
        snprintf(wdirname,sizeof(wdirname),"%s/sarg-general",outdirname);
        if((fp_gen=MY_FOPEN(wdirname,"w"))==NULL){

diff --git a/useragent.c b/useragent.c
index 1e654d392afdaf0494e60d3e2e62afdfebe524f5..73ff3991febffbe2b47d321a2044615067c9e870 100644 (file)
--- a/useragent.c
+++ b/useragent.c
@@ -30,60 +30,84 @@
 
 FileListObject UserAgentLog=NULL;
 
-void useragent(void)
+//! Log file where the user agent data are written.
+static char UserAgentTempLog[MAXLEN]="";
+
+static struct tm UserAgentStartDate;
+static struct tm UserAgentEndDate;
+
+/*!
+ * Open the temporary file to store the useragent entries to be
+ * reported.
+ *
+ * \return The file handle. It must be closed when the data have
+ * been written.
+ */
+FILE *UserAgent_Open(void)
+{
+       FILE *fp_ou;
+
+       if (UserAgentTempLog[0]) {
+               debuga(__FILE__,__LINE__,_("Useragent log already opened\n"));
+               exit(EXIT_FAILURE);
+       }
+       snprintf(UserAgentTempLog,sizeof(UserAgentTempLog),"%s/squagent.int_unsort",tmp);
+       if ((fp_ou=fopen(UserAgentTempLog,"w"))==NULL) {
+               debuga(__FILE__,__LINE__,_("Cannot open file \"%s\": %s\n"),UserAgentTempLog,strerror(errno));
+               exit(EXIT_FAILURE);
+       }
+       memset(&UserAgentStartDate,0,sizeof(UserAgentStartDate));
+       memset(&UserAgentEndDate,0,sizeof(UserAgentEndDate));
+       return(fp_ou);
+}
+
+/*!
+ * Write a user agent entry into the temporary log.
+ *
+ * \param fp The file opened by UserAgent_Open().
+ * \param Ip The IP address using this agent.
+ * \param User The user name.
+ * \param Agent The user agent string.
+ */
+void UserAgent_Write(FILE *fp,const char *Ip,const char *User,const char *Agent)
+{
+       fprintf(fp,"%s\t%s\t%s\n",Ip,Agent,User);
+       useragent_count++;
+}
+
+/*!
+ * Read the user provided useragent file and create
+ * a temporary file with the data to report.
+ */
+void UserAgent_Readlog(void)
 {
        FileObject *fp_log;
-       FILE *fp_in = NULL, *fp_ou = NULL, *fp_ht = NULL;
+       FILE *fp_ou = NULL;
        char *ptr;
-       char buf[MAXLEN];
-       char ip[MAXLEN], data[MAXLEN], agent[MAXLEN], user[MAXLEN];
-       char ipbefore[MAXLEN], namebefore[MAXLEN];
-       char tagent[MAXLEN];
-       char user_old[MAXLEN]="$#%0a3bc6";
-       char agent_old[MAXLEN]="$#%0a3bc6";
-       char hfile[MAXLEN];
-       char idate[MAXLEN], fdate[MAXLEN];
-       char tmp2[MAXLEN];
-       char tmp3[MAXLEN];
-       char day[4],month[5],year[5], wdate[20];
-       char csort[MAXLEN];
+       char ip[80], data[50], agent[MAXLEN], user[MAXLEN];
+       int day,month,year;
+       char monthname[5];
        const char *FileName;
-       int  agentot=0, agentot2=0, agentdif=0, cont=0, nagent;
        unsigned long totregsl=0;
-       int cstatus;
        int ndate;
-       double perc;
        struct getwordstruct gwarea, gwarea1;
        longline line;
        FileListIterator FIter;
+       struct tm logtime;
 
-       ip[0]='\0';
-       data[0]='\0';
-       agent[0]='\0';
-       user[0]='\0';
-       user_old[0]='\0';
-       agent_old[0]='\0';
-       ipbefore[0]='\0';
-       namebefore[0]='\0';
-
-       snprintf(tmp3,sizeof(tmp3),"%s/squagent.int_unsort",tmp);
-       snprintf(tmp2,sizeof(tmp2),"%s/squagent.int_log",tmp);
-
-       if((fp_ou=fopen(tmp3,"w"))==NULL) {
-               debuga(__FILE__,__LINE__,_("Cannot open file \"%s\": %s\n"),tmp3,strerror(errno));
-               exit(EXIT_FAILURE);
-       }
+       fp_ou=UserAgent_Open();
 
        if ((line=longline_create())==NULL) {
                debuga(__FILE__,__LINE__,_("Not enough memory to read useragent log\n"));
                exit(EXIT_FAILURE);
        }
+       memset(&logtime,0,sizeof(logtime));
 
        FIter=FileListIter_Open(UserAgentLog);
        while ((FileName=FileListIter_Next(FIter))!=NULL)
        {
                longline_reset(line);
-               if((fp_log=decomp(FileName))==NULL) {
+               if ((fp_log=decomp(FileName))==NULL) {
                        debuga(__FILE__,__LINE__,_("Cannot open file \"%s\": %s\n"),FileName,FileObject_GetLastOpenError());
                        exit(EXIT_FAILURE);
                }
@@ -92,49 +116,56 @@ void useragent(void)
                        debuga(__FILE__,__LINE__,_("Reading useragent log \"%s\"\n"),FileName);
                }
 
-               while((ptr=longline_read(fp_log,line))!=NULL) {
+               while ((ptr=longline_read(fp_log,line))!=NULL) {
                        totregsl++;
                        getword_start(&gwarea,ptr);
-                       if (getword(ip,sizeof(ip),&gwarea,' ')<0 || getword_skip(MAXLEN,&gwarea,'[')<0 ||
+                       if (getword(ip,sizeof(ip),&gwarea,' ')<0 || getword_skip(10,&gwarea,'[')<0 ||
                                getword(data,sizeof(data),&gwarea,' ')<0) {
                                debuga(__FILE__,__LINE__,_("Invalid record in file \"%s\"\n"),FileName);
                                exit(EXIT_FAILURE);
                        }
                        getword_start(&gwarea1,data);
-                       if (getword(day,sizeof(day),&gwarea1,'/')<0 || getword(month,sizeof(month),&gwarea1,'/')<0 ||
-                               getword(year,sizeof(year),&gwarea1,':')<0) {
+                       if (getword_atoi(&day,&gwarea1,'/')<0 || getword(monthname,sizeof(monthname),&gwarea1,'/')<0 ||
+                               getword_atoi(&year,&gwarea1,':')<0) {
                                debuga(__FILE__,__LINE__,_("Invalid date in file \"%s\"\n"),FileName);
                                exit(EXIT_FAILURE);
                        }
+                       month=month2num(monthname)+1;
+                       if (month>12) {
+                               debuga(__FILE__,__LINE__,_("Invalid month name \"%s\" found in user agent file \"%s\""),monthname,FileName);
+                               exit(EXIT_FAILURE);
+                       }
                        if (dfrom!=0 || duntil!=0){
-                               buildymd(day,month,year,wdate,sizeof(wdate));
-                               ndate=atoi(wdate);
+                               ndate=year*10000+month*100+day;
                                if (ndate<dfrom) continue;
                                if (ndate>duntil) break;
                        }
-                       if(totregsl == 1)
-                               strcpy(idate,data);
-                       strcpy(fdate,data);
+                       logtime.tm_year=year-1900;
+                       logtime.tm_mon=month-1;
+                       logtime.tm_mday=day;
+                       if (totregsl==1 || compare_date(&UserAgentStartDate,&logtime)>0)
+                               memcpy(&UserAgentStartDate,&logtime,sizeof(logtime));
+                       if (compare_date(&UserAgentEndDate,&logtime)<0)
+                               memcpy(&UserAgentEndDate,&logtime,sizeof(logtime));
                        if (getword_skip(MAXLEN,&gwarea,'"')<0 || getword(agent,sizeof(agent),&gwarea,'"')<0) {
                                debuga(__FILE__,__LINE__,_("Invalid useragent in file \"%s\"\n"),FileName);
                                exit(EXIT_FAILURE);
                        }
 
-                       if(gwarea.current[0]!='\0') {
+                       if (gwarea.current[0]!='\0') {
                                if (getword_skip(MAXLEN,&gwarea,' ')<0 || getword(user,sizeof(user),&gwarea,'\n')<0) {
                                        debuga(__FILE__,__LINE__,_("Invalid record in file \"%s\"\n"),FileName);
                                        exit(EXIT_FAILURE);
                                }
-                               if(user[0] == '-')
+                               if (user[0] == '-')
                                        strcpy(user,ip);
-                               if(user[0] == '\0')
+                               if (user[0] == '\0')
                                        strcpy(user,ip);
                        } else {
                                strcpy(user,ip);
                        }
 
-                       fprintf(fp_ou,"%s\t%s\t%s\n",ip,agent,user);
-                       useragent_count++;
+                       UserAgent_Write(fp_ou,ip,user,agent);
                }
 
                if (FileObject_Close(fp_log)==EOF) {
@@ -150,15 +181,40 @@ void useragent(void)
        }
 
        if (fclose(fp_ou)==EOF) {
-               debuga(__FILE__,__LINE__,_("Write error in \"%s\": %s\n"),tmp3,strerror(errno));
+               debuga(__FILE__,__LINE__,_("Write error in \"%s\": %s\n"),UserAgentTempLog,strerror(errno));
                exit(EXIT_FAILURE);
        }
-       if(debug) {
+}
+
+void UserAgent(void)
+{
+       FILE *fp_in = NULL, *fp_ou = NULL, *fp_ht = NULL;
+       char buf[MAXLEN];
+       char ip[80], agent[MAXLEN], user[MAXLEN];
+       char ipbefore[MAXLEN]="";
+       char namebefore[MAXLEN]="";
+       char tagent[MAXLEN];
+       char user_old[MAXLEN]="";
+       char agent_old[MAXLEN]="";
+       char hfile[MAXLEN];
+       char idate[100], fdate[100];
+       char tmp2[MAXLEN];
+       char tmp3[MAXLEN];
+       char csort[MAXLEN];
+       int  agentot=0, agentot2=0, agentdif=0, cont=0, nagent;
+       int cstatus;
+       double perc;
+       struct getwordstruct gwarea;
+
+       if (!UserAgentTempLog[0]) return;
+
+       snprintf(tmp2,sizeof(tmp2),"%s/squagent.int_log",tmp);
+       if (debug) {
                debuga(__FILE__,__LINE__,_("Sorting file \"%s\"\n"),tmp2);
        }
 
-       if (snprintf(csort,sizeof(csort),"sort -n -t \"\t\" -k 3,3 -k 2,2 -k 1,1 -o \"%s\" \"%s\"",tmp2,tmp3)>=sizeof(csort)) {
-               debuga(__FILE__,__LINE__,_("Sort command too long when sorting file \"%s\" to \"%s\"\n"),tmp2,tmp3);
+       if (snprintf(csort,sizeof(csort),"sort -n -t \"\t\" -k 3,3 -k 2,2 -k 1,1 -o \"%s\" \"%s\"",tmp2,UserAgentTempLog)>=sizeof(csort)) {
+               debuga(__FILE__,__LINE__,_("Sort command too long when sorting file \"%s\" to \"%s\"\n"),tmp2,UserAgentTempLog);
                exit(EXIT_FAILURE);
        }
        cstatus=system(csort);
@@ -167,28 +223,30 @@ void useragent(void)
                debuga(__FILE__,__LINE__,_("sort command: %s\n"),csort);
                exit(EXIT_FAILURE);
        }
-       if((fp_in=fopen(tmp2,"r"))==NULL) {
+       if ((fp_in=fopen(tmp2,"r"))==NULL) {
                debuga(__FILE__,__LINE__,_("Cannot open file \"%s\": %s\n"),tmp2,strerror(errno));
                debuga(__FILE__,__LINE__,_("sort command: %s\n"),csort);
                exit(EXIT_FAILURE);
        }
 
-       if (!KeepTempLog && unlink(tmp3)) {
-               debuga(__FILE__,__LINE__,_("Cannot delete \"%s\": %s\n"),tmp3,strerror(errno));
+       if (!KeepTempLog && unlink(UserAgentTempLog)) {
+               debuga(__FILE__,__LINE__,_("Cannot delete \"%s\": %s\n"),UserAgentTempLog,strerror(errno));
                exit(EXIT_FAILURE);
        }
 
        snprintf(hfile,sizeof(hfile),"%s/useragent.html", outdirname);
-       if((fp_ht=fopen(hfile,"w"))==NULL) {
+       if ((fp_ht=fopen(hfile,"w"))==NULL) {
                debuga(__FILE__,__LINE__,_("Cannot open file \"%s\": %s\n"),hfile,strerror(errno));
                exit(EXIT_FAILURE);
        }
 
-       if(debug)
+       if (debug)
                debuga(__FILE__,__LINE__,_("Making Useragent report\n"));
 
        write_html_header(fp_ht,(IndexTree == INDEX_TREE_DATE) ? 3 : 1,_("Squid Useragent's Report"),HTML_JS_NONE);
        fprintf(fp_ht,"<tr><th class=\"header_c\">%s</th></tr>\n",_("Squid Useragent's Report"));
+       strftime(idate,sizeof(idate),"%x",&UserAgentStartDate);
+       strftime(fdate,sizeof(fdate),"%x",&UserAgentEndDate);
        fprintf(fp_ht,"<tr><td class=\"header_c\">%s: %s - %s</td></tr>\n",_("Period"),idate,fdate);
        close_html_header(fp_ht);
 
@@ -199,15 +257,15 @@ void useragent(void)
 
        fprintf(fp_ht,"<tr><th class=\"header_l\">%s</th><th class=\"header_l\">%s</th></tr>\n",_("USERID"),_("AGENT"));
 
-       while(fgets(buf,sizeof(buf),fp_in)!=NULL) {
+       while (fgets(buf,sizeof(buf),fp_in)!=NULL) {
                getword_start(&gwarea,buf);
                if (getword(ip,sizeof(ip),&gwarea,'\t')<0) {
                        debuga(__FILE__,__LINE__,_("Invalid IP address in file \"%s\"\n"),tmp2);
                        exit(EXIT_FAILURE);
                }
 
-               if(Ip2Name) {
-                       if(strcmp(ip,ipbefore) != 0) {
+               if (Ip2Name) {
+                       if (strcmp(ip,ipbefore) != 0) {
                                strcpy(ipbefore,ip);
                                ip2name(ip,sizeof(ip));
                                strcpy(namebefore,ip);
@@ -223,13 +281,13 @@ void useragent(void)
                        exit(EXIT_FAILURE);
                }
 
-               if(strcmp(user,user_old) != 0) {
+               if (strcmp(user,user_old) != 0) {
                        fprintf(fp_ht,"<tr><td class=\"data2\">%s</td><td class=\"data2\">",user);
                        output_html_string(fp_ht,agent,250);
                        fputs("</td></tr>\n",fp_ht);
                        strcpy(user_old,user);
                        strcpy(agent_old,agent);
-               } else if(strcmp(agent,agent_old) != 0) {
+               } else if (strcmp(agent,agent_old) != 0) {
                        fputs("<tr><td></td><td class=\"data2\">",fp_ht);
                        output_html_string(fp_ht,agent,250);
                        fputs("</td></tr>\n",fp_ht);
@@ -243,6 +301,7 @@ void useragent(void)
                exit(EXIT_FAILURE);
        }
 
+       snprintf(tmp3,sizeof(tmp3),"%s/squagent2.int_log",tmp);
        if (snprintf(csort,sizeof(csort),"sort -t \"\t\" -k 2,2 -o \"%s\" \"%s\"",tmp3,tmp2)>=sizeof(csort)) {
                debuga(__FILE__,__LINE__,_("Sort command too long when sorting file \"%s\" to \"%s\"\n"),tmp2,tmp3);
                exit(EXIT_FAILURE);
@@ -253,7 +312,7 @@ void useragent(void)
                debuga(__FILE__,__LINE__,_("sort command: %s\n"),csort);
                exit(EXIT_FAILURE);
        }
-       if((fp_in=fopen(tmp3,"r"))==NULL) {
+       if ((fp_in=fopen(tmp3,"r"))==NULL) {
                debuga(__FILE__,__LINE__,_("Cannot open file \"%s\": %s\n"),tmp3,strerror(errno));
                debuga(__FILE__,__LINE__,_("sort command: %s\n"),csort);
                exit(EXIT_FAILURE);
@@ -264,7 +323,7 @@ void useragent(void)
                exit(EXIT_FAILURE);
        }
 
-       if((fp_ou=fopen(tmp2,"w"))==NULL) {
+       if ((fp_ou=fopen(tmp2,"w"))==NULL) {
                debuga(__FILE__,__LINE__,_("Cannot open file \"%s\": %s\n"),tmp2,strerror(errno));
                exit(EXIT_FAILURE);
        }
@@ -272,7 +331,7 @@ void useragent(void)
        agent_old[0]='\0';
        cont=0;
 
-       while(fgets(buf,sizeof(buf),fp_in)!=NULL) {
+       while (fgets(buf,sizeof(buf),fp_in)!=NULL) {
                getword_start(&gwarea,buf);
                if (getword(ip,sizeof(ip),&gwarea,'\t')<0) {
                        debuga(__FILE__,__LINE__,_("Invalid IP address in file \"%s\"\n"),tmp3);
@@ -283,12 +342,12 @@ void useragent(void)
                        exit(EXIT_FAILURE);
                }
 
-               if(!cont) {
+               if (!cont) {
                        cont++;
                        strcpy(agent_old,agent);
                }
 
-               if(strcmp(agent,agent_old) != 0) {
+               if (strcmp(agent,agent_old) != 0) {
                        agentdif++;
                        fprintf(fp_ou,"%06d %s\n",agentot,agent_old);
                        strcpy(agent_old,agent);