useragent.c

   1 /*
   2  * SARG Squid Analysis Report Generator      http://sarg.sourceforge.net
   3  *                                                            1998, 2015
   4  *
   5  * SARG donations:
   6  *      please look at http://sarg.sourceforge.net/donations.php
   7  * Support:
   8  *     http://sourceforge.net/projects/sarg/forums/forum/363374
   9  * ---------------------------------------------------------------------
  10  *
  11  *  This program is free software; you can redistribute it and/or modify
  12  *  it under the terms of the GNU General Public License as published by
  13  *  the Free Software Foundation; either version 2 of the License, or
  14  *  (at your option) any later version.
  15  *
  16  *  This program is distributed in the hope that it will be useful,
  17  *  but WITHOUT ANY WARRANTY; without even the implied warranty of
  18  *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  19  *  GNU General Public License for more details.
  20  *
  21  *  You should have received a copy of the GNU General Public License
  22  *  along with this program; if not, write to the Free Software
  23  *  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111, USA.
  24  *
  25  */
  26
  27 #include "include/conf.h"
  28 #include "include/defs.h"
  29 #include "include/filelist.h"
  30
  31 FileListObject UserAgentLog=NULL;
  32
  33 //! Log file where the user agent data are written.
  34 static char UserAgentTempLog[MAXLEN]="";
  35
  36 static struct tm UserAgentStartDate;
  37 static struct tm UserAgentEndDate;
  38
  39 /*!
  40  * Open the temporary file to store the useragent entries to be
  41  * reported.
  42  *
  43  * \return The file handle. It must be closed when the data have
  44  * been written.
  45  */
  46 FILE *UserAgent_Open(void)
  47 {
  48         FILE *fp_ou;
  49
  50         if (UserAgentTempLog[0]) {
  51                 debuga(__FILE__,__LINE__,_("Useragent log already opened\n"));
  52                 exit(EXIT_FAILURE);
  53         }
  54         snprintf(UserAgentTempLog,sizeof(UserAgentTempLog),"%s/squagent.int_unsort",tmp);
  55         if ((fp_ou=fopen(UserAgentTempLog,"w"))==NULL) {
  56                 debuga(__FILE__,__LINE__,_("Cannot open file \"%s\": %s\n"),UserAgentTempLog,strerror(errno));
  57                 exit(EXIT_FAILURE);
  58         }
  59         memset(&UserAgentStartDate,0,sizeof(UserAgentStartDate));
  60         memset(&UserAgentEndDate,0,sizeof(UserAgentEndDate));
  61         return(fp_ou);
  62 }
  63
  64 /*!
  65  * Write a user agent entry into the temporary log.
  66  *
  67  * \param fp The file opened by UserAgent_Open().
  68  * \param Ip The IP address using this agent.
  69  * \param User The user name.
  70  * \param Agent The user agent string.
  71  */
  72 void UserAgent_Write(FILE *fp,const char *Ip,const char *User,const char *Agent)
  73 {
  74         fprintf(fp,"%s\t%s\t%s\n",Ip,Agent,User);
  75         useragent_count++;
  76 }
  77
  78 /*!
  79  * Read the user provided useragent file and create
  80  * a temporary file with the data to report.
  81  */
  82 void UserAgent_Readlog(void)
  83 {
  84         FileObject *fp_log;
  85         FILE *fp_ou = NULL;
  86         char *ptr;
  87         char ip[80], data[50], agent[MAXLEN], user[MAXLEN];
  88         int day,month,year;
  89         char monthname[5];
  90         const char *FileName;
  91         unsigned long totregsl=0;
  92         int ndate;
  93         struct getwordstruct gwarea, gwarea1;
  94         longline line;
  95         FileListIterator FIter;
  96         struct tm logtime;
  97         int dfrom;
  98         int duntil;
  99
 100         fp_ou=UserAgent_Open();
 101
 102         if ((line=longline_create())==NULL) {
 103                 debuga(__FILE__,__LINE__,_("Not enough memory to read useragent log\n"));
 104                 exit(EXIT_FAILURE);
 105         }
 106         memset(&logtime,0,sizeof(logtime));
 107         getperiod_torange(&period,&dfrom,&duntil);
 108
 109         FIter=FileListIter_Open(UserAgentLog);
 110         while ((FileName=FileListIter_Next(FIter))!=NULL)
 111         {
 112                 longline_reset(line);
 113                 if ((fp_log=decomp(FileName))==NULL) {
 114                         debuga(__FILE__,__LINE__,_("Cannot open file \"%s\": %s\n"),FileName,FileObject_GetLastOpenError());
 115                         exit(EXIT_FAILURE);
 116                 }
 117
 118                 if(debug) {
 119                         debuga(__FILE__,__LINE__,_("Reading useragent log \"%s\"\n"),FileName);
 120                 }
 121
 122                 while ((ptr=longline_read(fp_log,line))!=NULL) {
 123                         totregsl++;
 124                         getword_start(&gwarea,ptr);
 125                         if (getword(ip,sizeof(ip),&gwarea,' ')<0 || getword_skip(10,&gwarea,'[')<0 ||
 126                                 getword(data,sizeof(data),&gwarea,' ')<0) {
 127                                 debuga(__FILE__,__LINE__,_("Invalid record in file \"%s\"\n"),FileName);
 128                                 exit(EXIT_FAILURE);
 129                         }
 130                         getword_start(&gwarea1,data);
 131                         if (getword_atoi(&day,&gwarea1,'/')<0 || getword(monthname,sizeof(monthname),&gwarea1,'/')<0 ||
 132                                 getword_atoi(&year,&gwarea1,':')<0) {
 133                                 debuga(__FILE__,__LINE__,_("Invalid date in file \"%s\"\n"),FileName);
 134                                 exit(EXIT_FAILURE);
 135                         }
 136                         month=month2num(monthname)+1;
 137                         if (month>12) {
 138                                 debuga(__FILE__,__LINE__,_("Invalid month name \"%s\" found in user agent file \"%s\""),monthname,FileName);
 139                                 exit(EXIT_FAILURE);
 140                         }
 141                         if (dfrom!=0 || duntil!=0){
 142                                 ndate=year*10000+month*100+day;
 143                                 if (ndate<dfrom) continue;
 144                                 if (ndate>duntil) break;
 145                         }
 146                         logtime.tm_year=year-1900;
 147                         logtime.tm_mon=month-1;
 148                         logtime.tm_mday=day;
 149                         if (totregsl==1 || compare_date(&UserAgentStartDate,&logtime)>0)
 150                                 memcpy(&UserAgentStartDate,&logtime,sizeof(logtime));
 151                         if (compare_date(&UserAgentEndDate,&logtime)<0)
 152                                 memcpy(&UserAgentEndDate,&logtime,sizeof(logtime));
 153                         if (getword_skip(MAXLEN,&gwarea,'"')<0 || getword(agent,sizeof(agent),&gwarea,'"')<0) {
 154                                 debuga(__FILE__,__LINE__,_("Invalid useragent in file \"%s\"\n"),FileName);
 155                                 exit(EXIT_FAILURE);
 156                         }
 157
 158                         if (gwarea.current[0]!='\0') {
 159                                 if (getword_skip(MAXLEN,&gwarea,' ')<0 || getword(user,sizeof(user),&gwarea,'\n')<0) {
 160                                         debuga(__FILE__,__LINE__,_("Invalid record in file \"%s\"\n"),FileName);
 161                                         exit(EXIT_FAILURE);
 162                                 }
 163                                 if (user[0] == '-')
 164                                         strcpy(user,ip);
 165                                 if (user[0] == '\0')
 166                                         strcpy(user,ip);
 167                         } else {
 168                                 strcpy(user,ip);
 169                         }
 170
 171                         UserAgent_Write(fp_ou,ip,user,agent);
 172                 }
 173
 174                 if (FileObject_Close(fp_log)==EOF) {
 175                         debuga(__FILE__,__LINE__,_("Read error in \"%s\": %s\n"),FileName,FileObject_GetLastCloseError());
 176                         exit(EXIT_FAILURE);
 177                 }
 178         }
 179         FileListIter_Close(FIter);
 180         longline_destroy(&line);
 181
 182         if(debug) {
 183                 debuga(__FILE__,__LINE__,_("   Records read: %ld\n"),totregsl);
 184         }
 185
 186         if (fclose(fp_ou)==EOF) {
 187                 debuga(__FILE__,__LINE__,_("Write error in \"%s\": %s\n"),UserAgentTempLog,strerror(errno));
 188                 exit(EXIT_FAILURE);
 189         }
 190 }
 191
 192 void UserAgent(void)
 193 {
 194         FILE *fp_in = NULL, *fp_ou = NULL, *fp_ht = NULL;
 195         char buf[MAXLEN];
 196         char ip[80], agent[MAXLEN], user[MAXLEN];
 197         char ipbefore[MAXLEN]="";
 198         char namebefore[MAXLEN]="";
 199         char tagent[MAXLEN];
 200         char user_old[MAXLEN]="";
 201         char agent_old[MAXLEN]="";
 202         char hfile[MAXLEN];
 203         char idate[100], fdate[100];
 204         char tmp2[MAXLEN];
 205         char tmp3[MAXLEN];
 206         char csort[MAXLEN];
 207         int  agentot=0, agentot2=0, agentdif=0, cont=0, nagent;
 208         int cstatus;
 209         double perc;
 210         struct getwordstruct gwarea;
 211
 212         if (!UserAgentTempLog[0]) return;
 213
 214         snprintf(tmp2,sizeof(tmp2),"%s/squagent.int_log",tmp);
 215         if (debug) {
 216                 debuga(__FILE__,__LINE__,_("Sorting file \"%s\"\n"),tmp2);
 217         }
 218
 219         if (snprintf(csort,sizeof(csort),"sort -n -t \"\t\" -k 3,3 -k 2,2 -k 1,1 -o \"%s\" \"%s\"",tmp2,UserAgentTempLog)>=sizeof(csort)) {
 220                 debuga(__FILE__,__LINE__,_("Sort command too long when sorting file \"%s\" to \"%s\"\n"),tmp2,UserAgentTempLog);
 221                 exit(EXIT_FAILURE);
 222         }
 223         cstatus=system(csort);
 224         if (!WIFEXITED(cstatus) || WEXITSTATUS(cstatus)) {
 225                 debuga(__FILE__,__LINE__,_("sort command return status %d\n"),WEXITSTATUS(cstatus));
 226                 debuga(__FILE__,__LINE__,_("sort command: %s\n"),csort);
 227                 exit(EXIT_FAILURE);
 228         }
 229         if ((fp_in=fopen(tmp2,"r"))==NULL) {
 230                 debuga(__FILE__,__LINE__,_("Cannot open file \"%s\": %s\n"),tmp2,strerror(errno));
 231                 debuga(__FILE__,__LINE__,_("sort command: %s\n"),csort);
 232                 exit(EXIT_FAILURE);
 233         }
 234
 235         if (!KeepTempLog && unlink(UserAgentTempLog)) {
 236                 debuga(__FILE__,__LINE__,_("Cannot delete \"%s\": %s\n"),UserAgentTempLog,strerror(errno));
 237                 exit(EXIT_FAILURE);
 238         }
 239
 240         snprintf(hfile,sizeof(hfile),"%s/useragent.html", outdirname);
 241         if ((fp_ht=fopen(hfile,"w"))==NULL) {
 242                 debuga(__FILE__,__LINE__,_("Cannot open file \"%s\": %s\n"),hfile,strerror(errno));
 243                 exit(EXIT_FAILURE);
 244         }
 245
 246         if (debug)
 247                 debuga(__FILE__,__LINE__,_("Making Useragent report\n"));
 248
 249         write_html_header(fp_ht,(IndexTree == INDEX_TREE_DATE) ? 3 : 1,_("Squid Useragent's Report"),HTML_JS_NONE);
 250         fprintf(fp_ht,"<tr><th class=\"header_c\">%s</th></tr>\n",_("Squid Useragent's Report"));
 251         strftime(idate,sizeof(idate),"%x",&UserAgentStartDate);
 252         strftime(fdate,sizeof(fdate),"%x",&UserAgentEndDate);
 253         fprintf(fp_ht,"<tr><td class=\"header_c\">%s: %s - %s</td></tr>\n",_("Period"),idate,fdate);
 254         close_html_header(fp_ht);
 255
 256         fputs("<br><br>\n",fp_ht);
 257
 258         fputs("<div class=\"report\"><table cellpadding=\"0\" cellspacing=\"0\">\n",fp_ht);
 259         fputs("<tr><td>&nbsp;</td><td>&nbsp;</td></tr>",fp_ht);
 260
 261         fprintf(fp_ht,"<tr><th class=\"header_l\">%s</th><th class=\"header_l\">%s</th></tr>\n",_("USERID"),_("AGENT"));
 262
 263         while (fgets(buf,sizeof(buf),fp_in)!=NULL) {
 264                 getword_start(&gwarea,buf);
 265                 if (getword(ip,sizeof(ip),&gwarea,'\t')<0) {
 266                         debuga(__FILE__,__LINE__,_("Invalid IP address in file \"%s\"\n"),tmp2);
 267                         exit(EXIT_FAILURE);
 268                 }
 269
 270                 if (Ip2Name) {
 271                         if (strcmp(ip,ipbefore) != 0) {
 272                                 strcpy(ipbefore,ip);
 273                                 ip2name(ip,sizeof(ip));
 274                                 strcpy(namebefore,ip);
 275                         } else strcpy(ip,namebefore);
 276                 }
 277
 278                 if (getword(agent,sizeof(agent),&gwarea,'\t')<0) {
 279                         debuga(__FILE__,__LINE__,_("Invalid useragent in file \"%s\"\n"),tmp2);
 280                         exit(EXIT_FAILURE);
 281                 }
 282                 if (getword(user,sizeof(user),&gwarea,'\t')<0) {
 283                         debuga(__FILE__,__LINE__,_("Invalid user ID in file \"%s\"\n"),tmp2);
 284                         exit(EXIT_FAILURE);
 285                 }
 286
 287                 if (strcmp(user,user_old) != 0) {
 288                         fprintf(fp_ht,"<tr><td class=\"data2\">%s</td><td class=\"data2\">",user);
 289                         output_html_string(fp_ht,agent,250);
 290                         fputs("</td></tr>\n",fp_ht);
 291                         strcpy(user_old,user);
 292                         strcpy(agent_old,agent);
 293                 } else if (strcmp(agent,agent_old) != 0) {
 294                         fputs("<tr><td></td><td class=\"data2\">",fp_ht);
 295                         output_html_string(fp_ht,agent,250);
 296                         fputs("</td></tr>\n",fp_ht);
 297                         strcpy(agent_old,agent);
 298                 }
 299         }
 300
 301         fputs("</table>\n",fp_ht);
 302         if (fclose(fp_in)==EOF) {
 303                 debuga(__FILE__,__LINE__,_("Read error in \"%s\": %s\n"),tmp2,strerror(errno));
 304                 exit(EXIT_FAILURE);
 305         }
 306
 307         snprintf(tmp3,sizeof(tmp3),"%s/squagent2.int_log",tmp);
 308         if (snprintf(csort,sizeof(csort),"sort -t \"\t\" -k 2,2 -o \"%s\" \"%s\"",tmp3,tmp2)>=sizeof(csort)) {
 309                 debuga(__FILE__,__LINE__,_("Sort command too long when sorting file \"%s\" to \"%s\"\n"),tmp2,tmp3);
 310                 exit(EXIT_FAILURE);
 311         }
 312         cstatus=system(csort);
 313         if (!WIFEXITED(cstatus) || WEXITSTATUS(cstatus)) {
 314                 debuga(__FILE__,__LINE__,_("sort command return status %d\n"),WEXITSTATUS(cstatus));
 315                 debuga(__FILE__,__LINE__,_("sort command: %s\n"),csort);
 316                 exit(EXIT_FAILURE);
 317         }
 318         if ((fp_in=fopen(tmp3,"r"))==NULL) {
 319                 debuga(__FILE__,__LINE__,_("Cannot open file \"%s\": %s\n"),tmp3,strerror(errno));
 320                 debuga(__FILE__,__LINE__,_("sort command: %s\n"),csort);
 321                 exit(EXIT_FAILURE);
 322         }
 323
 324         if (!KeepTempLog && unlink(tmp2)) {
 325                 debuga(__FILE__,__LINE__,_("Cannot delete \"%s\": %s\n"),tmp2,strerror(errno));
 326                 exit(EXIT_FAILURE);
 327         }
 328
 329         if ((fp_ou=fopen(tmp2,"w"))==NULL) {
 330                 debuga(__FILE__,__LINE__,_("Cannot open file \"%s\": %s\n"),tmp2,strerror(errno));
 331                 exit(EXIT_FAILURE);
 332         }
 333
 334         agent_old[0]='\0';
 335         cont=0;
 336
 337         while (fgets(buf,sizeof(buf),fp_in)!=NULL) {
 338                 getword_start(&gwarea,buf);
 339                 if (getword(ip,sizeof(ip),&gwarea,'\t')<0) {
 340                         debuga(__FILE__,__LINE__,_("Invalid IP address in file \"%s\"\n"),tmp3);
 341                         exit(EXIT_FAILURE);
 342                 }
 343                 if (getword(agent,sizeof(agent),&gwarea,'\t')<0) {
 344                         debuga(__FILE__,__LINE__,_("Invalid useragent in file \"%s\"\n"),tmp3);
 345                         exit(EXIT_FAILURE);
 346                 }
 347
 348                 if (!cont) {
 349                         cont++;
 350                         strcpy(agent_old,agent);
 351                 }
 352
 353                 if (strcmp(agent,agent_old) != 0) {
 354                         agentdif++;
 355                         fprintf(fp_ou,"%06d %s\n",agentot,agent_old);
 356                         strcpy(agent_old,agent);
 357                         agentot2+=agentot;
 358                         agentot=0;
 359                 }
 360                 agentot++;
 361         }
 362         agentdif++;
 363         fprintf(fp_ou,"%06d %s\n",agentot,agent);
 364         agentot2+=agentot;
 365
 366         if (fclose(fp_ou)==EOF) {
 367                 debuga(__FILE__,__LINE__,_("Write error in \"%s\": %s\n"),tmp2,strerror(errno));
 368                 exit(EXIT_FAILURE);
 369         }
 370         if (fclose(fp_in)==EOF) {
 371                 debuga(__FILE__,__LINE__,_("Read error in \"%s\": %s\n"),tmp3,strerror(errno));
 372                 exit(EXIT_FAILURE);
 373         }
 374
 375         if (!KeepTempLog && unlink(tmp3)) {
 376                 debuga(__FILE__,__LINE__,_("Cannot delete \"%s\": %s\n"),tmp3,strerror(errno));
 377                 exit(EXIT_FAILURE);
 378         }
 379
 380         if (snprintf(csort,sizeof(csort),"sort -n -r -k 1,1 -o \"%s\" \"%s\"",tmp3,tmp2)>=sizeof(csort)) {
 381                 debuga(__FILE__,__LINE__,_("Sort command too long when sorting file \"%s\" to \"%s\"\n"),tmp2,tmp3);
 382                 exit(EXIT_FAILURE);
 383         }
 384         cstatus=system(csort);
 385         if (!WIFEXITED(cstatus) || WEXITSTATUS(cstatus)) {
 386                 debuga(__FILE__,__LINE__,_("sort command return status %d\n"),WEXITSTATUS(cstatus));
 387                 debuga(__FILE__,__LINE__,_("sort command: %s\n"),csort);
 388                 exit(EXIT_FAILURE);
 389         }
 390         if((fp_in=fopen(tmp3,"r"))==NULL) {
 391                 debuga(__FILE__,__LINE__,_("Cannot open file \"%s\": %s\n"),tmp3,strerror(errno));
 392                 debuga(__FILE__,__LINE__,_("sort command: %s\n"),csort);
 393                 exit(EXIT_FAILURE);
 394         }
 395
 396         if (!KeepTempLog && unlink(tmp2)) {
 397                 debuga(__FILE__,__LINE__,_("Cannot delete \"%s\": %s\n"),tmp2,strerror(errno));
 398                 exit(EXIT_FAILURE);
 399         }
 400
 401         fputs("<br><br>\n",fp_ht);
 402
 403         fputs("<table cellpadding=\"0\" cellspacing=\"0\">\n",fp_ht);
 404         fprintf(fp_ht,"<tr><th class=\"header_l\">%s</th><th class=\"header_l\">%s</th><th class=\"header_c\">%%</th></tr>\n",_("AGENT"),_("TOTAL"));
 405
 406         perc=0.;
 407         while(fgets(buf,sizeof(buf),fp_in)!=NULL) {
 408                 fixendofline(buf);
 409                 getword_start(&gwarea,buf);
 410                 if (getword(tagent,sizeof(tagent),&gwarea,' ')<0) {
 411                         debuga(__FILE__,__LINE__,_("Invalid useragent in file \"%s\"\n"),tmp3);
 412                         exit(EXIT_FAILURE);
 413                 }
 414                 nagent=atoi(tagent);
 415                 perc=(agentot2>0) ? nagent * 100. / agentot2 : 0.;
 416
 417                 fputs("<tr><td class=\"data2\">",fp_ht);
 418                 output_html_string(fp_ht,gwarea.current,250);
 419                 fprintf(fp_ht,"</td><td class=\"data\">%d</td><td class=\"data\">%3.2lf</td></tr>\n",nagent,perc);
 420         }
 421         if (fclose(fp_in)==EOF) {
 422                 debuga(__FILE__,__LINE__,_("Read error in \"%s\": %s\n"),tmp3,strerror(errno));
 423                 exit(EXIT_FAILURE);
 424         }
 425
 426         fputs("</table></div>\n",fp_ht);
 427         write_html_trailer(fp_ht);
 428         if (fclose(fp_ht)==EOF) {
 429                 debuga(__FILE__,__LINE__,_("Write error in \"%s\": %s\n"),hfile,strerror(errno));
 430                 exit(EXIT_FAILURE);
 431         }
 432
 433         if (!KeepTempLog && unlink(tmp3)) {
 434                 debuga(__FILE__,__LINE__,_("Cannot delete \"%s\": %s\n"),tmp3,strerror(errno));
 435                 exit(EXIT_FAILURE);
 436         }
 437
 438         return;
 439 }