useragent.c

   1 /*
   2  * SARG Squid Analysis Report Generator      http://sarg.sourceforge.net
   3  *                                                            1998, 2015
   4  *
   5  * SARG donations:
   6  *      please look at http://sarg.sourceforge.net/donations.php
   7  * Support:
   8  *     http://sourceforge.net/projects/sarg/forums/forum/363374
   9  * ---------------------------------------------------------------------
  10  *
  11  *  This program is free software; you can redistribute it and/or modify
  12  *  it under the terms of the GNU General Public License as published by
  13  *  the Free Software Foundation; either version 2 of the License, or
  14  *  (at your option) any later version.
  15  *
  16  *  This program is distributed in the hope that it will be useful,
  17  *  but WITHOUT ANY WARRANTY; without even the implied warranty of
  18  *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  19  *  GNU General Public License for more details.
  20  *
  21  *  You should have received a copy of the GNU General Public License
  22  *  along with this program; if not, write to the Free Software
  23  *  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111, USA.
  24  *
  25  */
  26
  27 #include "include/conf.h"
  28 #include "include/defs.h"
  29 #include "include/filelist.h"
  30
  31 FileListObject UserAgentLog=NULL;
  32
  33 //! Log file where the user agent data are written.
  34 static char UserAgentTempLog[MAXLEN]="";
  35
  36 static struct tm UserAgentStartDate;
  37 static struct tm UserAgentEndDate;
  38
  39 /*!
  40  * Open the temporary file to store the useragent entries to be
  41  * reported.
  42  *
  43  * \return The file handle. It must be closed when the data have
  44  * been written.
  45  */
  46 FILE *UserAgent_Open(void)
  47 {
  48         FILE *fp_ou=NULL;
  49
  50         if (UserAgentTempLog[0]) {
  51                 debuga(__FILE__,__LINE__,_("Useragent log already opened\n"));
  52                 exit(EXIT_FAILURE);
  53         }
  54         if ((ReportType & REPORT_TYPE_USERAGENT)!=0) {
  55                 snprintf(UserAgentTempLog,sizeof(UserAgentTempLog),"%s/squagent.int_unsort",tmp);
  56                 if ((fp_ou=fopen(UserAgentTempLog,"w"))==NULL) {
  57                         debuga(__FILE__,__LINE__,_("Cannot open file \"%s\": %s\n"),UserAgentTempLog,strerror(errno));
  58                         exit(EXIT_FAILURE);
  59                 }
  60                 memset(&UserAgentStartDate,0,sizeof(UserAgentStartDate));
  61                 memset(&UserAgentEndDate,0,sizeof(UserAgentEndDate));
  62         }
  63         return(fp_ou);
  64 }
  65
  66 /*!
  67  * Write a user agent entry into the temporary log.
  68  *
  69  * \param fp The file opened by UserAgent_Open().
  70  * \param Ip The IP address using this agent.
  71  * \param User The user name.
  72  * \param Agent The user agent string.
  73  */
  74 void UserAgent_Write(FILE *fp,const struct tm *Time,const char *Ip,const char *User,const char *Agent)
  75 {
  76         if (fp) {
  77                 if (useragent_count==0 || compare_date(&UserAgentStartDate,Time)>0)
  78                         memcpy(&UserAgentStartDate,Time,sizeof(UserAgentStartDate));
  79                 if (useragent_count==0 || compare_date(&UserAgentEndDate,Time)<0)
  80                         memcpy(&UserAgentEndDate,Time,sizeof(UserAgentEndDate));
  81                 fprintf(fp,"%s\t%s\t%s\n",Ip,Agent,User);
  82                 useragent_count++;
  83         }
  84 }
  85
  86 /*!
  87  * Read the user provided useragent file and create
  88  * a temporary file with the data to report.
  89  */
  90 void UserAgent_Readlog(void)
  91 {
  92         FileObject *fp_log;
  93         FILE *fp_ou = NULL;
  94         char *ptr;
  95         char ip[80], data[50], agent[MAXLEN], user[MAXLEN];
  96         int day,month,year;
  97         char monthname[5];
  98         const char *FileName;
  99         unsigned long totregsl=0;
 100         int ndate;
 101         struct getwordstruct gwarea, gwarea1;
 102         longline line;
 103         FileListIterator FIter;
 104         struct tm logtime;
 105         int dfrom;
 106         int duntil;
 107
 108         fp_ou=UserAgent_Open();
 109
 110         if ((line=longline_create())==NULL) {
 111                 debuga(__FILE__,__LINE__,_("Not enough memory to read useragent log\n"));
 112                 exit(EXIT_FAILURE);
 113         }
 114         memset(&logtime,0,sizeof(logtime));
 115         getperiod_torange(&period,&dfrom,&duntil);
 116
 117         FIter=FileListIter_Open(UserAgentLog);
 118         while ((FileName=FileListIter_Next(FIter))!=NULL)
 119         {
 120                 longline_reset(line);
 121                 if ((fp_log=decomp(FileName))==NULL) {
 122                         debuga(__FILE__,__LINE__,_("Cannot open file \"%s\": %s\n"),FileName,FileObject_GetLastOpenError());
 123                         exit(EXIT_FAILURE);
 124                 }
 125
 126                 if(debug) {
 127                         debuga(__FILE__,__LINE__,_("Reading useragent log \"%s\"\n"),FileName);
 128                 }
 129
 130                 while ((ptr=longline_read(fp_log,line))!=NULL) {
 131                         totregsl++;
 132                         getword_start(&gwarea,ptr);
 133                         if (getword(ip,sizeof(ip),&gwarea,' ')<0 || getword_skip(10,&gwarea,'[')<0 ||
 134                                 getword(data,sizeof(data),&gwarea,' ')<0) {
 135                                 debuga(__FILE__,__LINE__,_("Invalid record in file \"%s\"\n"),FileName);
 136                                 exit(EXIT_FAILURE);
 137                         }
 138                         getword_start(&gwarea1,data);
 139                         if (getword_atoi(&day,&gwarea1,'/')<0 || getword(monthname,sizeof(monthname),&gwarea1,'/')<0 ||
 140                                 getword_atoi(&year,&gwarea1,':')<0) {
 141                                 debuga(__FILE__,__LINE__,_("Invalid date in file \"%s\"\n"),FileName);
 142                                 exit(EXIT_FAILURE);
 143                         }
 144                         month=month2num(monthname)+1;
 145                         if (month>12) {
 146                                 debuga(__FILE__,__LINE__,_("Invalid month name \"%s\" found in user agent file \"%s\""),monthname,FileName);
 147                                 exit(EXIT_FAILURE);
 148                         }
 149                         if (dfrom!=0 || duntil!=0){
 150                                 ndate=year*10000+month*100+day;
 151                                 if (ndate<dfrom) continue;
 152                                 if (ndate>duntil) break;
 153                         }
 154                         logtime.tm_year=year-1900;
 155                         logtime.tm_mon=month-1;
 156                         logtime.tm_mday=day;
 157                         if (getword_skip(MAXLEN,&gwarea,'"')<0 || getword(agent,sizeof(agent),&gwarea,'"')<0) {
 158                                 debuga(__FILE__,__LINE__,_("Invalid useragent in file \"%s\"\n"),FileName);
 159                                 exit(EXIT_FAILURE);
 160                         }
 161
 162                         if (gwarea.current[0]!='\0') {
 163                                 if (getword_skip(MAXLEN,&gwarea,' ')<0 || getword(user,sizeof(user),&gwarea,'\n')<0) {
 164                                         debuga(__FILE__,__LINE__,_("Invalid record in file \"%s\"\n"),FileName);
 165                                         exit(EXIT_FAILURE);
 166                                 }
 167                                 if (user[0] == '-')
 168                                         strcpy(user,ip);
 169                                 if (user[0] == '\0')
 170                                         strcpy(user,ip);
 171                         } else {
 172                                 strcpy(user,ip);
 173                         }
 174
 175                         UserAgent_Write(fp_ou,&logtime,ip,user,agent);
 176                 }
 177
 178                 if (FileObject_Close(fp_log)==EOF) {
 179                         debuga(__FILE__,__LINE__,_("Read error in \"%s\": %s\n"),FileName,FileObject_GetLastCloseError());
 180                         exit(EXIT_FAILURE);
 181                 }
 182         }
 183         FileListIter_Close(FIter);
 184         longline_destroy(&line);
 185
 186         if(debug) {
 187                 debuga(__FILE__,__LINE__,_("   Records read: %ld\n"),totregsl);
 188         }
 189
 190         if (fclose(fp_ou)==EOF) {
 191                 debuga(__FILE__,__LINE__,_("Write error in \"%s\": %s\n"),UserAgentTempLog,strerror(errno));
 192                 exit(EXIT_FAILURE);
 193         }
 194 }
 195
 196 void UserAgent(void)
 197 {
 198         FILE *fp_in = NULL, *fp_ou = NULL, *fp_ht = NULL;
 199         char buf[MAXLEN];
 200         char ip[80], agent[MAXLEN], user[MAXLEN];
 201         char ipbefore[MAXLEN]="";
 202         char namebefore[MAXLEN]="";
 203         char tagent[MAXLEN];
 204         char user_old[MAXLEN]="";
 205         char agent_old[MAXLEN]="";
 206         char hfile[MAXLEN];
 207         char idate[100], fdate[100];
 208         char tmp2[MAXLEN];
 209         char tmp3[MAXLEN];
 210         char csort[MAXLEN];
 211         int  agentot=0, agentot2=0, agentdif=0, cont=0, nagent;
 212         int cstatus;
 213         double perc;
 214         struct getwordstruct gwarea;
 215
 216         if (!UserAgentTempLog[0] || useragent_count==0) return;
 217
 218         snprintf(tmp2,sizeof(tmp2),"%s/squagent.int_log",tmp);
 219         if (debug) {
 220                 debuga(__FILE__,__LINE__,_("Sorting file \"%s\"\n"),tmp2);
 221         }
 222
 223         if (snprintf(csort,sizeof(csort),"sort -n -t \"\t\" -k 3,3 -k 2,2 -k 1,1 -o \"%s\" \"%s\"",tmp2,UserAgentTempLog)>=sizeof(csort)) {
 224                 debuga(__FILE__,__LINE__,_("Sort command too long when sorting file \"%s\" to \"%s\"\n"),tmp2,UserAgentTempLog);
 225                 exit(EXIT_FAILURE);
 226         }
 227         cstatus=system(csort);
 228         if (!WIFEXITED(cstatus) || WEXITSTATUS(cstatus)) {
 229                 debuga(__FILE__,__LINE__,_("sort command return status %d\n"),WEXITSTATUS(cstatus));
 230                 debuga(__FILE__,__LINE__,_("sort command: %s\n"),csort);
 231                 exit(EXIT_FAILURE);
 232         }
 233         if ((fp_in=fopen(tmp2,"r"))==NULL) {
 234                 debuga(__FILE__,__LINE__,_("Cannot open file \"%s\": %s\n"),tmp2,strerror(errno));
 235                 debuga(__FILE__,__LINE__,_("sort command: %s\n"),csort);
 236                 exit(EXIT_FAILURE);
 237         }
 238
 239         if (!KeepTempLog && unlink(UserAgentTempLog)) {
 240                 debuga(__FILE__,__LINE__,_("Cannot delete \"%s\": %s\n"),UserAgentTempLog,strerror(errno));
 241                 exit(EXIT_FAILURE);
 242         }
 243
 244         snprintf(hfile,sizeof(hfile),"%s/useragent.html", outdirname);
 245         if ((fp_ht=fopen(hfile,"w"))==NULL) {
 246                 debuga(__FILE__,__LINE__,_("Cannot open file \"%s\": %s\n"),hfile,strerror(errno));
 247                 exit(EXIT_FAILURE);
 248         }
 249
 250         if (debug)
 251                 debuga(__FILE__,__LINE__,_("Making Useragent report\n"));
 252
 253         write_html_header(fp_ht,(IndexTree == INDEX_TREE_DATE) ? 3 : 1,_("Squid Useragent's Report"),HTML_JS_NONE);
 254         fprintf(fp_ht,"<tr><th class=\"header_c\">%s</th></tr>\n",_("Squid Useragent's Report"));
 255         strftime(idate,sizeof(idate),"%x",&UserAgentStartDate);
 256         strftime(fdate,sizeof(fdate),"%x",&UserAgentEndDate);
 257         fprintf(fp_ht,"<tr><td class=\"header_c\">%s: %s - %s</td></tr>\n",_("Period"),idate,fdate);
 258         close_html_header(fp_ht);
 259
 260         fputs("<br><br>\n",fp_ht);
 261
 262         fputs("<div class=\"report\"><table cellpadding=\"0\" cellspacing=\"0\">\n",fp_ht);
 263         fputs("<tr><td>&nbsp;</td><td>&nbsp;</td></tr>",fp_ht);
 264
 265         fprintf(fp_ht,"<tr><th class=\"header_l\">%s</th><th class=\"header_l\">%s</th></tr>\n",_("USERID"),_("AGENT"));
 266
 267         while (fgets(buf,sizeof(buf),fp_in)!=NULL) {
 268                 getword_start(&gwarea,buf);
 269                 if (getword(ip,sizeof(ip),&gwarea,'\t')<0) {
 270                         debuga(__FILE__,__LINE__,_("Invalid IP address in file \"%s\"\n"),tmp2);
 271                         exit(EXIT_FAILURE);
 272                 }
 273
 274                 if (Ip2Name) {
 275                         if (strcmp(ip,ipbefore) != 0) {
 276                                 strcpy(ipbefore,ip);
 277                                 ip2name(ip,sizeof(ip));
 278                                 strcpy(namebefore,ip);
 279                         } else strcpy(ip,namebefore);
 280                 }
 281
 282                 if (getword(agent,sizeof(agent),&gwarea,'\t')<0) {
 283                         debuga(__FILE__,__LINE__,_("Invalid useragent in file \"%s\"\n"),tmp2);
 284                         exit(EXIT_FAILURE);
 285                 }
 286                 if (getword(user,sizeof(user),&gwarea,'\t')<0) {
 287                         debuga(__FILE__,__LINE__,_("Invalid user ID in file \"%s\"\n"),tmp2);
 288                         exit(EXIT_FAILURE);
 289                 }
 290
 291                 if (strcmp(user,user_old) != 0) {
 292                         fprintf(fp_ht,"<tr><td class=\"data2\">%s</td><td class=\"data2\">",user);
 293                         output_html_string(fp_ht,agent,250);
 294                         fputs("</td></tr>\n",fp_ht);
 295                         strcpy(user_old,user);
 296                         strcpy(agent_old,agent);
 297                 } else if (strcmp(agent,agent_old) != 0) {
 298                         fputs("<tr><td></td><td class=\"data2\">",fp_ht);
 299                         output_html_string(fp_ht,agent,250);
 300                         fputs("</td></tr>\n",fp_ht);
 301                         strcpy(agent_old,agent);
 302                 }
 303         }
 304
 305         fputs("</table>\n",fp_ht);
 306         if (fclose(fp_in)==EOF) {
 307                 debuga(__FILE__,__LINE__,_("Read error in \"%s\": %s\n"),tmp2,strerror(errno));
 308                 exit(EXIT_FAILURE);
 309         }
 310
 311         snprintf(tmp3,sizeof(tmp3),"%s/squagent2.int_log",tmp);
 312         if (snprintf(csort,sizeof(csort),"sort -t \"\t\" -k 2,2 -o \"%s\" \"%s\"",tmp3,tmp2)>=sizeof(csort)) {
 313                 debuga(__FILE__,__LINE__,_("Sort command too long when sorting file \"%s\" to \"%s\"\n"),tmp2,tmp3);
 314                 exit(EXIT_FAILURE);
 315         }
 316         cstatus=system(csort);
 317         if (!WIFEXITED(cstatus) || WEXITSTATUS(cstatus)) {
 318                 debuga(__FILE__,__LINE__,_("sort command return status %d\n"),WEXITSTATUS(cstatus));
 319                 debuga(__FILE__,__LINE__,_("sort command: %s\n"),csort);
 320                 exit(EXIT_FAILURE);
 321         }
 322         if ((fp_in=fopen(tmp3,"r"))==NULL) {
 323                 debuga(__FILE__,__LINE__,_("Cannot open file \"%s\": %s\n"),tmp3,strerror(errno));
 324                 debuga(__FILE__,__LINE__,_("sort command: %s\n"),csort);
 325                 exit(EXIT_FAILURE);
 326         }
 327
 328         if (!KeepTempLog && unlink(tmp2)) {
 329                 debuga(__FILE__,__LINE__,_("Cannot delete \"%s\": %s\n"),tmp2,strerror(errno));
 330                 exit(EXIT_FAILURE);
 331         }
 332
 333         if ((fp_ou=fopen(tmp2,"w"))==NULL) {
 334                 debuga(__FILE__,__LINE__,_("Cannot open file \"%s\": %s\n"),tmp2,strerror(errno));
 335                 exit(EXIT_FAILURE);
 336         }
 337
 338         agent_old[0]='\0';
 339         cont=0;
 340
 341         while (fgets(buf,sizeof(buf),fp_in)!=NULL) {
 342                 getword_start(&gwarea,buf);
 343                 if (getword(ip,sizeof(ip),&gwarea,'\t')<0) {
 344                         debuga(__FILE__,__LINE__,_("Invalid IP address in file \"%s\"\n"),tmp3);
 345                         exit(EXIT_FAILURE);
 346                 }
 347                 if (getword(agent,sizeof(agent),&gwarea,'\t')<0) {
 348                         debuga(__FILE__,__LINE__,_("Invalid useragent in file \"%s\"\n"),tmp3);
 349                         exit(EXIT_FAILURE);
 350                 }
 351
 352                 if (!cont) {
 353                         cont++;
 354                         strcpy(agent_old,agent);
 355                 }
 356
 357                 if (strcmp(agent,agent_old) != 0) {
 358                         agentdif++;
 359                         fprintf(fp_ou,"%06d %s\n",agentot,agent_old);
 360                         strcpy(agent_old,agent);
 361                         agentot2+=agentot;
 362                         agentot=0;
 363                 }
 364                 agentot++;
 365         }
 366         agentdif++;
 367         fprintf(fp_ou,"%06d %s\n",agentot,agent);
 368         agentot2+=agentot;
 369
 370         if (fclose(fp_ou)==EOF) {
 371                 debuga(__FILE__,__LINE__,_("Write error in \"%s\": %s\n"),tmp2,strerror(errno));
 372                 exit(EXIT_FAILURE);
 373         }
 374         if (fclose(fp_in)==EOF) {
 375                 debuga(__FILE__,__LINE__,_("Read error in \"%s\": %s\n"),tmp3,strerror(errno));
 376                 exit(EXIT_FAILURE);
 377         }
 378
 379         if (!KeepTempLog && unlink(tmp3)) {
 380                 debuga(__FILE__,__LINE__,_("Cannot delete \"%s\": %s\n"),tmp3,strerror(errno));
 381                 exit(EXIT_FAILURE);
 382         }
 383
 384         if (snprintf(csort,sizeof(csort),"sort -n -r -k 1,1 -o \"%s\" \"%s\"",tmp3,tmp2)>=sizeof(csort)) {
 385                 debuga(__FILE__,__LINE__,_("Sort command too long when sorting file \"%s\" to \"%s\"\n"),tmp2,tmp3);
 386                 exit(EXIT_FAILURE);
 387         }
 388         cstatus=system(csort);
 389         if (!WIFEXITED(cstatus) || WEXITSTATUS(cstatus)) {
 390                 debuga(__FILE__,__LINE__,_("sort command return status %d\n"),WEXITSTATUS(cstatus));
 391                 debuga(__FILE__,__LINE__,_("sort command: %s\n"),csort);
 392                 exit(EXIT_FAILURE);
 393         }
 394         if((fp_in=fopen(tmp3,"r"))==NULL) {
 395                 debuga(__FILE__,__LINE__,_("Cannot open file \"%s\": %s\n"),tmp3,strerror(errno));
 396                 debuga(__FILE__,__LINE__,_("sort command: %s\n"),csort);
 397                 exit(EXIT_FAILURE);
 398         }
 399
 400         if (!KeepTempLog && unlink(tmp2)) {
 401                 debuga(__FILE__,__LINE__,_("Cannot delete \"%s\": %s\n"),tmp2,strerror(errno));
 402                 exit(EXIT_FAILURE);
 403         }
 404
 405         fputs("<br><br>\n",fp_ht);
 406
 407         fputs("<table cellpadding=\"0\" cellspacing=\"0\">\n",fp_ht);
 408         fprintf(fp_ht,"<tr><th class=\"header_l\">%s</th><th class=\"header_l\">%s</th><th class=\"header_c\">%%</th></tr>\n",_("AGENT"),_("TOTAL"));
 409
 410         perc=0.;
 411         while(fgets(buf,sizeof(buf),fp_in)!=NULL) {
 412                 fixendofline(buf);
 413                 getword_start(&gwarea,buf);
 414                 if (getword(tagent,sizeof(tagent),&gwarea,' ')<0) {
 415                         debuga(__FILE__,__LINE__,_("Invalid useragent in file \"%s\"\n"),tmp3);
 416                         exit(EXIT_FAILURE);
 417                 }
 418                 nagent=atoi(tagent);
 419                 perc=(agentot2>0) ? nagent * 100. / agentot2 : 0.;
 420
 421                 fputs("<tr><td class=\"data2\">",fp_ht);
 422                 output_html_string(fp_ht,gwarea.current,250);
 423                 fprintf(fp_ht,"</td><td class=\"data\">%d</td><td class=\"data\">%3.2lf</td></tr>\n",nagent,perc);
 424         }
 425         if (fclose(fp_in)==EOF) {
 426                 debuga(__FILE__,__LINE__,_("Read error in \"%s\": %s\n"),tmp3,strerror(errno));
 427                 exit(EXIT_FAILURE);
 428         }
 429
 430         fputs("</table></div>\n",fp_ht);
 431         write_html_trailer(fp_ht);
 432         if (fclose(fp_ht)==EOF) {
 433                 debuga(__FILE__,__LINE__,_("Write error in \"%s\": %s\n"),hfile,strerror(errno));
 434                 exit(EXIT_FAILURE);
 435         }
 436
 437         if (!KeepTempLog && unlink(tmp3)) {
 438                 debuga(__FILE__,__LINE__,_("Cannot delete \"%s\": %s\n"),tmp3,strerror(errno));
 439                 exit(EXIT_FAILURE);
 440         }
 441
 442         return;
 443 }