useragent.c

   1 /*
   2  * SARG Squid Analysis Report Generator      http://sarg.sourceforge.net
   3  *                                                            1998, 2015
   4  *
   5  * SARG donations:
   6  *      please look at http://sarg.sourceforge.net/donations.php
   7  * Support:
   8  *     http://sourceforge.net/projects/sarg/forums/forum/363374
   9  * ---------------------------------------------------------------------
  10  *
  11  *  This program is free software; you can redistribute it and/or modify
  12  *  it under the terms of the GNU General Public License as published by
  13  *  the Free Software Foundation; either version 2 of the License, or
  14  *  (at your option) any later version.
  15  *
  16  *  This program is distributed in the hope that it will be useful,
  17  *  but WITHOUT ANY WARRANTY; without even the implied warranty of
  18  *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  19  *  GNU General Public License for more details.
  20  *
  21  *  You should have received a copy of the GNU General Public License
  22  *  along with this program; if not, write to the Free Software
  23  *  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111, USA.
  24  *
  25  */
  26
  27 #include "include/conf.h"
  28 #include "include/defs.h"
  29 #include "include/filelist.h"
  30
  31 FileListObject UserAgentLog=NULL;
  32
  33 void useragent(void)
  34 {
  35         FileObject *fp_log;
  36         FILE *fp_in = NULL, *fp_ou = NULL, *fp_ht = NULL;
  37         char *ptr;
  38         char buf[MAXLEN];
  39         char ip[MAXLEN], data[MAXLEN], agent[MAXLEN], user[MAXLEN];
  40         char ipbefore[MAXLEN], namebefore[MAXLEN];
  41         char tagent[MAXLEN];
  42         char user_old[MAXLEN]="$#%0a3bc6";
  43         char agent_old[MAXLEN]="$#%0a3bc6";
  44         char hfile[MAXLEN];
  45         char idate[MAXLEN], fdate[MAXLEN];
  46         char tmp2[MAXLEN];
  47         char tmp3[MAXLEN];
  48         char day[4],month[5],year[5], wdate[20];
  49         char csort[MAXLEN];
  50         const char *FileName;
  51         int  agentot=0, agentot2=0, agentdif=0, cont=0, nagent;
  52         unsigned long totregsl=0;
  53         int cstatus;
  54         int ndate;
  55         double perc;
  56         struct getwordstruct gwarea, gwarea1;
  57         longline line;
  58         FileListIterator FIter;
  59
  60         ip[0]='\0';
  61         data[0]='\0';
  62         agent[0]='\0';
  63         user[0]='\0';
  64         user_old[0]='\0';
  65         agent_old[0]='\0';
  66         ipbefore[0]='\0';
  67         namebefore[0]='\0';
  68
  69         snprintf(tmp3,sizeof(tmp3),"%s/squagent.int_unsort",tmp);
  70         snprintf(tmp2,sizeof(tmp2),"%s/squagent.int_log",tmp);
  71
  72         if((fp_ou=fopen(tmp3,"w"))==NULL) {
  73                 debuga(__FILE__,__LINE__,_("Cannot open file \"%s\": %s\n"),tmp3,strerror(errno));
  74                 exit(EXIT_FAILURE);
  75         }
  76
  77         if ((line=longline_create())==NULL) {
  78                 debuga(__FILE__,__LINE__,_("Not enough memory to read useragent log\n"));
  79                 exit(EXIT_FAILURE);
  80         }
  81
  82         FIter=FileListIter_Open(UserAgentLog);
  83         while ((FileName=FileListIter_Next(FIter))!=NULL)
  84         {
  85                 longline_reset(line);
  86                 if((fp_log=decomp(FileName))==NULL) {
  87                         debuga(__FILE__,__LINE__,_("Cannot open file \"%s\": %s\n"),FileName,FileObject_GetLastOpenError());
  88                         exit(EXIT_FAILURE);
  89                 }
  90
  91                 if(debug) {
  92                         debuga(__FILE__,__LINE__,_("Reading useragent log \"%s\"\n"),FileName);
  93                 }
  94
  95                 while((ptr=longline_read(fp_log,line))!=NULL) {
  96                         totregsl++;
  97                         getword_start(&gwarea,ptr);
  98                         if (getword(ip,sizeof(ip),&gwarea,' ')<0 || getword_skip(MAXLEN,&gwarea,'[')<0 ||
  99                                 getword(data,sizeof(data),&gwarea,' ')<0) {
 100                                 debuga(__FILE__,__LINE__,_("Invalid record in file \"%s\"\n"),FileName);
 101                                 exit(EXIT_FAILURE);
 102                         }
 103                         getword_start(&gwarea1,data);
 104                         if (getword(day,sizeof(day),&gwarea1,'/')<0 || getword(month,sizeof(month),&gwarea1,'/')<0 ||
 105                                 getword(year,sizeof(year),&gwarea1,':')<0) {
 106                                 debuga(__FILE__,__LINE__,_("Invalid date in file \"%s\"\n"),FileName);
 107                                 exit(EXIT_FAILURE);
 108                         }
 109                         if (dfrom!=0 || duntil!=0){
 110                                 buildymd(day,month,year,wdate,sizeof(wdate));
 111                                 ndate=atoi(wdate);
 112                                 if (ndate<dfrom) continue;
 113                                 if (ndate>duntil) break;
 114                         }
 115                         if(totregsl == 1)
 116                                 strcpy(idate,data);
 117                         strcpy(fdate,data);
 118                         if (getword_skip(MAXLEN,&gwarea,'"')<0 || getword(agent,sizeof(agent),&gwarea,'"')<0) {
 119                                 debuga(__FILE__,__LINE__,_("Invalid useragent in file \"%s\"\n"),FileName);
 120                                 exit(EXIT_FAILURE);
 121                         }
 122
 123                         if(gwarea.current[0]!='\0') {
 124                                 if (getword_skip(MAXLEN,&gwarea,' ')<0 || getword(user,sizeof(user),&gwarea,'\n')<0) {
 125                                         debuga(__FILE__,__LINE__,_("Invalid record in file \"%s\"\n"),FileName);
 126                                         exit(EXIT_FAILURE);
 127                                 }
 128                                 if(user[0] == '-')
 129                                         strcpy(user,ip);
 130                                 if(user[0] == '\0')
 131                                         strcpy(user,ip);
 132                         } else {
 133                                 strcpy(user,ip);
 134                         }
 135
 136                         fprintf(fp_ou,"%s\t%s\t%s\n",ip,agent,user);
 137                         useragent_count++;
 138                 }
 139
 140                 if (FileObject_Close(fp_log)==EOF) {
 141                         debuga(__FILE__,__LINE__,_("Read error in \"%s\": %s\n"),FileName,FileObject_GetLastCloseError());
 142                         exit(EXIT_FAILURE);
 143                 }
 144         }
 145         FileListIter_Close(FIter);
 146         longline_destroy(&line);
 147
 148         if(debug) {
 149                 debuga(__FILE__,__LINE__,_("   Records read: %ld\n"),totregsl);
 150         }
 151
 152         if (fclose(fp_ou)==EOF) {
 153                 debuga(__FILE__,__LINE__,_("Write error in \"%s\": %s\n"),tmp3,strerror(errno));
 154                 exit(EXIT_FAILURE);
 155         }
 156         if(debug) {
 157                 debuga(__FILE__,__LINE__,_("Sorting file \"%s\"\n"),tmp2);
 158         }
 159
 160         if (snprintf(csort,sizeof(csort),"sort -n -t \"\t\" -k 3,3 -k 2,2 -k 1,1 -o \"%s\" \"%s\"",tmp2,tmp3)>=sizeof(csort)) {
 161                 debuga(__FILE__,__LINE__,_("Sort command too long when sorting file \"%s\" to \"%s\"\n"),tmp2,tmp3);
 162                 exit(EXIT_FAILURE);
 163         }
 164         cstatus=system(csort);
 165         if (!WIFEXITED(cstatus) || WEXITSTATUS(cstatus)) {
 166                 debuga(__FILE__,__LINE__,_("sort command return status %d\n"),WEXITSTATUS(cstatus));
 167                 debuga(__FILE__,__LINE__,_("sort command: %s\n"),csort);
 168                 exit(EXIT_FAILURE);
 169         }
 170         if((fp_in=fopen(tmp2,"r"))==NULL) {
 171                 debuga(__FILE__,__LINE__,_("Cannot open file \"%s\": %s\n"),tmp2,strerror(errno));
 172                 debuga(__FILE__,__LINE__,_("sort command: %s\n"),csort);
 173                 exit(EXIT_FAILURE);
 174         }
 175
 176         if (!KeepTempLog && unlink(tmp3)) {
 177                 debuga(__FILE__,__LINE__,_("Cannot delete \"%s\": %s\n"),tmp3,strerror(errno));
 178                 exit(EXIT_FAILURE);
 179         }
 180
 181         snprintf(hfile,sizeof(hfile),"%s/useragent.html", outdirname);
 182         if((fp_ht=fopen(hfile,"w"))==NULL) {
 183                 debuga(__FILE__,__LINE__,_("Cannot open file \"%s\": %s\n"),hfile,strerror(errno));
 184                 exit(EXIT_FAILURE);
 185         }
 186
 187         if(debug)
 188                 debuga(__FILE__,__LINE__,_("Making Useragent report\n"));
 189
 190         write_html_header(fp_ht,(IndexTree == INDEX_TREE_DATE) ? 3 : 1,_("Squid Useragent's Report"),HTML_JS_NONE);
 191         fprintf(fp_ht,"<tr><th class=\"header_c\">%s</th></tr>\n",_("Squid Useragent's Report"));
 192         fprintf(fp_ht,"<tr><td class=\"header_c\">%s: %s - %s</td></tr>\n",_("Period"),idate,fdate);
 193         close_html_header(fp_ht);
 194
 195         fputs("<br><br>\n",fp_ht);
 196
 197         fputs("<div class=\"report\"><table cellpadding=\"0\" cellspacing=\"0\">\n",fp_ht);
 198         fputs("<tr><td>&nbsp;</td><td>&nbsp;</td></tr>",fp_ht);
 199
 200         fprintf(fp_ht,"<tr><th class=\"header_l\">%s</th><th class=\"header_l\">%s</th></tr>\n",_("USERID"),_("AGENT"));
 201
 202         while(fgets(buf,sizeof(buf),fp_in)!=NULL) {
 203                 getword_start(&gwarea,buf);
 204                 if (getword(ip,sizeof(ip),&gwarea,'\t')<0) {
 205                         debuga(__FILE__,__LINE__,_("Invalid IP address in file \"%s\"\n"),tmp2);
 206                         exit(EXIT_FAILURE);
 207                 }
 208
 209                 if(Ip2Name) {
 210                         if(strcmp(ip,ipbefore) != 0) {
 211                                 strcpy(ipbefore,ip);
 212                                 ip2name(ip,sizeof(ip));
 213                                 strcpy(namebefore,ip);
 214                         } else strcpy(ip,namebefore);
 215                 }
 216
 217                 if (getword(agent,sizeof(agent),&gwarea,'\t')<0) {
 218                         debuga(__FILE__,__LINE__,_("Invalid useragent in file \"%s\"\n"),tmp2);
 219                         exit(EXIT_FAILURE);
 220                 }
 221                 if (getword(user,sizeof(user),&gwarea,'\t')<0) {
 222                         debuga(__FILE__,__LINE__,_("Invalid user ID in file \"%s\"\n"),tmp2);
 223                         exit(EXIT_FAILURE);
 224                 }
 225
 226                 if(strcmp(user,user_old) != 0) {
 227                         fprintf(fp_ht,"<tr><td class=\"data2\">%s</td><td class=\"data2\">",user);
 228                         output_html_string(fp_ht,agent,250);
 229                         fputs("</td></tr>\n",fp_ht);
 230                         strcpy(user_old,user);
 231                         strcpy(agent_old,agent);
 232                 } else if(strcmp(agent,agent_old) != 0) {
 233                         fputs("<tr><td></td><td class=\"data2\">",fp_ht);
 234                         output_html_string(fp_ht,agent,250);
 235                         fputs("</td></tr>\n",fp_ht);
 236                         strcpy(agent_old,agent);
 237                 }
 238         }
 239
 240         fputs("</table>\n",fp_ht);
 241         if (fclose(fp_in)==EOF) {
 242                 debuga(__FILE__,__LINE__,_("Read error in \"%s\": %s\n"),tmp2,strerror(errno));
 243                 exit(EXIT_FAILURE);
 244         }
 245
 246         if (snprintf(csort,sizeof(csort),"sort -t \"\t\" -k 2,2 -o \"%s\" \"%s\"",tmp3,tmp2)>=sizeof(csort)) {
 247                 debuga(__FILE__,__LINE__,_("Sort command too long when sorting file \"%s\" to \"%s\"\n"),tmp2,tmp3);
 248                 exit(EXIT_FAILURE);
 249         }
 250         cstatus=system(csort);
 251         if (!WIFEXITED(cstatus) || WEXITSTATUS(cstatus)) {
 252                 debuga(__FILE__,__LINE__,_("sort command return status %d\n"),WEXITSTATUS(cstatus));
 253                 debuga(__FILE__,__LINE__,_("sort command: %s\n"),csort);
 254                 exit(EXIT_FAILURE);
 255         }
 256         if((fp_in=fopen(tmp3,"r"))==NULL) {
 257                 debuga(__FILE__,__LINE__,_("Cannot open file \"%s\": %s\n"),tmp3,strerror(errno));
 258                 debuga(__FILE__,__LINE__,_("sort command: %s\n"),csort);
 259                 exit(EXIT_FAILURE);
 260         }
 261
 262         if (!KeepTempLog && unlink(tmp2)) {
 263                 debuga(__FILE__,__LINE__,_("Cannot delete \"%s\": %s\n"),tmp2,strerror(errno));
 264                 exit(EXIT_FAILURE);
 265         }
 266
 267         if((fp_ou=fopen(tmp2,"w"))==NULL) {
 268                 debuga(__FILE__,__LINE__,_("Cannot open file \"%s\": %s\n"),tmp2,strerror(errno));
 269                 exit(EXIT_FAILURE);
 270         }
 271
 272         agent_old[0]='\0';
 273         cont=0;
 274
 275         while(fgets(buf,sizeof(buf),fp_in)!=NULL) {
 276                 getword_start(&gwarea,buf);
 277                 if (getword(ip,sizeof(ip),&gwarea,'\t')<0) {
 278                         debuga(__FILE__,__LINE__,_("Invalid IP address in file \"%s\"\n"),tmp3);
 279                         exit(EXIT_FAILURE);
 280                 }
 281                 if (getword(agent,sizeof(agent),&gwarea,'\t')<0) {
 282                         debuga(__FILE__,__LINE__,_("Invalid useragent in file \"%s\"\n"),tmp3);
 283                         exit(EXIT_FAILURE);
 284                 }
 285
 286                 if(!cont) {
 287                         cont++;
 288                         strcpy(agent_old,agent);
 289                 }
 290
 291                 if(strcmp(agent,agent_old) != 0) {
 292                         agentdif++;
 293                         fprintf(fp_ou,"%06d %s\n",agentot,agent_old);
 294                         strcpy(agent_old,agent);
 295                         agentot2+=agentot;
 296                         agentot=0;
 297                 }
 298                 agentot++;
 299         }
 300         agentdif++;
 301         fprintf(fp_ou,"%06d %s\n",agentot,agent);
 302         agentot2+=agentot;
 303
 304         if (fclose(fp_ou)==EOF) {
 305                 debuga(__FILE__,__LINE__,_("Write error in \"%s\": %s\n"),tmp2,strerror(errno));
 306                 exit(EXIT_FAILURE);
 307         }
 308         if (fclose(fp_in)==EOF) {
 309                 debuga(__FILE__,__LINE__,_("Read error in \"%s\": %s\n"),tmp3,strerror(errno));
 310                 exit(EXIT_FAILURE);
 311         }
 312
 313         if (!KeepTempLog && unlink(tmp3)) {
 314                 debuga(__FILE__,__LINE__,_("Cannot delete \"%s\": %s\n"),tmp3,strerror(errno));
 315                 exit(EXIT_FAILURE);
 316         }
 317
 318         if (snprintf(csort,sizeof(csort),"sort -n -r -k 1,1 -o \"%s\" \"%s\"",tmp3,tmp2)>=sizeof(csort)) {
 319                 debuga(__FILE__,__LINE__,_("Sort command too long when sorting file \"%s\" to \"%s\"\n"),tmp2,tmp3);
 320                 exit(EXIT_FAILURE);
 321         }
 322         cstatus=system(csort);
 323         if (!WIFEXITED(cstatus) || WEXITSTATUS(cstatus)) {
 324                 debuga(__FILE__,__LINE__,_("sort command return status %d\n"),WEXITSTATUS(cstatus));
 325                 debuga(__FILE__,__LINE__,_("sort command: %s\n"),csort);
 326                 exit(EXIT_FAILURE);
 327         }
 328         if((fp_in=fopen(tmp3,"r"))==NULL) {
 329                 debuga(__FILE__,__LINE__,_("Cannot open file \"%s\": %s\n"),tmp3,strerror(errno));
 330                 debuga(__FILE__,__LINE__,_("sort command: %s\n"),csort);
 331                 exit(EXIT_FAILURE);
 332         }
 333
 334         if (!KeepTempLog && unlink(tmp2)) {
 335                 debuga(__FILE__,__LINE__,_("Cannot delete \"%s\": %s\n"),tmp2,strerror(errno));
 336                 exit(EXIT_FAILURE);
 337         }
 338
 339         fputs("<br><br>\n",fp_ht);
 340
 341         fputs("<table cellpadding=\"0\" cellspacing=\"0\">\n",fp_ht);
 342         fprintf(fp_ht,"<tr><th class=\"header_l\">%s</th><th class=\"header_l\">%s</th><th class=\"header_c\">%%</th></tr>\n",_("AGENT"),_("TOTAL"));
 343
 344         perc=0.;
 345         while(fgets(buf,sizeof(buf),fp_in)!=NULL) {
 346                 fixendofline(buf);
 347                 getword_start(&gwarea,buf);
 348                 if (getword(tagent,sizeof(tagent),&gwarea,' ')<0) {
 349                         debuga(__FILE__,__LINE__,_("Invalid useragent in file \"%s\"\n"),tmp3);
 350                         exit(EXIT_FAILURE);
 351                 }
 352                 nagent=atoi(tagent);
 353                 perc=(agentot2>0) ? nagent * 100. / agentot2 : 0.;
 354
 355                 fputs("<tr><td class=\"data2\">",fp_ht);
 356                 output_html_string(fp_ht,gwarea.current,250);
 357                 fprintf(fp_ht,"</td><td class=\"data\">%d</td><td class=\"data\">%3.2lf</td></tr>\n",nagent,perc);
 358         }
 359         if (fclose(fp_in)==EOF) {
 360                 debuga(__FILE__,__LINE__,_("Read error in \"%s\": %s\n"),tmp3,strerror(errno));
 361                 exit(EXIT_FAILURE);
 362         }
 363
 364         fputs("</table></div>\n",fp_ht);
 365         write_html_trailer(fp_ht);
 366         if (fclose(fp_ht)==EOF) {
 367                 debuga(__FILE__,__LINE__,_("Write error in \"%s\": %s\n"),hfile,strerror(errno));
 368                 exit(EXIT_FAILURE);
 369         }
 370
 371         if (!KeepTempLog && unlink(tmp3)) {
 372                 debuga(__FILE__,__LINE__,_("Cannot delete \"%s\": %s\n"),tmp3,strerror(errno));
 373                 exit(EXIT_FAILURE);
 374         }
 375
 376         return;
 377 }