topsites.c

   1 /*
   2  * SARG Squid Analysis Report Generator      http://sarg.sourceforge.net
   3  *                                                            1998, 2015
   4  *
   5  * SARG donations:
   6  *      please look at http://sarg.sourceforge.net/donations.php
   7  * Support:
   8  *     http://sourceforge.net/projects/sarg/forums/forum/363374
   9  * ---------------------------------------------------------------------
  10  *
  11  *  This program is free software; you can redistribute it and/or modify
  12  *  it under the terms of the GNU General Public License as published by
  13  *  the Free Software Foundation; either version 2 of the License, or
  14  *  (at your option) any later version.
  15  *
  16  *  This program is distributed in the hope that it will be useful,
  17  *  but WITHOUT ANY WARRANTY; without even the implied warranty of
  18  *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  19  *  GNU General Public License for more details.
  20  *
  21  *  You should have received a copy of the GNU General Public License
  22  *  along with this program; if not, write to the Free Software
  23  *  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111, USA.
  24  *
  25  */
  26
  27 #include "include/conf.h"
  28 #include "include/defs.h"
  29
  30 #ifdef ENABLE_DOUBLE_CHECK_DATA
  31 extern struct globalstatstruct globstat;
  32 #endif
  33
  34 void topsites(void)
  35 {
  36         FileObject *fp_in;
  37         FILE *fp_ou;
  38
  39         char *buf;
  40         char *url;
  41         char *ourl=NULL;
  42         char csort[4096];
  43         char general[MAXLEN];
  44         char general2[MAXLEN];
  45         char general3[MAXLEN];
  46         char sites[MAXLEN];
  47         char report[MAXLEN];
  48         char ouser[MAX_USER_LEN]="";
  49         const char *sortf;
  50         const char *sortt;
  51         long long int nacc;
  52         long long int nbytes;
  53         long long int ntime;
  54         long long int tnacc=0;
  55         long long int tnbytes=0;
  56         long long int tntime=0;
  57         long long int twork1=0, twork2=0, twork3=0;
  58 #ifdef ENABLE_DOUBLE_CHECK_DATA
  59         long long int ttnacc=0;
  60         long long int ttnbytes=0;
  61         long long int ttntime=0;
  62 #endif
  63         int nusers=0;
  64         int regs=0;
  65         int cstatus;
  66         int url_len;
  67         int ourl_size=0;
  68         struct getwordstruct gwarea;
  69         longline line;
  70         struct generalitemstruct item;
  71
  72         if(Privacy) {
  73                 if (debugz>=LogLevel_Process) debugaz(__FILE__,__LINE__,_("Top sites report not produced because privacy option is on\n"));
  74                 return;
  75         }
  76         if (debugz>=LogLevel_Process)
  77                 debuga(__FILE__,__LINE__,_("Creating top sites report...\n"));
  78
  79         sprintf(general,"%s/sarg-general",outdirname);
  80         sprintf(sites,"%s/sarg-sites",outdirname);
  81         sprintf(general2,"%s/sarg-general2",outdirname);
  82         sprintf(general3,"%s/sarg-general3",outdirname);
  83
  84         sprintf(report,"%s/topsites.html",outdirname);
  85
  86         if (snprintf(csort,sizeof(csort),"sort -t \"\t\" -k 4,4 -k 1,1 -o \"%s\" \"%s\"",general2,general)>=sizeof(csort)) {
  87                 debuga(__FILE__,__LINE__,_("Sort command too long when sorting file \"%s\" to \"%s\"\n"),general,general2);
  88                 exit(EXIT_FAILURE);
  89         }
  90         cstatus=system(csort);
  91         if (!WIFEXITED(cstatus) || WEXITSTATUS(cstatus)) {
  92                 debuga(__FILE__,__LINE__,_("sort command return status %d\n"),WEXITSTATUS(cstatus));
  93                 debuga(__FILE__,__LINE__,_("sort command: %s\n"),csort);
  94                 exit(EXIT_FAILURE);
  95         }
  96
  97         if((fp_in=FileObject_Open(general2))==NULL) {
  98                 debuga(__FILE__,__LINE__,_("Cannot open file \"%s\": %s\n"),general2,FileObject_GetLastOpenError());
  99                 debuga(__FILE__,__LINE__,_("sort command: %s\n"),csort);
 100                 exit(EXIT_FAILURE);
 101         }
 102
 103         if((fp_ou=fopen(general3,"w"))==NULL) {
 104                 debuga(__FILE__,__LINE__,_("Cannot open file \"%s\": %s\n"),general3,strerror(errno));
 105                 exit(EXIT_FAILURE);
 106         }
 107
 108         if ((line=longline_create())==NULL) {
 109                 debuga(__FILE__,__LINE__,_("Not enough memory to read file \"%s\"\n"),general2);
 110                 exit(EXIT_FAILURE);
 111         }
 112
 113         while((buf=longline_read(fp_in,line))!=NULL) {
 114                 ger_read(buf,&item,general2);
 115                 if(item.total) continue;
 116
 117                 if(!regs) {
 118                         url_len=strlen(item.url);
 119                         if (!ourl || url_len>=ourl_size) {
 120                                 ourl_size=url_len+1;
 121                                 ourl=realloc(ourl,ourl_size);
 122                                 if (!ourl) {
 123                                         debuga(__FILE__,__LINE__,_("Not enough memory to store the url\n"));
 124                                         exit(EXIT_FAILURE);
 125                                 }
 126                         }
 127                         strcpy(ourl,item.url);
 128                         regs++;
 129                 }
 130
 131                 if(strcmp(item.url,ourl) != 0) {
 132                         /*
 133                         This complicated printf is due to Microsoft's inability to comply with any standard. Msvcrt is unable
 134                         to print a long long int unless it is exactly 64-bits long.
 135                         */
 136                         fprintf(fp_ou,"%"PRIu64"\t%"PRIu64"\t%"PRIu64"\t%d\t%s\n",(uint64_t)tnacc,(uint64_t)tnbytes,(uint64_t)tntime,nusers,ourl);
 137                         url_len=strlen(item.url);
 138                         if (url_len>=ourl_size) {
 139                                 ourl_size=url_len+1;
 140                                 ourl=realloc(ourl,ourl_size);
 141                                 if (!ourl) {
 142                                         debuga(__FILE__,__LINE__,_("Not enough memory to store the url\n"));
 143                                         exit(EXIT_FAILURE);
 144                                 }
 145                         }
 146                         strcpy(ourl,item.url);
 147                         strcpy(ouser,item.user);
 148                         tnacc=0;
 149                         tnbytes=0;
 150                         tntime=0;
 151                         nusers=1;
 152                 } else if (strcmp(item.user,ouser)!=0) {
 153                         strcpy(ouser,item.user);
 154                         nusers++;
 155                 }
 156
 157                 tnacc+=item.nacc;
 158                 tnbytes+=item.nbytes;
 159                 tntime+=item.nelap;
 160 #ifdef ENABLE_DOUBLE_CHECK_DATA
 161                 ttnacc+=item.nacc;
 162                 ttnbytes+=item.nbytes;
 163                 ttntime+=item.nelap;
 164 #endif
 165         }
 166         if (FileObject_Close(fp_in)) {
 167                 debuga(__FILE__,__LINE__,_("Read error in \"%s\": %s\n"),general2,FileObject_GetLastCloseError());
 168                 exit(EXIT_FAILURE);
 169         }
 170         longline_destroy(&line);
 171
 172         if (ourl) {
 173                 /*
 174                 This complicated printf is due to Microsoft's inability to comply with any standard. Msvcrt is unable
 175                 to print a long long int unless it is exactly 64-bits long.
 176                 */
 177                 fprintf(fp_ou,"%"PRIu64"\t%"PRIu64"\t%"PRIu64"\t%d\t%s\n",(uint64_t)tnacc,(uint64_t)tnbytes,(uint64_t)tntime,nusers,ourl);
 178                 free(ourl);
 179         }
 180
 181         if (fclose(fp_ou)==EOF) {
 182                 debuga(__FILE__,__LINE__,_("Write error in \"%s\": %s\n"),general3,strerror(errno));
 183                 exit(EXIT_FAILURE);
 184         }
 185
 186 #ifdef ENABLE_DOUBLE_CHECK_DATA
 187         if (ttnacc!=globstat.nacc || ttnbytes!=globstat.nbytes || ttntime!=globstat.elap) {
 188                 debuga(__FILE__,__LINE__,_("Total statistics mismatch when reading \"%s\" to produce the top sites\n"),general2);
 189                 exit(EXIT_FAILURE);
 190         }
 191 #endif
 192
 193         if (!KeepTempLog && unlink(general2)) {
 194                 debuga(__FILE__,__LINE__,_("Cannot delete \"%s\": %s\n"),general2,strerror(errno));
 195                 exit(EXIT_FAILURE);
 196         }
 197
 198         if((TopsitesSort & TOPSITE_SORT_CONNECT) != 0) {
 199                 sortf="-k 1,1 -k 2,2";
 200         } else if((TopsitesSort & TOPSITE_SORT_BYTES) != 0) {
 201                 sortf="-k 2,2 -k 1,1";
 202         } else if((TopsitesSort & TOPSITE_SORT_TIME) != 0) {
 203                 sortf="-k 3,3";
 204         } else if((TopsitesSort & TOPSITE_SORT_USER) != 0) {
 205                 sortf="-k 4,4 -k 1,1 -k 2,2";
 206         } else {
 207                 sortf="-k 2,2 -k 1,1"; //default is BYTES
 208         }
 209         if((TopsitesSort & TOPSITE_SORT_REVERSE) != 0) {
 210                 sortt="-r";
 211         } else {
 212                 sortt="";
 213         }
 214
 215         if (snprintf(csort,sizeof(csort),"sort -t \"\t\" %s -n %s -o \"%s\" \"%s\"",sortt,sortf,sites,general3)>=sizeof(csort)) {
 216                 debuga(__FILE__,__LINE__,_("Sort command too long when sorting file \"%s\" to \"%s\"\n"),general3,sites);
 217                 exit(EXIT_FAILURE);
 218         }
 219         cstatus=system(csort);
 220         if (!WIFEXITED(cstatus) || WEXITSTATUS(cstatus)) {
 221                 debuga(__FILE__,__LINE__,_("sort command return status %d\n"),WEXITSTATUS(cstatus));
 222                 debuga(__FILE__,__LINE__,_("sort command: %s\n"),csort);
 223                 exit(EXIT_FAILURE);
 224         }
 225         if((fp_in=FileObject_Open(sites))==NULL) {
 226                 debuga(__FILE__,__LINE__,_("Cannot open file \"%s\": %s\n"),sites,FileObject_GetLastOpenError());
 227                 debuga(__FILE__,__LINE__,_("sort command: %s\n"),csort);
 228                 exit(EXIT_FAILURE);
 229         }
 230
 231         if (!KeepTempLog && unlink(general3)) {
 232                 debuga(__FILE__,__LINE__,_("Cannot delete \"%s\": %s\n"),general3,strerror(errno));
 233                 exit(EXIT_FAILURE);
 234         }
 235
 236         if((fp_ou=fopen(report,"w"))==NULL) {
 237                 debuga(__FILE__,__LINE__,_("Cannot open file \"%s\": %s\n"),report,strerror(errno));
 238                 exit(EXIT_FAILURE);
 239         }
 240
 241         write_html_header(fp_ou,(IndexTree == INDEX_TREE_DATE) ? 3 : 1,_("Top sites"),HTML_JS_SORTTABLE);
 242         fputs("<tr><td class=\"header_c\">",fp_ou);
 243         fprintf(fp_ou,_("Period: %s"),period.html);
 244         fputs("</td></tr>\n",fp_ou);
 245         fputs("<tr><th class=\"header_c\">",fp_ou);
 246         fprintf(fp_ou,_("Top %d sites"),TopSitesNum);
 247         fputs("</th></tr>\n",fp_ou);
 248         close_html_header(fp_ou);
 249
 250         fputs("<div class=\"report\"><table cellpadding=\"1\" cellspacing=\"2\"",fp_ou);
 251         if (SortTableJs[0]) fputs(" class=\"sortable\"",fp_ou);
 252         fputs(">\n",fp_ou);
 253         fprintf(fp_ou,"<thead><tr><th class=\"header_l\">%s</th><th class=\"header_l",
 254         /* TRANSLATORS: This is a column header showing the position of the entry in the sorted list. */
 255         _("NUM"));
 256         if (SortTableJs[0]) fputs(" sorttable_alpha",fp_ou);
 257         fprintf(fp_ou,"\">%s</th><th class=\"header_l\">%s</th><th class=\"header_l\">%s</th><th class=\"header_l\">%s</th><th class=\"header_l\">%s</th></tr></thead>\n",
 258         /* TRANSLATORS: This is a column header showing the URL of the visited sites. */
 259         _("ACCESSED SITE"),
 260         /* TRANSLATORS: This is a column header showing the number of connections to a visited site. */
 261         _("CONNECT"),
 262         /* TRANSLATORS: This is a column header showing the number of transfered bytes. */
 263         _("BYTES"),
 264         /* TRANSLATORS: This is a column header showing the time spent by the proxy processing the requests. */
 265         pgettext("duration","TIME"),
 266         /* TRANSLATORS: This is a column header showing the number of users who visited a sites. */
 267         _("USERS"));
 268
 269         regs=0;
 270         ntopsites = 0;
 271
 272         if ((line=longline_create())==NULL) {
 273                 debuga(__FILE__,__LINE__,_("Not enough memory to read file \"%s\"\n"),sites);
 274                 exit(EXIT_FAILURE);
 275         }
 276
 277         while(regs<TopSitesNum && (buf=longline_read(fp_in,line))!=NULL) {
 278                 getword_start(&gwarea,buf);
 279                 if (getword_atoll(&nacc,&gwarea,'\t')<0) {
 280                         debuga(__FILE__,__LINE__,_("Invalid record in file \"%s\"\n"),sites);
 281                         exit(EXIT_FAILURE);
 282                 }
 283                 if (nacc == 0) continue;
 284                 if (getword_atoll(&nbytes,&gwarea,'\t')<0 || getword_atoll(&ntime,&gwarea,'\t')<0) {
 285                         debuga(__FILE__,__LINE__,_("Invalid record in file \"%s\"\n"),sites);
 286                         exit(EXIT_FAILURE);
 287                 }
 288                 if (getword_atoi(&nusers,&gwarea,'\t')<0) {
 289                         debuga(__FILE__,__LINE__,_("Invalid number of users in file \"%s\"\n"),sites);
 290                         exit(EXIT_FAILURE);
 291                 }
 292                 if (getword_ptr(buf,&url,&gwarea,'\t')<0) {
 293                         debuga(__FILE__,__LINE__,_("Invalid url in file \"%s\"\n"),sites);
 294                         exit(EXIT_FAILURE);
 295                 }
 296
 297                 twork1=nacc;
 298                 twork2=nbytes;
 299                 twork3=ntime;
 300
 301                 fprintf(fp_ou,"<tr><td class=\"data\">%d</td><td class=\"data2\">",++regs);
 302
 303                 if(BlockIt[0] != '\0' && url[0]!=ALIAS_PREFIX) {
 304                         fprintf(fp_ou,"<a href=\"%s%s?url=\"",wwwDocumentRoot,BlockIt);
 305                         output_html_url(fp_ou,url);
 306                         fputs("\"><img src=\"../images/sarg-squidguard-block.png\"></a>&nbsp;",fp_ou);
 307                 }
 308
 309                 output_html_link(fp_ou,url,100);
 310                 fputs("</td><td class=\"data\"",fp_ou);
 311                 if (SortTableJs[0]) fprintf(fp_ou," sorttable_customkey=\"%"PRId64"\"",(uint64_t)twork1);
 312                 fprintf(fp_ou,">%s</td>",fixnum(twork1,1));
 313                 fputs("<td class=\"data\"",fp_ou);
 314                 if (SortTableJs[0]) fprintf(fp_ou," sorttable_customkey=\"%"PRId64"\"",(uint64_t)twork2);
 315                 fprintf(fp_ou,">%s</td>",fixnum(twork2,1));
 316                 fputs("<td class=\"data\"",fp_ou);
 317                 if (SortTableJs[0]) fprintf(fp_ou," sorttable_customkey=\"%"PRId64"\"",(uint64_t)twork3);
 318                 fprintf(fp_ou,">%s</td>",fixtime(twork3));
 319                 fputs("<td class=\"data\"",fp_ou);
 320                 if (SortTableJs[0]) fprintf(fp_ou," sorttable_customkey=\"%d\"",nusers);
 321                 fprintf(fp_ou,">%s</td></tr>\n",fixnum(nusers,1));
 322         }
 323         if (FileObject_Close(fp_in)) {
 324                 debuga(__FILE__,__LINE__,_("Read error in \"%s\": %s\n"),sites,FileObject_GetLastCloseError());
 325                 exit(EXIT_FAILURE);
 326         }
 327         longline_destroy(&line);
 328
 329         fputs("</table></div>\n",fp_ou);
 330         write_html_trailer(fp_ou);
 331         if (fclose(fp_ou)==EOF) {
 332                 debuga(__FILE__,__LINE__,_("Write error in \"%s\": %s\n"),report,strerror(errno));
 333                 exit(EXIT_FAILURE);
 334         }
 335
 336         return;
 337 }