download.c

   1 /*
   2  * SARG Squid Analysis Report Generator      http://sarg.sourceforge.net
   3  *                                                            1998, 2013
   4  *
   5  * SARG donations:
   6  *      please look at http://sarg.sourceforge.net/donations.php
   7  * Support:
   8  *     http://sourceforge.net/projects/sarg/forums/forum/363374
   9  * ---------------------------------------------------------------------
  10  *
  11  *  This program is free software; you can redistribute it and/or modify
  12  *  it under the terms of the GNU General Public License as published by
  13  *  the Free Software Foundation; either version 2 of the License, or
  14  *  (at your option) any later version.
  15  *
  16  *  This program is distributed in the hope that it will be useful,
  17  *  but WITHOUT ANY WARRANTY; without even the implied warranty of
  18  *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  19  *  GNU General Public License for more details.
  20  *
  21  *  You should have received a copy of the GNU General Public License
  22  *  along with this program; if not, write to the Free Software
  23  *  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111, USA.
  24  *
  25  */
  26
  27 #include "include/conf.h"
  28 #include "include/defs.h"
  29
  30 /*!
  31 The buffer to store the list of the suffixes to take into account when generating
  32 the report of the downloaded files. The suffixes in the list are separated by the ASCII
  33 null.
  34 */
  35 /*@null@*/static char *DownloadSuffix=NULL;
  36
  37 /*!
  38 The index of all the suffixes stored in ::DownloadSuffix. The list is sorted alphabetically.
  39 to speed up the search.
  40 */
  41 /*@null@*/static char **DownloadSuffixIndex=NULL;
  42
  43 /*!
  44 The number of suffixes in ::DownloadSuffixIndex.
  45 */
  46 static int NDownloadSuffix=0;
  47
  48 //! Name of the file containing the unsorted downloaded entries.
  49 static char download_unsort[MAXLEN]="";
  50 //! The file handle to write the entries.
  51 static FILE *fp_download=NULL;
  52 //! \c True if at least one downloaded entry exists.
  53 static bool download_exists=false;
  54
  55 /*!
  56 Open a file to store the denied accesses.
  57
  58 \return The file handle or NULL if no file is necessary.
  59 */
  60 void download_open(void)
  61 {
  62         if ((ReportType & REPORT_TYPE_DOWNLOADS) == 0) {
  63                 if (debugz) debugaz(_("Download report not produced as it is not requested\n"));
  64                 return;
  65         }
  66         if (Privacy) {
  67                 if (debugz) debugaz(_("Download report not produced because privacy option is active\n"));
  68                 return;
  69         }
  70
  71         snprintf(download_unsort,sizeof(download_unsort),"%s/download.int_unsort",tmp);
  72         if ((fp_download=MY_FOPEN(download_unsort,"w"))==NULL) {
  73                 debuga(_("(log) Cannot open file %s: %s\n"),download_unsort,strerror(errno));
  74                 exit(EXIT_FAILURE);
  75         }
  76         return;
  77 }
  78
  79 /*!
  80 Write one entry in the unsorted downloaded file provided that it is required.
  81
  82 \param log_entry The entry to write into the log file.
  83 \param url The URL of the downloaded file.
  84 */
  85 void download_write(const struct ReadLogStruct *log_entry,const char *url)
  86 {
  87         char date[80];
  88
  89         if (fp_download && strstr(log_entry->HttpCode,"DENIED") != 0) {
  90                 strftime(date,sizeof(date),"%d/%m/%Y\t%H:%M:%S",&log_entry->EntryTime);
  91                 fprintf(fp_download,"%s\t%s\t%s\t%s\n",date,log_entry->User,log_entry->Ip,url);
  92                 download_exists=true;
  93         }
  94 }
  95
  96 /*!
  97 Close the file opened by denied_open().
  98 */
  99 void download_close(void)
 100 {
 101         if (fp_download)
 102         {
 103                 if (fclose(fp_download)==EOF) {
 104                         debuga(_("Write error in %s: %s\n"),download_unsort,strerror(errno));
 105                         exit(EXIT_FAILURE);
 106                 }
 107                 fp_download=NULL;
 108         }
 109 }
 110
 111 /*!
 112 Tell the caller if a download report exists.
 113
 114 \return \c True if the report is available or \c false if no report
 115 was generated.
 116 */
 117 bool is_download(void)
 118 {
 119         return(download_exists);
 120 }
 121
 122 /*!
 123 Sort the raw log file with the downloaded files.
 124
 125 \param report_in The name of the file where to store the sorted entries.
 126
 127 The file is sorted by columns 3, 1, 2 and 5 that are the columns of the user's ID, the
 128 date, the time and the URL.
 129 */
 130 static void download_sort(const char *report_in)
 131 {
 132         int clen;
 133         char csort[MAXLEN];
 134         int cstatus;
 135
 136         clen=snprintf(csort,sizeof(csort),"sort -T \"%s\" -t \"\t\" -k 3,3 -k 1,1 -k 2,2 -k 5,5 -o \"%s\" \"%s\"",
 137                         tmp, report_in, download_unsort);
 138         if (clen>=sizeof(csort)) {
 139                 debuga(_("Path too long to sort the file: %s\n"),download_unsort);
 140                 exit(EXIT_FAILURE);
 141         }
 142         cstatus=system(csort);
 143         if (!WIFEXITED(cstatus) || WEXITSTATUS(cstatus)) {
 144                 debuga(_("sort command return status %d\n"),WEXITSTATUS(cstatus));
 145                 debuga(_("sort command: %s\n"),csort);
 146                 exit(EXIT_FAILURE);
 147         }
 148         if (!KeepTempLog) {
 149                 if (unlink(download_unsort)) {
 150                         debuga(_("Cannot delete \"%s\": %s\n"),download_unsort,strerror(errno));
 151                         exit(EXIT_FAILURE);
 152                 }
 153                 download_unsort[0]='\0';
 154         }
 155 }
 156
 157 /*!
 158 Generate the report of the downloaded files. The list of the suffixes to take into account
 159 is set with set_download_suffix().
 160 */
 161 void download_report(void)
 162 {
 163         FILE *fp_in = NULL, *fp_ou = NULL;
 164
 165         char *buf;
 166         char *url;
 167         char report_in[MAXLEN];
 168         char report[MAXLEN];
 169         char ip[MAXLEN];
 170         char oip[MAXLEN];
 171         char user[MAXLEN];
 172         char ouser[MAXLEN];
 173         char ouser2[MAXLEN];
 174         char data[15];
 175         char hora[15];
 176         int  z=0;
 177         int  count=0;
 178         int i;
 179         int day,month,year;
 180         bool new_user;
 181         struct getwordstruct gwarea;
 182         longline line;
 183         struct userinfostruct *uinfo;
 184         struct tm t;
 185
 186         if (!download_exists) {
 187                 if (!KeepTempLog && download_unsort[0]!='\0' && unlink(download_unsort))
 188                         debuga(_("Cannot delete \"%s\": %s\n"),download_unsort,strerror(errno));
 189                 download_unsort[0]='\0';
 190                 if (debugz) debugaz(_("No downloaded files to report\n"));
 191                 return;
 192         }
 193
 194         ouser[0]='\0';
 195         ouser2[0]='\0';
 196
 197         // sort the raw file
 198         snprintf(report_in,sizeof(report_in),"%s/download.int_log",tmp);
 199         download_sort(report_in);
 200
 201         // produce the report.
 202         snprintf(report,sizeof(report),"%s/download.html",outdirname);
 203
 204         if((fp_in=MY_FOPEN(report_in,"r"))==NULL) {
 205                 debuga(_("(download) Cannot open log file %s: %s\n"),report_in,strerror(errno));
 206                 exit(EXIT_FAILURE);
 207         }
 208
 209         if((fp_ou=MY_FOPEN(report,"w"))==NULL) {
 210                 debuga(_("(download) Cannot open log file %s: %s\n"),report,strerror(errno));
 211                 exit(EXIT_FAILURE);
 212         }
 213
 214         write_html_header(fp_ou,(IndexTree == INDEX_TREE_DATE) ? 3 : 1,_("Downloads"),HTML_JS_NONE);
 215         fputs("<tr><td class=\"header_c\">",fp_ou);
 216         fprintf(fp_ou,_("Period: %s"),period.html);
 217         fputs("</td></tr>\n",fp_ou);
 218         fprintf(fp_ou,"<tr><th class=\"header_c\">%s</th></tr>\n",_("Downloads"));
 219         close_html_header(fp_ou);
 220
 221         fputs("<div class=\"report\"><table cellpadding=\"0\" cellspacing=\"2\">\n",fp_ou);
 222         fprintf(fp_ou,"<tr><th class=\"header_l\">%s</th><th class=\"header_l\">%s</th><th class=\"header_l\">%s</th><th class=\"header_l\">%s</th></tr>\n",_("USERID"),_("IP/NAME"),_("DATE/TIME"),_("ACCESSED SITE"));
 223
 224         if ((line=longline_create())==NULL) {
 225                 debuga(_("Not enough memory to read the downloaded files\n"));
 226                 exit(EXIT_FAILURE);
 227         }
 228
 229         while((buf=longline_read(fp_in,line))!=NULL) {
 230                 getword_start(&gwarea,buf);
 231                 if (getword(data,sizeof(data),&gwarea,'\t')<0 || getword(hora,sizeof(hora),&gwarea,'\t')<0 ||
 232                     getword(user,sizeof(user),&gwarea,'\t')<0 || getword(ip,sizeof(ip),&gwarea,'\t')<0) {
 233                         debuga(_("There is a broken record or garbage in file %s\n"),report_in);
 234                         exit(EXIT_FAILURE);
 235                 }
 236                 if (getword_ptr(buf,&url,&gwarea,'\t')<0) {
 237                         debuga(_("There is a broken url in file %s\n"),report_in);
 238                         exit(EXIT_FAILURE);
 239                 }
 240                 if (sscanf(data,"%d/%d/%d",&day,&month,&year)!=3) continue;
 241                 computedate(year,month,day,&t);
 242                 strftime(data,sizeof(data),"%x",&t);
 243
 244                 uinfo=userinfo_find_from_id(user);
 245                 if (!uinfo) {
 246                         debuga(_("Unknown user ID %s in file %s\n"),user,report_in);
 247                         exit(EXIT_FAILURE);
 248                 }
 249                 new_user=false;
 250                 if(!z) {
 251                         strcpy(ouser,user);
 252                         strcpy(oip,ip);
 253                         z++;
 254                         new_user=true;
 255                 } else {
 256                         if(strcmp(ouser,user) != 0) {
 257                                 strcpy(ouser,user);
 258                                 new_user=true;
 259                         }
 260                         if(strcmp(oip,ip) != 0) {
 261                                 strcpy(oip,ip);
 262                                 new_user=true;
 263                         }
 264                 }
 265
 266                 if(DownloadReportLimit) {
 267                         if(strcmp(ouser2,uinfo->label) == 0) {
 268                                 count++;
 269                         } else {
 270                                 count=1;
 271                                 strcpy(ouser2,uinfo->label);
 272                         }
 273                         if(count >= DownloadReportLimit)
 274                                 continue;
 275                 }
 276
 277                 for (i=strlen(url)-1 ; i>=0 && (unsigned char)url[i]<' ' ; i--) url[i]=0;
 278
 279                 fputs("<tr>",fp_ou);
 280                 if (new_user) {
 281                         if (uinfo->topuser)
 282                                 fprintf(fp_ou,"<td class=\"data\"><a href=\"%s/%s.html\">%s</a></td><td class=\"data\">%s</td>",uinfo->filename,uinfo->filename,uinfo->label,ip);
 283                         else
 284                                 fprintf(fp_ou,"<td class=\"data\">%s</td><td class=\"data\">%s</td>",uinfo->label,ip);
 285                 } else
 286                         fputs("<td class=\"data\"></td><td class=\"data\"></td>",fp_ou);
 287                 fprintf(fp_ou,"<td class=\"data\">%s-%s</td><td class=\"data2\">",data,hora);
 288                 if(BlockIt[0]!='\0' && url[0]!=ALIAS_PREFIX) {
 289                         fprintf(fp_ou,"<a href=\"%s%s?url=\"",wwwDocumentRoot,BlockIt);
 290                         output_html_url(fp_ou,url);
 291                         fprintf(fp_ou,"\"><img src=\"%s/sarg-squidguard-block.png\"></a>&nbsp;",ImageFile);
 292                 }
 293                 output_html_link(fp_ou,url,100);
 294                 fputs("</td></tr>\n",fp_ou);
 295         }
 296         fclose(fp_in);
 297         longline_destroy(&line);
 298
 299         fputs("</table></div>\n",fp_ou);
 300         if (write_html_trailer(fp_ou)<0)
 301                 debuga(_("Write error in file %s\n"),report);
 302         if (fclose(fp_ou)==EOF) {
 303                 debuga(_("Write error in %s: %s\n"),report,strerror(errno));
 304                 exit(EXIT_FAILURE);
 305         }
 306
 307         if (!KeepTempLog && unlink(report_in)) {
 308                 debuga(_("Cannot delete \"%s\": %s\n"),report_in,strerror(errno));
 309                 exit(EXIT_FAILURE);
 310         }
 311
 312         return;
 313 }
 314
 315 /*!
 316 Free the memory allocated by set_download_suffix().
 317 */
 318 void free_download(void)
 319 {
 320         if (DownloadSuffix) {
 321                 free(DownloadSuffix);
 322                 DownloadSuffix=NULL;
 323         }
 324         if (DownloadSuffixIndex) {
 325                 free(DownloadSuffixIndex);
 326                 DownloadSuffixIndex=NULL;
 327         }
 328         NDownloadSuffix=0;
 329 }
 330
 331 /*!
 332 Set the list of the suffixes corresponding to the download of files you want to detect with
 333 is_download_suffix(). The list is sorted to make the search faster.
 334
 335 \param list A comma separated list of the suffixes to set in ::DownloadSuffix.
 336
 337 \note The memory allocated by this function must be freed by free_download().
 338 */
 339 void set_download_suffix(const char *list)
 340 {
 341         char *str;
 342         int i, j, k;
 343         int cmp;
 344
 345         free_download();
 346
 347         DownloadSuffix=strdup(list);
 348         if (!DownloadSuffix) {
 349                 debuga(_("Download suffix list too long\n"));
 350                 exit(EXIT_FAILURE);
 351         }
 352         j = 1;
 353         for (i=0 ; list[i] ; i++)
 354                 if (list[i] == ',') j++;
 355         DownloadSuffixIndex=malloc(j*sizeof(char *));
 356         if (!DownloadSuffixIndex) {
 357                 debuga(_("Too many download suffixes\n"));
 358                 exit(EXIT_FAILURE);
 359         }
 360
 361         str = DownloadSuffix;
 362         for (i=0 ; DownloadSuffix[i] ; i++) {
 363                 if (DownloadSuffix[i] == ',') {
 364                         DownloadSuffix[i] = '\0';
 365                         if (*str) {
 366                                 cmp = -1;
 367                                 for (j=0 ; j<NDownloadSuffix && (cmp=strcasecmp(str,DownloadSuffixIndex[j]))>0 ; j++);
 368                                 if (cmp != 0) {
 369                                         for (k=NDownloadSuffix ; k>j ; k--)
 370                                                 DownloadSuffixIndex[k]=DownloadSuffixIndex[k-1];
 371                                         NDownloadSuffix++;
 372                                         DownloadSuffixIndex[j]=str;
 373                                 }
 374                         }
 375                         str=DownloadSuffix+i+1;
 376                 }
 377         }
 378
 379         if (*str) {
 380                 cmp = -1;
 381                 for (j=0 ; j<NDownloadSuffix && (cmp=strcasecmp(str,DownloadSuffixIndex[j]))>0 ; j++);
 382                 if (cmp != 0) {
 383                         for (k=NDownloadSuffix ; k>j ; k--)
 384                                 DownloadSuffixIndex[k]=DownloadSuffixIndex[k-1];
 385                         NDownloadSuffix++;
 386                         DownloadSuffixIndex[j]=str;
 387                 }
 388         }
 389 }
 390
 391 /*!
 392 Tell if the URL correspond to a downloaded file. The function takes the extension at the end of the
 393 URL with a maximum of 9 characters and compare it to the list of the download suffix in
 394 ::DownloadSuffix. If the suffix is found in the list, the function reports the URL as the download
 395 of a file.
 396
 397 \param url The URL to test.
 398
 399 \retval 1 The URL matches a suffix of a download.
 400 \retval 0 The URL is not a known download.
 401
 402 \note A downloaded file cannot be detected if the file name is embedded in a GET or POST request. Only requests
 403 that ends with the file name can be detected.
 404
 405 \note A URL embedding another web site's address ending by .com at the end of the URL will match the download
 406 extension com if it is defined in the ::DownloadSuffix.
 407 */
 408 bool is_download_suffix(const char *url)
 409 {
 410         int urllen;
 411         int i;
 412         int down, up, center;
 413         const char *suffix;
 414         int cmp;
 415         const int max_suffix=10;
 416
 417         if (DownloadSuffix == NULL || NDownloadSuffix == 0) return(false);
 418
 419         urllen=strlen(url)-1;
 420         if (urllen<=0) return(false);
 421         if (url[urllen] == '.') return(false); //reject a single trailing dot
 422         for (i=0 ; i<urllen && (url[i]!='/' || url[i+1]=='/') && url[i]!='?' ; i++);
 423         if (i>=urllen) return(false); // url is a hostname without any path or file to download
 424
 425         for (i=0 ; i<=max_suffix && i<urllen && url[urllen-i]!='.' ; i++)
 426                 if (url[urllen-i] == '/' || url[urllen-i] == '?') return(false);
 427         if (i>max_suffix || i>=urllen) return(false);
 428
 429         suffix=url+urllen-i+1;
 430         down=0;
 431         up=NDownloadSuffix-1;
 432         while (down<=up) {
 433                 center=(down+up)/2;
 434                 cmp=strcasecmp(suffix,DownloadSuffixIndex[center]);
 435                 if (cmp == 0) return(true);
 436                 if (cmp < 0)
 437                         up = center-1;
 438                 else
 439                         down = center+1;
 440         }
 441         return(false);
 442 }
 443
 444 /*!
 445 Remove any temporary file left by the download module.
 446 */
 447 void download_cleanup(void)
 448 {
 449         if (fp_download) {
 450                 if (fclose(fp_download)==EOF) {
 451                         debuga(_("Write error in %s: %s\n"),download_unsort,strerror(errno));
 452                         exit(EXIT_FAILURE);
 453                 }
 454                 fp_download=NULL;
 455         }
 456         if (download_unsort[0]) {
 457                 if (unlink(download_unsort)==-1)
 458                         debuga(_("Failed to delete %s: %s\n"),download_unsort,strerror(errno));
 459         }
 460 }