download.c

   1 /*
   2  * SARG Squid Analysis Report Generator      http://sarg.sourceforge.net
   3  *                                                            1998, 2015
   4  *
   5  * SARG donations:
   6  *      please look at http://sarg.sourceforge.net/donations.php
   7  * Support:
   8  *     http://sourceforge.net/projects/sarg/forums/forum/363374
   9  * ---------------------------------------------------------------------
  10  *
  11  *  This program is free software; you can redistribute it and/or modify
  12  *  it under the terms of the GNU General Public License as published by
  13  *  the Free Software Foundation; either version 2 of the License, or
  14  *  (at your option) any later version.
  15  *
  16  *  This program is distributed in the hope that it will be useful,
  17  *  but WITHOUT ANY WARRANTY; without even the implied warranty of
  18  *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  19  *  GNU General Public License for more details.
  20  *
  21  *  You should have received a copy of the GNU General Public License
  22  *  along with this program; if not, write to the Free Software
  23  *  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111, USA.
  24  *
  25  */
  26
  27 #include "include/conf.h"
  28 #include "include/defs.h"
  29 #include "include/readlog.h"
  30
  31 /*!
  32 The buffer to store the list of the suffixes to take into account when generating
  33 the report of the downloaded files. The suffixes in the list are separated by the ASCII
  34 null.
  35 */
  36 /*@null@*/static char *DownloadSuffix=NULL;
  37
  38 /*!
  39 The index of all the suffixes stored in ::DownloadSuffix. The list is sorted alphabetically.
  40 to speed up the search.
  41 */
  42 /*@null@*/static char **DownloadSuffixIndex=NULL;
  43
  44 /*!
  45 The number of suffixes in ::DownloadSuffixIndex.
  46 */
  47 static int NDownloadSuffix=0;
  48
  49 //! Name of the file containing the unsorted downloaded entries.
  50 static char download_unsort[MAXLEN]="";
  51 //! The file handle to write the entries.
  52 static FILE *fp_download=NULL;
  53 //! \c True if at least one downloaded entry exists.
  54 static bool download_exists=false;
  55
  56 /*!
  57 Open a file to store the denied accesses.
  58
  59 \return The file handle or NULL if no file is necessary.
  60 */
  61 void download_open(void)
  62 {
  63         if ((ReportType & REPORT_TYPE_DOWNLOADS) == 0) {
  64                 if (debugz>=LogLevel_Process) debugaz(__FILE__,__LINE__,_("Download report not produced as it is not requested\n"));
  65                 return;
  66         }
  67         if (Privacy) {
  68                 if (debugz>=LogLevel_Process) debugaz(__FILE__,__LINE__,_("Download report not produced because privacy option is active\n"));
  69                 return;
  70         }
  71
  72         snprintf(download_unsort,sizeof(download_unsort),"%s/download.int_unsort",tmp);
  73         if ((fp_download=MY_FOPEN(download_unsort,"w"))==NULL) {
  74                 debuga(__FILE__,__LINE__,_("Cannot open file \"%s\": %s\n"),download_unsort,strerror(errno));
  75                 exit(EXIT_FAILURE);
  76         }
  77         return;
  78 }
  79
  80 /*!
  81 Write one entry in the unsorted downloaded file provided that it is required.
  82
  83 \param log_entry The entry to write into the log file.
  84 \param url The URL of the downloaded file.
  85 */
  86 void download_write(const struct ReadLogStruct *log_entry,const char *url)
  87 {
  88         char date[80];
  89
  90         if (fp_download && strstr(log_entry->HttpCode,"DENIED") == 0) {
  91                 strftime(date,sizeof(date),"%d/%m/%Y\t%H:%M:%S",&log_entry->EntryTime);
  92                 fprintf(fp_download,"%s\t%s\t%s\t%s\n",date,log_entry->User,log_entry->Ip,url);
  93                 download_exists=true;
  94         }
  95 }
  96
  97 /*!
  98 Close the file opened by denied_open().
  99 */
 100 void download_close(void)
 101 {
 102         if (fp_download)
 103         {
 104                 if (fclose(fp_download)==EOF) {
 105                         debuga(__FILE__,__LINE__,_("Write error in \"%s\": %s\n"),download_unsort,strerror(errno));
 106                         exit(EXIT_FAILURE);
 107                 }
 108                 fp_download=NULL;
 109         }
 110 }
 111
 112 /*!
 113 Tell the caller if a download report exists.
 114
 115 \return \c True if the report is available or \c false if no report
 116 was generated.
 117 */
 118 bool is_download(void)
 119 {
 120         return(download_exists);
 121 }
 122
 123 /*!
 124 Sort the raw log file with the downloaded files.
 125
 126 \param report_in The name of the file where to store the sorted entries.
 127
 128 The file is sorted by columns 3, 1, 2 and 5 that are the columns of the user's ID, the
 129 date, the time and the URL.
 130 */
 131 static void download_sort(const char *report_in)
 132 {
 133         int clen;
 134         char csort[MAXLEN];
 135         int cstatus;
 136
 137         clen=snprintf(csort,sizeof(csort),"sort -T \"%s\" -t \"\t\" -k 3,3 -k 1,1 -k 2,2 -k 5,5 -o \"%s\" \"%s\"",
 138                         tmp, report_in, download_unsort);
 139         if (clen>=sizeof(csort)) {
 140                 debuga(__FILE__,__LINE__,_("Path too long to sort file \"%s\"\n"),download_unsort);
 141                 exit(EXIT_FAILURE);
 142         }
 143         cstatus=system(csort);
 144         if (!WIFEXITED(cstatus) || WEXITSTATUS(cstatus)) {
 145                 debuga(__FILE__,__LINE__,_("sort command return status %d\n"),WEXITSTATUS(cstatus));
 146                 debuga(__FILE__,__LINE__,_("sort command: %s\n"),csort);
 147                 exit(EXIT_FAILURE);
 148         }
 149         if (!KeepTempLog) {
 150                 if (unlink(download_unsort)) {
 151                         debuga(__FILE__,__LINE__,_("Cannot delete \"%s\": %s\n"),download_unsort,strerror(errno));
 152                         exit(EXIT_FAILURE);
 153                 }
 154                 download_unsort[0]='\0';
 155         }
 156 }
 157
 158 /*!
 159 Generate the report of the downloaded files. The list of the suffixes to take into account
 160 is set with set_download_suffix().
 161 */
 162 void download_report(void)
 163 {
 164         FileObject *fp_in = NULL;
 165         FILE *fp_ou = NULL;
 166
 167         char *buf;
 168         char *url;
 169         char report_in[MAXLEN];
 170         char report[MAXLEN];
 171         char ip[MAXLEN];
 172         char oip[MAXLEN];
 173         char user[MAXLEN];
 174         char ouser[MAXLEN];
 175         char ouser2[MAXLEN];
 176         char data[15];
 177         char hora[15];
 178         int  z=0;
 179         int  count=0;
 180         int i;
 181         int day,month,year;
 182         bool new_user;
 183         struct getwordstruct gwarea;
 184         longline line;
 185         struct userinfostruct *uinfo;
 186         struct tm t;
 187
 188         if (!download_exists) {
 189                 if (!KeepTempLog && download_unsort[0]!='\0' && unlink(download_unsort))
 190                         debuga(__FILE__,__LINE__,_("Cannot delete \"%s\": %s\n"),download_unsort,strerror(errno));
 191                 download_unsort[0]='\0';
 192                 if (debugz>=LogLevel_Process) debugaz(__FILE__,__LINE__,_("No downloaded files to report\n"));
 193                 return;
 194         }
 195
 196         if (debugz>=LogLevel_Process)
 197                 debuga(__FILE__,__LINE__,_("Creating download report...\n"));
 198         ouser[0]='\0';
 199         ouser2[0]='\0';
 200
 201         // sort the raw file
 202         snprintf(report_in,sizeof(report_in),"%s/download.int_log",tmp);
 203         download_sort(report_in);
 204
 205         // produce the report.
 206         snprintf(report,sizeof(report),"%s/download.html",outdirname);
 207
 208         if((fp_in=FileObject_Open(report_in))==NULL) {
 209                 debuga(__FILE__,__LINE__,_("Cannot open file \"%s\": %s\n"),report_in,FileObject_GetLastOpenError());
 210                 exit(EXIT_FAILURE);
 211         }
 212
 213         if((fp_ou=MY_FOPEN(report,"w"))==NULL) {
 214                 debuga(__FILE__,__LINE__,_("Cannot open file \"%s\": %s\n"),report,strerror(errno));
 215                 exit(EXIT_FAILURE);
 216         }
 217
 218         write_html_header(fp_ou,(IndexTree == INDEX_TREE_DATE) ? 3 : 1,_("Downloads"),HTML_JS_NONE);
 219         fputs("<tr><td class=\"header_c\">",fp_ou);
 220         fprintf(fp_ou,_("Period: %s"),period.html);
 221         fputs("</td></tr>\n",fp_ou);
 222         fprintf(fp_ou,"<tr><th class=\"header_c\">%s</th></tr>\n",_("Downloads"));
 223         close_html_header(fp_ou);
 224
 225         fputs("<div class=\"report\"><table cellpadding=\"0\" cellspacing=\"2\">\n",fp_ou);
 226         fprintf(fp_ou,"<tr><th class=\"header_l\">%s</th><th class=\"header_l\">%s</th><th class=\"header_l\">%s</th><th class=\"header_l\">%s</th></tr>\n",_("USERID"),_("IP/NAME"),_("DATE/TIME"),_("ACCESSED SITE"));
 227
 228         if ((line=longline_create())==NULL) {
 229                 debuga(__FILE__,__LINE__,_("Not enough memory to read file \"%s\"\n"),report_in);
 230                 exit(EXIT_FAILURE);
 231         }
 232
 233         while((buf=longline_read(fp_in,line))!=NULL) {
 234                 getword_start(&gwarea,buf);
 235                 if (getword(data,sizeof(data),&gwarea,'\t')<0 || getword(hora,sizeof(hora),&gwarea,'\t')<0 ||
 236                     getword(user,sizeof(user),&gwarea,'\t')<0 || getword(ip,sizeof(ip),&gwarea,'\t')<0) {
 237                         debuga(__FILE__,__LINE__,_("Invalid record in file \"%s\"\n"),report_in);
 238                         exit(EXIT_FAILURE);
 239                 }
 240                 if (getword_ptr(buf,&url,&gwarea,'\t')<0) {
 241                         debuga(__FILE__,__LINE__,_("Invalid url in file \"%s\"\n"),report_in);
 242                         exit(EXIT_FAILURE);
 243                 }
 244                 if (sscanf(data,"%d/%d/%d",&day,&month,&year)!=3) continue;
 245                 computedate(year,month,day,&t);
 246                 strftime(data,sizeof(data),"%x",&t);
 247
 248                 uinfo=userinfo_find_from_id(user);
 249                 if (!uinfo) {
 250                         debuga(__FILE__,__LINE__,_("Unknown user ID %s in file \"%s\"\n"),user,report_in);
 251                         exit(EXIT_FAILURE);
 252                 }
 253                 new_user=false;
 254                 if(!z) {
 255                         strcpy(ouser,user);
 256                         strcpy(oip,ip);
 257                         z++;
 258                         new_user=true;
 259                 } else {
 260                         if(strcmp(ouser,user) != 0) {
 261                                 strcpy(ouser,user);
 262                                 new_user=true;
 263                         }
 264                         if(strcmp(oip,ip) != 0) {
 265                                 strcpy(oip,ip);
 266                                 new_user=true;
 267                         }
 268                 }
 269
 270                 if(DownloadReportLimit) {
 271                         if(strcmp(ouser2,uinfo->label) == 0) {
 272                                 count++;
 273                         } else {
 274                                 count=1;
 275                                 strcpy(ouser2,uinfo->label);
 276                         }
 277                         if(count >= DownloadReportLimit)
 278                                 continue;
 279                 }
 280
 281                 for (i=strlen(url)-1 ; i>=0 && (unsigned char)url[i]<' ' ; i--) url[i]=0;
 282
 283                 fputs("<tr>",fp_ou);
 284                 if (new_user) {
 285                         if (uinfo->topuser)
 286                                 fprintf(fp_ou,"<td class=\"data\"><a href=\"%s/%s.html\">%s</a></td><td class=\"data\">%s</td>",uinfo->filename,uinfo->filename,uinfo->label,ip);
 287                         else
 288                                 fprintf(fp_ou,"<td class=\"data\">%s</td><td class=\"data\">%s</td>",uinfo->label,ip);
 289                 } else
 290                         fputs("<td class=\"data\"></td><td class=\"data\"></td>",fp_ou);
 291                 fprintf(fp_ou,"<td class=\"data\">%s-%s</td><td class=\"data2\">",data,hora);
 292                 if(BlockIt[0]!='\0' && url[0]!=ALIAS_PREFIX) {
 293                         fprintf(fp_ou,"<a href=\"%s%s?url=\"",wwwDocumentRoot,BlockIt);
 294                         output_html_url(fp_ou,url);
 295                         fprintf(fp_ou,"\"><img src=\"%s/sarg-squidguard-block.png\"></a>&nbsp;",ImageFile);
 296                 }
 297                 output_html_link(fp_ou,url,100);
 298                 fputs("</td></tr>\n",fp_ou);
 299         }
 300         if (FileObject_Close(fp_in)) {
 301                 debuga(__FILE__,__LINE__,_("Read error in \"%s\": %s\n"),report_in,FileObject_GetLastCloseError());
 302                 exit(EXIT_FAILURE);
 303         }
 304         longline_destroy(&line);
 305
 306         fputs("</table></div>\n",fp_ou);
 307         write_html_trailer(fp_ou);
 308         if (fclose(fp_ou)==EOF) {
 309                 debuga(__FILE__,__LINE__,_("Write error in \"%s\": %s\n"),report,strerror(errno));
 310                 exit(EXIT_FAILURE);
 311         }
 312
 313         if (!KeepTempLog && unlink(report_in)) {
 314                 debuga(__FILE__,__LINE__,_("Cannot delete \"%s\": %s\n"),report_in,strerror(errno));
 315                 exit(EXIT_FAILURE);
 316         }
 317
 318         return;
 319 }
 320
 321 /*!
 322 Free the memory allocated by set_download_suffix().
 323 */
 324 void free_download(void)
 325 {
 326         if (DownloadSuffix) {
 327                 free(DownloadSuffix);
 328                 DownloadSuffix=NULL;
 329         }
 330         if (DownloadSuffixIndex) {
 331                 free(DownloadSuffixIndex);
 332                 DownloadSuffixIndex=NULL;
 333         }
 334         NDownloadSuffix=0;
 335 }
 336
 337 /*!
 338 Set the list of the suffixes corresponding to the download of files you want to detect with
 339 is_download_suffix(). The list is sorted to make the search faster.
 340
 341 \param list A comma separated list of the suffixes to set in ::DownloadSuffix.
 342
 343 \note The memory allocated by this function must be freed by free_download().
 344 */
 345 void set_download_suffix(const char *list)
 346 {
 347         char *str;
 348         int i, j, k;
 349         int cmp;
 350
 351         free_download();
 352
 353         DownloadSuffix=strdup(list);
 354         if (!DownloadSuffix) {
 355                 debuga(__FILE__,__LINE__,_("Download suffix list too long\n"));
 356                 exit(EXIT_FAILURE);
 357         }
 358         j = 1;
 359         for (i=0 ; list[i] ; i++)
 360                 if (list[i] == ',') j++;
 361         DownloadSuffixIndex=malloc(j*sizeof(char *));
 362         if (!DownloadSuffixIndex) {
 363                 debuga(__FILE__,__LINE__,_("Too many download suffixes\n"));
 364                 exit(EXIT_FAILURE);
 365         }
 366
 367         str = DownloadSuffix;
 368         for (i=0 ; DownloadSuffix[i] ; i++) {
 369                 if (DownloadSuffix[i] == ',') {
 370                         DownloadSuffix[i] = '\0';
 371                         if (*str) {
 372                                 cmp = -1;
 373                                 for (j=0 ; j<NDownloadSuffix && (cmp=strcasecmp(str,DownloadSuffixIndex[j]))>0 ; j++);
 374                                 if (cmp != 0) {
 375                                         for (k=NDownloadSuffix ; k>j ; k--)
 376                                                 DownloadSuffixIndex[k]=DownloadSuffixIndex[k-1];
 377                                         NDownloadSuffix++;
 378                                         DownloadSuffixIndex[j]=str;
 379                                 }
 380                         }
 381                         str=DownloadSuffix+i+1;
 382                 }
 383         }
 384
 385         if (*str) {
 386                 cmp = -1;
 387                 for (j=0 ; j<NDownloadSuffix && (cmp=strcasecmp(str,DownloadSuffixIndex[j]))>0 ; j++);
 388                 if (cmp != 0) {
 389                         for (k=NDownloadSuffix ; k>j ; k--)
 390                                 DownloadSuffixIndex[k]=DownloadSuffixIndex[k-1];
 391                         NDownloadSuffix++;
 392                         DownloadSuffixIndex[j]=str;
 393                 }
 394         }
 395 }
 396
 397 /*!
 398 Tell if the URL correspond to a downloaded file. The function takes the extension at the end of the
 399 URL with a maximum of 9 characters and compare it to the list of the download suffix in
 400 ::DownloadSuffix. If the suffix is found in the list, the function reports the URL as the download
 401 of a file.
 402
 403 \param url The URL to test.
 404
 405 \retval 1 The URL matches a suffix of a download.
 406 \retval 0 The URL is not a known download.
 407
 408 \note A downloaded file cannot be detected if the file name is embedded in a GET or POST request. Only requests
 409 that ends with the file name can be detected.
 410
 411 \note A URL embedding another web site's address ending by .com at the end of the URL will match the download
 412 extension com if it is defined in the ::DownloadSuffix.
 413 */
 414 bool is_download_suffix(const char *url)
 415 {
 416         int urllen;
 417         int i;
 418         int down, up, center;
 419         const char *suffix;
 420         int cmp;
 421         const int max_suffix=10;
 422
 423         if (DownloadSuffix == NULL || NDownloadSuffix == 0) return(false);
 424
 425         urllen=strlen(url)-1;
 426         if (urllen<=0) return(false);
 427         if (url[urllen] == '.') return(false); //reject a single trailing dot
 428         for (i=0 ; i<urllen && (url[i]!='/' || url[i+1]=='/') && url[i]!='?' ; i++);
 429         if (i>=urllen) return(false); // url is a hostname without any path or file to download
 430
 431         for (i=0 ; i<=max_suffix && i<urllen && url[urllen-i]!='.' ; i++)
 432                 if (url[urllen-i] == '/' || url[urllen-i] == '?') return(false);
 433         if (i>max_suffix || i>=urllen) return(false);
 434
 435         suffix=url+urllen-i+1;
 436         down=0;
 437         up=NDownloadSuffix-1;
 438         while (down<=up) {
 439                 center=(down+up)/2;
 440                 cmp=strcasecmp(suffix,DownloadSuffixIndex[center]);
 441                 if (cmp == 0) return(true);
 442                 if (cmp < 0)
 443                         up = center-1;
 444                 else
 445                         down = center+1;
 446         }
 447         return(false);
 448 }
 449
 450 /*!
 451 Remove any temporary file left by the download module.
 452 */
 453 void download_cleanup(void)
 454 {
 455         if (fp_download) {
 456                 if (fclose(fp_download)==EOF) {
 457                         debuga(__FILE__,__LINE__,_("Write error in \"%s\": %s\n"),download_unsort,strerror(errno));
 458                         exit(EXIT_FAILURE);
 459                 }
 460                 fp_download=NULL;
 461         }
 462         if (download_unsort[0]) {
 463                 if (unlink(download_unsort)==-1)
 464                         debuga(__FILE__,__LINE__,_("Failed to delete \"%s\": %s\n"),download_unsort,strerror(errno));
 465         }
 466 }