log.c

   1 /*
   2  * SARG Squid Analysis Report Generator      http://sarg.sourceforge.net
   3  *                                                            1998, 2011
   4  *
   5  * SARG donations:
   6  *      please look at http://sarg.sourceforge.net/donations.php
   7  * Support:
   8  *     http://sourceforge.net/projects/sarg/forums/forum/363374
   9  * ---------------------------------------------------------------------
  10  *
  11  *  This program is free software; you can redistribute it and/or modify
  12  *  it under the terms of the GNU General Public License as published by
  13  *  the Free Software Foundation; either version 2 of the License, or
  14  *  (at your option) any later version.
  15  *
  16  *  This program is distributed in the hope that it will be useful,
  17  *  but WITHOUT ANY WARRANTY; without even the implied warranty of
  18  *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  19  *  GNU General Public License for more details.
  20  *
  21  *  You should have received a copy of the GNU General Public License
  22  *  along with this program; if not, write to the Free Software
  23  *  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111, USA.
  24  *
  25  */
  26
  27 #include "include/conf.h"
  28 #include "include/defs.h"
  29
  30 #ifdef HAVE_GETOPT_H
  31 #include <getopt.h>
  32 #endif
  33
  34 #define REPORT_EVERY_X_LINES 5000
  35 #define MAX_OPEN_USER_FILES 10
  36
  37 struct userfilestruct
  38 {
  39         struct userfilestruct *next;
  40         struct userinfostruct *user;
  41         FILE *file;
  42 };
  43
  44 /*@null@*/static char *userfile=NULL;
  45
  46 numlist weekdays = { { 0, 1, 2, 3, 4, 5, 6 }, 7 };
  47 numlist hours = { { 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23 }, 24 };
  48
  49 static void getusers(const char *pwdfile, int debug);
  50
  51 int main(int argc,char *argv[])
  52 {
  53         enum isa_col_id {
  54                 ISACOL_Ip,
  55                 ISACOL_UserName,
  56                 ISACOL_Date,
  57                 ISACOL_Time,
  58                 ISACOL_TimeTaken,
  59                 ISACOL_Bytes,
  60                 ISACOL_Uri,
  61                 ISACOL_Status,
  62                 ISACOL_Last //last entry of the list !
  63         };
  64         enum InputLogFormat {
  65                 ILF_Unknown,
  66                 ILF_Squid,
  67                 ILF_Common,
  68                 ILF_Sarg,
  69                 ILF_Isa,
  70                 ILF_Last //last entry of the list !
  71         };
  72
  73         FILE *fp_in = NULL, *fp_denied=NULL, *fp_authfail=NULL, *fp_log=NULL;
  74
  75         char sz_Download_Unsort[ 20000 ] ;
  76         FILE * fp_Download_Unsort = NULL ;
  77
  78         extern int optind;
  79         extern int optopt;
  80         extern char *optarg;
  81
  82         char data[255];
  83         char elap[255];
  84         char ip[MAXLEN];
  85         char tam[255];
  86         char fun[MAXLEN];
  87         char wuser[MAXLEN];
  88         char smartfilter[MAXLEN];
  89         char dia[128];
  90         char mes[30];
  91         char hora[30];
  92         char date[255];
  93         char arq[255];
  94         char arq_log[255];
  95         int hm, hmf, hmr;
  96         char hm_str[15];
  97         char uagent[MAXLEN];
  98         char hexclude[MAXLEN];
  99         char csort[MAXLEN];
 100         int cstatus;
 101         char tbuf2[128];
 102         char *str;
 103         char tmp3[MAXLEN];
 104         char denied_unsort[MAXLEN];
 105         char denied_sort[MAXLEN];
 106         char authfail_unsort[MAXLEN];
 107         char start_hour[128];
 108         char *linebuf;
 109         const char *url;
 110         char *full_url;
 111         char *urly;
 112         char user[MAX_USER_LEN];
 113         char splitprefix[MAXLEN];
 114         enum InputLogFormat ilf;
 115         int ilf_count[ILF_Last];
 116         int  ch;
 117         int  x;
 118         int  errflg=0;
 119         int  puser=0;
 120         bool  fhost=false;
 121         bool  dns=false;
 122         bool  fuser=false;
 123         int  idata=0;
 124         int  mindate=0;
 125         int  maxdate=0;
 126         int  iarq=0;
 127         int isa_ncols=0,isa_cols[ISACOL_Last];
 128         int lastlog=-1;
 129         bool from_stdin;
 130         bool from_pipe;
 131         int blen;
 132         int maxopenfiles;
 133         int nopen;
 134         bool id_is_ip;
 135         long totregsl=0;
 136         long totregsg=0;
 137         long totregsx=0;
 138         bool totper=false;
 139         long int max_elapsed=0;
 140         long long int iyear, imonth, iday;
 141         bool realt;
 142         bool userip;
 143         struct tm tt;
 144         struct tm *t;
 145         unsigned long recs1=0UL;
 146         unsigned long recs2=0UL;
 147         int OutputNonZero = REPORT_EVERY_X_LINES ;
 148         bool download_flag=false;
 149         char *download_url=NULL;
 150         struct getwordstruct gwarea;
 151         longline line;
 152         time_t tnum;
 153         struct stat logstat;
 154         struct userinfostruct *uinfo;
 155         struct userfilestruct *first_user_file, *ufile, *ufile1, *prev_ufile;
 156         static int split=0;
 157         static int convert=0;
 158         static int output_css=0;
 159         int option_index;
 160         static struct option long_options[]=
 161         {
 162                 {"convert",no_argument,&convert,1},
 163                 {"css",no_argument,&output_css,1},
 164                 {"lastlog",required_argument,NULL,2},
 165                 {"keeplogs",no_argument,NULL,3},
 166                 {"split",no_argument,&split,1},
 167                 {"splitprefix",required_argument,NULL,'P'},
 168                 {0,0,0,0}
 169         };
 170
 171 #ifdef HAVE_LOCALE_H
 172         setlocale(LC_TIME,"");
 173 #endif
 174
 175 #if defined(ENABLE_NLS) && defined(HAVE_LOCALE_H)
 176         if (!setlocale (LC_ALL, "")) {
 177                 fprintf(stderr,"SARG: Cannot set the locale LC_ALL to the environment variable\n");
 178                 exit(EXIT_FAILURE);
 179         }
 180         if (!bindtextdomain (PACKAGE_NAME, LOCALEDIR)) {
 181                 fprintf(stderr,"SARG: Cannot bind to text domain %s in directory %s (%s)\n",PACKAGE_NAME,LOCALEDIR,strerror(errno));
 182                 exit(EXIT_FAILURE);
 183         }
 184         if (!textdomain (PACKAGE_NAME)) {
 185                 fprintf(stderr,"SARG: Cannot set gettext domain for %s PACKAGE_NAME (%s)\n",PACKAGE_NAME,strerror(errno));
 186                 exit(EXIT_FAILURE);
 187         }
 188 #endif //ENABLE_NLS
 189
 190         BgImage[0]='\0';
 191         LogoImage[0]='\0';
 192         LogoText[0]='\0';
 193         PasswdFile[0]='\0';
 194         OutputEmail[0]='\0';
 195         UserAgentLog[0]='\0';
 196         ExcludeHosts[0]='\0';
 197         ExcludeUsers[0]='\0';
 198         ConfigFile[0]='\0';
 199         code[0]='\0';
 200         LastLog=0;
 201         ReportType=0UL;
 202         UserTabFile[0]='\0';
 203         BlockIt[0]='\0';
 204         ExternalCSSFile[0]='\0';
 205         RedirectorLogFormat[0]='\0';
 206         NRedirectorLogs=0;
 207         for (ilf=0 ; ilf<ILF_Last ; ilf++) ilf_count[ilf]=0;
 208
 209         snprintf(ExcludeCodes,sizeof(ExcludeCodes),"%s/exclude_codes",SYSCONFDIR);
 210         strcpy(GraphDaysBytesBarColor,"orange");
 211         strcpy(BgColor,"#ffffff");
 212         strcpy(TxColor,"#000000");
 213         strcpy(TxBgColor,"lavender");
 214         strcpy(TiColor,"darkblue");
 215         strcpy(Width,"80");
 216         strcpy(Height,"45");
 217         strcpy(LogoTextColor,"#000000");
 218         strcpy(HeaderColor,"darkblue");
 219         strcpy(HeaderBgColor,"#dddddd");
 220         strcpy(LogoTextColor,"#006699");
 221         strcpy(FontSize,"9px");
 222         strcpy(TempDir,"/tmp");
 223         strcpy(OutputDir,"/var/www/html/squid-reports");
 224         AnonymousOutputFiles=false;
 225         Ip2Name=false;
 226         strcpy(DateFormat,"u");
 227         OverwriteReport=false;
 228         RemoveTempFiles=true;
 229         strcpy(ReplaceIndex,"index.html");
 230         Index=INDEX_YES;
 231         RecordsWithoutUser=RECORDWITHOUTUSER_IP;
 232         UseComma=0;
 233         strcpy(MailUtility,"mailx");
 234         TopSitesNum=100;
 235         TopUsersNum=0;
 236         UserIp=0;
 237         TopuserSort=TOPUSER_SORT_BYTES | TOPUSER_SORT_REVERSE;
 238         UserSort=USER_SORT_BYTES | USER_SORT_REVERSE;
 239         TopsitesSort=TOPSITE_SORT_CONNECT | TOPSITE_SORT_REVERSE;
 240         LongUrl=0;
 241         strcpy(FontFace,"Verdana,Tahoma,Arial");
 242         datetimeby=DATETIME_BYTE;
 243         strcpy(CharSet,"ISO-8859-1");
 244         Privacy=0;
 245         strcpy(PrivacyString,"***.***.***.***");
 246         strcpy(PrivacyStringColor,"blue");
 247         SuccessfulMsg=true;
 248         TopUserFields=TOPUSERFIELDS_NUM | TOPUSERFIELDS_DATE_TIME | TOPUSERFIELDS_USERID | TOPUSERFIELDS_CONNECT |
 249               TOPUSERFIELDS_BYTES | TOPUSERFIELDS_SETYB | TOPUSERFIELDS_IN_CACHE_OUT |
 250               TOPUSERFIELDS_USED_TIME | TOPUSERFIELDS_MILISEC | TOPUSERFIELDS_PTIME |
 251               TOPUSERFIELDS_TOTAL | TOPUSERFIELDS_AVERAGE;
 252         UserReportFields=USERREPORTFIELDS_CONNECT | USERREPORTFIELDS_BYTES | USERREPORTFIELDS_SETYB |
 253               USERREPORTFIELDS_IN_CACHE_OUT | USERREPORTFIELDS_USED_TIME | USERREPORTFIELDS_MILISEC |
 254               USERREPORTFIELDS_PTIME | USERREPORTFIELDS_TOTAL | USERREPORTFIELDS_AVERAGE;
 255         strcpy(DataFileDelimiter,";");
 256         DataFileFields=DATA_FIELD_USER | DATA_FIELD_DATE | DATA_FIELD_TIME | DATA_FIELD_URL | DATA_FIELD_CONNECT |
 257               DATA_FIELD_BYTES | DATA_FIELD_IN_CACHE | DATA_FIELD_OUT_CACHE | DATA_FIELD_ELAPSED;
 258         ShowReadStatistics=true;
 259         strcpy(IndexSortOrder,"D");
 260         ShowSargInfo=true;
 261         ShowSargLogo=true;
 262         ParsedOutputLog[0]='\0';
 263         strcpy(ParsedOutputLogCompress,"/bin/gzip -f");
 264         DisplayedValues=DISPLAY_ABBREV;
 265         strcpy(HeaderFontSize,"9px");
 266         strcpy(TitleFontSize,"11px");
 267         strcpy(AuthUserTemplateFile,"sarg_htaccess");
 268         set_download_suffix("7z,ace,arj,avi,bat,bin,bz2,bzip,cab,com,cpio,dll,doc,dot,exe,gz,iso,lha,lzh,mdb,mov,mp3,mpeg,mpg,mso,nrg,ogg,ppt,rar,rtf,shs,src,sys,tar,tgz,vcd,vob,wma,wmv,zip");
 269         Graphs=true;
 270 #if defined(FONTDIR)
 271         strcpy(GraphFont,FONTDIR"/DejaVuSans.ttf");
 272 #else
 273         GraphFont[0]='\0';
 274 #endif
 275         strcpy(Ulimit,"20000");
 276         NtlmUserFormat=NTLMUSERFORMAT_DOMAINUSER;
 277         IndexTree=INDEX_TREE_FILE;
 278         IndexFields=INDEXFIELDS_DIRSIZE;
 279         strcpy(RealtimeTypes,"GET,PUT,CONNECT");
 280         RealtimeUnauthRec=REALTIME_UNAUTH_REC_SHOW;
 281         RedirectorFilterOutDate=true;
 282         DansguardianFilterOutDate=true;
 283         DataFileUrl=DATAFILEURL_IP;
 284         strcpy(MaxElapsed,"28800000");
 285         BytesInSitesUsersReport=0;
 286         UserAuthentication=0;
 287         strcpy(LDAPHost,"127.0.0.1");
 288         LDAPPort=389;
 289         LDAPProtocolVersion=3;
 290         LDAPBindDN[0]='\0';
 291         LDAPBindPW[0]='\0';
 292         LDAPBaseSearch[0]='\0';
 293         strcpy(LDAPFilterSearch, "(uid=%s)");
 294         strcpy(LDAPTargetAttr, "cn");
 295         SortTableJs[0]='\0';
 296
 297         dia[0]='\0';
 298         mes[0]='\0';
 299         hora[0]='\0';
 300         tmp[0]='\0';
 301         tmp3[0]='\0';
 302         us[0]='\0';
 303         date[0]='\0';
 304         df[0]='\0';
 305         uagent[0]='\0';
 306         hexclude[0]='\0';
 307         addr[0]='\0';
 308         hm=-1;
 309         hmf=-1;
 310         site[0]='\0';
 311         outdir[0]='\0';
 312         splitprefix[0]='\0';
 313         elap[0]='\0';
 314         email[0]='\0';
 315         UserInvalidChar[0]='\0';
 316         DataFile[0]='\0';
 317         SquidGuardConf[0]='\0';
 318         DansGuardianConf[0]='\0';
 319         start_hour[0]='\0';
 320         hm_str[0]='\0';
 321         HostAliasFile[0]='\0';
 322
 323         denied_count=0;
 324         download_count=0;
 325         authfail_count=0;
 326         dansguardian_count=0;
 327         redirector_count=0;
 328         useragent_count=0;
 329         DeniedReportLimit=10;
 330         AuthfailReportLimit=10;
 331         DansGuardianReportLimit=10;
 332         SquidGuardReportLimit=10;
 333         DownloadReportLimit=50;
 334         UserReportLimit=0;
 335         debug=0;
 336         debugz=0;
 337         debugm=0;
 338         iprel=false;
 339         userip=false;
 340         realt=false;
 341         realtime_refresh=3;
 342         realtime_access_log_lines=1000;
 343         cost=0.01;
 344         nocost=50000000;
 345         ndownload=0;
 346         squid24=false;
 347         dfrom=0;
 348         duntil=0;
 349
 350         bzero(IncludeUsers, sizeof(IncludeUsers));
 351         bzero(ExcludeString, sizeof(ExcludeString));
 352         first_user_file=NULL;
 353         memset(&period,0,sizeof(period));
 354
 355         NAccessLog=0;
 356         for(x=0; x<MAXLOGS; x++)
 357                 AccessLog[x][0]='\0';
 358         AccessLogFromCmdLine=0;
 359         RedirectorLogFromCmdLine=0;
 360
 361         strcpy(Title,_("Squid User Access Report"));
 362
 363         while((ch = getopt_long_only(argc, argv, "a:b:c:d:e:f:g:u:l:L:o:s:t:w:P:hijmnprvxyz",long_options,&option_index)) != -1){
 364                 switch(ch)
 365                 {
 366                         case 0:
 367                                 break;
 368                         case 2:
 369                                 lastlog=atoi(optarg);
 370                                 break;
 371                         case 3:
 372                                 lastlog=0;
 373                                 break;
 374                         case 'a':
 375                                 safe_strcpy(addr,optarg,sizeof(addr));
 376                                 break;
 377                         case 'b':
 378                                 safe_strcpy(uagent,optarg,sizeof(uagent));
 379                                 break;
 380                         case 'c':
 381                                 safe_strcpy(hexclude,optarg,sizeof(hexclude));
 382                                 break;
 383                         case 'd':
 384                                 safe_strcpy(date,optarg,sizeof(date));
 385                                 date_from(date, &dfrom, &duntil);
 386                                 break;
 387                         case 'e':
 388                                 safe_strcpy(email,optarg,sizeof(email));
 389                                 break;
 390                         case 'f':
 391                                 safe_strcpy(ConfigFile,optarg,sizeof(ConfigFile));
 392                                 break;
 393                         case 'g':
 394                                 safe_strcpy(df,optarg,sizeof(df));
 395                                 break;
 396                         case 'h':
 397                                 usage(argv[0]);
 398                                 exit(EXIT_SUCCESS);
 399                         case 'i':
 400                                 iprel=true;
 401                                 break;
 402                         case 'l':
 403                                 if (NAccessLog>=MAXLOGS) {
 404                                         debuga(_("Too many log files passed on command line with option -l.\n"));
 405                                         exit(EXIT_FAILURE);
 406                                 }
 407                                 if (strlen(optarg)>=MAX_LOG_FILELEN) {
 408                                         debuga(_("Log file name too long passed on command line with option -l: %s\n"),optarg);
 409                                         exit(EXIT_FAILURE);
 410                                 }
 411                                 strcpy(AccessLog[NAccessLog],optarg);
 412                                 NAccessLog++;
 413                                 AccessLogFromCmdLine++;
 414                                 break;
 415                         case 'L':
 416                                 if (NRedirectorLogs>MAX_REDIRECTOR_LOGS) {
 417                                         debuga(_("Too many redirector logs passed on command line with option -L.\n"));
 418                                         exit(EXIT_FAILURE);
 419                                 }
 420                                 if (strlen(optarg)>=MAX_REDIRECTOR_FILELEN) {
 421                                         debuga(_("Redirector log file name too long passed on command line with opton -L: %s\n"),optarg);
 422                                         exit(EXIT_FAILURE);
 423                                 }
 424                                 strcpy(RedirectorLogs[NRedirectorLogs],optarg);
 425                                 NRedirectorLogs++;
 426                                 RedirectorLogFromCmdLine++;
 427                                 break;
 428                         case 'm':
 429                                 debugm++;
 430                                 break;
 431                         case 'n':
 432                                 dns=true;
 433                                 break;
 434                         case 'o':
 435                                 safe_strcpy(outdir,optarg,sizeof(outdir));
 436                                 break;
 437                         case 'p':
 438                                 userip=true;
 439                                 break;
 440                         case 'P':
 441                                 safe_strcpy(splitprefix,optarg,sizeof(splitprefix));
 442                                 break;
 443                         case 'r':
 444                                 realt=true;
 445                                 break;
 446                         case 's':
 447                                 safe_strcpy(site,optarg,sizeof(site));
 448                                 break;
 449                         case 't':
 450                         {
 451                                 int h1,m1,h2,m2;
 452
 453                                 if(strstr(optarg,"-") == 0) {
 454                                         if(sscanf(optarg,"%d:%d",&h1,&m1)!=2) {
 455                                                 debuga(_("Time period passed on the command line with option -t must be MM:SS\n"));
 456                                                 exit(EXIT_FAILURE);
 457                                         }
 458                                         hm=h1*100+m1;
 459                                         hmf=hm;
 460                                         snprintf(hm_str,sizeof(hm_str),"%02d:%02d",h1,m1);
 461                                 } else {
 462                                         if(sscanf(optarg,"%d:%d-%d:%d",&h1,&m1,&h2,&m2)!=4) {
 463                                                 debuga(_("Time range passed on the command line with option -t must be MM:SS-MM:SS\n"));
 464                                                 exit(EXIT_FAILURE);
 465                                         }
 466                                         hm=h1*100+m1;
 467                                         hmf=h2*100+m2;
 468                                         snprintf(hm_str,sizeof(hm_str),"%02d:%02d-%02d:%02d",h1,m1,h2,m2);
 469                                 }
 470                                 break;
 471                         }
 472                         case 'u':
 473                                 safe_strcpy(us,optarg,sizeof(us));
 474                                 break;
 475                         case 'v':
 476                                 version();
 477                                 break;
 478                         case 'w':
 479                                 safe_strcpy(tmp,optarg,sizeof(tmp));
 480                                 break;
 481                         case 'x':
 482                                 debug++;
 483                                 break;
 484                         case 'y':
 485                                 langcode++;
 486                                 break;
 487                         case 'z':
 488                                 debugz++;
 489                                 break;
 490                         case ':':
 491                                 debuga(_("Option -%c requires an argument\n"),optopt);
 492                                 exit(EXIT_FAILURE);
 493                         case '?':
 494                                 usage(argv[0]);
 495                                 exit(EXIT_FAILURE);
 496                         default:
 497                                 abort();
 498                 }
 499         }
 500
 501         if (errflg>0) {
 502                 usage(argv[0]);
 503                 exit(2);
 504         }
 505
 506         if (optind<argc) {
 507                 for (iarq=optind ; iarq<argc ; iarq++) {
 508                         if (NAccessLog>=MAXLOGS) {
 509                                 debuga(_("Too many log files passed on command line.\n"));
 510                                 exit(EXIT_FAILURE);
 511                         }
 512                         if (strlen(argv[iarq])>=MAX_LOG_FILELEN) {
 513                                 debuga(_("Log file name too long passed on command line: %s\n"),argv[iarq]);
 514                                 exit(EXIT_FAILURE);
 515                         }
 516                         strcpy(AccessLog[NAccessLog],argv[iarq]);
 517                         NAccessLog++;
 518                         AccessLogFromCmdLine++;
 519                 }
 520         }
 521
 522         if(debug) debuga(_("Init\n"));
 523
 524         if(ConfigFile[0] == '\0') snprintf(ConfigFile,sizeof(ConfigFile),"%s/sarg.conf",SYSCONFDIR);
 525         if(access(ConfigFile, R_OK) != 0) {
 526                 debuga(_("Cannot open config file: %s - %s\n"),ConfigFile,strerror(errno));
 527                 exit(EXIT_FAILURE);
 528         }
 529
 530         if(access(ConfigFile, R_OK) == 0)
 531                 getconf();
 532
 533         if(userip) UserIp=true;
 534
 535         if(dns) Ip2Name=true;
 536
 537         if (lastlog>=0) LastLog=lastlog;
 538
 539         if(outdir[0] == '\0') strcpy(outdir,OutputDir);
 540         if(outdir[0] != '\0') strcat(outdir,"/");
 541
 542         if(realt) {
 543                 realtime();
 544                 exit(EXIT_SUCCESS);
 545         }
 546
 547         if(IndexTree == INDEX_TREE_FILE)
 548                 strcpy(ImageFile,"../images");
 549         else
 550                 strcpy(ImageFile,"../../../images");
 551
 552         dataonly=0;
 553         if(DataFile[0] != '\0')
 554                 dataonly++;
 555
 556         if(df[0] == '\0') strcpy(df,DateFormat);
 557         else strcpy(DateFormat,df);
 558
 559         if(df[0] == '\0') {
 560                 strcpy(df,"u");
 561                 strcpy(DateFormat,"u");
 562         }
 563         if (df[0]=='w')
 564                 IndexTree=INDEX_TREE_FILE;
 565
 566         if(NAccessLog == 0) {
 567                 strcpy(AccessLog[0],"/var/log/squid/access.log");
 568                 NAccessLog++;
 569         }
 570
 571         if(output_css) {
 572                 css_content(stdout);
 573                 exit(EXIT_SUCCESS);
 574         }
 575         if(split) {
 576                 for (iarq=0 ; iarq<NAccessLog ; iarq++)
 577                         splitlog(AccessLog[iarq], df, dfrom, duntil, convert, splitprefix);
 578                 exit(EXIT_SUCCESS);
 579         }
 580         if(convert) {
 581                 for (iarq=0 ; iarq<NAccessLog ; iarq++)
 582                         convlog(AccessLog[iarq], df, dfrom, duntil);
 583                 exit(EXIT_SUCCESS);
 584         }
 585
 586         load_excludecodes(ExcludeCodes);
 587
 588         if(access(PasswdFile, R_OK) == 0) {
 589                 getusers(PasswdFile,debug);
 590                 puser++;
 591         }
 592
 593         if(hexclude[0] == '\0')
 594                 strcpy(hexclude,ExcludeHosts);
 595         if(hexclude[0] != '\0') {
 596                 gethexclude(hexclude,debug);
 597                 fhost=true;
 598         }
 599
 600         if(ReportType == 0) {
 601                 ReportType=REPORT_TYPE_TOPUSERS | REPORT_TYPE_TOPSITES | REPORT_TYPE_USERS_SITES |
 602                            REPORT_TYPE_SITES_USERS | REPORT_TYPE_DATE_TIME | REPORT_TYPE_DENIED |
 603                            REPORT_TYPE_AUTH_FAILURES | REPORT_TYPE_SITE_USER_TIME_DATE | REPORT_TYPE_DOWNLOADS;
 604         }
 605
 606         if(access(ExcludeUsers, R_OK) == 0) {
 607                 getuexclude(ExcludeUsers,debug);
 608                 fuser=true;
 609         }
 610         if (HostAliasFile[0] != '\0')
 611                 read_hostalias(HostAliasFile);
 612
 613         indexonly=false;
 614         if(fuser) {
 615                 if(is_indexonly())
 616                         indexonly=true;
 617         }
 618         if(strcmp(ExcludeUsers,"indexonly") == 0) indexonly=true;
 619         if(Index == INDEX_ONLY) indexonly=true;
 620
 621         if(MaxElapsed[0] != '\0') max_elapsed=atol(MaxElapsed);
 622
 623         if(uagent[0] == '\0') strcpy(uagent,UserAgentLog);
 624
 625         if(tmp[0] == '\0') strcpy(tmp,TempDir);
 626         else strcpy(TempDir,tmp);
 627         /*
 628         For historical reasons, the temporary directory is the subdirectory "sarg" of the path
 629         provided by the user.
 630         */
 631         strcat(tmp,"/sarg");
 632
 633         if (tmp[0]!='\0' && strncmp(outdir,tmp,strlen(tmp))==0) {
 634                 debuga(_("The output directory \"%s\" must be outside of the temporary directory \"%s\"\n"),outdir,tmp);
 635                 exit(EXIT_FAILURE);
 636         }
 637
 638         if(email[0] == '\0' && OutputEmail[0] != '\0') strcpy(email,OutputEmail);
 639
 640         if(email[0] != '\0') {
 641                 my_mkdir(tmp);
 642                 strcpy(outdir,tmp);
 643                 strcat(outdir,"/");
 644         }
 645
 646         if(access(tmp, R_OK) == 0) {
 647                 unlinkdir(tmp,1);
 648         }
 649         my_mkdir(tmp);
 650         snprintf(denied_unsort,sizeof(denied_unsort),"%s/denied.log.unsort",tmp);
 651         snprintf(denied_sort,sizeof(denied_sort),"%s/denied.log",tmp);
 652         snprintf(authfail_unsort,sizeof(authfail_unsort),"%s/authfail.log.unsort",tmp);
 653
 654         if(debug) {
 655                 debuga(_("Parameters:\n"));
 656                 debuga(_("          Hostname or IP address (-a) = %s\n"),addr);
 657                 debuga(_("                   Useragent log (-b) = %s\n"),uagent);
 658                 debuga(_("                    Exclude file (-c) = %s\n"),hexclude);
 659                 debuga(_("                 Date from-until (-d) = %s\n"),date);
 660                 debuga(_("   Email address to send reports (-e) = %s\n"),email);
 661                 debuga(_("                     Config file (-f) = %s\n"),ConfigFile);
 662                 if(strcmp(df,"e") == 0)
 663                         debuga(_("                     Date format (-g) = Europe (dd/mm/yyyy)\n"));
 664                 if(strcmp(df,"u") == 0)
 665                         debuga(_("                     Date format (-g) = USA (mm/dd/yyyy)\n"));
 666                 if(strcmp(df,"w") == 0)
 667                         debuga(_("                     Date format (-g) = Sites & Users (yyyy/ww)\n"));
 668                 debuga(_("                       IP report (-i) = %s\n"),(iprel) ? _("Yes") : _("No"));
 669                 for (iarq=0 ; iarq<NAccessLog ; iarq++)
 670                         debuga(_("                       Input log (-l) = %s\n"),AccessLog[iarq]);
 671                 for (iarq=0 ; iarq<NRedirectorLogs ; iarq++)
 672                         debuga(_("                  Redirector log (-L) = %s\n"),RedirectorLogs[iarq]);
 673                 debuga(_("              Resolve IP Address (-n) = %s\n"),(Ip2Name) ? _("Yes") : _("No"));
 674                 debuga(_("                      Output dir (-o) = %s\n"),outdir);
 675                 debuga(_("Use Ip Address instead of userid (-p) = %s\n"),(UserIp) ? _("Yes") : _("No"));
 676                 debuga(_("                   Accessed site (-s) = %s\n"),site);
 677                 debuga(_("                            Time (-t) = %s\n"),hm_str);
 678                 debuga(_("                            User (-u) = %s\n"),us);
 679                 debuga(_("                   Temporary dir (-w) = %s\n"),tmp);
 680                 debuga(_("                  Debug messages (-x) = %s\n"),(debug) ? _("Yes") : _("No"));
 681                 debuga(_("                Process messages (-z) = %s\n"),(debugz) ? _("Yes") : _("No"));
 682                 debuga(_(" Previous reports to keep (--lastlog) = %d\n"),LastLog);
 683                 debuga("\n");
 684         }
 685
 686         if(debugm) {
 687                 printf(_("Parameters:\n"));
 688                 printf(_("          Hostname or IP address (-a) = %s\n"),addr);
 689                 printf(_("                   Useragent log (-b) = %s\n"),uagent);
 690                 printf(_("                    Exclude file (-c) = %s\n"),hexclude);
 691                 printf(_("                 Date from-until (-d) = %s\n"),date);
 692                 printf(_("   Email address to send reports (-e) = %s\n"),email);
 693                 printf(_("                     Config file (-f) = %s\n"),ConfigFile);
 694                 if(strcmp(df,"e") == 0)
 695                         printf(_("                     Date format (-g) = Europe (dd/mm/yyyy)\n"));
 696                 if(strcmp(df,"u") == 0)
 697                         printf(_("                     Date format (-g) = USA (mm/dd/yyyy)\n"));
 698                 if(strcmp(df,"w") == 0)
 699                         printf(_("                     Date format (-g) = Sites & Users (yyyy/ww)\n"));
 700                 printf(_("                       IP report (-i) = %s\n"),(iprel) ? _("Yes") : _("No"));
 701                 for (iarq=0 ; iarq<NAccessLog ; iarq++)
 702                         printf(_("                       Input log (-l) = %s\n"),AccessLog[iarq]);
 703                 for (iarq=0 ; iarq<NRedirectorLogs ; iarq++)
 704                         printf(_("                  Redirector log (-L) = %s\n"),RedirectorLogs[iarq]);
 705                 printf(_("              Resolve IP Address (-n) = %s\n"),(Ip2Name) ? _("Yes") : _("No"));
 706                 printf(_("                      Output dir (-o) = %s\n"),outdir);
 707                 printf(_("Use Ip Address instead of userid (-p) = %s\n"),(UserIp) ? _("Yes") : _("No"));
 708                 printf(_("                   Accessed site (-s) = %s\n"),site);
 709                 printf(_("                            Time (-t) = %s\n"),hm_str);
 710                 printf(_("                            User (-u) = %s\n"),us);
 711                 printf(_("                   Temporary dir (-w) = %s\n"),tmp);
 712                 printf(_("                  Debug messages (-x) = %s\n"),(debug) ? _("Yes") : _("No"));
 713                 printf(_("                Process messages (-z) = %s\n"),(debugz) ? _("Yes") : _("No"));
 714                 printf(_(" Previous reports to keep (--lastlog) = %d\n"),LastLog);
 715                 printf(_("sarg version: %s\n"),VERSION);
 716         }
 717
 718         if(debug)
 719                 debuga(_("sarg version: %s\n"),VERSION);
 720
 721 #ifdef ENABLE_DOUBLE_CHECK_DATA
 722         debuga(_("Sarg compiled to report warnings if the output is inconsistent\n"));
 723 #endif
 724
 725         maxopenfiles=MAX_OPEN_USER_FILES;
 726 #ifdef HAVE_RLIM_T
 727         if (Ulimit[0] != '\0') {
 728                 struct rlimit rl;
 729                 long l1, l2;
 730                 int rc=0;
 731
 732 #if defined(RLIMIT_NOFILE)
 733                 getrlimit (RLIMIT_NOFILE, &rl);
 734 #elif defined(RLIMIT_OFILE)
 735                 getrlimit (RLIMIT_OFILE, &rl);
 736 #else
 737 #warning "No rlimit resource for the number of open files"
 738 #endif
 739                 l1 = rl.rlim_cur;
 740                 l2 = rl.rlim_max;
 741
 742                 rl.rlim_cur = atol(Ulimit);
 743                 rl.rlim_max = atol(Ulimit);
 744 #if defined(RLIMIT_NOFILE)
 745                 rc=setrlimit (RLIMIT_NOFILE, &rl);
 746 #elif defined(RLIMIT_OFILE)
 747                 rc=setrlimit (RLIMIT_OFILE, &rl);
 748 #else
 749 #warning "No rlimit resource for the number of open files"
 750 #endif
 751                 if(rc == -1) {
 752                         debuga(_("setrlimit error - %s\n"),strerror(errno));
 753                 }
 754
 755                 if(debug)
 756                         debuga("Maximum file descriptor: cur=%ld max=%ld, changed to cur="RLIM_STRING" max="RLIM_STRING"\n",l1,l2,rl.rlim_cur,rl.rlim_max);
 757         }
 758 #endif
 759
 760         init_usertab(UserTabFile);
 761
 762         if ((line=longline_create())==NULL) {
 763                 debuga(_("Not enough memory to read a log file\n"));
 764                 exit(EXIT_FAILURE);
 765         }
 766
 767         snprintf(sz_Download_Unsort,sizeof(sz_Download_Unsort),"%s/download.unsort", tmp);
 768
 769         if(DataFile[0]=='\0') {
 770                 if((ReportType & REPORT_TYPE_DENIED) != 0) {
 771                         if((fp_denied=MY_FOPEN(denied_unsort,"w"))==NULL) {
 772                                 debuga(_("(log) Cannot open file: %s - %s\n"),denied_unsort,strerror(errno));
 773                                 exit(EXIT_FAILURE);
 774                         }
 775                 }
 776
 777                 if((ReportType & REPORT_TYPE_DENIED) != 0 || (ReportType & REPORT_TYPE_AUTH_FAILURES) != 0) {
 778                         if((fp_authfail=MY_FOPEN(authfail_unsort,"w"))==NULL) {
 779                                 debuga(_("(log) Cannot open file: %s - %s\n"),authfail_unsort,strerror(errno));
 780                                 exit(EXIT_FAILURE);
 781                         }
 782                 }
 783         }
 784
 785         for (iarq=0 ; iarq<NAccessLog ; iarq++) {
 786                 strcpy(arq,AccessLog[iarq]);
 787
 788                 strcpy(arqtt,arq);
 789
 790                 if(strcmp(arq,"-")==0) {
 791                         if(debug)
 792                                 debuga(_("Reading access log file: from stdin\n"));
 793                         fp_in=stdin;
 794                         from_stdin=true;
 795                 } else {
 796                         if (date[0]!='\0') {
 797                                 if (stat(arq,&logstat)!=0) {
 798                                         debuga(_("Cannot get the modification time of input log file %s (%s). Processing it anyway\n"),arq,strerror(errno));
 799                                 } else {
 800                                         struct tm *logtime=localtime(&logstat.st_mtime);
 801                                         if ((logtime->tm_year+1900)*10000+(logtime->tm_mon+1)*100+logtime->tm_mday<dfrom) {
 802                                                 debuga(_("Ignoring old log file %s\n"),arq);
 803                                                 continue;
 804                                         }
 805                                 }
 806                         }
 807                         fp_in=decomp(arq,&from_pipe);
 808                         if(fp_in==NULL) {
 809                                 debuga(_("(log) Cannot open log file: %s - %s\n"),arq,strerror(errno));
 810                                 exit(EXIT_FAILURE);
 811                         }
 812                         if(debug) debuga(_("Reading access log file: %s\n"),arq);
 813                         from_stdin=false;
 814                 }
 815                 ilf=ILF_Unknown;
 816                 download_flag=false;
 817                 // pre-read the file only if we have to show stats
 818                 if(ShowReadStatistics && !from_stdin && !from_pipe) {
 819                         size_t nread,i;
 820                         bool skipcr=false;
 821                         char tmp4[MAXLEN];
 822
 823                         recs1=0UL;
 824                         recs2=0UL;
 825
 826                         while ((nread=fread(tmp4,1,sizeof(tmp4),fp_in))>0) {
 827                                 for (i=0 ; i<nread ; i++)
 828                                         if (skipcr) {
 829                                                 if (tmp4[i]!='\n' && tmp4[i]!='\r') {
 830                                                         skipcr=false;
 831                                                 }
 832                                         } else {
 833                                                 if (tmp4[i]=='\n' || tmp4[i]=='\r') {
 834                                                         skipcr=true;
 835                                                         recs1++;
 836                                                 }
 837                                         }
 838                         }
 839                         rewind(fp_in);
 840                         printf(_("SARG: Records in file: %lu, reading: %3.2f%%"),recs1,(float) 0);
 841                         putchar('\r');
 842                         fflush( stdout ) ;
 843                 }
 844
 845                 longline_reset(line);
 846
 847                 while ((linebuf=longline_read(fp_in,line))!=NULL) {
 848                         blen=strlen(linebuf);
 849
 850                         if (ilf==ILF_Unknown) {
 851                                 if(strncmp(linebuf,"#Software: Mic",14) == 0) {
 852                                         fixendofline(linebuf);
 853                                         if (debug)
 854                                                 debuga(_("Log is from Microsoft ISA: %s\n"),linebuf);
 855                                         ilf=ILF_Isa;
 856                                         ilf_count[ilf]++;
 857                                         continue;
 858                                 }
 859
 860                                 if(strncmp(linebuf,"*** SARG Log ***",16) == 0) {
 861                                         if (getperiod_fromsarglog(arqtt,&period)<0) {
 862                                                 debuga(_("The name of the file is invalid: %s\n"),arq);
 863                                                 exit(EXIT_FAILURE);
 864                                         }
 865                                         ilf=ILF_Sarg;
 866                                         ilf_count[ilf]++;
 867                                         continue;
 868                                 }
 869                         }
 870
 871                         if(!fp_log && ParsedOutputLog[0] && ilf!=ILF_Sarg) {
 872                                 if(access(ParsedOutputLog,R_OK) != 0) {
 873                                         my_mkdir(ParsedOutputLog);
 874                                 }
 875                                 if (snprintf(arq_log,sizeof(arq_log),"%s/sarg_temp.log",ParsedOutputLog)>=sizeof(arq_log)) {
 876                                         debuga(_("File name too long: %s/sarg_temp.log\n"),ParsedOutputLog);
 877                                         exit(EXIT_FAILURE);
 878                                 }
 879                                 if((fp_log=MY_FOPEN(arq_log,"w"))==NULL) {
 880                                         debuga(_("(log) Cannot open log file: %s - %s\n"),arq_log,strerror(errno));
 881                                         exit(EXIT_FAILURE);
 882                                 }
 883                                 fputs("*** SARG Log ***\n",fp_log);
 884                         }
 885
 886                         recs2++;
 887                         if( ShowReadStatistics && !from_stdin && !from_pipe && --OutputNonZero<=0) {
 888                                 double perc = recs2 * 100. / recs1 ;
 889                                 printf(_("SARG: Records in file: %lu, reading: %3.2lf%%"),recs2,perc);
 890                                 putchar('\r');
 891                                 fflush (stdout);
 892                                 OutputNonZero = REPORT_EVERY_X_LINES ;
 893                         }
 894                         if(blen < 58) continue;
 895                         if(strstr(linebuf,"HTTP/0.0") != 0) continue;
 896                         if(strstr(linebuf,"logfile turned over") != 0) continue;
 897                         if(linebuf[0] == ' ') continue;
 898
 899                         // exclude_string
 900                         if(ExcludeString[0] != '\0') {
 901                                 bool exstring=false;
 902                                 getword_start(&gwarea,ExcludeString);
 903                                 while(strchr(gwarea.current,':') != 0) {
 904                                         if (getword_multisep(val1,sizeof(val1),&gwarea,':')<0) {
 905                                                 debuga(_("Maybe you have a broken record or garbage in your exclusion string\n"));
 906                                                 exit(EXIT_FAILURE);
 907                                         }
 908                                         if((str=(char *) strstr(linebuf,val1)) != (char *) NULL ) {
 909                                                 exstring=true;
 910                                                 break;
 911                                         }
 912                                 }
 913                                 if(!exstring && (str=(char *) strstr(linebuf,gwarea.current)) != (char *) NULL )
 914                                         exstring=true;
 915                                 if(exstring) continue;
 916                         }
 917
 918                         totregsl++;
 919                         if(debugm)
 920                                 printf("BUF=%s\n",linebuf);
 921
 922                         t=NULL;
 923                         if (ilf==ILF_Squid || ilf==ILF_Common || ilf==ILF_Unknown) {
 924                                 getword_start(&gwarea,linebuf);
 925                                 if (getword(data,sizeof(data),&gwarea,' ')<0) {
 926                                         debuga(_("Maybe you have a broken time in your access.log file\n"));
 927                                         exit(EXIT_FAILURE);
 928                                 }
 929                                 if((str=(char *) strchr(data, '.')) != (char *) NULL && (str=(char *) strchr(str+1, '.')) != (char *) NULL ) {
 930                                         strcpy(ip,data);
 931                                         strcpy(elap,"0");
 932                                         if(squid24) {
 933                                                 if (getword(user,sizeof(user),&gwarea,' ')<0 || getword_skip(255,&gwarea,' ')<0) {
 934                                                         debuga(_("Maybe you have a broken record or garbage in your %s file\n"),arq);
 935                                                         exit(EXIT_FAILURE);
 936                                                 }
 937                                         } else {
 938                                                 if (getword_skip(255,&gwarea,' ')<0 || getword(user,sizeof(user),&gwarea,' ')<0) {
 939                                                         debuga(_("Maybe you have a broken record or garbage in your %s file\n"),arq);
 940                                                         exit(EXIT_FAILURE);
 941                                                 }
 942                                         }
 943                                         if (getword(data,sizeof(data),&gwarea,']')<0 || getword_skip(MAXLEN,&gwarea,'"')<0 ||
 944                                             getword(fun,sizeof(fun),&gwarea,' ')<0) {
 945                                                 debuga(_("Maybe you have a broken record or garbage in your %s file\n"),arq);
 946                                                 exit(EXIT_FAILURE);
 947                                         }
 948                                         if (getword_ptr(linebuf,&full_url,&gwarea,' ')<0) {
 949                                                 debuga(_("Maybe you have a broken url in your %s file\n"),arq);
 950                                                 exit(EXIT_FAILURE);
 951                                         }
 952                                         if (getword_skip(MAXLEN,&gwarea,' ')<0) {
 953                                                 debuga(_("Maybe you have a broken record or garbage in your %s file\n"),arq);
 954                                                 exit(EXIT_FAILURE);
 955                                         }
 956                                         if (getword(code2,sizeof(code2),&gwarea,' ')<0) {
 957                                                 debuga(_("Maybe you have a broken record or garbage in your %s file\n"),arq);
 958                                                 exit(EXIT_FAILURE);
 959                                         }
 960                                         if (getword(tam,sizeof(tam),&gwarea,' ')<0) {
 961                                                 debuga(_("Maybe you have a broken record or garbage in your %s file\n"),arq);
 962                                                 exit(EXIT_FAILURE);
 963                                         }
 964                                         if((str=(char *) strchr(gwarea.current, ' ')) != (char *) NULL ) {
 965                                                 if (getword(code,sizeof(code),&gwarea,' ')<0) {
 966                                                         debuga(_("Maybe you have a broken record or garbage in your %s file\n"),arq);
 967                                                         exit(EXIT_FAILURE);
 968                                                 }
 969                                         } else {
 970                                                 if (getword(code,sizeof(code),&gwarea,'\0')<0) {
 971                                                         debuga(_("Maybe you have a broken record or garbage in your %s file\n"),arq);
 972                                                         exit(EXIT_FAILURE);
 973                                                 }
 974                                         }
 975
 976                                         if ((str = strchr(code, ':')) != NULL)
 977                                                 *str = '/';
 978
 979                                         if(strcmp(tam,"\0") == 0)
 980                                                 strcpy(tam,"0");
 981
 982                                         ilf=ILF_Common;
 983                                         ilf_count[ilf]++;
 984
 985                                         getword_start(&gwarea,data+1);
 986                                         if (getword_multisep(data,sizeof(data),&gwarea,':')<0){
 987                                                 debuga(_("Maybe you have a broken date in your %s file\n"),arq);
 988                                                 exit(EXIT_FAILURE);
 989                                         }
 990                                         if (getword_multisep(hora,sizeof(hora),&gwarea,' ')<0){
 991                                                 debuga(_("Maybe you have a broken date in your %s file\n"),arq);
 992                                                 exit(EXIT_FAILURE);
 993                                         }
 994                                         getword_start(&gwarea,data);
 995                                         if (getword_atoll(&iday,&gwarea,'/')<0){
 996                                                 debuga(_("Maybe you have a broken date in your %s file\n"),arq);
 997                                                 exit(EXIT_FAILURE);
 998                                         }
 999                                         if (getword(mes,sizeof(mes),&gwarea,'/')<0){
1000                                                 debuga(_("Maybe you have a broken date in your %s file\n"),arq);
1001                                                 exit(EXIT_FAILURE);
1002                                         }
1003                                         if (getword_atoll(&iyear,&gwarea,'/')<0){
1004                                                 debuga(_("Maybe you have a broken date in your %s file\n"),arq);
1005                                                 exit(EXIT_FAILURE);
1006                                         }
1007
1008                                         imonth=month2num(mes)+1;
1009                                         idata=builddia(iday,imonth,iyear);
1010                                         computedate(iyear,imonth,iday,&tt);
1011                                         if (sscanf(hora,"%d:%d:%d",&tt.tm_hour,&tt.tm_min,&tt.tm_sec)!=3 || tt.tm_hour<0 || tt.tm_hour>=24 ||
1012                                                         tt.tm_min<0 || tt.tm_min>=60 || tt.tm_sec<0 || tt.tm_sec>=60) {
1013                                                 debuga(_("Invalid time found in %s\n"),arq);
1014                                                 exit(EXIT_FAILURE);
1015                                         }
1016                                         t=&tt;
1017                                 }
1018
1019                                 if(ilf==ILF_Unknown || ilf==ILF_Squid) {
1020                                         if (getword(elap,sizeof(elap),&gwarea,' ')<0) {
1021                                                 debuga(_("Maybe you have a broken elapsed time in your %s file\n"),arq);
1022                                                 exit(EXIT_FAILURE);
1023                                         }
1024                                         while(strcmp(elap,"") == 0 && gwarea.current[0] != '\0')
1025                                                 if (getword(elap,sizeof(elap),&gwarea,' ')<0) {
1026                                                         debuga(_("Maybe you have a broken elapsed time in your %s file\n"),arq);
1027                                                         exit(EXIT_FAILURE);
1028                                                 }
1029                                         if(strlen(elap) < 1) continue;
1030                                         if (getword(ip,sizeof(ip),&gwarea,' ')<0){
1031                                                 debuga(_("Maybe you have a broken client IP address in your %s file\n"),arq);
1032                                                 exit(EXIT_FAILURE);
1033                                         }
1034                                         if (getword(code,sizeof(code),&gwarea,' ')<0){
1035                                                 debuga(_("Maybe you have a broken result code in your %s file\n"),arq);
1036                                                 exit(EXIT_FAILURE);
1037                                         }
1038                                         if (getword(tam,sizeof(tam),&gwarea,' ')<0){
1039                                                 debuga(_("Maybe you have a broken amount of data in your %s file\n"),arq);
1040                                                 exit(EXIT_FAILURE);
1041                                         }
1042                                         if (getword(fun,sizeof(fun),&gwarea,' ')<0){
1043                                                 debuga(_("Maybe you have a broken request method in your %s file\n"),arq);
1044                                                 exit(EXIT_FAILURE);
1045                                         }
1046                                         if (getword_ptr(linebuf,&full_url,&gwarea,' ')<0){
1047                                                 debuga(_("Maybe you have a broken url in your %s file\n"),arq);
1048                                                 exit(EXIT_FAILURE);
1049                                         }
1050                                         if (getword(user,sizeof(user),&gwarea,' ')<0){
1051                                                 debuga(_("Maybe you have a broken user ID in your %s file\n"),arq);
1052                                                 exit(EXIT_FAILURE);
1053                                         }
1054                                         ilf=ILF_Squid;
1055                                         ilf_count[ilf]++;
1056
1057                                         tnum=atoi(data);
1058                                         t=localtime(&tnum);
1059                                         if (t == NULL) {
1060                                                 debuga(_("Cannot convert the timestamp from the squid log file\n"));
1061                                                 exit(EXIT_FAILURE);
1062                                         }
1063
1064                                         strftime(tbuf2, sizeof(tbuf2), "%H%M", t);
1065
1066                                         idata=(t->tm_year+1900)*10000+(t->tm_mon+1)*100+t->tm_mday;
1067                                 }
1068                         }
1069                         if (ilf==ILF_Sarg) {
1070                                 getword_start(&gwarea,linebuf);
1071                                 if (getword(data,sizeof(data),&gwarea,'\t')<0){
1072                                         debuga(_("Maybe you have a broken record or garbage in your %s file\n"),arq);
1073                                         exit(EXIT_FAILURE);
1074                                 }
1075                                 if (getword(hora,sizeof(hora),&gwarea,'\t')<0) {
1076                                         debuga(_("Maybe you have a broken record or garbage in your %s file\n"),arq);
1077                                         exit(EXIT_FAILURE);
1078                                 }
1079                                 if (getword(user,sizeof(user),&gwarea,'\t')<0) {
1080                                         debuga(_("Maybe you have a broken record or garbage in your %s file\n"),arq);
1081                                         exit(EXIT_FAILURE);
1082                                 }
1083                                 if (getword(ip,sizeof(ip),&gwarea,'\t')<0) {
1084                                         debuga(_("Maybe you have a broken record or garbage in your %s file\n"),arq);
1085                                         exit(EXIT_FAILURE);
1086                                 }
1087                                 if (getword_ptr(linebuf,&full_url,&gwarea,'\t')<0){
1088                                         debuga(_("Maybe you have a broken record or garbage in your %s file\n"),arq);
1089                                         exit(EXIT_FAILURE);
1090                                 }
1091                                 if (getword(tam,sizeof(tam),&gwarea,'\t')<0){
1092                                         debuga(_("Maybe you have a broken record or garbage in your %s file\n"),arq);
1093                                         exit(EXIT_FAILURE);
1094                                 }
1095                                 if (getword(code,sizeof(code),&gwarea,'\t')<0){
1096                                         debuga(_("Maybe you have a broken record or garbage in your %s file\n"),arq);
1097                                         exit(EXIT_FAILURE);
1098                                 }
1099                                 if (getword(elap,sizeof(elap),&gwarea,'\t')<0){
1100                                         debuga(_("Maybe you have a broken record or garbage in your %s file\n"),arq);
1101                                         exit(EXIT_FAILURE);
1102                                 }
1103                                 if (getword(smartfilter,sizeof(smartfilter),&gwarea,'\0')<0){
1104                                         debuga(_("Maybe you have a broken record or garbage in your %s file\n"),arq);
1105                                         exit(EXIT_FAILURE);
1106                                 }
1107                                 getword_start(&gwarea,data);
1108                                 if (getword_atoll(&iday,&gwarea,'/')<0 || iday<1 || iday>31){
1109                                         debuga(_("Maybe you have a broken date in your %s file\n"),arq);
1110                                         exit(EXIT_FAILURE);
1111                                 }
1112                                 if (getword_atoll(&imonth,&gwarea,'/')<0 || imonth<1 || imonth>12){
1113                                         debuga(_("Maybe you have a broken date in your %s file\n"),arq);
1114                                         exit(EXIT_FAILURE);
1115                                 }
1116                                 if (getword_atoll(&iyear,&gwarea,'\0')<0){
1117                                         debuga(_("Maybe you have a broken date in your %s file\n"),arq);
1118                                         exit(EXIT_FAILURE);
1119                                 }
1120                                 idata=builddia(iday,imonth,iyear);
1121                                 computedate(iyear,imonth,iday,&tt);
1122                                 if (sscanf(hora,"%d:%d:%d",&tt.tm_hour,&tt.tm_min,&tt.tm_sec)!=3 || tt.tm_hour<0 || tt.tm_hour>=24 ||
1123                                                 tt.tm_min<0 || tt.tm_min>=60 || tt.tm_sec<0 || tt.tm_sec>=60) {
1124                                         debuga(_("Invalid time found in %s\n"),arq);
1125                                         exit(EXIT_FAILURE);
1126                                 }
1127                                 t=&tt;
1128                         }
1129                         if (ilf==ILF_Isa) {
1130                                 if (linebuf[0] == '#') {
1131                                         int ncols,cols[ISACOL_Last];
1132
1133                                         fixendofline(linebuf);
1134                                         getword_start(&gwarea,linebuf);
1135                                         // remove the #Fields: column at the beginning of the line
1136                                         if (getword_skip(1000,&gwarea,' ')<0){
1137                                                 debuga(_("Maybe you have a broken record or garbage in your %s file\n"),arq);
1138                                                 exit(EXIT_FAILURE);
1139                                         }
1140                                         for (ncols=0 ; ncols<ISACOL_Last ; ncols++) cols[ncols]=-1;
1141                                         ncols=0;
1142                                         while(gwarea.current[0] != '\0') {
1143                                                 if (getword(val1,sizeof(val1),&gwarea,'\t')<0){
1144                                                         debuga(_("Maybe you have a broken record or garbage in your %s file\n"),arq);
1145                                                         exit(EXIT_FAILURE);
1146                                                 }
1147                                                 if(strcmp(val1,"c-ip") == 0) cols[ISACOL_Ip]=ncols;
1148                                                 if(strcmp(val1,"cs-username") == 0) cols[ISACOL_UserName]=ncols;
1149                                                 if(strcmp(val1,"date") == 0) cols[ISACOL_Date]=ncols;
1150                                                 if(strcmp(val1,"time") == 0) cols[ISACOL_Time]=ncols;
1151                                                 if(strcmp(val1,"time-taken") == 0) cols[ISACOL_TimeTaken]=ncols;
1152                                                 if(strcmp(val1,"sc-bytes") == 0) cols[ISACOL_Bytes]=ncols;
1153                                                 if(strcmp(val1,"cs-uri") == 0) cols[ISACOL_Uri]=ncols;
1154                                                 if(strcmp(val1,"sc-status") == 0) cols[ISACOL_Status]=ncols;
1155                                                 ncols++;
1156                                         }
1157                                         if (cols[ISACOL_Ip]>=0) {
1158                                                 isa_ncols=ncols;
1159                                                 for (ncols=0 ; ncols<ISACOL_Last ; ncols++)
1160                                                         isa_cols[ncols]=cols[ncols];
1161                                         }
1162                                         continue;
1163                                 }
1164                                 if (!isa_ncols) continue;
1165                                 getword_start(&gwarea,linebuf);
1166                                 for (x=0 ; x<isa_ncols ; x++) {
1167                                         if (getword_ptr(linebuf,&str,&gwarea,'\t')<0) {
1168                                                 debuga(_("Maybe you have a broken record or garbage in your %s file\n"),arq);
1169                                                 exit(EXIT_FAILURE);
1170                                         }
1171                                         if (x==isa_cols[ISACOL_Ip]) {
1172                                                 if (strlen(str)>=sizeof(ip)) {
1173                                                         debuga(_("Maybe you have a broken IP in your %s file\n"),arq);
1174                                                         exit(EXIT_FAILURE);
1175                                                 }
1176                                                 strcpy(ip,str);
1177                                         } else if (x==isa_cols[ISACOL_UserName]) {
1178                                                 if (strlen(str)>=sizeof(user)) {
1179                                                         debuga(_("Maybe you have a broken user ID in your %s file\n"),arq);
1180                                                         exit(EXIT_FAILURE);
1181                                                 }
1182                                                 strcpy(user,str);
1183                                         } else if (x==isa_cols[ISACOL_Date]) {
1184                                                 if (strlen(str)>=sizeof(data)) {
1185                                                         debuga(_("Maybe you have a broken date in your %s file\n"),arq);
1186                                                         exit(EXIT_FAILURE);
1187                                                 }
1188                                                 strcpy(data,str);
1189                                         } else if (x==isa_cols[ISACOL_Time]) {
1190                                                 if (strlen(str)>=sizeof(hora)) {
1191                                                         debuga(_("Maybe you have a broken time in your %s file\n"),arq);
1192                                                         exit(EXIT_FAILURE);
1193                                                 }
1194                                                 strcpy(hora,str);
1195                                         } else if (x==isa_cols[ISACOL_TimeTaken]) {
1196                                                 if (strlen(str)>=sizeof(elap)) {
1197                                                         debuga(_("Maybe you have a broken download duration in your %s file\n"),arq);
1198                                                         exit(EXIT_FAILURE);
1199                                                 }
1200                                                 strcpy(elap,str);
1201                                         } else if (x==isa_cols[ISACOL_Bytes]) {
1202                                                 if (strlen(str)>=sizeof(tam)) {
1203                                                         debuga(_("Maybe you have a broken download size in your %s file\n"),arq);
1204                                                         exit(EXIT_FAILURE);
1205                                                 }
1206                                                 strcpy(tam,str);
1207                                         } else if (x==isa_cols[ISACOL_Uri]) {
1208                                                 full_url=str;
1209                                         } else if (x==isa_cols[ISACOL_Status]) {
1210                                                 if (strlen(str)>=sizeof(code)) {
1211                                                         debuga(_("Maybe you have a broken access code in your %s file\n"),arq);
1212                                                         exit(EXIT_FAILURE);
1213                                                 }
1214                                                 strcpy(code,str);
1215                                         }
1216                                 }
1217
1218                                 if(strcmp(code,"401") == 0 || strcmp(code,"403") == 0 || strcmp(code,"407") == 0) {
1219                                         sprintf(val1,"DENIED/%s",code);
1220                                         strcpy(code,val1);
1221                                 }
1222                                 getword_start(&gwarea,data);
1223                                 if (getword_atoll(&iyear,&gwarea,'-')<0){
1224                                         debuga(_("Maybe you have a broken year in your %s file\n"),arq);
1225                                         exit(EXIT_FAILURE);
1226                                 }
1227                                 if (getword_atoll(&imonth,&gwarea,'-')<0){
1228                                         debuga(_("Maybe you have a broken month in your %s file\n"),arq);
1229                                         exit(EXIT_FAILURE);
1230                                 }
1231                                 if (getword_atoll(&iday,&gwarea,'\0')<0){
1232                                         debuga(_("Maybe you have a broken day in your %s file\n"),arq);
1233                                         exit(EXIT_FAILURE);
1234                                 }
1235
1236                                 idata=builddia(iday,imonth,iyear);
1237                                 computedate(iyear,imonth,iday,&tt);
1238                                 if (isa_cols[ISACOL_Time]>=0) {
1239                                         if (sscanf(hora,"%d:%d:%d",&tt.tm_hour,&tt.tm_min,&tt.tm_sec)!=3 || tt.tm_hour<0 || tt.tm_hour>=24 ||
1240                                                         tt.tm_min<0 || tt.tm_min>=60 || tt.tm_sec<0 || tt.tm_sec>=60) {
1241                                                 debuga(_("Invalid time found in %s\n"),arq);
1242                                                 exit(EXIT_FAILURE);
1243                                         }
1244                                 }
1245                                 t=&tt;
1246                         }
1247                         if (t==NULL) {
1248                                 debuga(_("Unknown input log file format\n"));
1249                                 break;
1250                         }
1251
1252                         strftime(dia, sizeof(dia), "%d/%m/%Y", t);
1253                         snprintf(hora,sizeof(hora),"%02d:%02d:%02d",t->tm_hour,t->tm_min,t->tm_sec);
1254
1255                         if(debugm)
1256                                 printf("DATE=%s IDATA=%d DFROM=%d DUNTIL=%d\n",date,idata,dfrom,duntil);
1257
1258                         if(date[0] != '\0'){
1259                                 if(idata < dfrom || idata > duntil) continue;
1260                         }
1261
1262                         // Record only hours usage which is required
1263                         if (t) {
1264                                 if( bsearch( &( t -> tm_wday ), weekdays.list, weekdays.len, sizeof( int ), compar ) == NULL )
1265                                         continue;
1266
1267                                 if( bsearch( &( t -> tm_hour ), hours.list, hours.len, sizeof( int ), compar ) == NULL )
1268                                         continue;
1269                         }
1270
1271
1272                         if(strlen(user) > MAX_USER_LEN) {
1273                                 if (debugm) printf(_("User ID too long: %s\n"),user);
1274                                 totregsx++;
1275                                 continue;
1276                         }
1277
1278                         // include_users
1279                         if(IncludeUsers[0] != '\0') {
1280                                 snprintf(val1,sizeof(val1),":%s:",user);
1281                                 if((str=(char *) strstr(IncludeUsers,val1)) == (char *) NULL )
1282                                         continue;
1283                         }
1284
1285                         if(vercode(code)) {
1286                                 if (debugm) printf(_("Excluded code: %s\n"),code);
1287                                 totregsx++;
1288                                 continue;
1289                         }
1290
1291                         if(testvaliduserchar(user))
1292                                 continue;
1293
1294 #if 0
1295                         if((str = strstr(user,"%20")) != NULL) {
1296                                 /*
1297                                 This is a patch introduced to solve bug #1624251 reported at sourceforge but
1298                                 the side effect is to truncate the name at the first space and merge the reports
1299                                 of people whose name is identical up to the first space.
1300
1301                                 The old code used to truncate the user name at the first % if a %20 was
1302                                 found anywhere in the string. That means the string could be truncated
1303                                 at the wrong place if another % occured before the %20. This new code should
1304                                 avoid that problem and only truncate at the space. There is no bug
1305                                 report indicating that anybody noticed this.
1306                                 */
1307                                 *str='\0';
1308                         }
1309
1310                         /*
1311                         Code prior to 2.2.7 used to replace any %xx by a dot as long as a %5c was
1312                         found in the user name.
1313                         */
1314                         while((str = strstr(user,"%5c")) != NULL) {
1315                                 *str='.';
1316                                 for (x=3 ; str[x] ; x++) str[x-2]=str[x];
1317                         }
1318 #endif
1319                         // replace any tab by a single space
1320                         for (str=full_url ; *str ; str++)
1321                                 if (*str=='\t') *str=' ';
1322                         for (str=code ; *str ; str++)
1323                                 if (*str=='\t') *str=' ';
1324
1325                         urly=full_url;
1326
1327                         if(ilf!=ILF_Sarg) {
1328                                 /*
1329                                 The full URL is not saved in sarg log. There is no point in testing the URL to detect
1330                                 a downloaded file.
1331                                 */
1332                                 download_flag=is_download_suffix(full_url);
1333                                 if (download_flag) {
1334                                         download_url=full_url;
1335                                         download_count++;
1336                                 }
1337                         } else
1338                                 download_flag=false;
1339
1340                         url=process_url(full_url,LongUrl);
1341                         if (!url || url[0] == '\0') continue;
1342
1343                         if(addr[0] != '\0'){
1344                                 if(strcmp(addr,ip)!=0) continue;
1345                         }
1346                         if(fhost) {
1347                                 if(!vhexclude(url)) {
1348                                         if (debugm) printf(_("Excluded site: %s\n"),url);
1349                                         totregsx++;
1350                                         continue;
1351                                 }
1352                         }
1353
1354                         if(hm >= 0 && hmf >= 0) {
1355                                 hmr=t->tm_hour*100+t->tm_min;
1356                                 if(hmr < hm || hmr > hmf) continue;
1357                         }
1358
1359                         if(site[0] != '\0'){
1360                                 if(strstr(url,site)==0) continue;
1361                         }
1362
1363                         if(UserIp) {
1364                                 strcpy(user,ip);
1365                                 id_is_ip=true;
1366                         } else {
1367                                 id_is_ip=false;
1368                                 if(strcmp(user,"-") == 0 || strcmp(user," ") == 0 || strcmp(user,"") == 0) {
1369                                         if(RecordsWithoutUser == RECORDWITHOUTUSER_IP) {
1370                                                 strcpy(user,ip);
1371                                                 id_is_ip=true;
1372                                         }
1373                                         if(RecordsWithoutUser == RECORDWITHOUTUSER_IGNORE)
1374                                                 continue;
1375                                         if(RecordsWithoutUser == RECORDWITHOUTUSER_EVERYBODY)
1376                                                 strcpy(user,"everybody");
1377                                 } else {
1378                                         strlow(user);
1379                                         if(NtlmUserFormat == NTLMUSERFORMAT_USER) {
1380                                                 if((str = strchr(user,'_')) != 0) {
1381                                                         strcpy(warea,str+1);
1382                                                         strcpy(user,warea);
1383                                                 }
1384                                                 if((str = strchr(user,'+')) != 0) {
1385                                                         strcpy(warea,str+1);
1386                                                         strcpy(user,warea);
1387                                                 }
1388                                         }
1389                                 }
1390                         }
1391
1392                         if(us[0] != '\0'){
1393                                 if(strcmp(user,us)!=0) continue;
1394                         }
1395
1396                         if(puser) {
1397                                 snprintf(wuser,sizeof(wuser),":%s:",user);
1398                                 if(strstr(userfile, wuser) == 0)
1399                                         continue;
1400                         }
1401
1402                         if(fuser) {
1403                                 if(!vuexclude(user)) {
1404                                         if (debugm) printf(_("Excluded user: %s\n"),user);
1405                                         totregsx++;
1406                                         continue;
1407                                 }
1408                         }
1409
1410                         if(strcmp(user,"-") ==0 || strcmp(user," ") ==0 || strcmp(user,"") ==0 || strcmp(user,":") ==0)
1411                                 continue;
1412
1413                         if(max_elapsed) {
1414                                 if(atol(elap)>max_elapsed) {
1415                                         elap[0]='0';
1416                                         elap[1]='\0';
1417                                 }
1418                         }
1419
1420                         if((str=(char *) strstr(linebuf, "[SmartFilter:")) != (char *) NULL ) {
1421                                 fixendofline(str);
1422                                 snprintf(smartfilter,sizeof(smartfilter),"\"%s\"",str+1);
1423                         } else strcpy(smartfilter,"\"\"");
1424
1425                         nopen=0;
1426                         prev_ufile=NULL;
1427                         for (ufile=first_user_file ; ufile && strcmp(user,ufile->user->id)!=0 ; ufile=ufile->next) {
1428                                 prev_ufile=ufile;
1429                                 if (ufile->file) nopen++;
1430                         }
1431                         if (!ufile) {
1432                                 ufile=malloc(sizeof(*ufile));
1433                                 if (!ufile) {
1434                                         debuga(_("Not enough memory to store the user %s\n"),user);
1435                                         exit(EXIT_FAILURE);
1436                                 }
1437                                 memset(ufile,0,sizeof(*ufile));
1438                                 ufile->next=first_user_file;
1439                                 first_user_file=ufile;
1440                                 uinfo=userinfo_create(user);
1441                                 ufile->user=uinfo;
1442                                 uinfo->id_is_ip=id_is_ip;
1443                         } else {
1444                                 if (prev_ufile) {
1445                                         prev_ufile->next=ufile->next;
1446                                         ufile->next=first_user_file;
1447                                         first_user_file=ufile;
1448                                 }
1449                         }
1450
1451                         if (ufile->file==NULL) {
1452                                 if (nopen>=maxopenfiles) {
1453                                         x=0;
1454                                         for (ufile1=first_user_file ; ufile1 ; ufile1=ufile1->next) {
1455                                                 if (ufile1->file!=NULL) {
1456                                                         if (x>=maxopenfiles) {
1457                                                                 if (fclose(ufile1->file)==EOF) {
1458                                                                         debuga(_("Failed to close the log file of user %s - %s\n"),ufile1->user->id,strerror(errno));
1459                                                                         exit(EXIT_FAILURE);
1460                                                                 }
1461                                                                 ufile1->file=NULL;
1462                                                         }
1463                                                         x++;
1464                                                 }
1465                                         }
1466                                 }
1467                                 if (snprintf (tmp3, sizeof(tmp3), "%s/%s.unsort", tmp, ufile->user->filename)>=sizeof(tmp3)) {
1468                                         debuga(_("Temporary user file name too long: %s/%s.unsort\n"), tmp, ufile->user->filename);
1469                                         exit(EXIT_FAILURE);
1470                                 }
1471                                 if ((ufile->file = MY_FOPEN (tmp3, "a")) == NULL) {
1472                                         debuga(_("(log) Cannot open temporary file: %s - %s\n"), tmp3, strerror(errno));
1473                                         exit (1);
1474                                 }
1475                         }
1476
1477                         /*if ( strcmp ( user , sz_Last_User ) != 0 ) {
1478                                 if ( fp_Write_User )
1479                                         fclose( fp_Write_User ) ;
1480                                 sprintf (tmp3, "%s/%s.unsort", tmp, user);
1481
1482                                 if ((fp_Write_User = MY_FOPEN (tmp3, "a")) == NULL) {
1483                                         fprintf (stderr, "%s: (log) %s: %s - %s\n", argv[0], _("Cannot open temporary file"), tmp3, strerror(errno));
1484                                         exit (1);
1485                                 }
1486                                 strcpy( sz_Last_User , user ) ;
1487                         }*/
1488                         if (fprintf(ufile->file, "%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\n",dia,hora,ip,url,tam,code,elap,smartfilter)<=0) {
1489                                 debuga(_("Write error in the log file of user %s\n"),user);
1490                                 exit(EXIT_FAILURE);
1491                         }
1492
1493                         if(fp_log && ilf!=ILF_Sarg)
1494                                 fprintf(fp_log, "%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\n",dia,hora,user,ip,url,tam,code,elap,smartfilter);
1495
1496                         totregsg++;
1497
1498                         if(!dataonly && download_flag && download_url && strstr(code,"DENIED") == 0) {
1499                                 ndownload = 1;
1500
1501                                 if ( ! fp_Download_Unsort ) {
1502                                         if ((fp_Download_Unsort = MY_FOPEN ( sz_Download_Unsort, "a")) == NULL) {
1503                                                 debuga(_("(log) Cannot open temporary file: %s - %s\n"),sz_Download_Unsort, strerror(errno));
1504                                                 exit (1);
1505                                         }
1506                                 }
1507                                 fprintf(fp_Download_Unsort,"%s\t%s\t%s\t%s\t%s\n",dia,hora,user,ip,download_url);
1508                         }
1509
1510                         if((ReportType & REPORT_TYPE_DENIED) != 0) {
1511                                 if(fp_denied && strstr(code,"DENIED/403") != 0) {
1512                                         fprintf(fp_denied, "%s\t%s\t%s\t%s\t%s\n",dia,hora,user,ip,urly);
1513                                         denied_count++;
1514                                 }
1515                         }
1516                         if((ReportType & REPORT_TYPE_AUTH_FAILURES) != 0) {
1517                                 if(fp_authfail && (strstr(code,"DENIED/401") != 0 || strstr(code,"DENIED/407") != 0)) {
1518                                         fprintf(fp_authfail, "%s\t%s\t%s\t%s\t%s\n",dia,hora,user,ip,urly);
1519                                         authfail_count++;
1520                                 }
1521                         }
1522
1523                         if (ilf!=ILF_Sarg) {
1524                                 if(!totper || idata<mindate){
1525                                         mindate=idata;
1526                                         memcpy(&period.start,t,sizeof(*t));
1527                                         strcpy(start_hour,tbuf2);
1528                                 }
1529                                 if (!totper || idata>maxdate) {
1530                                         maxdate=idata;
1531                                         memcpy(&period.end,t,sizeof(*t));
1532                                 }
1533                                 totper=true;
1534                         }
1535
1536                         if(debugm){
1537                                 printf("IP=\t%s\n",ip);
1538                                 printf("USER=\t%s\n",user);
1539                                 printf("ELAP=\t%s\n",elap);
1540                                 printf("DATE=\t%s\n",dia);
1541                                 printf("TIME=\t%s\n",hora);
1542                                 printf("FUNC=\t%s\n",fun);
1543                                 printf("URL=\t%s\n",url);
1544                                 printf("CODE=\t%s\n",code);
1545                                 printf("LEN=\t%s\n",tam);
1546                         }
1547                 }
1548                 if (!from_stdin) {
1549                         if (from_pipe)
1550                                 pclose(fp_in);
1551                         else {
1552                                 fclose(fp_in);
1553                                 if( ShowReadStatistics )
1554                                         printf(_("SARG: Records in file: %lu, reading: %3.2f%%\n"),recs1, (float) 100 );
1555                         }
1556                 }
1557         }
1558
1559         if (debug)
1560                 debuga(_("   Records read: %ld, written: %ld, excluded: %ld\n"),totregsl,totregsg,totregsx);
1561
1562         longline_destroy(&line);
1563         if ( fp_Download_Unsort )
1564                 fclose (fp_Download_Unsort);
1565
1566         for (ufile=first_user_file ; ufile ; ufile=ufile1) {
1567                 ufile1=ufile->next;
1568                 if (ufile->file!=NULL) fclose(ufile->file);
1569                 free(ufile);
1570         }
1571
1572         free_download();
1573         free_excludecodes();
1574         free_exclude();
1575
1576         if(debug) {
1577                 int totalcount=0;
1578
1579                 for (ilf=0 ; ilf<ILF_Last ; ilf++) totalcount+=ilf_count[ilf];
1580
1581                 if(ilf_count[ILF_Common]>0 && ilf_count[ILF_Squid]>0)
1582                         debuga(_("Log with mixed records format (squid and common log)\n"));
1583
1584                 if(ilf_count[ILF_Common]>0 && ilf_count[ILF_Squid]==0)
1585                         debuga(_("Common log format\n"));
1586
1587                 if(ilf_count[ILF_Common]==0 && ilf_count[ILF_Squid]>0)
1588                         debuga(_("Squid log format\n"));
1589
1590                 if(ilf_count[ILF_Sarg]>0)
1591                         debuga(_("Sarg log format\n"));
1592
1593                 if(totalcount==0 && totregsg)
1594                         debuga(_("Log with invalid format\n"));
1595         }
1596
1597         if(!totregsg){
1598                 debuga(_("No records found\n"));
1599                 debuga(_("End\n"));
1600                 if(fp_denied) fclose(fp_denied);
1601                 if(fp_authfail) fclose(fp_authfail);
1602                 userinfo_free();
1603                 if(userfile) free(userfile);
1604                 close_usertab();
1605                 exit(EXIT_SUCCESS);
1606         }
1607
1608         if (date[0]!='\0') {
1609                 char date0[30], date1[30];
1610
1611                 strftime(date0,sizeof(date0),"%d/%m/%Y",&period.start);
1612                 strftime(date1,sizeof(date1),"%d/%m/%Y",&period.end);
1613                 debuga(_("Period covered by log files: %s-%s\n"),date0,date1);
1614                 getperiod_fromrange(&period,dfrom,duntil);
1615         }
1616         if (getperiod_buildtext(&period)<0) {
1617                 debuga(_("Failed to build the string representation of the date range\n"));
1618                 exit(EXIT_FAILURE);
1619         }
1620
1621         if(debugz){
1622                 debugaz(_("date=%s\n"),dia);
1623                 debugaz(_("period=%s\n"),period.text);
1624         }
1625
1626         if(debug)
1627                 debuga(_("Period: %s\n"),period.text);
1628
1629         if(fp_denied)
1630                 fclose(fp_denied);
1631         if(fp_authfail)
1632                 fclose(fp_authfail);
1633
1634         if(fp_log != NULL) {
1635                 char end_hour[128];
1636                 char val2[40];
1637                 char val4[255];//val4 must not be bigger than arq_log without fixing the strcpy below
1638
1639                 fclose(fp_log);
1640                 safe_strcpy(end_hour,tbuf2,sizeof(end_hour));
1641                 strftime(val2,sizeof(val2),"%d%m%Y",&period.start);
1642                 strftime(val1,sizeof(val1),"%d%m%Y",&period.end);
1643                 if (snprintf(val4,sizeof(val4),"%s/sarg-%s_%s-%s_%s.log",ParsedOutputLog,val2,start_hour,val1,end_hour)>=sizeof(val4)) {
1644                         debuga(_("File name too long: %s/sarg-%s_%s-%s_%s.log\n"),ParsedOutputLog,val2,start_hour,val1,end_hour);
1645                         exit(EXIT_FAILURE);
1646                 }
1647                 if (rename(arq_log,val4)) {
1648                         debuga(_("failed to rename %s to %s - %s\n"),arq_log,val4,strerror(errno));
1649                 } else {
1650                         strcpy(arq_log,val4);
1651
1652                         if(strcmp(ParsedOutputLogCompress,"nocompress") != 0 && ParsedOutputLogCompress[0] != '\0') {
1653                                 /*
1654                                 No double quotes around ParsedOutputLogCompress because it may contain command line options. If double quotes are
1655                                 necessary around the command name, put them in the configuration file.
1656                                 */
1657                                 if (snprintf(val1,sizeof(val1),"%s \"%s\"",ParsedOutputLogCompress,arq_log)>=sizeof(val1)) {
1658                                         debuga(_("Command too long: %s \"%s\"\n"),ParsedOutputLogCompress,arq_log);
1659                                         exit(EXIT_FAILURE);
1660                                 }
1661                                 cstatus=system(val1);
1662                                 if (!WIFEXITED(cstatus) || WEXITSTATUS(cstatus)) {
1663                                         debuga(_("command return status %d\n"),WEXITSTATUS(cstatus));
1664                                         debuga(_("command: %s\n"),val1);
1665                                         exit(EXIT_FAILURE);
1666                                 }
1667                         }
1668                 }
1669                 if(debug)
1670                         debuga(_("Sarg parsed log saved as %s\n"),arq_log);
1671         }
1672
1673         if(DataFile[0] == '\0' && (ReportType & REPORT_TYPE_DENIED) != 0) {
1674                 if (snprintf(csort,sizeof(csort),"sort -T \"%s\" -t \"\t\" -k 3,3 -k 5,5 -o \"%s\" \"%s\"",tmp,denied_sort,denied_unsort)>=sizeof(csort)) {
1675                         debuga(_("Sort command too long when sorting file \"%s\" to \"%s\"\n"),denied_unsort,denied_sort);
1676                         exit(EXIT_FAILURE);
1677                 }
1678                 cstatus=system(csort);
1679                 if (!WIFEXITED(cstatus) || WEXITSTATUS(cstatus)) {
1680                         debuga(_("sort command return status %d\n"),WEXITSTATUS(cstatus));
1681                         debuga(_("sort command: %s\n"),csort);
1682                         exit(EXIT_FAILURE);
1683                 }
1684                 if (unlink(denied_unsort)) {
1685                         debuga(_("Cannot delete %s - %s\n"),denied_unsort,strerror(errno));
1686                         exit(EXIT_FAILURE);
1687                 }
1688         }
1689
1690         sort_users_log(tmp, debug);
1691
1692         if(DataFile[0] != '\0')
1693                 data_file(tmp);
1694         else
1695                 gerarel();
1696
1697         if((ReportType & REPORT_TYPE_DENIED) != 0) {
1698                 unlink(denied_sort);
1699         }
1700
1701         if(strcmp(tmp,"/tmp") != 0) {
1702                 unlinkdir(tmp,0);
1703         }
1704
1705         free_hostalias();
1706         userinfo_free();
1707         if(userfile)
1708                 free(userfile);
1709         close_usertab();
1710
1711         if(debug)
1712                 debuga(_("End\n"));
1713
1714         exit(EXIT_SUCCESS);
1715 }
1716
1717
1718 static void getusers(const char *pwdfile, int debug)
1719 {
1720         FILE *fp_usr;
1721         char buf[255];
1722         char *str;
1723         long int nreg=0;
1724
1725         if(debug)
1726                 debuga(_("Loading password file from %s\n"),pwdfile);
1727
1728         if ((fp_usr = fopen(pwdfile, "r")) == NULL) {
1729                 debuga(_("(getusers) Cannot open file %s - %s\n"),pwdfile,strerror(errno));
1730                 exit(EXIT_FAILURE);
1731         }
1732
1733         if (fseek(fp_usr, 0, SEEK_END)==-1) {
1734                 debuga(_("Failed to move till the end of the users file %s: %s\n"),pwdfile,strerror(errno));
1735                 exit(EXIT_FAILURE);
1736         }
1737         nreg = ftell(fp_usr);
1738         if (nreg<0) {
1739                 debuga(_("Cannot get the size of file %s\n"),pwdfile);
1740                 exit(EXIT_FAILURE);
1741         }
1742         nreg = nreg+5000;
1743         if (fseek(fp_usr, 0, SEEK_SET)==-1) {
1744                 debuga(_("Failed to rewind the users file %s: %s\n"),pwdfile,strerror(errno));
1745                 exit(EXIT_FAILURE);
1746         }
1747
1748         if((userfile=(char *) malloc(nreg))==NULL){
1749                 debuga(_("malloc error (%ld)\n"),nreg);
1750                 exit(EXIT_FAILURE);
1751         }
1752
1753         bzero(userfile,nreg);
1754         strcpy(userfile,":");
1755
1756         while(fgets(buf,sizeof(buf),fp_usr)!=NULL) {
1757                 str=strchr(buf,':');
1758                 if (!str) {
1759                         debuga(_("You have an invalid user in your %s file\n"),pwdfile);
1760                         exit(EXIT_FAILURE);
1761                 }
1762                 str[1]='\0';
1763                 strcat(userfile,buf);
1764         }
1765
1766         fclose(fp_usr);
1767
1768         return;
1769 }