log.c

   1 /*
   2  * SARG Squid Analysis Report Generator      http://sarg.sourceforge.net
   3  *                                                            1998, 2011
   4  *
   5  * SARG donations:
   6  *      please look at http://sarg.sourceforge.net/donations.php
   7  * Support:
   8  *     http://sourceforge.net/projects/sarg/forums/forum/363374
   9  * ---------------------------------------------------------------------
  10  *
  11  *  This program is free software; you can redistribute it and/or modify
  12  *  it under the terms of the GNU General Public License as published by
  13  *  the Free Software Foundation; either version 2 of the License, or
  14  *  (at your option) any later version.
  15  *
  16  *  This program is distributed in the hope that it will be useful,
  17  *  but WITHOUT ANY WARRANTY; without even the implied warranty of
  18  *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  19  *  GNU General Public License for more details.
  20  *
  21  *  You should have received a copy of the GNU General Public License
  22  *  along with this program; if not, write to the Free Software
  23  *  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111, USA.
  24  *
  25  */
  26
  27 #include "include/conf.h"
  28 #include "include/defs.h"
  29
  30 #ifdef HAVE_GETOPT_H
  31 #include <getopt.h>
  32 #endif
  33
  34 #define REPORT_EVERY_X_LINES 5000
  35 #define MAX_OPEN_USER_FILES 10
  36
  37 struct userfilestruct
  38 {
  39         struct userfilestruct *next;
  40         struct userinfostruct *user;
  41         FILE *file;
  42 };
  43
  44 /*@null@*/static char *userfile=NULL;
  45
  46 numlist weekdays = { { 0, 1, 2, 3, 4, 5, 6 }, 7 };
  47 numlist hours = { { 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23 }, 24 };
  48
  49 static void getusers(const char *pwdfile, int debug);
  50
  51 int main(int argc,char *argv[])
  52 {
  53         enum isa_col_id {
  54                 ISACOL_Ip,
  55                 ISACOL_UserName,
  56                 ISACOL_Date,
  57                 ISACOL_Time,
  58                 ISACOL_TimeTaken,
  59                 ISACOL_Bytes,
  60                 ISACOL_Uri,
  61                 ISACOL_Status,
  62                 ISACOL_Last //last entry of the list !
  63         };
  64         enum InputLogFormat {
  65                 ILF_Unknown,
  66                 ILF_Squid,
  67                 ILF_Common,
  68                 ILF_Sarg,
  69                 ILF_Isa,
  70                 ILF_Last //last entry of the list !
  71         };
  72
  73         FILE *fp_in = NULL, *fp_denied=NULL, *fp_authfail=NULL, *fp_log=NULL;
  74
  75         char sz_Download_Unsort[ 20000 ] ;
  76         FILE * fp_Download_Unsort = NULL ;
  77
  78         extern int optind;
  79         extern int optopt;
  80         extern char *optarg;
  81
  82         char data[255];
  83         char elap[255];
  84         char ip[MAXLEN];
  85         char tam[255];
  86         char fun[MAXLEN];
  87         char wuser[MAXLEN];
  88         char smartfilter[MAXLEN];
  89         char dia[128];
  90         char mes[30];
  91         char ano[30];
  92         char hora[30];
  93         char wtemp[MAXLEN];
  94         char date[255];
  95         char arq[255];
  96         char arq_log[255];
  97         int hm, hmf, hmr;
  98         char hm_str[15];
  99         char uagent[MAXLEN];
 100         char hexclude[MAXLEN];
 101         char csort[MAXLEN];
 102         int cstatus;
 103         char tbuf2[128];
 104         char zip[20];
 105         char *str;
 106         char tmp3[MAXLEN];
 107         char denied_unsort[MAXLEN];
 108         char denied_sort[MAXLEN];
 109         char authfail_unsort[MAXLEN];
 110         char start_hour[128];
 111         char end_hour[128];
 112         char *linebuf;
 113         char hostname[512];
 114         char *url;
 115         char *urly;
 116         char user[MAX_USER_LEN];
 117         char splitprefix[MAXLEN];
 118         enum InputLogFormat ilf;
 119         int ilf_count[ILF_Last];
 120         int  ch;
 121         int  x;
 122         int  errflg=0;
 123         int  puser=0;
 124         bool  fhost=false;
 125         bool  dns=false;
 126         bool  fuser=false;
 127         int  idata=0;
 128         int  mindate=0;
 129         int  maxdate=0;
 130         int  iarq=0;
 131         int isa_ncols=0,isa_cols[ISACOL_Last];
 132         int lastlog=-1;
 133         bool from_stdin;
 134         bool from_pipe;
 135         int blen;
 136         int maxopenfiles;
 137         int nopen;
 138         bool id_is_ip;
 139         long totregsl=0;
 140         long totregsg=0;
 141         long totregsx=0;
 142         bool totper=false;
 143         long int max_elapsed=0;
 144         long long int iyear, imonth, iday;
 145         bool realt;
 146         bool userip;
 147         struct tm tt;
 148         struct tm *t;
 149         unsigned long recs1=0UL;
 150         unsigned long recs2=0UL;
 151         int OutputNonZero = REPORT_EVERY_X_LINES ;
 152         bool download_flag=false;
 153         char *download_url=NULL;
 154         struct getwordstruct gwarea;
 155         longline line;
 156         time_t tnum;
 157         struct stat logstat;
 158         struct userinfostruct *uinfo;
 159         struct userfilestruct *first_user_file, *ufile, *ufile1, *prev_ufile;
 160         static int split=0;
 161         static int convert=0;
 162         static int output_css=0;
 163         int option_index;
 164         static struct option long_options[]=
 165         {
 166                 {"convert",no_argument,&convert,1},
 167                 {"css",no_argument,&output_css,1},
 168                 {"lastlog",required_argument,NULL,2},
 169                 {"keeplogs",no_argument,NULL,3},
 170                 {"split",no_argument,&split,1},
 171                 {"splitprefix",required_argument,NULL,'P'},
 172                 {0,0,0,0}
 173         };
 174
 175 #ifdef HAVE_LOCALE_H
 176         setlocale(LC_TIME,"");
 177 #endif
 178
 179 #if defined(ENABLE_NLS) && defined(HAVE_LOCALE_H)
 180         if (!setlocale (LC_ALL, "")) {
 181                 fprintf(stderr,"SARG: Cannot set the locale LC_ALL to the environment variable\n");
 182                 exit(EXIT_FAILURE);
 183         }
 184         if (!bindtextdomain (PACKAGE_NAME, LOCALEDIR)) {
 185                 fprintf(stderr,"SARG: Cannot bind to text domain %s in directory %s (%s)\n",PACKAGE_NAME,LOCALEDIR,strerror(errno));
 186                 exit(EXIT_FAILURE);
 187         }
 188         if (!textdomain (PACKAGE_NAME)) {
 189                 fprintf(stderr,"SARG: Cannot set gettext domain for %s PACKAGE_NAME (%s)\n",PACKAGE_NAME,strerror(errno));
 190                 exit(EXIT_FAILURE);
 191         }
 192 #endif //ENABLE_NLS
 193
 194         BgImage[0]='\0';
 195         LogoImage[0]='\0';
 196         LogoText[0]='\0';
 197         PasswdFile[0]='\0';
 198         OutputEmail[0]='\0';
 199         UserAgentLog[0]='\0';
 200         ExcludeHosts[0]='\0';
 201         ExcludeUsers[0]='\0';
 202         ConfigFile[0]='\0';
 203         code[0]='\0';
 204         LastLog=0;
 205         ReportType=0UL;
 206         UserTabFile[0]='\0';
 207         BlockIt[0]='\0';
 208         ExternalCSSFile[0]='\0';
 209         RedirectorLogFormat[0]='\0';
 210         NRedirectorLogs=0;
 211         for (ilf=0 ; ilf<ILF_Last ; ilf++) ilf_count[ilf]=0;
 212
 213         sprintf(ExcludeCodes,"%s/exclude_codes",SYSCONFDIR);
 214         strcpy(GraphDaysBytesBarColor,"orange");
 215         strcpy(BgColor,"#ffffff");
 216         strcpy(TxColor,"#000000");
 217         strcpy(TxBgColor,"lavender");
 218         strcpy(TiColor,"darkblue");
 219         strcpy(Width,"80");
 220         strcpy(Height,"45");
 221         strcpy(LogoTextColor,"#000000");
 222         strcpy(HeaderColor,"darkblue");
 223         strcpy(HeaderBgColor,"#dddddd");
 224         strcpy(LogoTextColor,"#006699");
 225         strcpy(FontSize,"9px");
 226         strcpy(TempDir,"/tmp");
 227         strcpy(OutputDir,"/var/www/html/squid-reports");
 228         AnonymousOutputFiles=false;
 229         Ip2Name=false;
 230         strcpy(DateFormat,"u");
 231         OverwriteReport=false;
 232         RemoveTempFiles=true;
 233         strcpy(ReplaceIndex,"index.html");
 234         Index=INDEX_YES;
 235         RecordsWithoutUser=RECORDWITHOUTUSER_IP;
 236         UseComma=0;
 237         strcpy(MailUtility,"mailx");
 238         TopSitesNum=100;
 239         TopUsersNum=0;
 240         UserIp=0;
 241         TopuserSort=TOPUSER_SORT_BYTES | TOPUSER_SORT_REVERSE;
 242         UserSort=USER_SORT_BYTES | USER_SORT_REVERSE;
 243         TopsitesSort=TOPSITE_SORT_CONNECT | TOPSITE_SORT_REVERSE;
 244         LongUrl=0;
 245         strcpy(FontFace,"Verdana,Tahoma,Arial");
 246         datetimeby=DATETIME_BYTE;
 247         strcpy(CharSet,"ISO-8859-1");
 248         Privacy=0;
 249         strcpy(PrivacyString,"***.***.***.***");
 250         strcpy(PrivacyStringColor,"blue");
 251         SuccessfulMsg=true;
 252         TopUserFields=TOPUSERFIELDS_NUM | TOPUSERFIELDS_DATE_TIME | TOPUSERFIELDS_USERID | TOPUSERFIELDS_CONNECT |
 253               TOPUSERFIELDS_BYTES | TOPUSERFIELDS_SETYB | TOPUSERFIELDS_IN_CACHE_OUT |
 254               TOPUSERFIELDS_USED_TIME | TOPUSERFIELDS_MILISEC | TOPUSERFIELDS_PTIME |
 255               TOPUSERFIELDS_TOTAL | TOPUSERFIELDS_AVERAGE;
 256         UserReportFields=USERREPORTFIELDS_CONNECT | USERREPORTFIELDS_BYTES | USERREPORTFIELDS_SETYB |
 257               USERREPORTFIELDS_IN_CACHE_OUT | USERREPORTFIELDS_USED_TIME | USERREPORTFIELDS_MILISEC |
 258               USERREPORTFIELDS_PTIME | USERREPORTFIELDS_TOTAL | USERREPORTFIELDS_AVERAGE;
 259         strcpy(DataFileDelimiter,";");
 260         DataFileFields=DATA_FIELD_USER | DATA_FIELD_DATE | DATA_FIELD_TIME | DATA_FIELD_URL | DATA_FIELD_CONNECT |
 261               DATA_FIELD_BYTES | DATA_FIELD_IN_CACHE | DATA_FIELD_OUT_CACHE | DATA_FIELD_ELAPSED;
 262         ShowReadStatistics=true;
 263         strcpy(IndexSortOrder,"D");
 264         ShowSargInfo=true;
 265         ShowSargLogo=true;
 266         ParsedOutputLog[0]='\0';
 267         strcpy(ParsedOutputLogCompress,"/bin/gzip -f");
 268         DisplayedValues=DISPLAY_ABBREV;
 269         strcpy(HeaderFontSize,"9px");
 270         strcpy(TitleFontSize,"11px");
 271         strcpy(AuthUserTemplateFile,"sarg_htaccess");
 272         set_download_suffix("7z,ace,arj,avi,bat,bin,bz2,bzip,cab,com,cpio,dll,doc,dot,exe,gz,iso,lha,lzh,mdb,mov,mp3,mpeg,mpg,mso,nrg,ogg,ppt,rar,rtf,shs,src,sys,tar,tgz,vcd,vob,wma,wmv,zip");
 273         Graphs=true;
 274 #if defined(FONTDIR)
 275         strcpy(GraphFont,FONTDIR"/DejaVuSans.ttf");
 276 #else
 277         GraphFont[0]='\0';
 278 #endif
 279         strcpy(Ulimit,"20000");
 280         NtlmUserFormat=NTLMUSERFORMAT_DOMAINUSER;
 281         IndexTree=INDEX_TREE_FILE;
 282         IndexFields=INDEXFIELDS_DIRSIZE;
 283         strcpy(RealtimeTypes,"GET,PUT,CONNECT");
 284         RealtimeUnauthRec=REALTIME_UNAUTH_REC_SHOW;
 285         RedirectorFilterOutDate=true;
 286         DansguardianFilterOutDate=true;
 287         DataFileUrl=DATAFILEURL_IP;
 288         strcpy(MaxElapsed,"28800000");
 289         BytesInSitesUsersReport=0;
 290         UserAuthentication=0;
 291         strcpy(LDAPHost,"127.0.0.1");
 292         LDAPPort=389;
 293         LDAPProtocolVersion=3;
 294         LDAPBindDN[0]='\0';
 295         LDAPBindPW[0]='\0';
 296         LDAPBaseSearch[0]='\0';
 297         strcpy(LDAPFilterSearch, "(uid=%s)");
 298         strcpy(LDAPTargetAttr, "cn");
 299         SortTableJs[0]='\0';
 300
 301         dia[0]='\0';
 302         mes[0]='\0';
 303         ano[0]='\0';
 304         hora[0]='\0';
 305         tmp[0]='\0';
 306         tmp3[0]='\0';
 307         wtemp[0]='\0';
 308         us[0]='\0';
 309         date[0]='\0';
 310         df[0]='\0';
 311         uagent[0]='\0';
 312         hexclude[0]='\0';
 313         addr[0]='\0';
 314         hm=-1;
 315         hmf=-1;
 316         site[0]='\0';
 317         outdir[0]='\0';
 318         splitprefix[0]='\0';
 319         elap[0]='\0';
 320         email[0]='\0';
 321         zip[0]='\0';
 322         UserInvalidChar[0]='\0';
 323         DataFile[0]='\0';
 324         SquidGuardConf[0]='\0';
 325         DansGuardianConf[0]='\0';
 326         start_hour[0]='\0';
 327         end_hour[0]='\0';
 328         hm_str[0]='\0';
 329
 330         denied_count=0;
 331         download_count=0;
 332         authfail_count=0;
 333         dansguardian_count=0;
 334         squidguard_count=0;
 335         useragent_count=0;
 336         DeniedReportLimit=10;
 337         AuthfailReportLimit=10;
 338         DansGuardianReportLimit=10;
 339         SquidGuardReportLimit=10;
 340         DownloadReportLimit=50;
 341         UserReportLimit=0;
 342         debug=0;
 343         debugz=0;
 344         debugm=0;
 345         iprel=false;
 346         userip=false;
 347         realt=false;
 348         realtime_refresh=3;
 349         realtime_access_log_lines=1000;
 350         cost=0.01;
 351         nocost=50000000;
 352         ndownload=0;
 353         squid24=false;
 354         dfrom=0;
 355         duntil=0;
 356
 357         bzero(IncludeUsers, sizeof(IncludeUsers));
 358         bzero(ExcludeString, sizeof(ExcludeString));
 359         first_user_file=NULL;
 360         memset(&period,0,sizeof(period));
 361
 362         NAccessLog=0;
 363         for(x=0; x<MAXLOGS; x++)
 364                 AccessLog[x][0]='\0';
 365         AccessLogFromCmdLine=0;
 366         RedirectorLogFromCmdLine=0;
 367
 368         strcpy(Title,_("Squid User Access Report"));
 369
 370         while((ch = getopt_long_only(argc, argv, "a:b:c:d:e:f:g:u:l:L:o:s:t:w:P:hijmnprvxyz",long_options,&option_index)) != -1){
 371                 switch(ch)
 372                 {
 373                         case 0:
 374                                 break;
 375                         case 2:
 376                                 lastlog=atoi(optarg);
 377                                 break;
 378                         case 3:
 379                                 lastlog=0;
 380                                 break;
 381                         case 'a':
 382                                 strcpy(addr,optarg);
 383                                 break;
 384                         case 'b':
 385                                 strcpy(uagent,optarg);
 386                                 break;
 387                         case 'c':
 388                                 strcpy(hexclude,optarg);
 389                                 break;
 390                         case 'd':
 391                                 strncpy(date,optarg,sizeof(date)-1);
 392                                 date[sizeof(date)-1]='\0';
 393                                 date_from(date, &dfrom, &duntil);
 394                                 break;
 395                         case 'e':
 396                                 strcpy(email,optarg);
 397                                 break;
 398                         case 'f':
 399                                 strcpy(ConfigFile,optarg);
 400                                 break;
 401                         case 'g':
 402                                 strcpy(df,optarg);
 403                                 break;
 404                         case 'h':
 405                                 usage(argv[0]);
 406                                 exit(EXIT_SUCCESS);
 407                         case 'i':
 408                                 iprel=true;
 409                                 break;
 410                         case 'l':
 411                                 if (NAccessLog>=MAXLOGS) {
 412                                         debuga(_("Too many log files passed on command line with option -l.\n"));
 413                                         exit(EXIT_FAILURE);
 414                                 }
 415                                 if (strlen(optarg)>=MAX_LOG_FILELEN) {
 416                                         debuga(_("Log file name too long passed on command line with option -l: %s\n"),optarg);
 417                                         exit(EXIT_FAILURE);
 418                                 }
 419                                 strcpy(AccessLog[NAccessLog],optarg);
 420                                 NAccessLog++;
 421                                 AccessLogFromCmdLine++;
 422                                 break;
 423                         case 'L':
 424                                 if (NRedirectorLogs>MAX_REDIRECTOR_LOGS) {
 425                                         debuga(_("Too many redirector logs passed on command line with option -L.\n"));
 426                                         exit(EXIT_FAILURE);
 427                                 }
 428                                 if (strlen(optarg)>=MAX_REDIRECTOR_FILELEN) {
 429                                         debuga(_("Redirector log file name too long passed on command line with opton -L: %s\n"),optarg);
 430                                         exit(EXIT_FAILURE);
 431                                 }
 432                                 strcpy(RedirectorLogs[NRedirectorLogs],optarg);
 433                                 NRedirectorLogs++;
 434                                 RedirectorLogFromCmdLine++;
 435                                 break;
 436                         case 'm':
 437                                 debugm++;
 438                                 break;
 439                         case 'n':
 440                                 dns=true;
 441                                 break;
 442                         case 'o':
 443                                 strcpy(outdir,optarg);
 444                                 break;
 445                         case 'p':
 446                                 userip=true;
 447                                 break;
 448                         case 'P':
 449                                 strcpy(splitprefix,optarg);
 450                                 break;
 451                         case 'r':
 452                                 realt=true;
 453                                 break;
 454                         case 's':
 455                                 strcpy(site,optarg);
 456                                 break;
 457                         case 't':
 458                         {
 459                                 int h1,m1,h2,m2;
 460
 461                                 if(strstr(optarg,"-") == 0) {
 462                                         if(sscanf(optarg,"%d:%d",&h1,&m1)!=2) {
 463                                                 debuga(_("Time period passed on the command line with option -t must be MM:SS\n"));
 464                                                 exit(EXIT_FAILURE);
 465                                         }
 466                                         hm=h1*100+m1;
 467                                         hmf=hm;
 468                                         snprintf(hm_str,sizeof(hm_str),"%02d:%02d",h1,m1);
 469                                 } else {
 470                                         if(sscanf(optarg,"%d:%d-%d:%d",&h1,&m1,&h2,&m2)!=4) {
 471                                                 debuga(_("Time range passed on the command line with option -t must be MM:SS-MM:SS\n"));
 472                                                 exit(EXIT_FAILURE);
 473                                         }
 474                                         hm=h1*100+m1;
 475                                         hmf=h2*100+m2;
 476                                         snprintf(hm_str,sizeof(hm_str),"%02d:%02d-%02d:%02d",h1,m1,h2,m2);
 477                                 }
 478                                 break;
 479                         }
 480                         case 'u':
 481                                 strcpy(us,optarg);
 482                                 break;
 483                         case 'v':
 484                                 version();
 485                                 break;
 486                         case 'w':
 487                                 strcpy(tmp,optarg);
 488                                 break;
 489                         case 'x':
 490                                 debug++;
 491                                 break;
 492                         case 'y':
 493                                 langcode++;
 494                                 break;
 495                         case 'z':
 496                                 debugz++;
 497                                 break;
 498                         case ':':
 499                                 debuga(_("Option -%c requires an argument\n"),optopt);
 500                                 exit(EXIT_FAILURE);
 501                         case '?':
 502                                 usage(argv[0]);
 503                                 exit(EXIT_FAILURE);
 504                         default:
 505                                 abort();
 506                 }
 507         }
 508
 509         if (errflg>0) {
 510                 usage(argv[0]);
 511                 exit(2);
 512         }
 513
 514         if (optind<argc) {
 515                 for (iarq=optind ; iarq<argc ; iarq++) {
 516                         if (NAccessLog>=MAXLOGS) {
 517                                 debuga(_("Too many log files passed on command line.\n"));
 518                                 exit(EXIT_FAILURE);
 519                         }
 520                         if (strlen(argv[iarq])>=MAX_LOG_FILELEN) {
 521                                 debuga(_("Log file name too long passed on command line: %s\n"),argv[iarq]);
 522                                 exit(EXIT_FAILURE);
 523                         }
 524                         strcpy(AccessLog[NAccessLog],argv[iarq]);
 525                         NAccessLog++;
 526                         AccessLogFromCmdLine++;
 527                 }
 528         }
 529
 530         if(debug) debuga(_("Init\n"));
 531
 532         if(ConfigFile[0] == '\0') snprintf(ConfigFile,sizeof(ConfigFile),"%s/sarg.conf",SYSCONFDIR);
 533         if(access(ConfigFile, R_OK) != 0) {
 534                 debuga(_("Cannot open config file: %s - %s\n"),ConfigFile,strerror(errno));
 535                 exit(EXIT_FAILURE);
 536         }
 537
 538         if(access(ConfigFile, R_OK) == 0)
 539                 getconf();
 540
 541         if(userip) UserIp=true;
 542
 543         if(dns) Ip2Name=true;
 544
 545         if (lastlog>=0) LastLog=lastlog;
 546
 547         if(outdir[0] == '\0') strcpy(outdir,OutputDir);
 548         if(outdir[0] != '\0') strcat(outdir,"/");
 549
 550         if(realt) {
 551                 realtime();
 552                 exit(EXIT_SUCCESS);
 553         }
 554
 555         if(IndexTree == INDEX_TREE_FILE)
 556                 strcpy(ImageFile,"../images");
 557         else
 558                 strcpy(ImageFile,"../../../images");
 559
 560         dataonly=0;
 561         if(DataFile[0] != '\0')
 562                 dataonly++;
 563
 564         if(df[0] == '\0') strcpy(df,DateFormat);
 565         else strcpy(DateFormat,df);
 566
 567         if(df[0] == '\0') {
 568                 strcpy(df,"u");
 569                 strcpy(DateFormat,"u");
 570         }
 571         if (df[0]=='w')
 572                 IndexTree=INDEX_TREE_FILE;
 573
 574         if(NAccessLog == 0) {
 575                 strcpy(AccessLog[0],"/var/log/squid/access.log");
 576                 NAccessLog++;
 577         }
 578
 579         if(output_css) {
 580                 css_content(stdout);
 581                 exit(EXIT_SUCCESS);
 582         }
 583         if(split) {
 584                 for (iarq=0 ; iarq<NAccessLog ; iarq++)
 585                         splitlog(AccessLog[iarq], df, dfrom, duntil, convert, splitprefix);
 586                 exit(EXIT_SUCCESS);
 587         }
 588         if(convert) {
 589                 for (iarq=0 ; iarq<NAccessLog ; iarq++)
 590                         convlog(AccessLog[iarq], df, dfrom, duntil);
 591                 exit(EXIT_SUCCESS);
 592         }
 593
 594         load_excludecodes(ExcludeCodes);
 595
 596         if(access(PasswdFile, R_OK) == 0) {
 597                 getusers(PasswdFile,debug);
 598                 puser++;
 599         }
 600
 601         if(hexclude[0] == '\0')
 602                 strcpy(hexclude,ExcludeHosts);
 603         if(hexclude[0] != '\0') {
 604                 gethexclude(hexclude,debug);
 605                 fhost=true;
 606         }
 607
 608         if(ReportType == 0) {
 609                 ReportType=REPORT_TYPE_TOPUSERS | REPORT_TYPE_TOPSITES | REPORT_TYPE_USERS_SITES |
 610                            REPORT_TYPE_SITES_USERS | REPORT_TYPE_DATE_TIME | REPORT_TYPE_DENIED |
 611                            REPORT_TYPE_AUTH_FAILURES | REPORT_TYPE_SITE_USER_TIME_DATE | REPORT_TYPE_DOWNLOADS;
 612         }
 613
 614         if(access(ExcludeUsers, R_OK) == 0) {
 615                 getuexclude(ExcludeUsers,debug);
 616                 fuser=true;
 617         }
 618
 619         indexonly=false;
 620         if(fuser) {
 621                 if(is_indexonly())
 622                         indexonly=true;
 623         }
 624         if(strcmp(ExcludeUsers,"indexonly") == 0) indexonly=true;
 625         if(Index == INDEX_ONLY) indexonly=true;
 626
 627         if(MaxElapsed[0] != '\0') max_elapsed=atol(MaxElapsed);
 628
 629         if(uagent[0] == '\0') strcpy(uagent,UserAgentLog);
 630
 631         if(tmp[0] == '\0') strcpy(tmp,TempDir);
 632         else strcpy(TempDir,tmp);
 633         /*
 634         For historical reasons, the temporary directory is the subdirectory "sarg" of the path
 635         provided by the user.
 636         */
 637         strcat(tmp,"/sarg");
 638
 639         if (tmp[0]!='\0' && strncmp(outdir,tmp,strlen(tmp))==0) {
 640                 debuga(_("The output directory \"%s\" must be outside of the temporary directory \"%s\"\n"),outdir,tmp);
 641                 exit(EXIT_FAILURE);
 642         }
 643
 644         if(email[0] == '\0' && OutputEmail[0] != '\0') strcpy(email,OutputEmail);
 645
 646         if(email[0] != '\0') {
 647                 my_mkdir(tmp);
 648                 strcpy(outdir,tmp);
 649                 strcat(outdir,"/");
 650         }
 651
 652         if(access(tmp, R_OK) == 0) {
 653                 unlinkdir(tmp,1);
 654         }
 655         my_mkdir(tmp);
 656         snprintf(denied_unsort,sizeof(denied_unsort),"%s/denied.log.unsort",tmp);
 657         snprintf(denied_sort,sizeof(denied_sort),"%s/denied.log",tmp);
 658         snprintf(authfail_unsort,sizeof(authfail_unsort),"%s/authfail.log.unsort",tmp);
 659
 660         if(debug) {
 661                 debuga(_("Parameters:\n"));
 662                 debuga(_("          Hostname or IP address (-a) = %s\n"),addr);
 663                 debuga(_("                   Useragent log (-b) = %s\n"),uagent);
 664                 debuga(_("                    Exclude file (-c) = %s\n"),hexclude);
 665                 debuga(_("                 Date from-until (-d) = %s\n"),date);
 666                 debuga(_("   Email address to send reports (-e) = %s\n"),email);
 667                 debuga(_("                     Config file (-f) = %s\n"),ConfigFile);
 668                 if(strcmp(df,"e") == 0)
 669                         debuga(_("                     Date format (-g) = Europe (dd/mm/yyyy)\n"));
 670                 if(strcmp(df,"u") == 0)
 671                         debuga(_("                     Date format (-g) = USA (mm/dd/yyyy)\n"));
 672                 if(strcmp(df,"w") == 0)
 673                         debuga(_("                     Date format (-g) = Sites & Users (yyyy/ww)\n"));
 674                 debuga(_("                       IP report (-i) = %s\n"),(iprel) ? _("Yes") : _("No"));
 675                 for (iarq=0 ; iarq<NAccessLog ; iarq++)
 676                         debuga(_("                       Input log (-l) = %s\n"),AccessLog[iarq]);
 677                 for (iarq=0 ; iarq<NRedirectorLogs ; iarq++)
 678                         debuga(_("                  Redirector log (-L) = %s\n"),RedirectorLogs[iarq]);
 679                 debuga(_("              Resolve IP Address (-n) = %s\n"),(Ip2Name) ? _("Yes") : _("No"));
 680                 debuga(_("                      Output dir (-o) = %s\n"),outdir);
 681                 debuga(_("Use Ip Address instead of userid (-p) = %s\n"),(UserIp) ? _("Yes") : _("No"));
 682                 debuga(_("                   Accessed site (-s) = %s\n"),site);
 683                 debuga(_("                            Time (-t) = %s\n"),hm_str);
 684                 debuga(_("                            User (-u) = %s\n"),us);
 685                 debuga(_("                   Temporary dir (-w) = %s\n"),tmp);
 686                 debuga(_("                  Debug messages (-x) = %s\n"),(debug) ? _("Yes") : _("No"));
 687                 debuga(_("                Process messages (-z) = %s\n"),(debugz) ? _("Yes") : _("No"));
 688                 debuga(_(" Previous reports to keep (--lastlog) = %d\n"),LastLog);
 689                 debuga("\n");
 690         }
 691
 692         if(debugm) {
 693                 printf(_("Parameters:\n"));
 694                 printf(_("          Hostname or IP address (-a) = %s\n"),addr);
 695                 printf(_("                   Useragent log (-b) = %s\n"),uagent);
 696                 printf(_("                    Exclude file (-c) = %s\n"),hexclude);
 697                 printf(_("                 Date from-until (-d) = %s\n"),date);
 698                 printf(_("   Email address to send reports (-e) = %s\n"),email);
 699                 printf(_("                     Config file (-f) = %s\n"),ConfigFile);
 700                 if(strcmp(df,"e") == 0)
 701                         printf(_("                     Date format (-g) = Europe (dd/mm/yyyy)\n"));
 702                 if(strcmp(df,"u") == 0)
 703                         printf(_("                     Date format (-g) = USA (mm/dd/yyyy)\n"));
 704                 if(strcmp(df,"w") == 0)
 705                         printf(_("                     Date format (-g) = Sites & Users (yyyy/ww)\n"));
 706                 printf(_("                       IP report (-i) = %s\n"),(iprel) ? _("Yes") : _("No"));
 707                 for (iarq=0 ; iarq<NAccessLog ; iarq++)
 708                         printf(_("                       Input log (-l) = %s\n"),AccessLog[iarq]);
 709                 for (iarq=0 ; iarq<NRedirectorLogs ; iarq++)
 710                         printf(_("                  Redirector log (-L) = %s\n"),RedirectorLogs[iarq]);
 711                 printf(_("              Resolve IP Address (-n) = %s\n"),(Ip2Name) ? _("Yes") : _("No"));
 712                 printf(_("                      Output dir (-o) = %s\n"),outdir);
 713                 printf(_("Use Ip Address instead of userid (-p) = %s\n"),(UserIp) ? _("Yes") : _("No"));
 714                 printf(_("                   Accessed site (-s) = %s\n"),site);
 715                 printf(_("                            Time (-t) = %s\n"),hm_str);
 716                 printf(_("                            User (-u) = %s\n"),us);
 717                 printf(_("                   Temporary dir (-w) = %s\n"),tmp);
 718                 printf(_("                  Debug messages (-x) = %s\n"),(debug) ? _("Yes") : _("No"));
 719                 printf(_("                Process messages (-z) = %s\n"),(debugz) ? _("Yes") : _("No"));
 720                 printf(_(" Previous reports to keep (--lastlog) = %d\n"),LastLog);
 721                 printf(_("sarg version: %s\n"),VERSION);
 722         }
 723
 724         if(debug)
 725                 debuga(_("sarg version: %s\n"),VERSION);
 726
 727 #ifdef ENABLE_DOUBLE_CHECK_DATA
 728         debuga(_("Sarg compiled to report warnings if the output is inconsistent\n"));
 729 #endif
 730
 731         maxopenfiles=MAX_OPEN_USER_FILES;
 732 #ifdef HAVE_RLIM_T
 733         if (Ulimit[0] != '\0') {
 734                 struct rlimit rl;
 735                 long l1, l2;
 736                 int rc=0;
 737
 738 #if defined(RLIMIT_NOFILE)
 739                 getrlimit (RLIMIT_NOFILE, &rl);
 740 #elif defined(RLIMIT_OFILE)
 741                 getrlimit (RLIMIT_OFILE, &rl);
 742 #else
 743 #warning "No rlimit resource for the number of open files"
 744 #endif
 745                 l1 = rl.rlim_cur;
 746                 l2 = rl.rlim_max;
 747
 748                 rl.rlim_cur = atol(Ulimit);
 749                 rl.rlim_max = atol(Ulimit);
 750 #if defined(RLIMIT_NOFILE)
 751                 rc=setrlimit (RLIMIT_NOFILE, &rl);
 752 #elif defined(RLIMIT_OFILE)
 753                 rc=setrlimit (RLIMIT_OFILE, &rl);
 754 #else
 755 #warning "No rlimit resource for the number of open files"
 756 #endif
 757                 if(rc == -1) {
 758                         debuga(_("setrlimit error - %s\n"),strerror(errno));
 759                 }
 760
 761                 if(debug)
 762                         debuga("Maximum file descriptor: cur=%ld max=%ld, changed to cur="RLIM_STRING" max="RLIM_STRING,l1,l2,rl.rlim_cur,rl.rlim_max);
 763         }
 764 #endif
 765
 766         init_usertab(UserTabFile);
 767
 768         if ((line=longline_create())==NULL) {
 769                 debuga(_("Not enough memory to read a log file\n"));
 770                 exit(EXIT_FAILURE);
 771         }
 772
 773         snprintf(sz_Download_Unsort,sizeof(sz_Download_Unsort),"%s/download.unsort", tmp);
 774
 775         if(DataFile[0]=='\0') {
 776                 if((ReportType & REPORT_TYPE_DENIED) != 0) {
 777                         if((fp_denied=MY_FOPEN(denied_unsort,"w"))==NULL) {
 778                                 debuga(_("(log) Cannot open file: %s - %s\n"),denied_unsort,strerror(errno));
 779                                 exit(EXIT_FAILURE);
 780                         }
 781                 }
 782
 783                 if((ReportType & REPORT_TYPE_DENIED) != 0 || (ReportType & REPORT_TYPE_AUTH_FAILURES) != 0) {
 784                         if((fp_authfail=MY_FOPEN(authfail_unsort,"w"))==NULL) {
 785                                 debuga(_("(log) Cannot open file: %s - %s\n"),authfail_unsort,strerror(errno));
 786                                 exit(EXIT_FAILURE);
 787                         }
 788                 }
 789         }
 790
 791         for (iarq=0 ; iarq<NAccessLog ; iarq++) {
 792                 strcpy(arq,AccessLog[iarq]);
 793
 794                 strcpy(arqtt,arq);
 795
 796                 if(strcmp(arq,"-")==0) {
 797                         if(debug)
 798                                 debuga(_("Reading access log file: from stdin\n"));
 799                         fp_in=stdin;
 800                         from_stdin=true;
 801                 } else {
 802                         if (date[0]!='\0') {
 803                                 if (stat(arq,&logstat)!=0) {
 804                                         debuga(_("Cannot get the modification time of input log file %s (%s). Processing it anyway\n"),arq,strerror(errno));
 805                                 } else {
 806                                         struct tm *logtime=localtime(&logstat.st_mtime);
 807                                         if ((logtime->tm_year+1900)*10000+(logtime->tm_mon+1)*100+logtime->tm_mday<dfrom) {
 808                                                 debuga(_("Ignoring old log file %s\n"),arq);
 809                                                 continue;
 810                                         }
 811                                 }
 812                         }
 813                         fp_in=decomp(arq,&from_pipe);
 814                         if(fp_in==NULL) {
 815                                 debuga(_("(log) Cannot open log file: %s - %s\n"),arq,strerror(errno));
 816                                 exit(EXIT_FAILURE);
 817                         }
 818                         if(debug) debuga(_("Reading access log file: %s\n"),arq);
 819                         from_stdin=false;
 820                 }
 821                 ilf=ILF_Unknown;
 822                 download_flag=false;
 823                 // pre-read the file only if we have to show stats
 824                 if(ShowReadStatistics && !from_stdin && !from_pipe) {
 825                         size_t nread,i;
 826                         bool skipcr=false;
 827                         char tmp4[MAXLEN];
 828
 829                         recs1=0UL;
 830                         recs2=0UL;
 831
 832                         while ((nread=fread(tmp4,1,sizeof(tmp4),fp_in))>0) {
 833                                 for (i=0 ; i<nread ; i++)
 834                                         if (skipcr) {
 835                                                 if (tmp4[i]!='\n' && tmp4[i]!='\r') {
 836                                                         skipcr=false;
 837                                                 }
 838                                         } else {
 839                                                 if (tmp4[i]=='\n' || tmp4[i]=='\r') {
 840                                                         skipcr=true;
 841                                                         recs1++;
 842                                                 }
 843                                         }
 844                         }
 845                         rewind(fp_in);
 846                         printf(_("SARG: Records in file: %lu, reading: %3.2f%%"),recs1,(float) 0);
 847                         putchar('\r');
 848                         fflush( stdout ) ;
 849                 }
 850
 851                 longline_reset(line);
 852
 853                 while ((linebuf=longline_read(fp_in,line))!=NULL) {
 854                         blen=strlen(linebuf);
 855
 856                         if (ilf==ILF_Unknown) {
 857                                 if(strncmp(linebuf,"#Software: Mic",14) == 0) {
 858                                         fixendofline(linebuf);
 859                                         if (debug)
 860                                                 debuga(_("Log is from Microsoft ISA: %s\n"),linebuf);
 861                                         ilf=ILF_Isa;
 862                                         ilf_count[ilf]++;
 863                                         continue;
 864                                 }
 865
 866                                 if(strncmp(linebuf,"*** SARG Log ***",16) == 0) {
 867                                         if (getperiod_fromsarglog(arqtt,&period)<0) {
 868                                                 debuga(_("The name of the file is invalid: %s\n"),arq);
 869                                                 exit(EXIT_FAILURE);
 870                                         }
 871                                         ilf=ILF_Sarg;
 872                                         ilf_count[ilf]++;
 873                                         continue;
 874                                 }
 875                         }
 876
 877                         if(!fp_log && ParsedOutputLog[0] && ilf!=ILF_Sarg) {
 878                                 if(access(ParsedOutputLog,R_OK) != 0) {
 879                                         my_mkdir(ParsedOutputLog);
 880                                 }
 881                                 sprintf(arq_log,"%s/sarg_temp.log",ParsedOutputLog);
 882                                 if((fp_log=MY_FOPEN(arq_log,"w"))==NULL) {
 883                                         debuga(_("(log) Cannot open log file: %s - %s\n"),arq_log,strerror(errno));
 884                                         exit(EXIT_FAILURE);
 885                                 }
 886                                 fputs("*** SARG Log ***\n",fp_log);
 887                         }
 888
 889                         recs2++;
 890                         if( ShowReadStatistics && !from_stdin && !from_pipe && --OutputNonZero<=0) {
 891                                 double perc = recs2 * 100. / recs1 ;
 892                                 printf(_("SARG: Records in file: %lu, reading: %3.2lf%%"),recs2,perc);
 893                                 putchar('\r');
 894                                 fflush (stdout);
 895                                 OutputNonZero = REPORT_EVERY_X_LINES ;
 896                         }
 897                         if(blen < 58) continue;
 898                         if(strstr(linebuf,"HTTP/0.0") != 0) continue;
 899                         if(strstr(linebuf,"logfile turned over") != 0) continue;
 900                         if(linebuf[0] == ' ') continue;
 901
 902                         // exclude_string
 903                         if(ExcludeString[0] != '\0') {
 904                                 bool exstring=false;
 905                                 getword_start(&gwarea,ExcludeString);
 906                                 while(strchr(gwarea.current,':') != 0) {
 907                                         if (getword_multisep(val1,sizeof(val1),&gwarea,':')<0) {
 908                                                 debuga(_("Maybe you have a broken record or garbage in your exclusion string\n"));
 909                                                 exit(EXIT_FAILURE);
 910                                         }
 911                                         if((str=(char *) strstr(linebuf,val1)) != (char *) NULL ) {
 912                                                 exstring=true;
 913                                                 break;
 914                                         }
 915                                 }
 916                                 if(!exstring && (str=(char *) strstr(linebuf,gwarea.current)) != (char *) NULL )
 917                                         exstring=true;
 918                                 if(exstring) continue;
 919                         }
 920
 921                         totregsl++;
 922                         if(debugm)
 923                                 printf("BUF=%s\n",linebuf);
 924
 925                         t=NULL;
 926                         if (ilf==ILF_Squid || ilf==ILF_Common || ilf==ILF_Unknown) {
 927                                 getword_start(&gwarea,linebuf);
 928                                 if (getword(data,sizeof(data),&gwarea,' ')<0) {
 929                                         debuga(_("Maybe you have a broken time in your access.log file\n"));
 930                                         exit(EXIT_FAILURE);
 931                                 }
 932                                 if((str=(char *) strchr(data, '.')) != (char *) NULL && (str=(char *) strchr(str+1, '.')) != (char *) NULL ) {
 933                                         strcpy(ip,data);
 934                                         strcpy(elap,"0");
 935                                         if(squid24) {
 936                                                 if (getword(user,sizeof(user),&gwarea,' ')<0 || getword_skip(255,&gwarea,' ')<0) {
 937                                                         debuga(_("Maybe you have a broken record or garbage in your %s file\n"),arq);
 938                                                         exit(EXIT_FAILURE);
 939                                                 }
 940                                         } else {
 941                                                 if (getword_skip(255,&gwarea,' ')<0 || getword(user,sizeof(user),&gwarea,' ')<0) {
 942                                                         debuga(_("Maybe you have a broken record or garbage in your %s file\n"),arq);
 943                                                         exit(EXIT_FAILURE);
 944                                                 }
 945                                         }
 946                                         if (getword(data,sizeof(data),&gwarea,']')<0 || getword_skip(MAXLEN,&gwarea,'"')<0 ||
 947                                             getword(fun,sizeof(fun),&gwarea,' ')<0) {
 948                                                 debuga(_("Maybe you have a broken record or garbage in your %s file\n"),arq);
 949                                                 exit(EXIT_FAILURE);
 950                                         }
 951                                         if (getword_ptr(linebuf,&url,&gwarea,' ')<0) {
 952                                                 debuga(_("Maybe you have a broken url in your %s file\n"),arq);
 953                                                 exit(EXIT_FAILURE);
 954                                         }
 955                                         if (getword_skip(MAXLEN,&gwarea,' ')<0) {
 956                                                 debuga(_("Maybe you have a broken record or garbage in your %s file\n"),arq);
 957                                                 exit(EXIT_FAILURE);
 958                                         }
 959                                         if (getword(code2,sizeof(code2),&gwarea,' ')<0) {
 960                                                 debuga(_("Maybe you have a broken record or garbage in your %s file\n"),arq);
 961                                                 exit(EXIT_FAILURE);
 962                                         }
 963                                         if (getword(tam,sizeof(tam),&gwarea,' ')<0) {
 964                                                 debuga(_("Maybe you have a broken record or garbage in your %s file\n"),arq);
 965                                                 exit(EXIT_FAILURE);
 966                                         }
 967                                         if((str=(char *) strchr(gwarea.current, ' ')) != (char *) NULL ) {
 968                                                 if (getword(code,sizeof(code),&gwarea,' ')<0) {
 969                                                         debuga(_("Maybe you have a broken record or garbage in your %s file\n"),arq);
 970                                                         exit(EXIT_FAILURE);
 971                                                 }
 972                                         } else {
 973                                                 if (getword(code,sizeof(code),&gwarea,'\0')<0) {
 974                                                         debuga(_("Maybe you have a broken record or garbage in your %s file\n"),arq);
 975                                                         exit(EXIT_FAILURE);
 976                                                 }
 977                                         }
 978
 979                                         if ((str = strchr(code, ':')) != NULL)
 980                                                 *str = '/';
 981
 982                                         if(strcmp(tam,"\0") == 0)
 983                                                 strcpy(tam,"0");
 984
 985                                         ilf=ILF_Common;
 986                                         ilf_count[ilf]++;
 987
 988                                         getword_start(&gwarea,data+1);
 989                                         if (getword_multisep(data,sizeof(data),&gwarea,':')<0){
 990                                                 debuga(_("Maybe you have a broken date in your %s file\n"),arq);
 991                                                 exit(EXIT_FAILURE);
 992                                         }
 993                                         if (getword_multisep(hora,sizeof(hora),&gwarea,' ')<0){
 994                                                 debuga(_("Maybe you have a broken date in your %s file\n"),arq);
 995                                                 exit(EXIT_FAILURE);
 996                                         }
 997                                         getword_start(&gwarea,data);
 998                                         if (getword_atoll(&iday,&gwarea,'/')<0){
 999                                                 debuga(_("Maybe you have a broken date in your %s file\n"),arq);
1000                                                 exit(EXIT_FAILURE);
1001                                         }
1002                                         if (getword(mes,sizeof(mes),&gwarea,'/')<0){
1003                                                 debuga(_("Maybe you have a broken date in your %s file\n"),arq);
1004                                                 exit(EXIT_FAILURE);
1005                                         }
1006                                         if (getword_atoll(&iyear,&gwarea,'/')<0){
1007                                                 debuga(_("Maybe you have a broken date in your %s file\n"),arq);
1008                                                 exit(EXIT_FAILURE);
1009                                         }
1010
1011                                         imonth=month2num(mes)+1;
1012                                         idata=builddia(iday,imonth,iyear);
1013                                         computedate(iyear,imonth,iday,&tt);
1014                                         if (sscanf(hora,"%d:%d:%d",&tt.tm_hour,&tt.tm_min,&tt.tm_sec)!=3 || tt.tm_hour<0 || tt.tm_hour>=24 ||
1015                                                         tt.tm_min<0 || tt.tm_min>=60 || tt.tm_sec<0 || tt.tm_sec>=60) {
1016                                                 debuga(_("Invalid time found in %s\n"),arq);
1017                                                 exit(EXIT_FAILURE);
1018                                         }
1019                                         t=&tt;
1020                                 }
1021
1022                                 if(ilf==ILF_Unknown || ilf==ILF_Squid) {
1023                                         if (getword(elap,sizeof(elap),&gwarea,' ')<0) {
1024                                                 debuga(_("Maybe you have a broken elapsed time in your %s file\n"),arq);
1025                                                 exit(EXIT_FAILURE);
1026                                         }
1027                                         while(strcmp(elap,"") == 0 && gwarea.current[0] != '\0')
1028                                                 if (getword(elap,sizeof(elap),&gwarea,' ')<0) {
1029                                                         debuga(_("Maybe you have a broken elapsed time in your %s file\n"),arq);
1030                                                         exit(EXIT_FAILURE);
1031                                                 }
1032                                         if(strlen(elap) < 1) continue;
1033                                         if (getword(ip,sizeof(ip),&gwarea,' ')<0){
1034                                                 debuga(_("Maybe you have a broken client IP address in your %s file\n"),arq);
1035                                                 exit(EXIT_FAILURE);
1036                                         }
1037                                         if (getword(code,sizeof(code),&gwarea,' ')<0){
1038                                                 debuga(_("Maybe you have a broken result code in your %s file\n"),arq);
1039                                                 exit(EXIT_FAILURE);
1040                                         }
1041                                         if (getword(tam,sizeof(tam),&gwarea,' ')<0){
1042                                                 debuga(_("Maybe you have a broken amount of data in your %s file\n"),arq);
1043                                                 exit(EXIT_FAILURE);
1044                                         }
1045                                         if (getword(fun,sizeof(fun),&gwarea,' ')<0){
1046                                                 debuga(_("Maybe you have a broken request method in your %s file\n"),arq);
1047                                                 exit(EXIT_FAILURE);
1048                                         }
1049                                         if (getword_ptr(linebuf,&url,&gwarea,' ')<0){
1050                                                 debuga(_("Maybe you have a broken url in your %s file\n"),arq);
1051                                                 exit(EXIT_FAILURE);
1052                                         }
1053                                         if (getword(user,sizeof(user),&gwarea,' ')<0){
1054                                                 debuga(_("Maybe you have a broken user ID in your %s file\n"),arq);
1055                                                 exit(EXIT_FAILURE);
1056                                         }
1057                                         ilf=ILF_Squid;
1058                                         ilf_count[ilf]++;
1059
1060                                         tnum=atoi(data);
1061                                         t=localtime(&tnum);
1062                                         if (t == NULL) {
1063                                                 debuga(_("Cannot convert the timestamp from the squid log file\n"));
1064                                                 exit(EXIT_FAILURE);
1065                                         }
1066
1067                                         strftime(tbuf2, sizeof(tbuf2), "%H%M", t);
1068
1069                                         idata=(t->tm_year+1900)*10000+(t->tm_mon+1)*100+t->tm_mday;
1070                                 }
1071                         }
1072                         if (ilf==ILF_Sarg) {
1073                                 getword_start(&gwarea,linebuf);
1074                                 if (getword(data,sizeof(data),&gwarea,'\t')<0){
1075                                         debuga(_("Maybe you have a broken record or garbage in your %s file\n"),arq);
1076                                         exit(EXIT_FAILURE);
1077                                 }
1078                                 if (getword(hora,sizeof(hora),&gwarea,'\t')<0) {
1079                                         debuga(_("Maybe you have a broken record or garbage in your %s file\n"),arq);
1080                                         exit(EXIT_FAILURE);
1081                                 }
1082                                 if (getword(user,sizeof(user),&gwarea,'\t')<0) {
1083                                         debuga(_("Maybe you have a broken record or garbage in your %s file\n"),arq);
1084                                         exit(EXIT_FAILURE);
1085                                 }
1086                                 if (getword(ip,sizeof(ip),&gwarea,'\t')<0) {
1087                                         debuga(_("Maybe you have a broken record or garbage in your %s file\n"),arq);
1088                                         exit(EXIT_FAILURE);
1089                                 }
1090                                 if (getword_ptr(linebuf,&url,&gwarea,'\t')<0){
1091                                         debuga(_("Maybe you have a broken record or garbage in your %s file\n"),arq);
1092                                         exit(EXIT_FAILURE);
1093                                 }
1094                                 if (getword(tam,sizeof(tam),&gwarea,'\t')<0){
1095                                         debuga(_("Maybe you have a broken record or garbage in your %s file\n"),arq);
1096                                         exit(EXIT_FAILURE);
1097                                 }
1098                                 if (getword(code,sizeof(code),&gwarea,'\t')<0){
1099                                         debuga(_("Maybe you have a broken record or garbage in your %s file\n"),arq);
1100                                         exit(EXIT_FAILURE);
1101                                 }
1102                                 if (getword(elap,sizeof(elap),&gwarea,'\t')<0){
1103                                         debuga(_("Maybe you have a broken record or garbage in your %s file\n"),arq);
1104                                         exit(EXIT_FAILURE);
1105                                 }
1106                                 if (getword(smartfilter,sizeof(smartfilter),&gwarea,'\0')<0){
1107                                         debuga(_("Maybe you have a broken record or garbage in your %s file\n"),arq);
1108                                         exit(EXIT_FAILURE);
1109                                 }
1110                                 getword_start(&gwarea,data);
1111                                 if (getword_atoll(&iday,&gwarea,'/')<0 || iday<1 || iday>31){
1112                                         debuga(_("Maybe you have a broken date in your %s file\n"),arq);
1113                                         exit(EXIT_FAILURE);
1114                                 }
1115                                 if (getword_atoll(&imonth,&gwarea,'/')<0 || imonth<1 || imonth>12){
1116                                         debuga(_("Maybe you have a broken date in your %s file\n"),arq);
1117                                         exit(EXIT_FAILURE);
1118                                 }
1119                                 if (getword_atoll(&iyear,&gwarea,'\0')<0){
1120                                         debuga(_("Maybe you have a broken date in your %s file\n"),arq);
1121                                         exit(EXIT_FAILURE);
1122                                 }
1123                                 idata=builddia(iday,imonth,iyear);
1124                                 computedate(iyear,imonth,iday,&tt);
1125                                 if (sscanf(hora,"%d:%d:%d",&tt.tm_hour,&tt.tm_min,&tt.tm_sec)!=3 || tt.tm_hour<0 || tt.tm_hour>=24 ||
1126                                                 tt.tm_min<0 || tt.tm_min>=60 || tt.tm_sec<0 || tt.tm_sec>=60) {
1127                                         debuga(_("Invalid time found in %s\n"),arq);
1128                                         exit(EXIT_FAILURE);
1129                                 }
1130                                 t=&tt;
1131                         }
1132                         if (ilf==ILF_Isa) {
1133                                 if (linebuf[0] == '#') {
1134                                         int ncols,cols[ISACOL_Last];
1135
1136                                         fixendofline(linebuf);
1137                                         getword_start(&gwarea,linebuf);
1138                                         // remove the #Fields: column at the beginning of the line
1139                                         if (getword_skip(1000,&gwarea,' ')<0){
1140                                                 debuga(_("Maybe you have a broken record or garbage in your %s file\n"),arq);
1141                                                 exit(EXIT_FAILURE);
1142                                         }
1143                                         for (ncols=0 ; ncols<ISACOL_Last ; ncols++) cols[ncols]=-1;
1144                                         ncols=0;
1145                                         while(gwarea.current[0] != '\0') {
1146                                                 if (getword(val1,sizeof(val1),&gwarea,'\t')<0){
1147                                                         debuga(_("Maybe you have a broken record or garbage in your %s file\n"),arq);
1148                                                         exit(EXIT_FAILURE);
1149                                                 }
1150                                                 if(strcmp(val1,"c-ip") == 0) cols[ISACOL_Ip]=ncols;
1151                                                 if(strcmp(val1,"cs-username") == 0) cols[ISACOL_UserName]=ncols;
1152                                                 if(strcmp(val1,"date") == 0) cols[ISACOL_Date]=ncols;
1153                                                 if(strcmp(val1,"time") == 0) cols[ISACOL_Time]=ncols;
1154                                                 if(strcmp(val1,"time-taken") == 0) cols[ISACOL_TimeTaken]=ncols;
1155                                                 if(strcmp(val1,"sc-bytes") == 0) cols[ISACOL_Bytes]=ncols;
1156                                                 if(strcmp(val1,"cs-uri") == 0) cols[ISACOL_Uri]=ncols;
1157                                                 if(strcmp(val1,"sc-status") == 0) cols[ISACOL_Status]=ncols;
1158                                                 ncols++;
1159                                         }
1160                                         if (cols[ISACOL_Ip]>=0) {
1161                                                 isa_ncols=ncols;
1162                                                 for (ncols=0 ; ncols<ISACOL_Last ; ncols++)
1163                                                         isa_cols[ncols]=cols[ncols];
1164                                         }
1165                                         continue;
1166                                 }
1167                                 if (!isa_ncols) continue;
1168                                 getword_start(&gwarea,linebuf);
1169                                 for (x=0 ; x<isa_ncols ; x++) {
1170                                         if (getword_ptr(linebuf,&str,&gwarea,'\t')<0) {
1171                                                 debuga(_("Maybe you have a broken record or garbage in your %s file\n"),arq);
1172                                                 exit(EXIT_FAILURE);
1173                                         }
1174                                         if (x==isa_cols[ISACOL_Ip]) {
1175                                                 if (strlen(str)>=sizeof(ip)) {
1176                                                         debuga(_("Maybe you have a broken IP in your %s file\n"),arq);
1177                                                         exit(EXIT_FAILURE);
1178                                                 }
1179                                                 strcpy(ip,str);
1180                                         } else if (x==isa_cols[ISACOL_UserName]) {
1181                                                 if (strlen(str)>=sizeof(user)) {
1182                                                         debuga(_("Maybe you have a broken user ID in your %s file\n"),arq);
1183                                                         exit(EXIT_FAILURE);
1184                                                 }
1185                                                 strcpy(user,str);
1186                                         } else if (x==isa_cols[ISACOL_Date]) {
1187                                                 if (strlen(str)>=sizeof(data)) {
1188                                                         debuga(_("Maybe you have a broken date in your %s file\n"),arq);
1189                                                         exit(EXIT_FAILURE);
1190                                                 }
1191                                                 strcpy(data,str);
1192                                         } else if (x==isa_cols[ISACOL_Time]) {
1193                                                 if (strlen(str)>=sizeof(hora)) {
1194                                                         debuga(_("Maybe you have a broken time in your %s file\n"),arq);
1195                                                         exit(EXIT_FAILURE);
1196                                                 }
1197                                                 strcpy(hora,str);
1198                                         } else if (x==isa_cols[ISACOL_TimeTaken]) {
1199                                                 if (strlen(str)>=sizeof(elap)) {
1200                                                         debuga(_("Maybe you have a broken download duration in your %s file\n"),arq);
1201                                                         exit(EXIT_FAILURE);
1202                                                 }
1203                                                 strcpy(elap,str);
1204                                         } else if (x==isa_cols[ISACOL_Bytes]) {
1205                                                 if (strlen(str)>=sizeof(tam)) {
1206                                                         debuga(_("Maybe you have a broken download size in your %s file\n"),arq);
1207                                                         exit(EXIT_FAILURE);
1208                                                 }
1209                                                 strcpy(tam,str);
1210                                         } else if (x==isa_cols[ISACOL_Uri]) {
1211                                                 url=str;
1212                                         } else if (x==isa_cols[ISACOL_Status]) {
1213                                                 if (strlen(str)>=sizeof(code)) {
1214                                                         debuga(_("Maybe you have a broken access code in your %s file\n"),arq);
1215                                                         exit(EXIT_FAILURE);
1216                                                 }
1217                                                 strcpy(code,str);
1218                                         }
1219                                 }
1220
1221                                 if(strcmp(code,"401") == 0 || strcmp(code,"403") == 0 || strcmp(code,"407") == 0) {
1222                                         sprintf(val1,"DENIED/%s",code);
1223                                         strcpy(code,val1);
1224                                 }
1225                                 getword_start(&gwarea,data);
1226                                 if (getword_atoll(&iyear,&gwarea,'-')<0){
1227                                         debuga(_("Maybe you have a broken year in your %s file\n"),arq);
1228                                         exit(EXIT_FAILURE);
1229                                 }
1230                                 if (getword_atoll(&imonth,&gwarea,'-')<0){
1231                                         debuga(_("Maybe you have a broken month in your %s file\n"),arq);
1232                                         exit(EXIT_FAILURE);
1233                                 }
1234                                 if (getword_atoll(&iday,&gwarea,'\0')<0){
1235                                         debuga(_("Maybe you have a broken day in your %s file\n"),arq);
1236                                         exit(EXIT_FAILURE);
1237                                 }
1238
1239                                 idata=builddia(iday,imonth,iyear);
1240                                 computedate(iyear,imonth,iday,&tt);
1241                                 if (isa_cols[ISACOL_Time]>=0) {
1242                                         if (sscanf(hora,"%d:%d:%d",&tt.tm_hour,&tt.tm_min,&tt.tm_sec)!=3 || tt.tm_hour<0 || tt.tm_hour>=24 ||
1243                                                         tt.tm_min<0 || tt.tm_min>=60 || tt.tm_sec<0 || tt.tm_sec>=60) {
1244                                                 debuga(_("Invalid time found in %s\n"),arq);
1245                                                 exit(EXIT_FAILURE);
1246                                         }
1247                                 }
1248                                 t=&tt;
1249                         }
1250                         if (t==NULL) {
1251                                 debuga(_("Unknown input log file format\n"));
1252                                 break;
1253                         }
1254
1255                         strftime(dia, sizeof(dia), "%d/%m/%Y", t);
1256                         snprintf(hora,sizeof(hora),"%02d:%02d:%02d",t->tm_hour,t->tm_min,t->tm_sec);
1257
1258                         if(debugm)
1259                                 printf("DATE=%s IDATA=%d DFROM=%d DUNTIL=%d\n",date,idata,dfrom,duntil);
1260
1261                         if(date[0] != '\0'){
1262                                 if(idata < dfrom || idata > duntil) continue;
1263                         }
1264
1265                         // Record only hours usage which is required
1266                         if (t) {
1267                                 if( bsearch( &( t -> tm_wday ), weekdays.list, weekdays.len, sizeof( int ), compar ) == NULL )
1268                                         continue;
1269
1270                                 if( bsearch( &( t -> tm_hour ), hours.list, hours.len, sizeof( int ), compar ) == NULL )
1271                                         continue;
1272                         }
1273
1274
1275                         if(strlen(user) > MAX_USER_LEN) {
1276                                 if (debugm) printf(_("User ID too long: %s\n"),user);
1277                                 totregsx++;
1278                                 continue;
1279                         }
1280
1281                         // include_users
1282                         if(IncludeUsers[0] != '\0') {
1283                                 sprintf(val1,":%s:",user);
1284                                 if((str=(char *) strstr(IncludeUsers,val1)) == (char *) NULL )
1285                                         continue;
1286                         }
1287
1288                         if(vercode(code)) {
1289                                 if (debugm) printf(_("Excluded code: %s\n"),code);
1290                                 totregsx++;
1291                                 continue;
1292                         }
1293
1294                         if(testvaliduserchar(user))
1295                                 continue;
1296
1297 #if 0
1298                         if((str = strstr(user,"%20")) != NULL) {
1299                                 /*
1300                                 This is a patch introduced to solve bug #1624251 reported at sourceforge but
1301                                 the side effect is to truncate the name at the first space and merge the reports
1302                                 of people whose name is identical up to the first space.
1303
1304                                 The old code used to truncate the user name at the first % if a %20 was
1305                                 found anywhere in the string. That means the string could be truncated
1306                                 at the wrong place if another % occured before the %20. This new code should
1307                                 avoid that problem and only truncate at the space. There is no bug
1308                                 report indicating that anybody noticed this.
1309                                 */
1310                                 *str='\0';
1311                         }
1312
1313                         /*
1314                         Code prior to 2.2.7 used to replace any %xx by a dot as long as a %5c was
1315                         found in the user name.
1316                         */
1317                         while((str = strstr(user,"%5c")) != NULL) {
1318                                 *str='.';
1319                                 for (x=3 ; str[x] ; x++) str[x-2]=str[x];
1320                         }
1321 #endif
1322
1323                         urly=url;
1324
1325                         if(ilf!=ILF_Sarg) {
1326                                 /*
1327                                 The full URL is not saved in sarg log. There is no point in testing the URL to detect
1328                                 a downloaded file.
1329                                 */
1330                                 download_flag=is_download_suffix(url);
1331                                 if (download_flag) {
1332                                         download_url=url;
1333                                         download_count++;
1334                                 }
1335                         } else
1336                                 download_flag=false;
1337
1338                         // remove any protocol:// at the beginning of the URL
1339                         if ((str = strchr(url,'/')) != NULL && str[1] == '/') {
1340                                 int i;
1341
1342                                 str+=2;
1343                                 for (i=0 ; str[i] ; i++)
1344                                         url[i]=str[i];
1345                                 url[i]='\0';
1346                         }
1347
1348                         if(!LongUrl) {
1349                                 url_hostname(url,hostname,sizeof(hostname));
1350                                 url=hostname;
1351                         }
1352
1353                         if(url[0] == '\0') continue;
1354
1355                         if(addr[0] != '\0'){
1356                                 if(strcmp(addr,ip)!=0) continue;
1357                         }
1358                         if(fhost) {
1359                                 if(!vhexclude(url)) {
1360                                         if (debugm) printf(_("Excluded site: %s\n"),url);
1361                                         totregsx++;
1362                                         continue;
1363                                 }
1364                         }
1365
1366                         if(hm >= 0 && hmf >= 0) {
1367                                 hmr=t->tm_hour*100+t->tm_min;
1368                                 if(hmr < hm || hmr > hmf) continue;
1369                         }
1370
1371                         if(site[0] != '\0'){
1372                                 if(strstr(url,site)==0) continue;
1373                         }
1374
1375                         if(UserIp) {
1376                                 strcpy(user,ip);
1377                                 id_is_ip=true;
1378                         } else {
1379                                 id_is_ip=false;
1380                                 if(strcmp(user,"-") == 0 || strcmp(user," ") == 0 || strcmp(user,"") == 0) {
1381                                         if(RecordsWithoutUser == RECORDWITHOUTUSER_IP) {
1382                                                 strcpy(user,ip);
1383                                                 id_is_ip=true;
1384                                         }
1385                                         if(RecordsWithoutUser == RECORDWITHOUTUSER_IGNORE)
1386                                                 continue;
1387                                         if(RecordsWithoutUser == RECORDWITHOUTUSER_EVERYBODY)
1388                                                 strcpy(user,"everybody");
1389                                 } else {
1390                                         strlow(user);
1391                                         if(NtlmUserFormat == NTLMUSERFORMAT_USER) {
1392                                                 if((str = strchr(user,'_')) != 0) {
1393                                                         strcpy(warea,str+1);
1394                                                         strcpy(user,warea);
1395                                                 }
1396                                                 if((str = strchr(user,'+')) != 0) {
1397                                                         strcpy(warea,str+1);
1398                                                         strcpy(user,warea);
1399                                                 }
1400                                         }
1401                                 }
1402                         }
1403
1404                         if(us[0] != '\0'){
1405                                 if(strcmp(user,us)!=0) continue;
1406                         }
1407
1408                         if(puser) {
1409                                 sprintf(wuser,":%s:",user);
1410                                 if(strstr(userfile, wuser) == 0)
1411                                         continue;
1412                         }
1413
1414                         if(fuser) {
1415                                 if(!vuexclude(user)) {
1416                                         if (debugm) printf(_("Excluded user: %s\n"),user);
1417                                         totregsx++;
1418                                         continue;
1419                                 }
1420                         }
1421
1422                         if(strcmp(user,"-") ==0 || strcmp(user," ") ==0 || strcmp(user,"") ==0 || strcmp(user,":") ==0)
1423                                 continue;
1424
1425                         if(max_elapsed) {
1426                                 if(atol(elap)>max_elapsed) {
1427                                         elap[0]='0';
1428                                         elap[1]='\0';
1429                                 }
1430                         }
1431
1432                         if((str=(char *) strstr(linebuf, "[SmartFilter:")) != (char *) NULL ) {
1433                                 fixendofline(str);
1434                                 sprintf(smartfilter,"\"%s\"",str+1);
1435                         } else sprintf(smartfilter,"\"\"");
1436
1437                         nopen=0;
1438                         prev_ufile=NULL;
1439                         for (ufile=first_user_file ; ufile && strcmp(user,ufile->user->id)!=0 ; ufile=ufile->next) {
1440                                 prev_ufile=ufile;
1441                                 if (ufile->file) nopen++;
1442                         }
1443                         if (!ufile) {
1444                                 ufile=malloc(sizeof(*ufile));
1445                                 if (!ufile) {
1446                                         debuga(_("Not enough memory to store the user %s\n"),user);
1447                                         exit(EXIT_FAILURE);
1448                                 }
1449                                 memset(ufile,0,sizeof(*ufile));
1450                                 ufile->next=first_user_file;
1451                                 first_user_file=ufile;
1452                                 uinfo=userinfo_create(user);
1453                                 ufile->user=uinfo;
1454                                 uinfo->id_is_ip=id_is_ip;
1455                         } else {
1456                                 if (prev_ufile) {
1457                                         prev_ufile->next=ufile->next;
1458                                         ufile->next=first_user_file;
1459                                         first_user_file=ufile;
1460                                 }
1461                         }
1462
1463                         if (ufile->file==NULL) {
1464                                 if (nopen>=maxopenfiles) {
1465                                         x=0;
1466                                         for (ufile1=first_user_file ; ufile1 ; ufile1=ufile1->next) {
1467                                                 if (ufile1->file!=NULL) {
1468                                                         if (x>=maxopenfiles) {
1469                                                                 if (fclose(ufile1->file)==EOF) {
1470                                                                         debuga(_("Failed to close the log file of user %s - %s\n"),ufile1->user->id,strerror(errno));
1471                                                                         exit(EXIT_FAILURE);
1472                                                                 }
1473                                                                 ufile1->file=NULL;
1474                                                         }
1475                                                         x++;
1476                                                 }
1477                                         }
1478                                 }
1479                                 if (snprintf (tmp3, sizeof(tmp3), "%s/%s.unsort", tmp, ufile->user->filename)>=sizeof(tmp3)) {
1480                                         debuga(_("Temporary user file name too long: %s/%s.unsort\n"), tmp, ufile->user->filename);
1481                                         exit(EXIT_FAILURE);
1482                                 }
1483                                 if ((ufile->file = MY_FOPEN (tmp3, "a")) == NULL) {
1484                                         debuga(_("(log) Cannot open temporary file: %s - %s\n"), tmp3, strerror(errno));
1485                                         exit (1);
1486                                 }
1487                         }
1488
1489                         /*if ( strcmp ( user , sz_Last_User ) != 0 ) {
1490                                 if ( fp_Write_User )
1491                                         fclose( fp_Write_User ) ;
1492                                 sprintf (tmp3, "%s/%s.unsort", tmp, user);
1493
1494                                 if ((fp_Write_User = MY_FOPEN (tmp3, "a")) == NULL) {
1495                                         fprintf (stderr, "%s: (log) %s: %s - %s\n", argv[0], _("Cannot open temporary file"), tmp3, strerror(errno));
1496                                         exit (1);
1497                                 }
1498                                 strcpy( sz_Last_User , user ) ;
1499                         }*/
1500                         if (fprintf(ufile->file, "%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\n",dia,hora,ip,url,tam,code,elap,smartfilter)<=0) {
1501                                 debuga(_("Write error in the log file of user %s\n"),user);
1502                                 exit(EXIT_FAILURE);
1503                         }
1504
1505                         if(fp_log && ilf!=ILF_Sarg)
1506                                 fprintf(fp_log, "%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\n",dia,hora,user,ip,url,tam,code,elap,smartfilter);
1507
1508                         totregsg++;
1509
1510                         if(!dataonly && download_flag && download_url && strstr(code,"DENIED") == 0) {
1511                                 ndownload = 1;
1512
1513                                 if ( ! fp_Download_Unsort ) {
1514                                         if ((fp_Download_Unsort = MY_FOPEN ( sz_Download_Unsort, "a")) == NULL) {
1515                                                 debuga(_("(log) Cannot open temporary file: %s - %s\n"),sz_Download_Unsort, strerror(errno));
1516                                                 exit (1);
1517                                         }
1518                                 }
1519                                 fprintf(fp_Download_Unsort,"%s\t%s\t%s\t%s\t%s\n",dia,hora,user,ip,download_url);
1520                         }
1521
1522                         if((ReportType & REPORT_TYPE_DENIED) != 0 || (ReportType & REPORT_TYPE_AUTH_FAILURES) != 0) {
1523                                 if(fp_denied && strstr(code,"DENIED/403") != 0) {
1524                                         fprintf(fp_denied, "%s\t%s\t%s\t%s\t%s\n",dia,hora,user,ip,urly);
1525                                         denied_count++;
1526                                 }
1527                                 if(fp_authfail && (strstr(code,"DENIED/401") != 0 || strstr(code,"DENIED/407") != 0)) {
1528                                         fprintf(fp_authfail, "%s\t%s\t%s\t%s\t%s\n",dia,hora,user,ip,urly);
1529                                         authfail_count++;
1530                                 }
1531                         }
1532
1533                         if (ilf!=ILF_Sarg) {
1534                                 if(!totper || idata<mindate){
1535                                         mindate=idata;
1536                                         memcpy(&period.start,t,sizeof(*t));
1537                                         strcpy(start_hour,tbuf2);
1538                                 }
1539                                 if (!totper || idata>maxdate) {
1540                                         maxdate=idata;
1541                                         memcpy(&period.end,t,sizeof(*t));
1542                                 }
1543                                 totper=true;
1544                         }
1545
1546                         if(debugm){
1547                                 printf("IP=\t%s\n",ip);
1548                                 printf("USER=\t%s\n",user);
1549                                 printf("ELAP=\t%s\n",elap);
1550                                 printf("DATE=\t%s\n",dia);
1551                                 printf("TIME=\t%s\n",hora);
1552                                 printf("FUNC=\t%s\n",fun);
1553                                 printf("URL=\t%s\n",url);
1554                                 printf("CODE=\t%s\n",code);
1555                                 printf("LEN=\t%s\n",tam);
1556                         }
1557                 }
1558                 if (!from_stdin) {
1559                         if (from_pipe)
1560                                 pclose(fp_in);
1561                         else {
1562                                 fclose(fp_in);
1563                                 if( ShowReadStatistics )
1564                                         printf(_("SARG: Records in file: %lu, reading: %3.2f%%\n"),recs1, (float) 100 );
1565                         }
1566                 }
1567         }
1568
1569         if (debug)
1570                 debuga(_("   Records read: %ld, written: %ld, excluded: %ld\n"),totregsl,totregsg,totregsx);
1571
1572         longline_destroy(&line);
1573         if ( fp_Download_Unsort )
1574                 fclose (fp_Download_Unsort);
1575
1576         for (ufile=first_user_file ; ufile ; ufile=ufile1) {
1577                 ufile1=ufile->next;
1578                 if (ufile->file!=NULL) fclose(ufile->file);
1579                 free(ufile);
1580         }
1581
1582         free_download();
1583         free_excludecodes();
1584         free_exclude();
1585
1586         if(debug) {
1587                 int totalcount=0;
1588
1589                 for (ilf=0 ; ilf<ILF_Last ; ilf++) totalcount+=ilf_count[ilf];
1590
1591                 if(ilf_count[ILF_Common]>0 && ilf_count[ILF_Squid]>0)
1592                         debuga(_("Log with mixed records format (squid and common log)\n"));
1593
1594                 if(ilf_count[ILF_Common]>0 && ilf_count[ILF_Squid]==0)
1595                         debuga(_("Common log format\n"));
1596
1597                 if(ilf_count[ILF_Common]==0 && ilf_count[ILF_Squid]>0)
1598                         debuga(_("Squid log format\n"));
1599
1600                 if(ilf_count[ILF_Sarg]>0)
1601                         debuga(_("Sarg log format\n"));
1602
1603                 if(totalcount==0 && totregsg)
1604                         debuga(_("Log with invalid format\n"));
1605         }
1606
1607         if(!totregsg){
1608                 debuga(_("No records found\n"));
1609                 debuga(_("End\n"));
1610                 if(fp_denied) fclose(fp_denied);
1611                 if(fp_authfail) fclose(fp_authfail);
1612                 userinfo_free();
1613                 if(userfile) free(userfile);
1614                 close_usertab();
1615                 exit(EXIT_SUCCESS);
1616         }
1617
1618         if (date[0]!='\0') {
1619                 char date0[30], date1[30];
1620
1621                 strftime(date0,sizeof(date0),"%d/%m/%Y",&period.start);
1622                 strftime(date1,sizeof(date1),"%d/%m/%Y",&period.end);
1623                 debuga(_("Period covered by log files: %s-%s\n"),date0,date1);
1624                 getperiod_fromrange(&period,dfrom,duntil);
1625         }
1626         if (getperiod_buildtext(&period)<0) {
1627                 debuga(_("Failed to build the string representation of the date range\n"));
1628                 exit(EXIT_FAILURE);
1629         }
1630
1631         if(debugz){
1632                 debugaz("data",dia);
1633                 debugaz("period",period.text);
1634         }
1635
1636         if(debug)
1637                 debuga(_("Period: %s\n"),period.text);
1638
1639         if(fp_denied)
1640                 fclose(fp_denied);
1641         if(fp_authfail)
1642                 fclose(fp_authfail);
1643
1644         if(fp_log != NULL) {
1645                 fclose(fp_log);
1646                 strcpy(end_hour,tbuf2);
1647                 strftime(val2,sizeof(val2),"%d%m%Y",&period.start);
1648                 strftime(val1,sizeof(val1),"%d%m%Y",&period.end);
1649                 sprintf(val4,"%s/sarg-%s_%s-%s_%s.log",ParsedOutputLog,val2,start_hour,val1,end_hour);
1650                 if (rename(arq_log,val4)) {
1651                         debuga(_("failed to rename %s to %s - %s\n"),arq_log,val4,strerror(errno));
1652                 } else {
1653                         strcpy(arq_log,val4);
1654
1655                         if(strcmp(ParsedOutputLogCompress,"nocompress") != 0 && ParsedOutputLogCompress[0] != '\0') {
1656                                 /*
1657                                 No double quotes around ParsedOutputLogCompress because it may contain command line options. If double quotes are
1658                                 necessary around the command name, put them in the configuration file.
1659                                 */
1660                                 sprintf(val1,"%s \"%s\"",ParsedOutputLogCompress,arq_log);
1661                                 cstatus=system(val1);
1662                                 if (!WIFEXITED(cstatus) || WEXITSTATUS(cstatus)) {
1663                                         debuga(_("command return status %d\n"),WEXITSTATUS(cstatus));
1664                                         debuga(_("command: %s\n"),val1);
1665                                         exit(EXIT_FAILURE);
1666                                 }
1667                         }
1668                 }
1669                 if(debug)
1670                         debuga(_("Sarg parsed log saved as %s\n"),arq_log);
1671         }
1672
1673         if(DataFile[0] == '\0' && (ReportType & REPORT_TYPE_DENIED) != 0) {
1674                 sprintf(csort,"sort -T \"%s\" -k 3,3 -k 5,5 -o \"%s\" \"%s\"",tmp,denied_sort,denied_unsort);
1675                 cstatus=system(csort);
1676                 if (!WIFEXITED(cstatus) || WEXITSTATUS(cstatus)) {
1677                         debuga(_("sort command return status %d\n"),WEXITSTATUS(cstatus));
1678                         debuga(_("sort command: %s\n"),csort);
1679                         exit(EXIT_FAILURE);
1680                 }
1681                 if (unlink(denied_unsort)) {
1682                         debuga(_("Cannot delete %s - %s\n"),denied_unsort,strerror(errno));
1683                         exit(EXIT_FAILURE);
1684                 }
1685         }
1686
1687         sort_users_log(tmp, debug);
1688
1689         if(DataFile[0] != '\0')
1690                 data_file(tmp);
1691         else
1692                 gerarel();
1693
1694         if((ReportType & REPORT_TYPE_DENIED) != 0) {
1695                 unlink(denied_sort);
1696         }
1697
1698         if(strcmp(tmp,"/tmp") != 0) {
1699                 unlinkdir(tmp,0);
1700         }
1701
1702         userinfo_free();
1703         if(userfile)
1704                 free(userfile);
1705         close_usertab();
1706
1707         if(debug)
1708                 debuga(_("End\n"));
1709
1710         exit(EXIT_SUCCESS);
1711 }
1712
1713
1714 static void getusers(const char *pwdfile, int debug)
1715 {
1716         FILE *fp_usr;
1717         char buf[255];
1718         char *str;
1719         long int nreg=0;
1720
1721         if(debug)
1722                 debuga(_("Loading password file from %s\n"),pwdfile);
1723
1724         if ((fp_usr = fopen(pwdfile, "r")) == NULL) {
1725                 debuga(_("(getusers) Cannot open file %s - %s\n"),pwdfile,strerror(errno));
1726                 exit(EXIT_FAILURE);
1727         }
1728
1729         if (fseek(fp_usr, 0, SEEK_END)==-1) {
1730                 debuga(_("Failed to move till the end of the users file %s: %s\n"),pwdfile,strerror(errno));
1731                 exit(EXIT_FAILURE);
1732         }
1733         nreg = ftell(fp_usr);
1734         if (nreg<0) {
1735                 debuga(_("Cannot get the size of file %s\n"),pwdfile);
1736                 exit(EXIT_FAILURE);
1737         }
1738         nreg = nreg+5000;
1739         if (fseek(fp_usr, 0, SEEK_SET)==-1) {
1740                 debuga(_("Failed to rewind the users file %s: %s\n"),pwdfile,strerror(errno));
1741                 exit(EXIT_FAILURE);
1742         }
1743
1744         if((userfile=(char *) malloc(nreg))==NULL){
1745                 debuga(_("malloc error (%ld)\n"),nreg);
1746                 exit(EXIT_FAILURE);
1747         }
1748
1749         bzero(userfile,nreg);
1750         strcpy(userfile,":");
1751
1752         while(fgets(buf,sizeof(buf),fp_usr)!=NULL) {
1753                 str=strchr(buf,':');
1754                 if (!str) {
1755                         debuga(_("You have an invalid user in your %s file\n"),pwdfile);
1756                         exit(EXIT_FAILURE);
1757                 }
1758                 str[1]='\0';
1759                 strcat(userfile,buf);
1760         }
1761
1762         fclose(fp_usr);
1763
1764         return;
1765 }