log.c

   1 /*
   2  * AUTHOR: Pedro Lineu Orso                         pedro.orso@gmail.com
   3  *                                                            1998, 2008
   4  * SARG Squid Analysis Report Generator      http://sarg.sourceforge.net
   5  *
   6  * SARG donations:
   7  *      please look at http://sarg.sourceforge.net/donations.php
   8  * ---------------------------------------------------------------------
   9  *
  10  *  This program is free software; you can redistribute it and/or modify
  11  *  it under the terms of the GNU General Public License as published by
  12  *  the Free Software Foundation; either version 2 of the License, or
  13  *  (at your option) any later version.
  14  *
  15  *  This program is distributed in the hope that it will be useful,
  16  *  but WITHOUT ANY WARRANTY; without even the implied warranty of
  17  *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  18  *  GNU General Public License for more details.
  19  *
  20  *  You should have received a copy of the GNU General Public License
  21  *  along with this program; if not, write to the Free Software
  22  *  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111, USA.
  23  *
  24  */
  25
  26 #include "include/conf.h"
  27 #include "include/defs.h"
  28
  29 #define LEGACY_WRITE_USER
  30 #define LEGACY_WRITE_DOWNLOAD
  31 #define REPORT_EVERY_X_LINES 5000
  32
  33
  34 char *userfile;
  35 char *excludefile;
  36 char *excludeuser;
  37
  38 char sz_Last_User[ MAXLEN ] = { 0 } ;
  39 int bool_ShowReadStatistics ;
  40
  41 numlist weekdays = { { 0, 1, 2, 3, 4, 5, 6 }, 7 };
  42 numlist hours = { { 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12,
  43              13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23 }, 24 };
  44
  45 static void getusers(const char *pwdfile, int debug);
  46 static void gethexclude(const char *hexfile, int debug);
  47 static void getuexclude(const char *uexfile, int debug);
  48
  49 int main(int argc,char *argv[])
  50 {
  51
  52    FILE *fp_in = NULL, *fp_ou = NULL, *fp_denied=NULL, *fp_authfail=NULL, *fp_log=NULL;
  53
  54    char sz_Download_Unsort[ 20000 ] ;
  55    FILE * fp_Download_Unsort = NULL ;
  56    FILE * fp_Write_User = NULL ;
  57
  58    extern int optind;
  59    extern int optopt;
  60    extern char *optarg;
  61
  62    char w[MAXLEN];
  63    char data[255];
  64    char elap[255];
  65    char none[255];
  66    char ip[MAXLEN];
  67    char msg[MAXLEN];
  68    char tam[255];
  69    char fun[MAXLEN];
  70    char wuser[MAXLEN];
  71    char smartfilter[MAXLEN];
  72    char dia[128];
  73    char wdata[128];
  74    char mes[30];
  75    char ano[30];
  76    char hora[30];
  77    char wtemp[MAXLEN];
  78    char wtemp2[255];
  79    char date[255];
  80    char arq[255];
  81    char arq_log[255];
  82    char warq[255][255];
  83    char hm[15], hmf[15], hmr[15], whm[15];
  84    int  chm=0;
  85    char uagent[MAXLEN];
  86    char hexclude[MAXLEN];
  87    char csort[MAXLEN];
  88    int cstatus;
  89    char tbuf[128];
  90    char tbuf2[128];
  91    char zip[20];
  92    char *str;
  93    char bufz[MAXLEN];
  94    char bufy[MAXLEN];
  95    int  common;
  96    int  common_log=0;
  97    int  squid_log=0;
  98    int  ch;
  99    int  x, l;
 100    int  errflg=0;
 101    int  puser=0;
 102    int  fhost=0;
 103    int  dns=0;
 104    int  fuser=0;
 105    int  idata=0;
 106    int  mindate=0;
 107    int  narq=0;
 108    int  iarq=0;
 109    int  exstring=0;
 110    int  i0=0,i1=-1,i2=-1,i3=-1,i4=-1,i5=-1,i6=-1,i7=-1,i8=-1;
 111    long totregsl=0;
 112    long totregsg=0;
 113    long totregsx=0;
 114    long totper=0;
 115    long int  max_elapsed=0;
 116    time_t tt;
 117    struct tm *t;
 118    off_t recs1=0;
 119    unsigned long recs2=0;
 120    struct rlimit rl;
 121    int OutputNonZero = REPORT_EVERY_X_LINES ;
 122    int download_flag;
 123    char download_url[MAXLEN];
 124
 125    BgImage[0]='\0';
 126    LogoImage[0]='\0';
 127    LogoText[0]='\0';
 128    PasswdFile[0]='\0';
 129    OutputEmail[0]='\0';
 130    Ip2Name[0]='\0';
 131    UserAgentLog[0]='\0';
 132    ExcludeHosts[0]='\0';
 133    ExcludeUsers[0]='\0';
 134    ConfigFile[0]='\0';
 135    code[0]='\0';
 136    LastLog[0]='\0';
 137    UserIp[0]='\0';
 138    ReportType[0]='\0';
 139    UserTabFile[0]='\0';
 140    BlockIt[0]='\0';
 141    ExternalCSSFile[0]='\0';
 142    SquidGuardLogFormat[0]='\0';
 143    SquidGuardLogAlternate[0]='\0';
 144    arq[0]='\0';
 145
 146    strcpy(AccessLog,"/usr/local/squid/var/logs/access.log");
 147    sprintf(ExcludeCodes,"%s/exclude_codes",SYSCONFDIR);
 148    strcpy(GraphDaysBytesBarColor,"orange");
 149    strcpy(BgColor,"#ffffff");
 150    strcpy(TxColor,"#000000");
 151    strcpy(TxBgColor,"lavender");
 152    strcpy(TiColor,"darkblue");
 153    strcpy(Width,"80");
 154    strcpy(Height,"45");
 155    strcpy(LogoTextColor,"#000000");
 156    strcpy(HeaderColor,"darkblue");
 157    strcpy(HeaderBgColor,"#dddddd");
 158    strcpy(LogoTextColor,"#006699");
 159    strcpy(FontSize,"9px");
 160    strcpy(TempDir,"/tmp");
 161    strcpy(OutputDir,"/var/www/html/squid-reports");
 162    strcpy(Ip2Name,"no");
 163    strcpy(DateFormat,"u");
 164    strcpy(OverwriteReport,"no");
 165    strcpy(RemoveTempFiles,"yes");
 166    strcpy(ReplaceIndex,"index.html");
 167    strcpy(Index,"yes");
 168    strcpy(RecordsWithoutUser,"ip");
 169    strcpy(UseComma,"no");
 170    strcpy(MailUtility,"mailx");
 171    strcpy(TopSitesNum,"100");
 172    strcpy(UserIp,"no");
 173    strcpy(TopuserSortField,"BYTES");
 174    strcpy(UserSortField,"BYTES");
 175    strcpy(TopuserSortOrder,"reverse");
 176    strcpy(UserSortOrder,"reverse");
 177    strcpy(TopsitesSortField,"CONNECT");
 178    strcpy(TopsitesSortType,"D");
 179    strcpy(LongUrl,"no");
 180    strcpy(language,"English");
 181    strcpy(FontFace,"Verdana,Tahoma,Arial");
 182    strcpy(datetimeby,"elap");
 183    strcpy(CharSet,"ISO-8859-1");
 184    strcpy(Privacy,"no");
 185    strcpy(PrivacyString,"***.***.***.***");
 186    strcpy(PrivacyStringColor,"blue");
 187    strcpy(SuccessfulMsg,"yes");
 188    strcpy(TopUserFields,"NUM DATE_TIME USERID CONNECT BYTES %BYTES IN-CACHE-OUT USED_TIME MILISEC %TIME TOTAL AVERAGE");
 189    strcpy(UserReportFields,"CONNECT BYTES %BYTES IN-CACHE-OUT USED_TIME MILISEC %TIME TOTAL AVERAGE");
 190    strcpy(DataFileDelimiter,";");
 191    strcpy(DataFileFields,"user;date;time;url;connect;bytes;in_cache;out_cache;elapsed");
 192    strcpy(SiteUserTimeDateType,"table");
 193    strcpy(ShowReadStatistics,"yes");
 194    strcpy(IndexSortOrder,"D");
 195    strcpy(ShowSargInfo,"yes");
 196    strcpy(ShowSargLogo,"yes");
 197    strcpy(ParsedOutputLog,"no");
 198    strcpy(ParsedOutputLogCompress,"/bin/gzip");
 199    strcpy(DisplayedValues,"abbreviation");
 200    strcpy(HeaderFontSize,"9px");
 201    strcpy(TitleFontSize,"11px");
 202    strcpy(AuthUserFile,"/usr/local/sarg/passwd");
 203    strcpy(AuthName,"SARG, Restricted Access");
 204    strcpy(AuthType,"basic");
 205    strcpy(Require,"require user admin %u");
 206    strcpy(DownloadSuffix,"7z,ace,arj,avi,bat,bin,bz2,bzip,cab,com,cpio,dll,doc,dot,exe,gz,iso,lha,lzh,mdb,mov,mp3,mpeg,mpg,mso,nrg,ogg,ppt,rar,rtf,shs,src,sys,tar,tgz,vcd,vob,wma,wmv,zip");
 207    strcpy(Graphs,"yes");
 208    strcpy(Ulimit,"20000");
 209    strcpy(NtlmUserFormat,"domainname+username");
 210    strcpy(IndexTree,"file");
 211    strcpy(RealtimeTypes,"GET,PUT,CONNECT");
 212    strcpy(RealtimeUnauthRec,"show");
 213    strcpy(SquidguardIgnoreDate,"off");
 214    strcpy(DansguardianIgnoreDate,"off");
 215    strcpy(DataFileUrl,"ip");
 216    strcpy(MaxElapsed,"28800000");
 217    strcpy(BytesInSitesUsersReport,"no");
 218
 219    dia[0]='\0';
 220    mes[0]='\0';
 221    ano[0]='\0';
 222    hora[0]='\0';
 223    tmp[0]='\0';
 224    tmp2[0]='\0';
 225    tmp3[0]='\0';
 226    wtemp[0]='\0';
 227    wtemp2[0]='\0';
 228    us[0]='\0';
 229    date[0]='\0';
 230    df[0]='\0';
 231    uagent[0]='\0';
 232    hexclude[0]='\0';
 233    addr[0]='\0';
 234    hm[0]='\0';
 235    hmf[0]='\0';
 236    site[0]='\0';
 237    outdir[0]='\0';
 238    elap[0]='\0';
 239    email[0]='\0';
 240    zip[0]='\0';
 241    UserInvalidChar[0]='\0';
 242    DataFile[0]='\0';
 243    SquidGuardConf[0]='\0';
 244    DansGuardianConf[0]='\0';
 245
 246    denied_count=0;
 247    download_count=0;
 248    authfail_count=0;
 249    dansguardian_count=0;
 250    squidguard_count=0;
 251    DeniedReportLimit=10;
 252    AuthfailReportLimit=10;
 253    DansGuardianReportLimit=10;
 254    SquidGuardReportLimit=10;
 255    DownloadReportLimit=50;
 256    UserReportLimit=0;
 257    debug=0;
 258    debugz=0;
 259    debugm=0;
 260    iprel=0;
 261    userip=0;
 262    color1=0;
 263    color2=0;
 264    color3=0;
 265    sarglog=0;
 266    isalog=0;
 267    dotinuser=0;
 268    realt=0;
 269    realtime_refresh=3;
 270    realtime_access_log_lines=1000;
 271    cost=0.01;
 272    nocost=50000000;
 273    ndownload=0;
 274    squid24=0;
 275
 276    bzero(IncludeUsers, MAXLEN);
 277    bzero(ExcludeString, MAXLEN);
 278
 279    for(x=0; x<=254; x++)
 280       warq[x][0]='\0';
 281
 282    language_load(language);
 283    strcpy(Title,text[88]);
 284
 285    while((ch = getopt(argc, argv, "a:b:c:d:e:f:g:u:l:L:o:s:t:w:hijmnprvxyz")) != -1){
 286       switch(ch)
 287       {
 288          case 'a':
 289             strcpy(addr,optarg);
 290             break;
 291          case 'b':
 292             strcpy(uagent,optarg);
 293             break;
 294          case 'c':
 295             strcpy(hexclude,optarg);
 296             break;
 297          case 'd':
 298             strcpy(date,optarg);
 299             strcpy(cduntil,optarg);
 300             if (getword(cdfrom,sizeof(cdfrom),cduntil,'-')<0) {
 301                printf("SARG: Maybe you have a broken record or garbage in your date range.\n");
 302                exit(1);
 303             }
 304             date_from(date, cdfrom, cduntil);
 305             dfrom=atoi(cdfrom);
 306             duntil=atoi(cduntil);
 307             break;
 308          case 'e':
 309             strcpy(email,optarg);
 310             break;
 311          case 'f':
 312             strcpy(ConfigFile,optarg);
 313             break;
 314          case 'g':
 315             strcpy(df,optarg);
 316             break;
 317          case 'h':
 318             usage(argv[0]);
 319             exit(0);
 320             break;
 321          case 'i':
 322             iprel++;
 323             break;
 324          case 'l':
 325             if (narq>=sizeof(warq)/sizeof(warq[0])) {
 326                printf("SARG: Too many log files.\n");
 327                exit(1);
 328             }
 329             strcpy(warq[narq],optarg);
 330             narq++;
 331             break;
 332          case 'L':
 333             strcpy(SquidGuardLogAlternate,optarg);
 334             break;
 335          case 'm':
 336             debugm++;
 337             break;
 338          case 'n':
 339             dns++;
 340             break;
 341          case 'o':
 342             strcpy(outdir,optarg);
 343             break;
 344          case 'p':
 345             userip++;
 346             break;
 347          case 'r':
 348             realt++;
 349             break;
 350          case 's':
 351             strcpy(site,optarg);
 352             break;
 353          case 't':
 354             if(strstr(optarg,"-") == 0) {
 355               strcpy(hm,optarg);
 356               strcpy(hmf,optarg);
 357             } else {
 358                if (getword(hm,sizeof(hm),optarg,'-')<0) {
 359                   printf("SARG: Maybe you have a broken record or garbage in your time range.\n");
 360                   exit(1);
 361                }
 362                strcpy(hmf,optarg);
 363             }
 364             if(strlen(hm) > 5) {
 365                printf("SARG: time period must be MM or MM:SS. Exit.\n");
 366                exit(1);
 367             }
 368             bzero(whm,15);
 369             if(strstr(hm,":") != 0) {
 370                if (getword(warea,sizeof(warea),hm,':')<0) {
 371                   printf("SARG: Maybe you have a broken record or garbage in your time range.\n");
 372                   exit(1);
 373                }
 374                sprintf(whm,"%s%s",warea,hm);
 375                strcpy(hm,whm);
 376             }
 377             bzero(whm,15);
 378             if(strstr(hmf,":") != 0) {
 379                if (getword(warea,sizeof(warea),hmf,':')<0) {
 380                   printf("SARG: Maybe you have a broken record or garbage in your time range.\n");
 381                   exit(1);
 382                }
 383                sprintf(whm,"%s%s",warea,hmf);
 384                strcpy(hmf,whm);
 385             }
 386             break;
 387          case 'u':
 388             strcpy(us,optarg);
 389             break;
 390          case 'v':
 391             version();
 392             break;
 393          case 'w':
 394             strcpy(tmp,optarg);
 395             break;
 396          case 'x':
 397             debug++;
 398             break;
 399          case 'y':
 400             langcode++;
 401             break;
 402          case 'z':
 403             debugz++;
 404             break;
 405          case ':':
 406             fprintf(stderr, "Option -%c require an argument\n",optopt);
 407             errflg++;
 408             break;
 409          case '?':
 410             usage(argv[0]);
 411             exit(1);
 412             break;
 413       }
 414
 415    }
 416
 417    if (errflg) {
 418       usage(argv[0]);
 419       exit(2);
 420    }
 421
 422    if(debug) debuga("Init");
 423
 424    if(ConfigFile[0] == '\0') sprintf(ConfigFile,"%s/sarg.conf",SYSCONFDIR);
 425    if(access(ConfigFile, R_OK) != 0) {
 426       debuga("Cannot open config file: %s - %s",ConfigFile,strerror(errno));
 427       exit(1);
 428    }
 429
 430    if(access(ConfigFile, R_OK) == 0)
 431       getconf();
 432
 433    if(realt) {
 434       realtime();
 435       exit(0);
 436    }
 437
 438    if(strcmp(IndexTree,"file") == 0)
 439       strcpy(ImageFile,"../images");
 440    else
 441       strcpy(ImageFile,"../../../images");
 442
 443    dataonly=0;
 444    if(DataFile[0] != '\0');
 445       dataonly++;
 446
 447    subs(TopUserFields,sizeof(TopUserFields),"%BYTES","SETYB");
 448
 449    subs(UserReportFields,sizeof(UserReportFields),"%BYTES","SETYB");
 450
 451    if(!narq) {
 452       strcpy(warq[0],AccessLog);
 453       narq++;
 454    }
 455
 456    if(strcmp(hexclude,"onvert") == 0 && strcmp(site,"plit") != 0) {
 457       convlog(warq[0], df, dfrom, duntil);
 458       exit(0);
 459    }
 460
 461    if(strcmp(site,"plit") == 0) {
 462       splitlog(warq[0], df, dfrom, duntil, hexclude);
 463       exit(0);
 464    }
 465
 466    load_excludecodes(ExcludeCodes);
 467
 468    if(access(PasswdFile, R_OK) == 0) {
 469       getusers(PasswdFile,debug);
 470       puser++;
 471    }
 472
 473    if(hexclude[0] == '\0')
 474       strcpy(hexclude,ExcludeHosts);
 475    if(strlen(hexclude) > 0) {
 476       if(access(hexclude, R_OK) != 0) {
 477          debuga("Cannot open exclude_hosts file: %s - %s",hexclude,strerror(errno));
 478          exit(1);
 479       }
 480       gethexclude(hexclude,debug);
 481       fhost++;
 482    }
 483
 484    if(ReportType[0] == '\0')
 485       strcpy(ReportType,"topusers topsites users_sites sites_users date_time denied auth_failures site_user_time_date downloads");
 486
 487    if(access(ExcludeUsers, R_OK) == 0) {
 488       getuexclude(ExcludeUsers,debug);
 489       fuser++;
 490    }
 491
 492    indexonly=0;
 493    if(fuser) {
 494       if(strstr(excludeuser,"indexonly") != 0)
 495          indexonly++;
 496    }
 497    if(strcmp(ExcludeUsers,"indexonly") == 0) indexonly++;
 498    if(strcmp(Index,"only") == 0) indexonly++;
 499
 500    if(dns) strcpy(Ip2Name,"yes");
 501
 502    if(strcmp(UserIp,"yes") == 0) userip++;
 503
 504    if(strlen(MaxElapsed)>1) max_elapsed=atol(MaxElapsed);
 505
 506    if(strlen(outdir)<1) strcpy(outdir,OutputDir);
 507    strcat(outdir,"/");
 508
 509
 510    if(arq[0] == '\0') strcpy(arq,AccessLog);
 511
 512    if(uagent[0] == '\0') strcpy(uagent,UserAgentLog);
 513
 514    if(tmp[0] == '\0') strcpy(tmp,TempDir);
 515    else strcpy(TempDir,tmp);
 516
 517    if(df[0] == '\0') strcpy(df,DateFormat);
 518    else strcpy(DateFormat,df);
 519
 520    if(df[0] == '\0') {
 521       strcpy(df,"u");
 522       strcpy(DateFormat,"u");
 523    }
 524
 525    if(email[0] == '\0' && OutputEmail[0] != '\0') strcpy(email,OutputEmail);
 526
 527    strcpy(tmp2,tmp);
 528
 529    if(strlen(email) > 0) {
 530       sprintf(wtemp2,"%s/sarg",tmp2);
 531       my_mkdir(wtemp2);
 532       strcat(tmp2,"/sarg");
 533       strcpy(outdir,tmp2);
 534       strcat(outdir,"/");
 535    }
 536
 537    strcat(tmp2,"/sarg.log");
 538
 539    sprintf(warea,"%s/sarg",tmp);
 540    if(access(warea, R_OK) == 0) {
 541       sprintf(tmp3,"rm -rf %s",warea);
 542       system(tmp3);
 543    }
 544
 545    sprintf(tmp3,"%s/sarg",tmp);
 546    my_mkdir(tmp3);
 547    strcpy(tmp4,tmp3);
 548    strcpy(tmp5,tmp3);
 549    strcpy(tmp6,tmp3);
 550    strcat(tmp4,"/denied.log.unsort");
 551    strcat(tmp5,"/denied.log");
 552    strcat(tmp6,"/authfail.log.unsort");
 553
 554    if(debug) {
 555       fprintf(stderr, "SARG: %s:\nSARG:\n",text[22]);
 556       fprintf(stderr, "SARG: %35s (-a) = %s\n",text[23],addr);
 557       fprintf(stderr, "SARG: %35s (-b) = %s\n",text[71],uagent);
 558       fprintf(stderr, "SARG: %35s (-c) = %s\n",text[69],hexclude);
 559       fprintf(stderr, "SARG: %35s (-d) = %s\n",text[24],date);
 560       fprintf(stderr, "SARG: %35s (-e) = %s\n",text[41],email);
 561       fprintf(stderr, "SARG: %35s (-f) = %s\n",text[70],ConfigFile);
 562       if(strcmp(df,"e") == 0)
 563          fprintf(stderr, "SARG: %35s (-g) = %s (dd/mm/yyyy)\n",text[25],text[26]);
 564       if(strcmp(df,"u") == 0)
 565          fprintf(stderr, "SARG: %35s (-g) = %s (mm/dd/yyyy)\n",text[25],text[27]);
 566       if(strcmp(df,"w") == 0)
 567          fprintf(stderr, "SARG: %35s (-g) = %s (yyyy/ww)\n",text[25],text[85]);
 568       if(iprel)
 569          fprintf(stderr, "SARG: %35s (-i) = %s\n",text[28],text[1]);
 570        else
 571          fprintf(stderr, "SARG: %35s (-i) = %s\n",text[28],text[2]);
 572       fprintf(stderr, "SARG: %35s (-l) = %s\n",text[37],arq);
 573       if(strcmp(Ip2Name,"yes") == 0)
 574          fprintf(stderr, "SARG: %35s (-n) = %s\n",text[65],text[1]);
 575        else
 576          fprintf(stderr, "SARG: %35s (-n) = %s\n",text[65],text[2]);
 577       fprintf(stderr, "SARG: %35s (-o) = %s\n",text[38],outdir);
 578       if(strcmp(UserIp,"yes") == 0)
 579          fprintf(stderr, "SARG: %35s (-p) = %s\n",text[29],text[1]);
 580        else
 581          fprintf(stderr, "SARG: %35s (-p) = %s\n",text[29],text[2]);
 582       fprintf(stderr, "SARG: %35s (-s) = %s\n",text[30],site);
 583       fprintf(stderr, "SARG: %35s (-t) = %s\n",text[31],hm);
 584       fprintf(stderr, "SARG: %35s (-u) = %s\n",text[32],us);
 585       fprintf(stderr, "SARG: %35s (-w) = %s\n",text[34],tmp);
 586       if(debug)
 587          fprintf(stderr, "SARG: %35s (-x) = %s\n",text[35],text[1]);
 588        else
 589          fprintf(stderr, "SARG: %35s (-x) = %s\n",text[35],text[2]);
 590       if(debugz)
 591          fprintf(stderr, "SARG: %35s (-z) = %s\n",text[36],text[1]);
 592        else
 593          fprintf(stderr, "SARG: %35s (-z) = %s\n",text[36],text[2]);
 594       fprintf(stderr, "SARG:\n");
 595    }
 596
 597    if(debugm) {
 598       printf("%s:\nSARG:\n",text[22]);
 599       printf("%35s (-a) = %s\n",text[23],addr);
 600       printf("%35s (-b) = %s\n",text[71],uagent);
 601       printf("%35s (-c) = %s\n",text[69],hexclude);
 602       printf("%35s (-d) = %s\n",text[24],date);
 603       printf("%35s (-e) = %s\n",text[41],email);
 604       printf("%35s (-f) = %s\n",text[70],ConfigFile);
 605       if(strcmp(df,"e") == 0)
 606          printf("%35s (-g) = %s (dd/mm/yyyy)\n",text[25],text[26]);
 607       if(strcmp(df,"u") == 0)
 608          printf("%35s (-g) = %s (mm/dd/yyyy)\n",text[25],text[27]);
 609       if(strcmp(df,"w") == 0)
 610          printf("%35s (-g) = %s (yyyy/ww)\n",text[25],text[85]);
 611       if(iprel)
 612          printf("%35s (-i) = %s\n",text[28],text[1]);
 613        else
 614          printf("%35s (-i) = %s\n",text[28],text[2]);
 615       printf("%35s (-l) = %s\n",text[37],arq);
 616       if(strcmp(Ip2Name,"yes") == 0)
 617          printf("%35s (-n) = %s\n",text[65],text[1]);
 618        else
 619          printf("%35s (-n) = %s\n",text[65],text[2]);
 620       printf("%35s (-o) = %s\n",text[38],outdir);
 621       if(strcmp(UserIp,"yes") == 0)
 622          printf("%35s (-p) = %s\n",text[29],text[1]);
 623        else
 624          printf("%35s (-p) = %s\n",text[29],text[2]);
 625       printf("%35s (-s) = %s\n",text[30],site);
 626       printf("%35s (-t) = %s\n",text[31],hm);
 627       printf("%35s (-u) = %s\n",text[32],us);
 628       printf("%35s (-w) = %s\n",text[34],tmp);
 629       if(debug)
 630          printf("%35s (-x) = %s\n",text[35],text[1]);
 631        else
 632          printf("%35s (-x) = %s\n",text[35],text[2]);
 633       if(debugz)
 634          printf("%35s (-z) = %s\n",text[36],text[1]);
 635        else
 636          printf("%35s (-z) = %s\n",text[36],text[2]);
 637       printf("sarg %s: %s\n",text[73],VERSION);
 638       printf("Language=%s\n\n",text[3]);
 639    }
 640
 641    if(debug)
 642       debuga("sarg %s: %s",text[73],VERSION);
 643
 644    if (Ulimit[0] != '\0') {
 645       long l1, l2;
 646       int rc=0;
 647
 648 #if defined(RLIMIT_NOFILE)
 649       getrlimit (RLIMIT_NOFILE, &rl);
 650 #elif defined(RLIMIT_OFILE)
 651       getrlimit (RLIMIT_OFILE, &rl);
 652 #else
 653 #warning "No rlimit resource for the number of open files"
 654 #endif
 655       l1 = rl.rlim_cur;
 656       l2 = rl.rlim_max;
 657
 658       rl.rlim_cur = atol(Ulimit);
 659       rl.rlim_max = atol(Ulimit);
 660 #if defined(RLIMIT_NOFILE)
 661       rc=setrlimit (RLIMIT_NOFILE, &rl);
 662 #elif defined(RLIMIT_OFILE)
 663       rc=setrlimit (RLIMIT_OFILE, &rl);
 664 #else
 665 #warning "No rlimit resource for the number of open files"
 666 #endif
 667       if(rc == -1) {
 668             debuga("setrlimit error - %s\n",strerror(errno));
 669       }
 670
 671       if(debug)
 672          debuga("Maximum file descriptor: cur=%ld max=%ld, changed to cur="RLIM_STRING" max="RLIM_STRING,l1,l2,rl.rlim_cur,rl.rlim_max);
 673    }
 674
 675    read_usertab(UserTabFile);
 676
 677    sprintf ( sz_Download_Unsort , "%s/sarg/download.unsort", tmp);
 678    bool_ShowReadStatistics = ( strcmp(ShowReadStatistics,"yes") == 0 ) ;
 679
 680    while(narq--) {
 681       strcpy(arq,warq[iarq]);
 682       iarq++;
 683
 684       strcpy(arqtt,arq);
 685       decomp(arq,zip,tmp);
 686       if(debug)
 687          debuga("%s: %s",text[7],arq);
 688
 689       if((fp_in=MY_FOPEN(arq,"r"))==NULL) {
 690          fprintf(stderr, "%s: (log) %s: %s - %s\n",argv[0],text[8],arq,strerror(errno));
 691          exit(1);
 692       }
 693       fgets(bufz,sizeof(bufz),fp_in);
 694       if(!isalog && strncmp(bufz,"#Software: Mic",14) == 0) isalog++;
 695
 696       if(strncmp(bufz,"*** SARG Log ***",16) == 0) {
 697          if (getword(val2,sizeof(val2),arqtt,'-')<0 || getword(val2,sizeof(val2),arqtt,'_')<0 ||
 698             getword(val3,sizeof(val3),arqtt,'-')<0 || getword(val3,sizeof(val3),arqtt,'_')<0) {
 699             printf("SARG: Maybe you have a broken record or garbage in your %s file.\n",arq);
 700             exit(1);
 701          }
 702          sprintf(period,"%s-%s",val2,val3);
 703          sarglog=1;
 704       } else fseek(fp_in, 0, SEEK_SET);
 705
 706       if(strcmp(ParsedOutputLog, "no") != 0 && !sarglog) {
 707          if(access(ParsedOutputLog,R_OK) != 0) {
 708             sprintf(csort,"%s",ParsedOutputLog);
 709             my_mkdir(csort);
 710          }
 711          sprintf(arq_log,"%s/sarg_temp.log",ParsedOutputLog);
 712          if((fp_log=MY_FOPEN(arq_log,"w"))==NULL) {
 713             fprintf(stderr, "%s: (log) %s: %s - %s\n",argv[0],text[8],arq_log,strerror(errno));
 714             exit(1);
 715          }
 716          fputs("*** SARG Log ***\n",fp_log);
 717       }
 718
 719       if(strstr(ReportType,"denied") != 0) {
 720          if((fp_denied=MY_FOPEN(tmp4,"w"))==NULL) {
 721             fprintf(stderr, "%s: (log) %s: %s - %s\n",argv[0],text[45],tmp4,strerror(errno));
 722             exit(1);
 723          }
 724       }
 725
 726       if(DataFile[0]=='\0') {
 727          if(strstr(ReportType,"denied") != 0 || strstr(ReportType,"auth_failures") != 0) {
 728             if((fp_authfail=MY_FOPEN(tmp6,"w"))==NULL) {
 729                fprintf(stderr, "%s: (log) %s: %s - %s\n",argv[0],text[45],tmp6,strerror(errno));
 730                exit(1);
 731             }
 732          }
 733       }
 734
 735       // pre-Read the file only if I have to show stats
 736       if(bool_ShowReadStatistics) {
 737          rewind(fp_in);
 738          recs1=0;
 739          recs2=0;
 740
 741          while( fgets(bufz,sizeof(bufz),fp_in) != NULL ) recs1++;
 742          rewind(fp_in);
 743          printf("SARG: Records in file: " OFFSET_STRING ", reading: %3.2f%%\r",recs1,(float) 0);
 744          fflush( stdout ) ;
 745       }
 746
 747       while(fgets(bufz,sizeof(bufz),fp_in)!=NULL) {
 748          recs2++;
 749          if( bool_ShowReadStatistics && ! --OutputNonZero) {
 750            perc = recs2 * 100 ;
 751            perc = perc / recs1 ;
 752            printf("SARG: Records in file: " OFFSET_STRING", reading: %3.2f%%\r",recs1,perc);
 753            fflush (stdout);
 754            OutputNonZero = REPORT_EVERY_X_LINES ;
 755          }
 756          if(strlen(bufz) > MAXLEN-1) continue;
 757          if(!bufz[0]) continue;
 758          if(strstr(bufz,"HTTP/0.0") != 0) continue;
 759          if(strstr(bufz,"logfile turned over") != 0) continue;
 760          if(bufz[0] == ' ') continue;
 761          if(strlen(bufz) < 58) continue;
 762
 763          // Record only hours usage which is required
 764          tt = (time_t) strtoul( bufz, NULL, 10 );
 765          t = localtime( &tt );
 766
 767          if( bsearch( &( t -> tm_wday ), weekdays.list, weekdays.len,
 768                                           sizeof( int ), compar ) == NULL )
 769             continue;
 770
 771          if( bsearch( &( t -> tm_hour ), hours.list, hours.len,
 772                                           sizeof( int ), compar ) == NULL )
 773             continue;
 774
 775          // exclude_string
 776          exstring=0;
 777          if(strlen(ExcludeString) > 0) {
 778             strcpy(warea,bufz);
 779             strcpy(html,ExcludeString);
 780             while(strstr(html,":") != 0) {
 781                if (getword_multisep(val1,sizeof(val1),html,':')<0) {
 782                   printf("SARG: Maybe you have a broken record or garbage in your exclusion string.\n");
 783                   exit(1);
 784                }
 785                if((str=(char *) strstr(warea,val1)) != (char *) NULL )
 786                   exstring++;
 787             }
 788             if((str=(char *) strstr(warea,html)) != (char *) NULL )
 789                   exstring++;
 790          }
 791          if(exstring) continue;
 792
 793          strcpy(bufy,bufz);
 794          if ((str = strchr(bufz, '\n')) != NULL)
 795             *str = '\0';          /* strip \n */
 796
 797          totregsl++;
 798          common=0;
 799          if(debugm)
 800             printf("BUF=%s\n",bufz);
 801
 802          if(!sarglog && !isalog) {
 803             if (getword(data,sizeof(data),bufz,' ')<0) {
 804                printf("SARG: Maybe you have a broken record or garbage in your access.log file.\n");
 805                exit(1);
 806             }
 807             if((str=(char *) strstr(data, ".")) != (char *) NULL ) {
 808                if((str=(char *) strstr(str+1, ".")) != (char *) NULL ) {
 809                   strcpy(ip,data);
 810                   strcpy(elap,"0");
 811                   if(squid24) {
 812                      if (getword(user,sizeof(user),bufz,' ')<0 || getword(none,sizeof(none),bufz,' ')<0) {
 813                         printf("SARG: Maybe you have a broken record or garbage in your %s file.\n",arq);
 814                         exit(1);
 815                      }
 816                   } else {
 817                      if (getword(none,sizeof(none),bufz,' ')<0 || getword(user,sizeof(user),bufz,' ')<0) {
 818                         printf("SARG: Maybe you have a broken record or garbage in your %s file.\n",arq);
 819                         exit(1);
 820                      }
 821                   }
 822                   if (getword(data,sizeof(data),bufz,']')<0 || getword(fun,sizeof(fun),bufz,'"')<0 ||
 823                       getword(fun,sizeof(fun),bufz,' ')<0 || getword(url,sizeof(url),bufz,' ')<0 ||
 824                       getword(code2,sizeof(code2),bufz,' ')<0 || getword(code2,sizeof(code2),bufz,' ')<0 ||
 825                       getword(tam,sizeof(tam),bufz,' ')<0) {
 826                      printf("SARG: Maybe you have a broken record or garbage in your %s file.\n",arq);
 827                      exit(1);
 828                   }
 829                   if((str=(char *) strstr(bufz, " ")) != (char *) NULL ) {
 830                      if (getword(code,sizeof(code),bufz,' ')<0) {
 831                         printf("SARG: Maybe you have a broken record or garbage in your %s file.\n",arq);
 832                         exit(1);
 833                      }
 834                   }
 835                   else strcpy(code,bufz);
 836
 837                   if ((str = strchr(code, ':')) != NULL)
 838                      *str = '/';
 839
 840                   if(strcmp(tam,"\0") == 0)
 841                      strcpy(tam,"0");
 842
 843                   common++;
 844                   common_log=1;
 845                }
 846             }
 847
 848             if(!common) {
 849                if (getword(elap,sizeof(elap),bufz,' ')<0) {
 850                   printf("SARG: Maybe you have a broken record or garbage in your %s file.\n",arq);
 851                   exit(1);
 852                }
 853                while(strcmp(elap,"") == 0 && strlen(bufz) > 0)
 854                   if (getword(elap,sizeof(elap),bufz,' ')<0) {
 855                      printf("SARG: Maybe you have a broken record or garbage in your %s file.\n",arq);
 856                      exit(1);
 857                   }
 858                if(strlen(elap) < 1) continue;
 859                if (getword(ip,sizeof(ip),bufz,' ')<0){
 860                   printf("SARG: Maybe you have a broken record or garbage in your %s file.\n",arq);
 861                   exit(1);
 862                }
 863                if (getword(code,sizeof(code),bufz,' ')<0){
 864                   printf("SARG: Maybe you have a broken record or garbage in your %s file.\n",arq);
 865                   exit(1);
 866                }
 867                if (getword(tam,sizeof(tam),bufz,' ')<0){
 868                   printf("SARG: Maybe you have a broken record or garbage in your %s file.\n",arq);
 869                   exit(1);
 870                }
 871                if (getword(fun,sizeof(fun),bufz,' ')<0){
 872                   printf("SARG: Maybe you have a broken record or garbage in your %s file.\n",arq);
 873                   exit(1);
 874                }
 875                if (getword(url,sizeof(url),bufz,' ')<0){
 876                   printf("SARG: Maybe you have a broken record or garbage in your %s file.\n",arq);
 877                   exit(1);
 878                }
 879 //              while (strstr(bufz,"%20") != 0) {
 880 //                 getword(warea,bufz,' ');
 881 //                 strcat(url,warea);
 882 //              }
 883                if (getword(user,sizeof(user),bufz,' ')<0){
 884                   printf("SARG: Maybe you have a broken record or garbage in your %s file.\n",arq);
 885                   exit(1);
 886                }
 887                squid_log=1;
 888             }
 889          } else if(!isalog) {
 890             if (getword(data,sizeof(data),bufz,' ')<0){
 891                printf("SARG: Maybe you have a broken record or garbage in your %s file.\n",arq);
 892                exit(1);
 893             }
 894             if (getword(hora,sizeof(hora),bufz,' ')<0) {
 895                printf("SARG: Maybe you have a broken record or garbage in your %s file.\n",arq);
 896                exit(1);
 897             }
 898             if (getword(user,sizeof(user),bufz,' ')<0) {
 899                printf("SARG: Maybe you have a broken record or garbage in your %s file.\n",arq);
 900                exit(1);
 901             }
 902             if (getword(ip,sizeof(ip),bufz,' ')<0) {
 903                printf("SARG: Maybe you have a broken record or garbage in your %s file.\n",arq);
 904                exit(1);
 905             }
 906             if (getword(url,sizeof(url),bufz,' ')<0){
 907                printf("SARG: Maybe you have a broken record or garbage in your %s file.\n",arq);
 908                exit(1);
 909             }
 910             if (getword(tam,sizeof(tam),bufz,' ')<0){
 911                printf("SARG: Maybe you have a broken record or garbage in your %s file.\n",arq);
 912                exit(1);
 913             }
 914             if (getword(code,sizeof(code),bufz,' ')<0){
 915                printf("SARG: Maybe you have a broken record or garbage in your %s file.\n",arq);
 916                exit(1);
 917             }
 918             if (getword(elap,sizeof(elap),bufz,' ')<0){
 919                printf("SARG: Maybe you have a broken record or garbage in your %s file.\n",arq);
 920                exit(1);
 921             }
 922             if (getword(smartfilter,sizeof(smartfilter),bufz,' ')<0){
 923                printf("SARG: Maybe you have a broken record or garbage in your %s file.\n",arq);
 924                exit(1);
 925             }
 926          } else if(isalog) {
 927             if(!i0) {
 928                if (getword(val1,sizeof(val1),bufz,' ')<0){
 929                   printf("SARG: Maybe you have a broken record or garbage in your %s file.\n",arq);
 930                   exit(1);
 931                }
 932                fixendofline(bufz);
 933                while(bufz[0] != '\0') {
 934                      if (getword(val1,sizeof(val1),bufz,'\t')<0){
 935                      printf("SARG: Maybe you have a broken record or garbage in your %s file.\n",arq);
 936                      exit(1);
 937                   }
 938                   if(strcmp(val1,"c-ip") == 0) i1=i0;
 939                   if(strcmp(val1,"cs-username") == 0) i2=i0;
 940                   if(strcmp(val1,"date") == 0) i3=i0;
 941                   if(strcmp(val1,"time") == 0) i4=i0;
 942                   if(strcmp(val1,"time-taken") == 0) i5=i0;
 943                   if(strcmp(val1,"sc-bytes") == 0) i6=i0;
 944                   if(strcmp(val1,"cs-uri") == 0) i7=i0;
 945                   if(strcmp(val1,"sc-status") == 0) i8=i0;
 946                   i0++;
 947                }
 948                if (i1<0) {
 949                   // not the line we are looking for
 950                   i0=0;
 951                }
 952             }
 953             fgets(bufz,sizeof(bufz),fp_in);
 954             for (x=0 ; x<i0 ; x++) {
 955                if (getword3(val1,sizeof(val1),bufz,'\t')<0) {
 956                   printf("SARG: Maybe you have a broken record or garbage in your %s file.\n",arq);
 957                   exit(1);
 958                }
 959                if (x==i1) {
 960                   if (strlen(val1)>=sizeof(ip)) {
 961                      printf("SARG: Maybe you have a broken IP in your %s file.\n",arq);
 962                      exit(1);
 963                   }
 964                   strcpy(ip,val1);
 965                } else if (x==i2) {
 966                   if (strlen(val1)>=sizeof(user)) {
 967                      printf("SARG: Maybe you have a broken user in your %s file.\n",arq);
 968                      exit(1);
 969                   }
 970                   strcpy(user,val1);
 971                } else if (x==i3) {
 972                   if (strlen(val1)>=sizeof(data)) {
 973                      printf("SARG: Maybe you have a broken date in your %s file.\n",arq);
 974                      exit(1);
 975                   }
 976                   strcpy(data,val1);
 977                } else if (x==i4) {
 978                   if (strlen(val1)>=sizeof(hora)) {
 979                      printf("SARG: Maybe you have a broken time in your %s file.\n",arq);
 980                      exit(1);
 981                   }
 982                   strcpy(hora,val1);
 983                } else if (x==i5) {
 984                   if (strlen(val1)>=sizeof(elap)) {
 985                      printf("SARG: Maybe you have a broken download duration in your %s file.\n",arq);
 986                      exit(1);
 987                   }
 988                   strcpy(elap,val1);
 989                } else if (x==i6) {
 990                   if (strlen(val1)>=sizeof(tam)) {
 991                      printf("SARG: Maybe you have a broken download size in your %s file.\n",arq);
 992                      exit(1);
 993                   }
 994                   strcpy(tam,val1);
 995                } else if (x==i7) {
 996                   if (strlen(val1)>=sizeof(url)) {
 997                      printf("SARG: Maybe you have a broken URL in your %s file.\n",arq);
 998                      exit(1);
 999                   }
1000                   strcpy(url,val1);
1001                } else if (x==i8) {
1002                   if (strlen(val1)>=sizeof(code)) {
1003                      printf("SARG: Maybe you have a broken access code in your %s file.\n",arq);
1004                      exit(1);
1005                   }
1006                   strcpy(code,val1);
1007                }
1008             }
1009
1010             if(strcmp(code,"401") == 0 || strcmp(code,"403") == 0 || strcmp(code,"407") == 0) {
1011                sprintf(val1,"DENIED/%s",code);
1012                strcpy(code,val1);
1013             }
1014             if (getword(ano,sizeof(ano),data,'-')<0){
1015                printf("SARG: Maybe you have a broken record or garbage in your %s file.\n",arq);
1016                exit(1);
1017             }
1018             if (getword(mes,sizeof(mes),data,'-')<0){
1019                printf("SARG: Maybe you have a broken record or garbage in your %s file.\n",arq);
1020                exit(1);
1021             }
1022             strcpy(dia,data);
1023             conv_month_name(mes);
1024             sprintf(data," %s/%s/%s:%s",dia,mes,ano,hora);
1025          }
1026
1027          if(strlen(user) > 150) {
1028             if (debugm) printf("User too long: %s\n",user);
1029             totregsx++;
1030             continue;
1031          }
1032
1033          // include_users
1034          if(IncludeUsers[0] != '\0') {
1035             sprintf(val1,":%s:",user);
1036             if((str=(char *) strstr(IncludeUsers,val1)) == (char *) NULL )
1037                continue;
1038          }
1039
1040          if(vercode(code)) {
1041             if (debugm) printf("Excluded code: %s\n",code);
1042             totregsx++;
1043             continue;
1044          }
1045
1046          if(testvaliduserchar(user))
1047             continue;
1048
1049          if(strstr(user,"%20") != 0) {
1050             if (getword(w,sizeof(w),user,'%')<0) {
1051                printf("SARG: Maybe you have a broken user in your %s file.\n",arq);
1052                exit(1);
1053             }
1054             strcpy(user,w);
1055          }
1056
1057          while(strstr(user,"%5c") != 0) {
1058             if (getword_multisep(w,sizeof(w),user,'%')<0){
1059                printf("SARG: Maybe you have a broken user in your %s file.\n",arq);
1060                exit(1);
1061             }
1062             strcpy(wuser,user+2);
1063             sprintf(user,"%s.%s",w,wuser);
1064          }
1065
1066          for(str=user; *str; str++) {
1067             if(*str=='.') dotinuser++;
1068             if(*str=='?' || *str=='.' || *str==':' || *str=='/' || *str=='\\')
1069                *str='_';
1070          }
1071
1072          strlow(user);
1073          if(strncmp(NtlmUserFormat,"user",4) == 0) {
1074             if(strchr(user,'_') != 0)
1075                if (getword_multisep(warea,sizeof(warea),user,'_')<0){
1076                   printf("SARG: Maybe you have a broken user in your %s file.\n",arq);
1077                   exit(1);
1078                }
1079             if(strchr(user,'+') != 0)
1080                if (getword_multisep(warea,sizeof(warea),user,'+')<0){
1081                   printf("SARG: Maybe you have a broken user in your %s file.\n",arq);
1082                   exit(1);
1083                }
1084          }
1085
1086          if(strstr(ReportType,"denied") != 0)
1087             strcpy(urly,url);
1088
1089          download_flag=is_download_suffix(url);
1090          if (download_flag) {
1091             strcpy(download_url,url);
1092             download_count++;
1093          }
1094
1095          if (strchr(url,'/')) {
1096             // remove any protocol:// at the beginning of the URL
1097             if (getword(w,sizeof(w),url,'/')<0){
1098                printf("SARG: Maybe you have a broken url in your %s file.\n",arq);
1099                exit(1);
1100             }
1101             if (getword(w,sizeof(w),url,'/')<0){
1102                printf("SARG: Maybe you have a broken url in your %s file.\n",arq);
1103                exit(1);
1104             }
1105             if (!strchr(url,'/')) {
1106                if (debugm) printf("URL without directory: %s\n",url);
1107                totregsx++;
1108                continue;
1109             }
1110          }
1111
1112          if(strcmp(LongUrl,"no") == 0) {
1113             char *endofhost=strchr(url,'/');
1114             if (endofhost)
1115                *endofhost='\0';
1116             if(strlen(url) > 512 && (endofhost=strchr(url,'%')) != NULL) {
1117                *endofhost='\0';
1118             }
1119          }
1120
1121          if(!sarglog) {
1122             if(!common && !isalog) {
1123                tt=atoi(data);
1124                t=localtime(&tt);
1125
1126                strftime(tbuf2, sizeof(tbuf2), "%H%M", t);
1127                if(strncmp(df,"u",1) == 0)
1128                   strftime(tbuf, sizeof(tbuf), "%Y%b%d", t);
1129                if(strncmp(df,"e",1) == 0)
1130                   strftime(tbuf, sizeof(tbuf), "%d%b%Y", t);
1131                if(strncmp(df,"w",1) == 0) {
1132                   strcpy(IndexTree,"file");
1133                   strftime(tbuf, sizeof(tbuf), "%Y.%U", t);
1134                }
1135
1136                strftime(wdata, sizeof(wdata), "%Y%m%d", t);
1137                idata=atoi(wdata);
1138
1139                if(strncmp(df,"u",1)==0)
1140                   strftime(dia, sizeof(dia), "%m/%d/%Y", t);
1141                else
1142                   strftime(dia, sizeof(dia), "%d/%m/%Y", t);
1143                sprintf(hora,"%02d:%02d:%02d",t->tm_hour,t->tm_min,t->tm_sec);
1144             } else {
1145                strcpy(wtemp,data+1);
1146                if (getword_multisep(data,sizeof(data),wtemp,':')<0){
1147                   printf("SARG: Maybe you have a broken date in your %s file.\n",arq);
1148                   exit(1);
1149                }
1150                if (getword_multisep(hora,sizeof(hora),wtemp,' ')<0){
1151                   printf("SARG: Maybe you have a broken date in your %s file.\n",arq);
1152                   exit(1);
1153                }
1154                if (getword_multisep(dia,sizeof(dia),data,'/')<0){
1155                   printf("SARG: Maybe you have a broken date in your %s file.\n",arq);
1156                   exit(1);
1157                }
1158                if (getword_multisep(mes,sizeof(mes),data,'/')<0){
1159                   printf("SARG: Maybe you have a broken date in your %s file.\n",arq);
1160                   exit(1);
1161                }
1162                if (getword_multisep(ano,sizeof(ano),data,'/')<0){
1163                   printf("SARG: Maybe you have a broken date in your %s file.\n",arq);
1164                   exit(1);
1165                }
1166
1167                if(strcmp(df,"u") == 0)
1168                   snprintf(tbuf,sizeof(tbuf),"%s%s%s",ano,mes,dia);
1169                if(strcmp(df,"e") == 0)
1170                   snprintf(tbuf,sizeof(tbuf),"%s%s%s",dia,mes,ano);
1171                builddia(dia,mes,ano,df,wdata);
1172                idata=atoi(wdata);
1173             }
1174          } else {
1175             if (getword_multisep(mes,sizeof(mes),data,'/')<0){
1176                printf("SARG: Maybe you have a broken date in your %s file.\n",arq);
1177                exit(1);
1178             }
1179             if (getword_multisep(dia,sizeof(dia),data,'/')<0){
1180                printf("SARG: Maybe you have a broken date in your %s file.\n",arq);
1181                exit(1);
1182             }
1183             if (getword_multisep(ano,sizeof(ano),data,0)<0){
1184                printf("SARG: Maybe you have a broken date in your %s file.\n",arq);
1185                exit(1);
1186             }
1187          }
1188
1189          if(debugm)
1190             printf("DATE=%s IDATA=%d DFROM=%d DUNTIL=%d\n",date,idata,dfrom,duntil);
1191
1192          l=1;
1193          if(us[0] != '\0'){
1194             if(strcmp(user,us)==0)
1195                l=1;
1196             else
1197                l=0;
1198          }
1199
1200          if(l){
1201             if(addr[0] != '\0'){
1202                if(strcmp(addr,ip)==0)
1203                   l=1;else l=0;
1204             }
1205             if(fhost) {
1206 //              l=vhexclude(excludefile,ip);
1207                l=vhexclude(excludefile,url);
1208                if(!l) {
1209                   if (debugm) printf("Excluded site: %s\n",url);
1210                   totregsx++;
1211                }
1212             }
1213          }
1214
1215          if(l){
1216             if(date[0] != '\0'){
1217                if(idata >= dfrom && idata <= duntil)
1218                   l=1;else l=0;
1219             }
1220          }
1221          if(l){
1222             if(hm[0] != '\0') {
1223                strcpy(whm,hora);
1224                bzero(hmr,15);
1225                chm++;
1226                while(chm) {
1227                   if (getword_multisep(warea,sizeof(warea),whm,':')<0){
1228                      printf("SARG: Maybe you have a broken time in your %s file.\n",arq);
1229                      exit(1);
1230                   }
1231                   strncat(hmr,warea,2);
1232                   chm--;
1233                }
1234                strncat(hmr,whm,2);
1235
1236                if(atoi(hmr) >= atoi(hm) && atoi(hmr) <= atoi(hmf))
1237                   l=1;else l=0;
1238             }
1239          }
1240          if(l){
1241             if(site[0] != '\0'){
1242                if(strstr(url,site)!=0)
1243                   l=1;else l=0;
1244             }
1245          }
1246
1247          if(userip)
1248             strcpy(user,ip);
1249
1250          if(strcmp(user,"-") == 0 || strcmp(user," ") == 0 || strcmp(user,"") == 0) {
1251             if(strcmp(RecordsWithoutUser,"ip") == 0)
1252                strcpy(user,ip);
1253             if(strcmp(RecordsWithoutUser,"ignore") == 0)
1254                continue;
1255             if(strcmp(RecordsWithoutUser,"everybody") == 0)
1256                strcpy(user,"everybody");
1257          }
1258
1259          if(dotinuser) {
1260             subs(user,sizeof(user),"_",".");
1261             dotinuser=0;
1262          }
1263
1264          if(puser) {
1265             sprintf(wuser,":%s:",user);
1266             if(strstr(userfile, wuser) == 0)
1267                continue;
1268          }
1269
1270          if(l) {
1271             if(fuser) {
1272                l=vuexclude(excludeuser,user);
1273                if(!l) {
1274                   if (debugm) printf("Excluded user: %s\n",user);
1275                   totregsx++;
1276                }
1277             }
1278          }
1279
1280          if(l) {
1281             if(userip)
1282                fixip(user);
1283          }
1284
1285          if(l && max_elapsed) {
1286             if(atol(elap)>max_elapsed) {
1287                elap[0]='0';
1288                elap[1]='\0';
1289             }
1290          }
1291
1292          if(l) {
1293             if(strcmp(user,"-") !=0 && url[0] != '\0' && strcmp(user," ") !=0 && strcmp(user,"") !=0 && strcmp(user,":") !=0){
1294                if((str=(char *) strstr(bufz, "[SmartFilter:")) != (char *) NULL ) {
1295                   str[strlen(str)-1]='\0';
1296                   sprintf(smartfilter,"\"%s\"",str+1);
1297                } else sprintf(smartfilter,"\"\"");
1298
1299                sprintf(bufz, "%s %s %s %s %s %s %s %s %s\n",dia,hora,user,ip,url,tam,code,elap,smartfilter);
1300
1301 #ifdef LEGACY_WRITE_USER
1302                sprintf(tmp3,"%s/sarg/%s.unsort",tmp,user);
1303                if((fp_ou=MY_FOPEN(tmp3,"a"))==NULL) {
1304                   fprintf(stderr, "%s: (zzzlog) %s: %s - %s\n",argv[0],text[9],tmp3,strerror(errno));
1305                   exit(1);
1306                }
1307                fputs(bufz,fp_ou);
1308
1309 #else
1310                if ( strcmp ( user , sz_Last_User ) != 0 ) {
1311                   if ( fp_Write_User )
1312                      fclose( fp_Write_User ) ;
1313                   sprintf (tmp3, "%s/sarg/%s.unsort", tmp, user);
1314
1315                   if ((fp_Write_User = MY_FOPEN (tmp3, "a")) == NULL) {
1316                      fprintf (stderr, "%s: (log) %s: %s - %s\n", argv[0], text[9], tmp3, strerror(errno));
1317                      exit (1);
1318                   }
1319                   strcpy( sz_Last_User , user ) ;
1320                }
1321                fputs (bufz, fp_Write_User);
1322 #endif
1323
1324                if(strcmp(ParsedOutputLog, "no") != 0 && !sarglog)
1325                   fputs(bufz,fp_log);
1326
1327 #ifdef LEGACY_WRITE_USER
1328                fclose(fp_ou);
1329 #endif
1330                totregsg++;
1331
1332                if(download_flag && strstr(code,"DENIED") == 0) {
1333                   ndownload = 1;
1334                   sprintf(bufz, "%s %s %s %s %s\n",dia,hora,user,ip,download_url);
1335
1336 #ifdef LEGACY_WRITE_DOWNLOAD
1337                   sprintf(tmp3,"%s/sarg/download.unsort",tmp);
1338                   if((fp_ou=MY_FOPEN(tmp3,"a"))==NULL) {
1339                      fprintf(stderr, "%s: (log) %s: %s - %s\n",argv[0],text[9],tmp3,strerror(errno));
1340                      exit(1);
1341                   }
1342                   fputs(bufz,fp_ou);
1343                   fclose(fp_ou);
1344 #else
1345                   if ( ! fp_Download_Unsort ) {
1346                      if ((fp_Download_Unsort = MY_FOPEN ( sz_Download_Unsort, "a")) == NULL) {
1347                         fprintf (stderr, "%s: (log) %s: %s - %s\n", argv[0], text[9], tmp3, strerror(errno));
1348                         exit (1);
1349                      }
1350                   }
1351                   fputs (bufz, fp_Download_Unsort);
1352 #endif
1353                }
1354
1355                if(strstr(ReportType,"denied") != 0 || strstr(ReportType,"auth_failures") != 0) {
1356                   if(strstr(code,"DENIED/403") != 0) {
1357                      fprintf(fp_denied, "%s %s %s %s %s\n",dia,hora,user,ip,urly);
1358                      denied_count++;
1359                   }
1360                   if(strstr(code,"DENIED/401") != 0 || strstr(code,"DENIED/407") != 0) {
1361                      if(fp_authfail)
1362                         fprintf(fp_authfail, "%s %s %s %s %s\n",dia,hora,user,ip,urly);
1363                      authfail_count++;
1364                   }
1365                }
1366
1367                if((!totper || idata<mindate) && !sarglog){
1368                   totper++;
1369                   mindate=idata;
1370                   sprintf(period,"%s-",tbuf);
1371                   sprintf(per_hour,"%s-",tbuf2);
1372                   if(strlen(date)>0)
1373                      fixper(tbuf, period, cduntil);
1374                   if(debugz){
1375                      debugaz("tbuf",tbuf);
1376                      debugaz("period",period);
1377                   }
1378                }
1379             }
1380
1381             if(debugm){
1382                printf("IP=\t%s\n",ip);
1383                printf("USER=\t%s\n",user);
1384                printf("ELAP=\t%s\n",elap);
1385                printf("DATE=\t%s\n",dia);
1386                printf("TIME=\t%s\n",hora);
1387                printf("FUNC=\t%s\n",fun);
1388                printf("URL=\t%s\n",url);
1389                printf("CODE=\t%s\n",code);
1390                printf("LEN=\t%s\n",tam);
1391             }
1392          }
1393       }
1394       if( bool_ShowReadStatistics )
1395         printf("SARG: Records in file: " OFFSET_STRING ", reading: %3.2f%%\n",recs1, (float) 100 );
1396    }
1397
1398    if ( fp_Download_Unsort )
1399      fclose (fp_Download_Unsort);
1400
1401    if (fp_Write_User)
1402      fclose (fp_Write_User);
1403
1404    if(debug) {
1405       debuga("   %s: %ld, %s: %ld, %s: %ld",text[10],totregsl,text[11],totregsg,text[68],totregsx);
1406
1407       if((common_log) && (squid_log))
1408          debuga("%s",text[12]);
1409
1410       if((common_log) && (!squid_log))
1411          debuga("%s",text[13]);
1412
1413       if((!common_log) && (squid_log))
1414          debuga("%s",text[14]);
1415
1416       if(sarglog)
1417          debuga("%s",text[124]);
1418
1419       if((!common_log) && (!squid_log) && (!sarglog) && (!isalog)) {
1420          if(!totregsg) {
1421             fprintf(stderr, "SARG: %s\n",text[16]);
1422             fprintf(stderr, "SARG: %s\n",text[21]);
1423          } else fprintf(stderr, "SARG: %s\n",text[15]);
1424          bzero(msg,sizeof(msg));
1425          fclose(fp_in);
1426 //         fclose(fp_ou);
1427          if(fp_denied)
1428             fclose(fp_denied);
1429          if(fp_authfail)
1430             fclose(fp_authfail);
1431          unlink(tmp4);
1432          unlink(tmp6);
1433          unlink(tmp3);
1434          exit(0);
1435       }
1436    }
1437
1438    if(!totregsg){
1439       fprintf(stderr, "SARG: %s\n",text[16]);
1440       fprintf(stderr, "SARG: %s\n",text[21]);
1441       fclose(fp_in);
1442 //      fclose(fp_ou);
1443       if(fp_denied)
1444          fclose(fp_denied);
1445       if(fp_authfail)
1446          fclose(fp_authfail);
1447       exit(0);
1448    }
1449
1450    if(date[0] == '\0' && !sarglog) {
1451       strcat(period,tbuf);
1452       strcat(per_hour,tbuf2);
1453    }
1454
1455    if(debugz){
1456       debugaz("data",dia);
1457       debugaz("tbuf",tbuf);
1458       debugaz("period",period);
1459    }
1460
1461    if(debug)
1462       debuga("%s: %s",text[17],period);
1463
1464    fclose(fp_in);
1465 //   fclose(fp_ou);
1466    if(fp_denied)
1467       fclose(fp_denied);
1468    if(fp_authfail)
1469       fclose(fp_authfail);
1470
1471    if(strcmp(ParsedOutputLog, "no") != 0 && !sarglog) {
1472       fclose(fp_log);
1473       strcpy(val1,period);
1474       if (getword_multisep(val2,sizeof(val2),val1,'-')<0){
1475          printf("SARG: Maybe you have a broken date range definition.\n");
1476          exit(1);
1477       }
1478       if (getword_multisep(val3,sizeof(val3),per_hour,'-')<0){
1479          printf("SARG: Maybe you have a broken date range definition.\n");
1480          exit(1);
1481       }
1482       sprintf(val4,"%s/sarg-%s_%s-%s_%s.log",ParsedOutputLog,val2,val3,val1,per_hour);
1483       rename(arq_log,val4);
1484       strcpy(arq_log,val4);
1485
1486       if(strcmp(ParsedOutputLogCompress,"nocompress") != 0) {
1487          sprintf(val1,"%s %s",ParsedOutputLogCompress,arq_log);
1488          cstatus=system(val1);
1489          if (!WIFEXITED(cstatus) || WEXITSTATUS(cstatus)) {
1490             fprintf(stderr, "SARG: command return status %d\n",WEXITSTATUS(cstatus));
1491             fprintf(stderr, "SARG: command: %s\n",val1);
1492             exit(1);
1493          }
1494       }
1495
1496       if(debug)
1497          debuga("%s %s",text[123],arq_log);
1498    }
1499
1500    if(strstr(ReportType,"denied") != 0) {
1501       sprintf(csort,"sort -T %s -k 3,3 -k 5,5 -o '%s' '%s'",tmp,tmp5,tmp4);
1502       cstatus=system(csort);
1503       if (!WIFEXITED(cstatus) || WEXITSTATUS(cstatus)) {
1504          fprintf(stderr, "SARG: sort command return status %d\n",WEXITSTATUS(cstatus));
1505          fprintf(stderr, "SARG: sort command: %s\n",csort);
1506          exit(1);
1507       }
1508       unlink(tmp4);
1509    }
1510
1511    sort_users_log(tmp, debug);
1512
1513    if(DataFile[0] != '\0')
1514       data_file(tmp);
1515    else
1516       gerarel();
1517
1518    unlink(tmp2);
1519    if(strstr(ReportType,"denied") != 0)
1520       unlink(tmp5);
1521
1522    if(zip[0] != '\0' && strcmp(zip,"zcat") !=0) {
1523       recomp(arq, zip);
1524    }
1525 //   else  unlink(arq);
1526
1527    if(strcmp(tmp,"/tmp") != 0) {
1528       sprintf(tmp4,"rm -rf %s",tmp);
1529       system(tmp4);
1530    }
1531
1532    free_excludecodes();
1533    if(userfile)
1534       free(userfile);
1535    if(excludefile)
1536       free(excludefile);
1537    if(excludeuser)
1538       free(excludeuser);
1539
1540    if(debug)
1541       debuga("%s",text[21]);
1542
1543    exit(0);
1544
1545 }
1546
1547
1548 static void getusers(const char *pwdfile, int debug)
1549 {
1550
1551    FILE *fp_usr;
1552    char buf[255];
1553    char *str;
1554    unsigned long int nreg=0;
1555
1556    if(debug)
1557       debuga("%s: %s",text[60],pwdfile);
1558
1559    if ((fp_usr = fopen(pwdfile, "r")) == NULL) {
1560       fprintf(stderr, "SARG: (getusers) %s: %s - %s\n",text[45],pwdfile,strerror(errno));
1561       exit(1);
1562    }
1563
1564    fseek(fp_usr, (off_t)0, SEEK_END);
1565    nreg = ftell(fp_usr);
1566    if (nreg<0) {
1567       printf("SARG: Cannot get the size of file %s",pwdfile);
1568       exit(1);
1569    }
1570    nreg = nreg+5000;
1571    fseek(fp_usr, (off_t)0, SEEK_SET);
1572
1573    if((userfile=(char *) malloc(nreg))==NULL){
1574       fprintf(stderr, "SARG: %s (%ld):\n",text[59],nreg);
1575       exit(1);
1576    }
1577
1578    bzero(userfile,nreg);
1579    strcpy(userfile,":");
1580
1581    while(fgets(buf,sizeof(buf),fp_usr)!=NULL) {
1582       str=strchr(buf,':');
1583       if (!str) {
1584          printf("SARG: You have an invalid user in your %s file.\n",pwdfile);
1585          exit(1);
1586       }
1587       str[1]=0;
1588       strcat(userfile,buf);
1589    }
1590
1591    fclose(fp_usr);
1592
1593    return;
1594 }
1595
1596
1597 static void gethexclude(const char *hexfile, int debug)
1598 {
1599
1600    FILE *fp_ex;
1601    char buf[255];
1602    unsigned long int nreg=0;
1603
1604    if(debug)
1605       debuga("%s: %s",text[67],hexfile);
1606
1607    if ((fp_ex = fopen(hexfile, "r")) == NULL) {
1608       fprintf(stderr, "SARG: (gethexclude) %s: %s - %s\n",text[45],hexfile,strerror(errno));
1609       exit(1);
1610    }
1611
1612    fseek(fp_ex, (off_t)0, SEEK_END);
1613    nreg = ftell(fp_ex);
1614    if (nreg<0) {
1615       printf("SARG: Cannot get the size of file %s",hexfile);
1616       exit(1);
1617    }
1618    nreg += 11;
1619    fseek(fp_ex, (off_t)0, SEEK_SET);
1620
1621    if((excludefile=(char *) malloc(nreg))==NULL){
1622       fprintf(stderr, "SARG: %s (%ld):\n",text[59],nreg);
1623       exit(1);
1624    }
1625
1626    bzero(excludefile,nreg);
1627
1628    while(fgets(buf,sizeof(buf),fp_ex)!=NULL){
1629      if(strchr(buf,'#') != NULL)
1630         continue;
1631      fixendofline(buf);
1632      strcat(excludefile,buf);
1633      strcat(excludefile," ");
1634    }
1635
1636    strcat(excludefile,"*END* ");
1637
1638    fclose(fp_ex);
1639
1640    return;
1641 }
1642
1643
1644 static void getuexclude(const char *uexfile, int debug)
1645 {
1646
1647    FILE *fp_ex;
1648    char buf[255];
1649    unsigned long int nreg=0;
1650
1651    if(debug)
1652       debuga("%s: %s",text[67],uexfile);
1653
1654    if ((fp_ex = fopen(uexfile, "r")) == NULL) {
1655       fprintf(stderr, "SARG: (gethexclude) %s: %s - %s\n",text[45],uexfile,strerror(errno));
1656       exit(1);
1657    }
1658
1659    fseek(fp_ex, (off_t)0, SEEK_END);
1660    nreg = ftell(fp_ex);
1661    if (nreg<0) {
1662       printf("SARG: Cannot get the size of file %s",uexfile);
1663       exit(1);
1664    }
1665    nreg += 11;
1666    fseek(fp_ex, (off_t)0, SEEK_SET);
1667
1668    if((excludeuser=(char *) malloc(nreg))==NULL){
1669       fprintf(stderr, "SARG: %s (%ld):\n",text[59],nreg);
1670       exit(1);
1671    }
1672
1673    bzero(excludeuser,nreg);
1674
1675    while(fgets(buf,sizeof(buf),fp_ex)!=NULL){
1676      if(strchr(buf,'#') != NULL)
1677         continue;
1678      fixendofline(buf);
1679      strcat(excludeuser,buf);
1680      strcat(excludeuser," ");
1681    }
1682
1683    strcat(excludeuser,"*END* ");
1684
1685    fclose(fp_ex);
1686
1687    return;
1688 }