log.c

   1 /*
   2  * AUTHOR: Pedro Lineu Orso                         pedro.orso@gmail.com
   3  *                                                            1998, 2008
   4  * SARG Squid Analysis Report Generator      http://sarg.sourceforge.net
   5  *
   6  * SARG donations:
   7  *      please look at http://sarg.sourceforge.net/donations.php
   8  * ---------------------------------------------------------------------
   9  *
  10  *  This program is free software; you can redistribute it and/or modify
  11  *  it under the terms of the GNU General Public License as published by
  12  *  the Free Software Foundation; either version 2 of the License, or
  13  *  (at your option) any later version.
  14  *
  15  *  This program is distributed in the hope that it will be useful,
  16  *  but WITHOUT ANY WARRANTY; without even the implied warranty of
  17  *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  18  *  GNU General Public License for more details.
  19  *
  20  *  You should have received a copy of the GNU General Public License
  21  *  along with this program; if not, write to the Free Software
  22  *  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111, USA.
  23  *
  24  */
  25
  26 #include "include/conf.h"
  27
  28 #define LEGACY_WRITE_USER
  29 #define LEGACY_WRITE_DOWNLOAD
  30 #define REPORT_EVERY_X_LINES 5000
  31
  32
  33 char *userfile;
  34 char *excludefile;
  35 char *excludeuser;
  36
  37 char sz_Last_User[ MAXLEN ] = { 0 } ;
  38 int bool_ShowReadStatistics ;
  39
  40 numlist weekdays = { { 0, 1, 2, 3, 4, 5, 6 }, 7 };
  41 numlist hours = { { 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12,
  42              13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23 }, 24 };
  43
  44 void getusers(char *pwdfile, int debug);
  45 void gethexclude(char *hexfile, int debug);
  46 void getuexclude(char *uexfile, int debug);
  47 void ttx(char *user);
  48 int compar( const void *, const void * );
  49
  50 #define _FILE_OFFSET_BITS 64
  51
  52 int main(argc,argv)
  53    int argc;
  54    char *argv[];
  55 {
  56
  57    FILE *fp_in = NULL, *fp_ou = NULL, *fp_denied, *fp_usr, *fp_authfail, *fp_log;
  58
  59    char sz_Download_Unsort[ 20000 ] ;
  60    FILE * fp_Download_Unsort = NULL ;
  61    FILE * fp_Write_User = NULL ;
  62
  63    extern int optind;
  64    extern int optopt;
  65    extern char *optarg;
  66
  67    char w[MAXLEN];
  68    char data[255];
  69    char elap[255];
  70    char none[255];
  71    char ip[MAXLEN];
  72    char msg[MAXLEN];
  73    char tam[255];
  74    char fun[255];
  75    char wuser[MAXLEN];
  76    char smartfilter[MAXLEN];
  77    char dia[128];
  78    char wdata[128];
  79    char mes[30];
  80    char ano[30];
  81    char hora[30];
  82    char wtemp[MAXLEN];
  83    char wtemp2[255];
  84    char date[255];
  85    char arq[255];
  86    char arq_log[255];
  87    char warq[255][255];
  88    char hm[15], hmf[15], hmr[15], whm[15];
  89    int  chm=0;
  90    char uagent[MAXLEN];
  91    char hexclude[MAXLEN];
  92    char csort[MAXLEN];
  93    char tbuf[128];
  94    char tbuf2[128];
  95    char zip[20];
  96    char *str;
  97    char bufz[MAXLEN];
  98    char bufy[MAXLEN];
  99    int  common;
 100    int  common_log=0;
 101    int  squid_log=0;
 102    int  ch;
 103    int  x, l;
 104    int  errflg=0;
 105    int  puser=0;
 106    int  fhost=0;
 107    int  dns=0;
 108    int  fuser=0;
 109    int  idata=0;
 110    int  narq=0;
 111    int  iarq=0;
 112    int  exstring=0;
 113    int  i0=0,i1=0,i2=0,i3=0,i4=0,i5=0,i6=0,i7=0,i8=0;
 114    long totregsl=0;
 115    long totregsg=0;
 116    long totregsx=0;
 117    long totper=0;
 118    long int  max_elapsed=0;
 119    time_t tt;
 120    struct tm *t;
 121    unsigned long nreg=0;
 122    off_t recs1=0;
 123    unsigned long recs2=0;
 124    struct rlimit rl;
 125    int OutputNonZero = REPORT_EVERY_X_LINES ;
 126
 127    BgImage[0]='\0';
 128    LogoImage[0]='\0';
 129    LogoText[0]='\0';
 130    PasswdFile[0]='\0';
 131    OutputEmail[0]='\0';
 132    Ip2Name[0]='\0';
 133    UserAgentLog[0]='\0';
 134    ExcludeHosts[0]='\0';
 135    ExcludeUsers[0]='\0';
 136    ConfigFile[0]='\0';
 137    code[0]='\0';
 138    LastLog[0]='\0';
 139    UserIp[0]='\0';
 140    ReportType[0]='\0';
 141    UserTabFile[0]='\0';
 142    BlockIt[0]='\0';
 143    ExternalCSSFile[0]='\0';
 144    SquidGuardLogFormat[0]='\0';
 145    SquidGuardLogAlternate[0]='\0';
 146    arq[0]='\0';
 147
 148    strcpy(AccessLog,"/usr/local/squid/var/logs/access.log");
 149    sprintf(ExcludeCodes,"%s/exclude_codes",SYSCONFDIR);
 150    strcpy(GraphDaysBytesBarColor,"orange");
 151    strcpy(BgColor,"#ffffff");
 152    strcpy(TxColor,"#000000");
 153    strcpy(TxBgColor,"lavender");
 154    strcpy(TiColor,"darkblue");
 155    strcpy(Width,"80");
 156    strcpy(Height,"45");
 157    strcpy(LogoTextColor,"#000000");
 158    strcpy(HeaderColor,"darkblue");
 159    strcpy(HeaderBgColor,"#dddddd");
 160    strcpy(LogoTextColor,"#006699");
 161    strcpy(FontSize,"9px");
 162    strcpy(TempDir,"/tmp");
 163    strcpy(OutputDir,"/var/www/html/squid-reports");
 164    strcpy(Ip2Name,"no");
 165    strcpy(DateFormat,"u");
 166    strcpy(OverwriteReport,"no");
 167    strcpy(RemoveTempFiles,"yes");
 168    strcpy(ReplaceIndex,"index.html");
 169    strcpy(Index,"yes");
 170    strcpy(RecordsWithoutUser,"ip");
 171    strcpy(UseComma,"no");
 172    strcpy(MailUtility,"mailx");
 173    strcpy(TopSitesNum,"100");
 174    strcpy(UserIp,"no");
 175    strcpy(TopuserSortField,"BYTES");
 176    strcpy(UserSortField,"BYTES");
 177    strcpy(TopuserSortOrder,"reverse");
 178    strcpy(UserSortOrder,"reverse");
 179    strcpy(TopsitesSortField,"CONNECT");
 180    strcpy(TopsitesSortType,"D");
 181    strcpy(LongUrl,"no");
 182    strcpy(language,"English");
 183    strcpy(FontFace,"Verdana,Tahoma,Arial");
 184    strcpy(datetimeby,"elap");
 185    strcpy(CharSet,"ISO-8859-1");
 186    strcpy(Privacy,"no");
 187    strcpy(PrivacyString,"***.***.***.***");
 188    strcpy(PrivacyStringColor,"blue");
 189    strcpy(SuccessfulMsg,"yes");
 190    strcpy(TopUserFields,"NUM DATE_TIME USERID CONNECT BYTES %BYTES IN-CACHE-OUT USED_TIME MILISEC %TIME TOTAL AVERAGE");
 191    strcpy(UserReportFields,"CONNECT BYTES %BYTES IN-CACHE-OUT USED_TIME MILISEC %TIME TOTAL AVERAGE");
 192    strcpy(DataFileDelimiter,";");
 193    strcpy(DataFileFields,"user;date;time;url;connect;bytes;in_cache;out_cache;elapsed");
 194    strcpy(SiteUserTimeDateType,"table");
 195    strcpy(ShowReadStatistics,"yes");
 196    strcpy(IndexSortOrder,"D");
 197    strcpy(ShowSargInfo,"yes");
 198    strcpy(ShowSargLogo,"yes");
 199    strcpy(ParsedOutputLog,"no");
 200    strcpy(ParsedOutputLogCompress,"/bin/gzip");
 201    strcpy(DisplayedValues,"abbreviation");
 202    strcpy(HeaderFontSize,"9px");
 203    strcpy(TitleFontSize,"11px");
 204    strcpy(AuthUserFile,"/usr/local/sarg/passwd");
 205    strcpy(AuthName,"SARG, Restricted Access");
 206    strcpy(AuthType,"basic");
 207    strcpy(Require,"require user admin %u");
 208    strcpy(DownloadSuffix,"7z,ace,arj,avi,bat,bin,bz2,bzip,cab,com,cpio,dll,doc,dot,exe,gz,iso,lha,lzh,mdb,mov,mp3,mpeg,mpg,mso,nrg,ogg,ppt,rar,rtf,shs,src,sys,tar,tgz,vcd,vob,wma,wmv,zip");
 209    strcpy(Graphs,"yes");
 210    strcpy(Ulimit,"20000");
 211    strcpy(NtlmUserFormat,"domainname+username");
 212    strcpy(IndexTree,"file");
 213    strcpy(RealtimeTypes,"GET,PUT,CONNECT");
 214    strcpy(RealtimeUnauthRec,"show");
 215    strcpy(SquidguardIgnoreDate,"off");
 216    strcpy(DansguardianIgnoreDate,"off");
 217    strcpy(DataFileUrl,"ip");
 218    strcpy(MaxElapsed,"28800000");
 219    strcpy(BytesInSitesUsersReport,"no");
 220
 221    dia[0]='\0';
 222    mes[0]='\0';
 223    ano[0]='\0';
 224    hora[0]='\0';
 225    tmp[0]='\0';
 226    tmp2[0]='\0';
 227    tmp3[0]='\0';
 228    wtemp[0]='\0';
 229    wtemp2[0]='\0';
 230    us[0]='\0';
 231    date[0]='\0';
 232    df[0]='\0';
 233    uagent[0]='\0';
 234    hexclude[0]='\0';
 235    addr[0]='\0';
 236    hm[0]='\0';
 237    hmf[0]='\0';
 238    site[0]='\0';
 239    outdir[0]='\0';
 240    elap[0]='\0';
 241    email[0]='\0';
 242    zip[0]='\0';
 243    UserInvalidChar[0]='\0';
 244    DataFile[0]='\0';
 245    SquidGuardConf[0]='\0';
 246    DansGuardianConf[0]='\0';
 247
 248    excode=0;
 249    denied_count=0;
 250    download_count=0;
 251    authfail_count=0;
 252    dansguardian_count=0;
 253    squidguard_count=0;
 254    DeniedReportLimit=10;
 255    AuthfailReportLimit=10;
 256    DansGuardianReportLimit=10;
 257    SquidGuardReportLimit=10;
 258    DownloadReportLimit=50;
 259    UserReportLimit=0;
 260    debug=0;
 261    debugz=0;
 262    debugm=0;
 263    iprel=0;
 264    userip=0;
 265    color1=0;
 266    color2=0;
 267    color3=0;
 268    sarglog=0;
 269    isalog=0;
 270    dotinuser=0;
 271    realt=0;
 272    realtime_refresh=3;
 273    realtime_access_log_lines=1000;
 274    cost=0.01;
 275    nocost=50000000;
 276    ndownload=0;
 277    squid24=0;
 278
 279    bzero(IncludeUsers, MAXLEN);
 280    bzero(ExcludeString, MAXLEN);
 281
 282    for(x=0; x<=254; x++)
 283       warq[x][0]='\0';
 284
 285    language_load(language);
 286    strcpy(Title,text[88]);
 287
 288    while((ch = getopt(argc, argv, "a:b:c:d:e:f:g:u:l:L:o:s:t:w:hijmnprvxyz")) != -1){
 289       switch(ch)
 290       {
 291          case 'a':
 292             strcpy(addr,optarg);
 293             break;
 294          case 'b':
 295             strcpy(uagent,optarg);
 296             break;
 297          case 'c':
 298             strcpy(hexclude,optarg);
 299             break;
 300          case 'd':
 301             strcpy(date,optarg);
 302             strcpy(cduntil,optarg);
 303             if (getword(cdfrom,sizeof(cdfrom),cduntil,'-')<0) {
 304                printf("SARG: Maybe you have a broken record or garbage in your date range.\n");
 305                exit(1);
 306             }
 307             date_from(date, cdfrom, cduntil);
 308             dfrom=atoi(cdfrom);
 309             duntil=atoi(cduntil);
 310             break;
 311          case 'e':
 312             strcpy(email,optarg);
 313             break;
 314          case 'f':
 315             strcpy(ConfigFile,optarg);
 316             break;
 317          case 'g':
 318             strcpy(df,optarg);
 319             break;
 320          case 'h':
 321             usage(argv[0]);
 322             exit(0);
 323             break;
 324          case 'i':
 325             iprel++;
 326             break;
 327          case 'l':
 328             strcpy(warq[narq],optarg);
 329             narq++;
 330             break;
 331          case 'L':
 332             strcpy(SquidGuardLogAlternate,optarg);
 333             break;
 334          case 'm':
 335             debugm++;
 336             break;
 337          case 'n':
 338             dns++;
 339             break;
 340          case 'o':
 341             strcpy(outdir,optarg);
 342             break;
 343          case 'p':
 344             userip++;
 345             break;
 346          case 'r':
 347             realt++;
 348             break;
 349          case 's':
 350             strcpy(site,optarg);
 351             break;
 352          case 't':
 353             if(strstr(optarg,"-") == 0) {
 354               strcpy(hm,optarg);
 355               strcpy(hmf,optarg);
 356             } else {
 357                if (getword(hm,sizeof(hm),optarg,'-')<0) {
 358                   printf("SARG: Maybe you have a broken record or garbage in your time range.\n");
 359                   exit(1);
 360                }
 361                strcpy(hmf,optarg);
 362             }
 363             if(strlen(hm) > 5) {
 364                printf("SARG: time period must be MM or MM:SS. Exit.\n");
 365                exit(1);
 366             }
 367             bzero(whm,15);
 368             if(strstr(hm,":") != 0) {
 369                if (getword(warea,sizeof(warea),hm,':')<0) {
 370                   printf("SARG: Maybe you have a broken record or garbage in your time range.\n");
 371                   exit(1);
 372                }
 373                sprintf(whm,"%s%s",warea,hm);
 374                strcpy(hm,whm);
 375             }
 376             bzero(whm,15);
 377             if(strstr(hmf,":") != 0) {
 378                if (getword(warea,sizeof(warea),hmf,':')<0) {
 379                   printf("SARG: Maybe you have a broken record or garbage in your time range.\n");
 380                   exit(1);
 381                }
 382                sprintf(whm,"%s%s",warea,hmf);
 383                strcpy(hmf,whm);
 384             }
 385             break;
 386          case 'u':
 387             strcpy(us,optarg);
 388             break;
 389          case 'v':
 390             version();
 391             break;
 392          case 'w':
 393             strcpy(tmp,optarg);
 394             break;
 395          case 'x':
 396             debug++;
 397             break;
 398          case 'y':
 399             langcode++;
 400             break;
 401          case 'z':
 402             debugz++;
 403             break;
 404          case ':':
 405             fprintf(stderr, "Option -%c require an argument\n",optopt);
 406             errflg++;
 407             break;
 408          case '?':
 409             usage(argv[0]);
 410             exit(1);
 411             break;
 412       }
 413
 414    }
 415
 416    if (errflg) {
 417       usage(argv[0]);
 418       exit(2);
 419    }
 420
 421    if(debug) debuga("Init");
 422
 423    if(ConfigFile[0] == '\0') sprintf(ConfigFile,"%s/sarg.conf",SYSCONFDIR);
 424     else if(access(ConfigFile, R_OK) != 0) {
 425             sprintf(msg,"Cannot open config file: %s - %s",ConfigFile,strerror(errno));
 426             debuga(msg);
 427             exit(1);
 428          }
 429
 430    if(access(ConfigFile, R_OK) == 0)
 431       getconf();
 432
 433    if(realt) {
 434       realtime();
 435       exit(0);
 436    }
 437
 438    if(strcmp(IndexTree,"file") == 0)
 439       strcpy(ImageFile,"../images");
 440    else
 441       strcpy(ImageFile,"../../../images");
 442
 443    dataonly=0;
 444    if(DataFile[0] != '\0');
 445       dataonly++;
 446
 447    str2=(char *)subs(TopUserFields,"%BYTES","SETYB");
 448    strcpy(TopUserFields,str2);
 449
 450    str2=(char *)subs(UserReportFields,"%BYTES","SETYB");
 451    strcpy(UserReportFields,str2);
 452
 453    if(!narq) {
 454       strcpy(warq[0],AccessLog);
 455       narq++;
 456    }
 457
 458    if(strcmp(hexclude,"onvert") == 0 && strcmp(site,"plit") != 0) {
 459       convlog(warq[0], df, dfrom, duntil);
 460       exit(0);
 461    }
 462
 463    if(strcmp(site,"plit") == 0) {
 464       splitlog(warq[0], df, dfrom, duntil, hexclude);
 465       exit(0);
 466    }
 467
 468    if(ExcludeCodes[0] != '\0') {
 469       if((excludecode=(char *) malloc(1024))==NULL) {
 470          fprintf(stderr, "SARG: %s (1024):\n",text[59]);
 471          exit(1);
 472       }
 473       bzero(excludecode,1024);
 474       load_excludecodes();
 475    }
 476
 477    if(access(PasswdFile, R_OK) == 0) {
 478       getusers(PasswdFile,debug);
 479       puser++;
 480    }
 481
 482    if(hexclude[0] == '\0')
 483       strcpy(hexclude,ExcludeHosts);
 484    if(strlen(hexclude) > 0) {
 485       if(access(hexclude, R_OK) != 0) {
 486          sprintf(msg,"Cannot open exclude_hosts file: %s - %s",hexclude,strerror(errno));
 487          debuga(msg);
 488          exit(1);
 489       } else {
 490          gethexclude(hexclude,debug);
 491          fhost++;
 492       }
 493    }
 494
 495    if(ReportType[0] == '\0')
 496       strcpy(ReportType,"topusers topsites users_sites sites_users date_time denied auth_failures site_user_time_date downloads");
 497
 498    if(access(ExcludeUsers, R_OK) == 0) {
 499       getuexclude(ExcludeUsers,debug);
 500       fuser++;
 501    }
 502
 503    indexonly=0;
 504    if(fuser) {
 505       if(strstr(excludeuser,"indexonly") != 0)
 506          indexonly++;
 507    }
 508    if(strcmp(ExcludeUsers,"indexonly") == 0) indexonly++;
 509    if(strcmp(Index,"only") == 0) indexonly++;
 510
 511    if(dns) strcpy(Ip2Name,"yes");
 512
 513    if(strcmp(UserIp,"yes") == 0) userip++;
 514
 515    if(strlen(MaxElapsed)>1) max_elapsed=atol(MaxElapsed);
 516
 517    if(strlen(outdir)<1) strcpy(outdir,OutputDir);
 518    strcat(outdir,"/");
 519
 520
 521    if(arq[0] == '\0') strcpy(arq,AccessLog);
 522
 523    if(uagent[0] == '\0') strcpy(uagent,UserAgentLog);
 524
 525    if(tmp[0] == '\0') strcpy(tmp,TempDir);
 526    else strcpy(TempDir,tmp);
 527
 528    if(df[0] == '\0') strcpy(df,DateFormat);
 529    else strcpy(DateFormat,df);
 530
 531    if(df[0] == '\0') {
 532       strcpy(df,"u");
 533       strcpy(DateFormat,"u");
 534    }
 535
 536    if(strlen(email)<1 && strlen(OutputEmail)>0) strcpy(email,OutputEmail);
 537
 538    strcpy(tmp2,tmp);
 539
 540    if(strlen(email) > 0) {
 541       sprintf(wtemp2,"%s/sarg",tmp2);
 542       my_mkdir(wtemp2);
 543       strcat(tmp2,"/sarg");
 544       strcpy(outdir,tmp2);
 545       strcat(outdir,"/");
 546    }
 547
 548    strcat(tmp2,"/sarg.log");
 549
 550    sprintf(warea,"%s/sarg",tmp);
 551    if(access(warea, R_OK) == 0) {
 552       sprintf(tmp3,"rm -rf %s",warea);
 553       system(tmp3);
 554    }
 555
 556    sprintf(tmp3,"%s/sarg",tmp);
 557    my_mkdir(tmp3);
 558    strcpy(tmp4,tmp3);
 559    strcpy(tmp5,tmp3);
 560    strcpy(tmp6,tmp3);
 561    strcat(tmp4,"/denied.log.unsort");
 562    strcat(tmp5,"/denied.log");
 563    strcat(tmp6,"/authfail.log.unsort");
 564
 565    if(debug) {
 566       fprintf(stderr, "SARG: %s:\nSARG:\n",text[22]);
 567       fprintf(stderr, "SARG: %35s (-a) = %s\n",text[23],addr);
 568       fprintf(stderr, "SARG: %35s (-b) = %s\n",text[71],uagent);
 569       fprintf(stderr, "SARG: %35s (-c) = %s\n",text[69],hexclude);
 570       fprintf(stderr, "SARG: %35s (-d) = %s\n",text[24],date);
 571       fprintf(stderr, "SARG: %35s (-e) = %s\n",text[41],email);
 572       fprintf(stderr, "SARG: %35s (-f) = %s\n",text[70],ConfigFile);
 573       if(strcmp(df,"e") == 0)
 574          fprintf(stderr, "SARG: %35s (-g) = %s (dd/mm/yyyy)\n",text[25],text[26]);
 575       if(strcmp(df,"u") == 0)
 576          fprintf(stderr, "SARG: %35s (-g) = %s (mm/dd/yyyy)\n",text[25],text[27]);
 577       if(strcmp(df,"w") == 0)
 578          fprintf(stderr, "SARG: %35s (-g) = %s (yyyy/ww)\n",text[25],text[85]);
 579       if(iprel)
 580          fprintf(stderr, "SARG: %35s (-i) = %s\n",text[28],text[1]);
 581        else
 582          fprintf(stderr, "SARG: %35s (-i) = %s\n",text[28],text[2]);
 583       fprintf(stderr, "SARG: %35s (-l) = %s\n",text[37],arq);
 584       if(strcmp(Ip2Name,"yes") == 0)
 585          fprintf(stderr, "SARG: %35s (-n) = %s\n",text[65],text[1]);
 586        else
 587          fprintf(stderr, "SARG: %35s (-n) = %s\n",text[65],text[2]);
 588       fprintf(stderr, "SARG: %35s (-o) = %s\n",text[38],outdir);
 589       if(strcmp(UserIp,"yes") == 0)
 590          fprintf(stderr, "SARG: %35s (-p) = %s\n",text[29],text[1]);
 591        else
 592          fprintf(stderr, "SARG: %35s (-p) = %s\n",text[29],text[2]);
 593       fprintf(stderr, "SARG: %35s (-s) = %s\n",text[30],site);
 594       fprintf(stderr, "SARG: %35s (-t) = %s\n",text[31],hm);
 595       fprintf(stderr, "SARG: %35s (-u) = %s\n",text[32],us);
 596       fprintf(stderr, "SARG: %35s (-w) = %s\n",text[34],tmp);
 597       if(debug)
 598          fprintf(stderr, "SARG: %35s (-x) = %s\n",text[35],text[1]);
 599        else
 600          fprintf(stderr, "SARG: %35s (-x) = %s\n",text[35],text[2]);
 601       if(debugz)
 602          fprintf(stderr, "SARG: %35s (-z) = %s\n",text[36],text[1]);
 603        else
 604          fprintf(stderr, "SARG: %35s (-z) = %s\n",text[36],text[2]);
 605       fprintf(stderr, "SARG:\n");
 606    }
 607
 608    if(debugm) {
 609       printf("%s:\nSARG:\n",text[22]);
 610       printf("%35s (-a) = %s\n",text[23],addr);
 611       printf("%35s (-b) = %s\n",text[71],uagent);
 612       printf("%35s (-c) = %s\n",text[69],hexclude);
 613       printf("%35s (-d) = %s\n",text[24],date);
 614       printf("%35s (-e) = %s\n",text[41],email);
 615       printf("%35s (-f) = %s\n",text[70],ConfigFile);
 616       if(strcmp(df,"e") == 0)
 617          printf("%35s (-g) = %s (dd/mm/yyyy)\n",text[25],text[26]);
 618       if(strcmp(df,"u") == 0)
 619          printf("%35s (-g) = %s (mm/dd/yyyy)\n",text[25],text[27]);
 620       if(strcmp(df,"w") == 0)
 621          printf("%35s (-g) = %s (yyyy/ww)\n",text[25],text[85]);
 622       if(iprel)
 623          printf("%35s (-i) = %s\n",text[28],text[1]);
 624        else
 625          printf("%35s (-i) = %s\n",text[28],text[2]);
 626       printf("%35s (-l) = %s\n",text[37],arq);
 627       if(strcmp(Ip2Name,"yes") == 0)
 628          printf("%35s (-n) = %s\n",text[65],text[1]);
 629        else
 630          printf("%35s (-n) = %s\n",text[65],text[2]);
 631       printf("%35s (-o) = %s\n",text[38],outdir);
 632       if(strcmp(UserIp,"yes") == 0)
 633          printf("%35s (-p) = %s\n",text[29],text[1]);
 634        else
 635          printf("%35s (-p) = %s\n",text[29],text[2]);
 636       printf("%35s (-s) = %s\n",text[30],site);
 637       printf("%35s (-t) = %s\n",text[31],hm);
 638       printf("%35s (-u) = %s\n",text[32],us);
 639       printf("%35s (-w) = %s\n",text[34],tmp);
 640       if(debug)
 641          printf("%35s (-x) = %s\n",text[35],text[1]);
 642        else
 643          printf("%35s (-x) = %s\n",text[35],text[2]);
 644       if(debugz)
 645          printf("%35s (-z) = %s\n",text[36],text[1]);
 646        else
 647          printf("%35s (-z) = %s\n",text[36],text[2]);
 648       printf("sarg %s: %s\n",text[73],VERSION);
 649       printf("Language=%s\n\n",text[3]);
 650    }
 651
 652    if(debug){
 653       sprintf(msg,"sarg %s: %s",text[73],VERSION);
 654       debuga(msg);
 655    }
 656
 657    if (strlen(Ulimit) > 0) {
 658 #if defined(RLIMIT_NOFILE)
 659       getrlimit (RLIMIT_NOFILE, &rl);
 660 #else #if defined(RLIMIT_OFILE)
 661       getrlimit (RLIMIT_OFILE, &rl);
 662 #endif
 663       l1 = rl.rlim_cur;
 664       l2 = rl.rlim_max;
 665
 666       if(Ulimit[0] != '\0') {
 667          rl.rlim_cur = atol(Ulimit);
 668          rl.rlim_max = atol(Ulimit);
 669  #if defined(RLIMIT_NOFILE)
 670          if(setrlimit (RLIMIT_NOFILE, &rl) == -1) {
 671  #else #if defined(RLIMIT_OFILE)
 672          if(setrlimit (RLIMIT_OFILE, &rl) == -1) {
 673  #endif
 674             sprintf(msg,"setrlimit error - %s\n",strerror(errno));
 675             debuga(msg);
 676          }
 677       }
 678
 679       if(debug) {
 680          sprintf(msg,"Maximum file descriptor: cur=%ld max=%ld, changed to cur=%ld max=%ld",l1,l2,rl.rlim_cur,rl.rlim_max);
 681          debuga(msg);
 682       }
 683    }
 684
 685    if (UserTabFile[0] != '\0') {
 686       if(debug) {
 687          sprintf(msg,"%s: %s",text[86],UserTabFile);
 688          debuga(msg);
 689       }
 690       if((fp_usr=fopen(UserTabFile,"r"))==NULL) {
 691         fprintf(stderr, "%s: (log) %s: %s - %s\n",argv[0],text[45],UserTabFile,strerror(errno));
 692         exit(1);
 693       }
 694       nreg = lseek(fileno(fp_usr), 0, SEEK_END);
 695       lseek(fileno(fp_usr), 0, 0);
 696       if((userfile=(char *) malloc(nreg+100))==NULL){
 697          fprintf(stderr, "%s ERROR: %s",argv[0],text[87]);
 698          exit(1);
 699       }
 700       bzero(userfile,nreg+100);
 701       strncat(userfile,":",1);
 702       z1=0;
 703       z2=1;
 704       while(fgets(buf,MAXLEN,fp_usr)!=NULL) {
 705          buf[strlen(buf)-1]='\0';
 706          if(strstr(buf,"\r") != 0) buf[strlen(buf)-1]='\0';
 707         if (getword_multisep(bufy,sizeof(bufy),buf,' ')<0) {
 708            printf("SARG: Maybe you have a broken record or garbage in your %s file.\n",UserTabFile);
 709            exit(1);
 710         }
 711          for(z1=0; z1<=strlen(bufy); z1++) {
 712             userfile[z2]=bufy[z1];
 713             z2++;
 714          }
 715          strncat(userfile,":",1);
 716          for(z1=0; z1<=strlen(buf); z1++) {
 717             userfile[z2]=buf[z1];
 718             z2++;
 719          }
 720          strncat(userfile,":",1);
 721       }
 722       fclose(fp_usr);
 723    }
 724
 725    sprintf ( sz_Download_Unsort , "%s/sarg/download.unsort", tmp);
 726    bool_ShowReadStatistics = ( strcmp(ShowReadStatistics,"yes") == 0 ) ;
 727
 728    while(narq--) {
 729       strcpy(arq,warq[iarq]);
 730       iarq++;
 731
 732    strcpy(arqtt,arq);
 733    decomp(arq,zip,tmp);
 734    if(debug) {
 735       sprintf(msg, "%s: %s",text[7],arq);
 736       debuga(msg);
 737    }
 738
 739 #if defined(HAVE_FOPEN64)
 740    if((fp_in=(long)fopen64(arq,"r"))==NULL) {
 741 #else
 742    if((fp_in=fopen(arq,"r"))==NULL) {
 743 #endif
 744      fprintf(stderr, "%s: (log) %s: %s - %s\n",argv[0],text[8],arq,strerror(errno));
 745      exit(1);
 746    }
 747    fgets(bufz,sizeof(bufz),fp_in);
 748    if(!isalog && strncmp(bufz,"#Software: Mic",14) == 0) isalog++;
 749
 750    if(strncmp(bufz,"*** SARG Log ***",16) == 0) {
 751       if (getword(val2,sizeof(val2),arqtt,'-')<0 || getword(val2,sizeof(val2),arqtt,'_')<0 ||
 752           getword(val3,sizeof(val3),arqtt,'-')<0 || getword(val3,sizeof(val3),arqtt,'_')<0) {
 753          printf("SARG: Maybe you have a broken record or garbage in your %s file.\n",arq);
 754          exit(1);
 755       }
 756       sprintf(period,"%s-%s",val2,val3);
 757       sarglog=1;
 758    } else lseek(fileno(fp_in), 0, 0);
 759
 760    if(strcmp(ParsedOutputLog, "no") != 0 && !sarglog) {
 761       if(access(ParsedOutputLog,R_OK) != 0) {
 762          sprintf(csort,"%s",ParsedOutputLog);
 763          my_mkdir(csort);
 764       }
 765       sprintf(arq_log,"%s/sarg_temp.log",ParsedOutputLog);
 766       if((fp_log=fopen(arq_log,"w"))==NULL) {
 767         fprintf(stderr, "%s: (log) %s: %s - %s\n",argv[0],text[8],arq_log,strerror(errno));
 768         exit(1);
 769       }
 770       fputs("*** SARG Log ***\n",fp_log);
 771    }
 772
 773    if(strstr(ReportType,"denied") != 0) {
 774       if((fp_denied=fopen(tmp4,"w"))==NULL) {
 775          fprintf(stderr, "%s: (log) %s: %s - %s\n",argv[0],text[45],tmp4,strerror(errno));
 776          exit(1);
 777       }
 778    }
 779
 780    if(DataFile[0]=='\0') {
 781       if(strstr(ReportType,"denied") != 0 || strstr(ReportType,"auth_failures") != 0) {
 782          if((fp_authfail=fopen(tmp6,"w"))==NULL) {
 783             fprintf(stderr, "%s: (log) %s: %s - %s\n",argv[0],text[45],tmp6,strerror(errno));
 784             exit(1);
 785          }
 786       }
 787    }
 788
 789    // pre-Read the file only if I have to show stats
 790    if(bool_ShowReadStatistics) {
 791       rewind(fp_in);
 792       recs1=0;
 793       recs2=0;
 794
 795       while( fgets(bufz,sizeof(bufz),fp_in) != NULL ) recs1++;
 796       rewind(fp_in);
 797       printf("SARG: Records in file: " OFFSET_STRING ", reading: %3.2f%%\r",recs1,(float) 0);
 798       fflush( stdout ) ;
 799    }
 800
 801    while(fgets(bufz,sizeof(bufz),fp_in)!=NULL) {
 802         recs2++;
 803         if( bool_ShowReadStatistics && ! --OutputNonZero) {
 804            perc = recs2 * 100 ;
 805            perc = perc / recs1 ;
 806            printf("SARG: Records in file: " OFFSET_STRING ", reading: %3.2f%%\r",recs1,perc);
 807            fflush (stdout);
 808            OutputNonZero = REPORT_EVERY_X_LINES ;
 809         }
 810         if(strlen(bufz) > MAXLEN-1) continue;
 811         if(!bufz[0]) continue;
 812         if(strstr(bufz,"HTTP/0.0") != 0) continue;
 813         if(strstr(bufz,"logfile turned over") != 0) continue;
 814         if(bufz[0] == ' ') continue;
 815         if(strlen(bufz) < 58) continue;
 816
 817         // Record only hours usage which is required
 818         tt = (time_t) strtoul( bufz, NULL, 10 );
 819         t = localtime( &tt );
 820
 821         if( bsearch( &( t -> tm_wday ), weekdays.list, weekdays.len,
 822                                         sizeof( int ), compar ) == NULL )
 823           continue;
 824
 825         if( bsearch( &( t -> tm_hour ), hours.list, hours.len,
 826                                         sizeof( int ), compar ) == NULL )
 827           continue;
 828
 829         // exclude_string
 830         exstring=0;
 831         if(strlen(ExcludeString) > 0) {
 832            strcpy(warea,bufz);
 833            strcpy(html,ExcludeString);
 834            while(strstr(html,":") != 0) {
 835               if (getword_multisep(val1,sizeof(val1),html,':')<0) {
 836                  printf("SARG: Maybe you have a broken record or garbage in your exclusion string.\n");
 837                  exit(1);
 838               }
 839               if((str=(char *) strstr(warea,val1)) != (char *) NULL )
 840                  exstring++;
 841            }
 842            if((str=(char *) strstr(warea,html)) != (char *) NULL )
 843                  exstring++;
 844         }
 845         if(exstring) continue;
 846
 847         strcpy(bufy,bufz);
 848         if ((str = strchr(bufz, '\n')) != NULL)
 849            *str = '\0';          /* strip \n */
 850
 851         totregsl++;
 852         common=0;
 853         if(debugm)
 854            printf("BUF=%s\n",bufz);
 855
 856         if(!sarglog && !isalog) {
 857            if (getword(data,sizeof(data),bufz,' ')<0) {
 858              printf("SARG: Maybe you have a broken record or garbage in your access.log file.\n");
 859              exit(1);
 860            }
 861            if((str=(char *) strstr(data, ".")) != (char *) NULL ) {
 862               if((str=(char *) strstr(str+1, ".")) != (char *) NULL ) {
 863                  strcpy(ip,data);
 864                  strcpy(elap,"0");
 865                          if(squid24) {
 866                     if (getword(user,sizeof(user),bufz,' ')<0 || getword(none,sizeof(none),bufz,' ')<0) {
 867                        printf("SARG: Maybe you have a broken record or garbage in your %s file.\n",arq);
 868                        exit(1);
 869                     }
 870                          } else {
 871                         if (getword(none,sizeof(none),bufz,' ')<0 || getword(user,sizeof(user),bufz,' ')<0) {
 872                        printf("SARG: Maybe you have a broken record or garbage in your %s file.\n",arq);
 873                        exit(1);
 874                     }
 875                  }
 876                      if (getword(data,sizeof(data),bufz,']')<0 || getword(fun,sizeof(fun),bufz,'"')<0 ||
 877                          getword(fun,sizeof(fun),bufz,' ')<0 || getword(url,sizeof(url),bufz,' ')<0 ||
 878                      getword(code2,sizeof(code2),bufz,' ')<0 || getword(code2,sizeof(code2),bufz,' ')<0 ||
 879                      getword(tam,sizeof(tam),bufz,' ')<0) {
 880                     printf("SARG: Maybe you have a broken record or garbage in your %s file.\n",arq);
 881                     exit(1);
 882                  }
 883                  if((str=(char *) strstr(bufz, " ")) != (char *) NULL ) {
 884                     if (getword(code,sizeof(code),bufz,' ')<0) {
 885                        printf("SARG: Maybe you have a broken record or garbage in your %s file.\n",arq);
 886                        exit(1);
 887                     }
 888                  }
 889                  else strcpy(code,bufz);
 890
 891                  if ((str = strchr(code, ':')) != NULL)
 892                     *str = '/';
 893
 894                  if(strcmp(tam,"\0") == 0)
 895                     strcpy(tam,"0");
 896
 897                  common++;
 898                  common_log=1;
 899               }
 900            }
 901
 902            if(!common) {
 903               if (getword(elap,sizeof(elap),bufz,' ')<0) {
 904              printf("SARG: Maybe you have a broken record or garbage in your %s file.\n",arq);
 905              exit(1);
 906           }
 907               while(strcmp(elap,"") == 0 && strlen(bufz) > 0)
 908                  if (getword(elap,sizeof(elap),bufz,' ')<0) {
 909                 printf("SARG: Maybe you have a broken record or garbage in your %s file.\n",arq);
 910                 exit(1);
 911              }
 912               if(strlen(elap) < 1) continue;
 913                   if (getword(ip,sizeof(ip),bufz,' ')<0){
 914                  printf("SARG: Maybe you have a broken record or garbage in your %s file.\n",arq);
 915                  exit(1);
 916               }
 917               if (getword(code,sizeof(code),bufz,' ')<0){
 918                  printf("SARG: Maybe you have a broken record or garbage in your %s file.\n",arq);
 919                  exit(1);
 920               }
 921               if (getword(tam,sizeof(tam),bufz,' ')<0){
 922                  printf("SARG: Maybe you have a broken record or garbage in your %s file.\n",arq);
 923                  exit(1);
 924               }
 925               if (getword(fun,sizeof(fun),bufz,' ')<0){
 926                  printf("SARG: Maybe you have a broken record or garbage in your %s file.\n",arq);
 927                  exit(1);
 928               }
 929               if (getword(url,sizeof(url),bufz,' ')<0){
 930                  printf("SARG: Maybe you have a broken record or garbage in your %s file.\n",arq);
 931                  exit(1);
 932               }
 933 //              while (strstr(bufz,"%20") != 0) {
 934 //                 getword(warea,bufz,' ');
 935 //                 strcat(url,warea);
 936 //              }
 937               if (getword(user,sizeof(user),bufz,' ')<0){
 938                  printf("SARG: Maybe you have a broken record or garbage in your %s file.\n",arq);
 939                  exit(1);
 940               }
 941               squid_log=1;
 942                }
 943         } else if(!isalog) {
 944                if (getword(data,sizeof(data),bufz,' ')<0){
 945               printf("SARG: Maybe you have a broken record or garbage in your %s file.\n",arq);
 946               exit(1);
 947            }
 948            if (getword(hora,sizeof(hora),bufz,' ')<0) {
 949               printf("SARG: Maybe you have a broken record or garbage in your %s file.\n",arq);
 950               exit(1);
 951            }
 952            if (getword(user,sizeof(user),bufz,' ')<0) {
 953               printf("SARG: Maybe you have a broken record or garbage in your %s file.\n",arq);
 954               exit(1);
 955            }
 956            if (getword(ip,sizeof(ip),bufz,' ')<0) {
 957               printf("SARG: Maybe you have a broken record or garbage in your %s file.\n",arq);
 958               exit(1);
 959            }
 960            if (getword(url,sizeof(url),bufz,' ')<0){
 961               printf("SARG: Maybe you have a broken record or garbage in your %s file.\n",arq);
 962               exit(1);
 963            }
 964            if (getword(tam,sizeof(tam),bufz,' ')<0){
 965               printf("SARG: Maybe you have a broken record or garbage in your %s file.\n",arq);
 966               exit(1);
 967            }
 968            if (getword(code,sizeof(code),bufz,' ')<0){
 969               printf("SARG: Maybe you have a broken record or garbage in your %s file.\n",arq);
 970               exit(1);
 971            }
 972            if (getword(elap,sizeof(elap),bufz,' ')<0){
 973               printf("SARG: Maybe you have a broken record or garbage in your %s file.\n",arq);
 974               exit(1);
 975            }
 976            if (getword(smartfilter,sizeof(smartfilter),bufz,' ')<0){
 977               printf("SARG: Maybe you have a broken record or garbage in your %s file.\n",arq);
 978               exit(1);
 979            }
 980         } else if(isalog) {
 981            if(!i0) {
 982               if (getword(val1,sizeof(val1),bufz,' ')<0){
 983                  printf("SARG: Maybe you have a broken record or garbage in your %s file.\n",arq);
 984                  exit(1);
 985               }
 986               while(strstr(bufz,"\t") != 0) {
 987                  if (getword(val1,sizeof(val1),bufz,'\t')<0){
 988                     printf("SARG: Maybe you have a broken record or garbage in your %s file.\n",arq);
 989                     exit(1);
 990                  }
 991                  i0++;
 992                  if(strcmp(val1,"c-ip") == 0) i1=i0;
 993                  if(strcmp(val1,"cs-username") == 0) i2=i0;
 994                  if(strcmp(val1,"date") == 0) i3=i0;
 995                  if(strcmp(val1,"time") == 0) i4=i0;
 996                  if(strcmp(val1,"time-taken") == 0) i5=i0;
 997                  if(strcmp(val1,"sc-bytes") == 0) i6=i0;
 998                  if(strcmp(val1,"cs-uri") == 0) i7=i0;
 999                  if(strcmp(val1,"sc-status") == 0) i8=i0;
1000               }
1001            }
1002            fgets(bufz,sizeof(bufz),fp_in);
1003            strcpy(val1,bufz);
1004            for(x=0; x<=i1-1; x++) {
1005               if (getword3(ip,sizeof(ip),val1,'\t')<0) {
1006                  printf("SARG: Maybe you have a broken record or garbage in your %s file.\n",arq);
1007                  exit(1);
1008               }
1009            }
1010            strcpy(val1,bufz);
1011            for(x=0; x<=i2-1; x++) {
1012               if (getword3(user,sizeof(user),val1,'\t')<0) {
1013                  printf("SARG: Maybe you have a broken record or garbage in your %s file.\n",arq);
1014                  exit(1);
1015               }
1016            }
1017            strcpy(val1,bufz);
1018            for(x=0; x<=i3-1; x++) {
1019               if (getword3(data,sizeof(data),val1,'\t')<0) {
1020                  printf("SARG: Maybe you have a broken record or garbage in your %s file.\n",arq);
1021                  exit(1);
1022               }
1023            }
1024            strcpy(val1,bufz);
1025            for(x=0; x<=i4-1; x++) {
1026               if (getword3(hora,sizeof(hora),val1,'\t')<0) {
1027                  printf("SARG: Maybe you have a broken record or garbage in your %s file.\n",arq);
1028                  exit(1);
1029               }
1030            }
1031            strcpy(val1,bufz);
1032            for(x=0; x<=i5-1; x++) {
1033               if (getword3(elap,sizeof(elap),val1,'\t')<0) {
1034                  printf("SARG: Maybe you have a broken record or garbage in your %s file.\n",arq);
1035                  exit(1);
1036               }
1037            }
1038            strcpy(val1,bufz);
1039            for(x=0; x<=i6-1; x++) {
1040               if (getword3(tam,sizeof(tam),val1,'\t')<0) {
1041                  printf("SARG: Maybe you have a broken record or garbage in your %s file.\n",arq);
1042                  exit(1);
1043               }
1044            }
1045            strcpy(val1,bufz);
1046            for(x=0; x<=i7-1; x++) {
1047               if (getword3(url,sizeof(url),val1,'\t')<0) {
1048                  printf("SARG: Maybe you have a broken record or garbage in your %s file.\n",arq);
1049                  exit(1);
1050               }
1051            }
1052            strcpy(val1,bufz);
1053            for(x=0; x<=i8-1; x++) {
1054               if (getword3(code,sizeof(code),val1,'\t')<0) {
1055                  printf("SARG: Maybe you have a broken record or garbage in your %s file.\n",arq);
1056                  exit(1);
1057               }
1058            }
1059
1060            if(strcmp(code,"401") == 0 || strcmp(code,"403") == 0 || strcmp(code,"407") == 0) {
1061               sprintf(val1,"DENIED/%s",code);
1062               strcpy(code,val1);
1063            }
1064            if (getword(ano,sizeof(ano),data,'-')<0){
1065               printf("SARG: Maybe you have a broken record or garbage in your %s file.\n",arq);
1066               exit(1);
1067            }
1068            if (getword(mes,sizeof(mes),data,'-')<0){
1069               printf("SARG: Maybe you have a broken record or garbage in your %s file.\n",arq);
1070               exit(1);
1071            }
1072            strcpy(dia,data);
1073            conv_month_name(mes);
1074            sprintf(data," %s/%s/%s:%s",dia,mes,ano,hora);
1075         }
1076
1077         if(strlen(user) > 150) {
1078            totregsx++;
1079            continue;
1080         }
1081
1082         // include_users
1083         if(strlen(IncludeUsers) > 0) {
1084            sprintf(val1,":%s:",user);
1085            if((str=(char *) strstr(IncludeUsers,val1)) == (char *) NULL )
1086               continue;
1087         }
1088
1089         if(excode) {
1090            if(vercode(code)) {
1091               totregsx++;
1092               continue;
1093            }
1094         }
1095
1096         if(testvaliduserchar(user))
1097            continue;
1098
1099         if(strstr(user,"%20") != 0) {
1100            if (getword(w,sizeof(w),user,'%')<0) {
1101               printf("SARG: Maybe you have a broken user in your %s file.\n",arq);
1102               exit(1);
1103            }
1104            strcpy(user,w);
1105         }
1106
1107         while(strstr(user,"%5c") != 0) {
1108            if (getword_multisep(w,sizeof(w),user,'%')<0){
1109               printf("SARG: Maybe you have a broken user in your %s file.\n",arq);
1110               exit(1);
1111            }
1112            strcpy(wuser,user+2);
1113            sprintf(user,"%s.%s",w,wuser);
1114         }
1115
1116         str=user;
1117         for(z1=0; z1<strlen(str); z1++) {
1118            if(isalnum(str[z1]) || ispunct(str[z1])) {
1119            if(str[z1]=='.') dotinuser++;
1120            if(str[z1]=='?' || str[z1]=='.' || str[z1]==':' || str[z1]=='/' || str[z1]=='\\')
1121               str[z1]='_';
1122            }
1123         }
1124
1125         strlow(user);
1126         if(strncmp(NtlmUserFormat,"user",4) == 0) {
1127            if(strstr(user,"_") != 0)
1128               if (getword_multisep(warea,sizeof(warea),user,'_')<0){
1129                  printf("SARG: Maybe you have a broken user in your %s file.\n",arq);
1130                  exit(1);
1131               }
1132            if(strstr(user,"+") != 0)
1133               if (getword_multisep(warea,sizeof(warea),user,'+')<0){
1134                  printf("SARG: Maybe you have a broken user in your %s file.\n",arq);
1135                  exit(1);
1136               }
1137         }
1138
1139         if(strstr(ReportType,"denied") != 0)
1140            strcpy(urly,url);
1141
1142         if(strlen(DownloadSuffix)) {
1143            suffix[0]='\0';
1144            download_flag=0;
1145            if(strncmp(url+strlen(url)-4,".",1) == 0)
1146               strcpy(suffix,url+strlen(url)-3);
1147            else strcpy(suffix,url+strlen(url)-4);
1148            if(strstr(DownloadSuffix,suffix) != 0) {
1149               strcpy(download_url,url);
1150               download_flag=1;
1151               download_count++;
1152            }
1153         }
1154
1155         if (strchr(url,'/')) {
1156            if (getword(w,sizeof(w),url,'/')<0){
1157               printf("SARG: Maybe you have a broken url in your %s file.\n",arq);
1158               exit(1);
1159            }
1160            if (getword(w,sizeof(w),url,'/')<0){
1161               printf("SARG: Maybe you have a broken url in your %s file.\n",arq);
1162               exit(1);
1163            }
1164            if (!strchr(url,'/')) {
1165               totregsx++;
1166               continue;
1167           }
1168         }
1169
1170         if(strcmp(LongUrl,"no") == 0) {
1171            if (getword_multisep(w,sizeof(w),url,'/')<0){
1172               printf("SARG: Maybe you have a broken url in your %s file.\n",arq);
1173               exit(1);
1174            }
1175            strcpy(url,w);
1176            if(strlen(url) > 512 && strstr(url,"%") != 0) {
1177               if (getword_multisep(w,sizeof(w),url,'%')<0){
1178                  printf("SARG: Maybe you have a broken url in your %s file.\n",arq);
1179                  exit(1);
1180               }
1181               strcpy(url,w);
1182            }
1183         }
1184
1185         if(!sarglog) {
1186            if(!common && !isalog) {
1187               tt=atoi(data);
1188               t=localtime(&tt);
1189
1190               strftime(tbuf2, 127, "%H%M", t);
1191               if(strncmp(df,"u",1) == 0)
1192                  strftime(tbuf, 127, "%Y%b%d", t);
1193               if(strncmp(df,"e",1) == 0)
1194                  strftime(tbuf, 127, "%d%b%Y", t);
1195               if(strncmp(df,"w",1) == 0) {
1196                  strcpy(IndexTree,"file");
1197                  strftime(tbuf, 127, "%Y.%U", t);
1198               }
1199
1200               strftime(dia, 127, "%d/%m/%Y", t);
1201               strftime(wdata, 127, "%Y%m%d", t);
1202
1203               idata=atoi(wdata);
1204
1205               if(strncmp(df,"u",1)==0)
1206                  strftime(dia, 127, "%m/%d/%Y", t);
1207               sprintf(hora,"%02d:%02d:%02d",t->tm_hour,t->tm_min,t->tm_sec);
1208                } else {
1209                   strcpy(wtemp,data+1);
1210                   if (getword_multisep(data,sizeof(data),wtemp,':')<0){
1211                  printf("SARG: Maybe you have a broken date in your %s file.\n",arq);
1212                  exit(1);
1213               }
1214               if (getword_multisep(hora,sizeof(hora),wtemp,' ')<0){
1215                  printf("SARG: Maybe you have a broken date in your %s file.\n",arq);
1216                  exit(1);
1217               }
1218               if (getword_multisep(dia,sizeof(dia),data,'/')<0){
1219                  printf("SARG: Maybe you have a broken date in your %s file.\n",arq);
1220                  exit(1);
1221               }
1222               if (getword_multisep(mes,sizeof(mes),data,'/')<0){
1223                  printf("SARG: Maybe you have a broken date in your %s file.\n",arq);
1224                  exit(1);
1225               }
1226               if (getword_multisep(ano,sizeof(ano),data,'/')<0){
1227                  printf("SARG: Maybe you have a broken date in your %s file.\n",arq);
1228                  exit(1);
1229               }
1230
1231               if(strcmp(df,"u") == 0)
1232                  sprintf(tbuf,"%s%s%s",ano,mes,dia);
1233               if(strcmp(df,"e") == 0)
1234                  sprintf(tbuf,"%s%s%s",dia,mes,ano);
1235               builddia(dia,mes,ano,df,wdata);
1236               idata=atoi(wdata);
1237            }
1238         } else {
1239                if (getword_multisep(mes,sizeof(mes),data,'/')<0){
1240               printf("SARG: Maybe you have a broken date in your %s file.\n",arq);
1241               exit(1);
1242            }
1243            if (getword_multisep(dia,sizeof(dia),data,'/')<0){
1244               printf("SARG: Maybe you have a broken date in your %s file.\n",arq);
1245               exit(1);
1246            }
1247            if (getword_multisep(ano,sizeof(ano),data,0)<0){
1248               printf("SARG: Maybe you have a broken date in your %s file.\n",arq);
1249               exit(1);
1250            }
1251         }
1252
1253         if(debugm)
1254            printf("DATE=%s IDATA=%d DFROM=%d DUNTIL=%d\n",date,idata,dfrom,duntil);
1255
1256         l=1;
1257         if(strlen(us)>0){
1258            if(strcmp(user,us)==0)
1259               l=1;else l=0;
1260         }
1261
1262         if(l){
1263            if(strlen(addr)>0){
1264               if(strcmp(addr,ip)==0)
1265                  l=1;else l=0;
1266            }
1267            if(fhost) {
1268 //              l=vhexclude(excludefile,ip);
1269               l=vhexclude(excludefile,url);
1270               if(!l)
1271                  totregsx++;
1272            }
1273         }
1274
1275         if(l){
1276            if(strlen(date) > 0){
1277               if(idata >= dfrom && idata <= duntil)
1278                  l=1;else l=0;
1279            }
1280         }
1281         if(l){
1282            if(strlen(hm)>0) {
1283               strcpy(whm,hora);
1284               bzero(hmr,15);
1285               chm++;
1286               while(chm) {
1287                  if (getword_multisep(warea,sizeof(warea),whm,':')<0){
1288                     printf("SARG: Maybe you have a broken time in your %s file.\n",arq);
1289                     exit(1);
1290                  }
1291                  strncat(hmr,warea,2);
1292                  chm--;
1293               }
1294               strncat(hmr,whm,2);
1295
1296               if(atoi(hmr) >= atoi(hm) && atoi(hmr) <= atoi(hmf))
1297                  l=1;else l=0;
1298            }
1299         }
1300         if(l){
1301            if(strlen(site)>0){
1302               if(strstr(url,site)!=0)
1303                  l=1;else l=0;
1304            }
1305         }
1306
1307         if(userip)
1308            strcpy(user,ip);
1309
1310         if(strcmp(user,"-") == 0 || strcmp(user," ") == 0 || strcmp(user,"") == 0) {
1311            if(strcmp(RecordsWithoutUser,"ip") == 0)
1312               strcpy(user,ip);
1313            if(strcmp(RecordsWithoutUser,"ignore") == 0)
1314               continue;
1315            if(strcmp(RecordsWithoutUser,"everybody") == 0)
1316               strcpy(user,"everybody");
1317         }
1318
1319         if(dotinuser) {
1320            str2=(char *)subs(user,"_",".");
1321            strcpy(user,str2);
1322            dotinuser=0;
1323         }
1324
1325         if(puser) {
1326            sprintf(wuser,":%s:",user);
1327            if(strstr(userfile, wuser) == 0)
1328               continue;
1329         }
1330
1331         if(l) {
1332            if(fuser) {
1333               l=vuexclude(excludeuser,user);
1334               if(!l)
1335                  totregsx++;
1336            }
1337         }
1338
1339         if(l) {
1340            if(userip)
1341               fixip(user);
1342         }
1343
1344         if(l&&max_elapsed) {
1345            if(atol(elap)>max_elapsed) {
1346               elap[0]='0';
1347               elap[1]='\0';
1348             }
1349         }
1350
1351         if(l) {
1352            if(strcmp(user,"-") !=0 && strlen(url) > 0 && strcmp(user," ") !=0 && strcmp(user,"") !=0 && strcmp(user,":") !=0){
1353               if((str=(char *) strstr(bufz, "[SmartFilter:")) != (char *) NULL ) {
1354                  str[strlen(str)-1]='\0';
1355                  sprintf(smartfilter,"\"%s\"",str+1);
1356               } else sprintf(smartfilter,"\"\"");
1357
1358               sprintf(bufz, "%s %s %s %s %s %s %s %s %s\n",dia,hora,user,ip,url,tam,code,elap,smartfilter);
1359
1360 #ifdef LEGACY_WRITE_USER
1361               sprintf(tmp3,"%s/sarg/%s.unsort",tmp,user);
1362 #if defined(HAVE_FOPEN64)
1363               if((fp_ou=fopen64(tmp3,"a"))==NULL) {
1364 #else
1365               if((fp_ou=fopen(tmp3,"a"))==NULL) {
1366 #endif
1367                 fprintf(stderr, "%s: (zzzlog) %s: %s - %s\n",argv[0],text[9],tmp3,strerror(errno));
1368                 exit(1);
1369               }
1370               fputs(bufz,fp_ou);
1371
1372 #else
1373                   if ( strcmp ( user , sz_Last_User ) != 0 ) {
1374                         if ( fp_Write_User )
1375                                 fclose( fp_Write_User ) ;
1376                         sprintf (tmp3, "%s/sarg/%s.unsort", tmp, user);
1377
1378 #if defined(HAVE_FOPEN64)
1379                         if ((fp_Write_User = fopen64 (tmp3, "a")) == NULL) {
1380 #else
1381                         if ((fp_Write_User = fopen (tmp3, "a")) == NULL) {
1382 #endif
1383                               fprintf (stderr, "%s: (log) %s: %s - %s\n", argv[0], text[9], tmp3, strerror(errno));
1384                                 exit (1);
1385                         }
1386                         strcpy( sz_Last_User , user ) ;
1387                   }
1388                   fputs (bufz, fp_Write_User);
1389 #endif
1390
1391               if(strcmp(ParsedOutputLog, "no") != 0 && !sarglog)
1392                  fputs(bufz,fp_log);
1393
1394 #ifdef LEGACY_WRITE_USER
1395               fclose(fp_ou);
1396 #endif
1397               totregsg++;
1398
1399               if(download_flag && strstr(code,"DENIED") == 0) {
1400                  ndownload = 1;
1401                  sprintf(bufz, "%s %s %s %s %s %s\n",dia,hora,user,ip,download_url);
1402
1403 #ifdef LEGACY_WRITE_DOWNLOAD
1404                  sprintf(tmp3,"%s/sarg/download.unsort",tmp);
1405 #if defined(HAVE_FOPEN64)
1406                  if((fp_ou=fopen64(tmp3,"a"))==NULL) {
1407 #else
1408                  if((fp_ou=fopen(tmp3,"a"))==NULL) {
1409 #endif
1410                     fprintf(stderr, "%s: (log) %s: %s - %s\n",argv[0],text[9],tmp3,strerror(errno));
1411                     exit(1);
1412                  }
1413                  fputs(bufz,fp_ou);
1414                  fclose(fp_ou);
1415 #else
1416                  if ( ! fp_Download_Unsort ) {
1417 #if defined(HAVE_FOPEN64)
1418                       if ((fp_Download_Unsort = fopen64 ( sz_Download_Unsort, "a")) == NULL) {
1419 #else
1420                       if ((fp_Download_Unsort = fopen (sz_Download_Unsort, "a")) == NULL) {
1421 #endif
1422                           fprintf (stderr, "%s: (log) %s: %s - %s\n", argv[0], text[9], tmp3, strerror(errno));
1423                           exit (1);
1424                       }
1425                  }
1426                  fputs (bufz, fp_Download_Unsort);
1427 #endif
1428               }
1429
1430               if(strstr(ReportType,"denied") != 0 || strstr(ReportType,"auth_failures") != 0) {
1431                  if(strstr(code,"DENIED/403") != 0) {
1432                     sprintf(bufz, "%s %s %s %s %s\n",dia,hora,user,ip,urly);
1433                     fputs(bufz,fp_denied);
1434                     denied_count++;
1435                  }
1436                  if(strstr(code,"DENIED/401") != 0 || strstr(code,"DENIED/407") != 0) {
1437                     sprintf(bufz, "%s %s %s %s %s\n",dia,hora,user,ip,urly);
1438                     if(fp_authfail)
1439                        fputs(bufz,fp_authfail);
1440                     authfail_count++;
1441                  }
1442               }
1443
1444               if(!totper && !sarglog){
1445                  totper++;
1446                  sprintf(period,"%s-",tbuf);
1447                  sprintf(per_hour,"%s-",tbuf2);
1448                  if(strlen(date)>0)
1449                     fixper(tbuf, period, cduntil);
1450                  if(debugz){
1451                     debugaz("tbuf",tbuf);
1452                     debugaz("period",period);
1453                  }
1454               }
1455            }
1456
1457            if(debugm){
1458               printf("IP=\t%s\n",ip);
1459               printf("USER=\t%s\n",user);
1460               printf("ELAP=\t%s\n",elap);
1461               printf("DATE=\t%s\n",dia);
1462               printf("TIME=\t%s\n",hora);
1463               printf("FUNC=\t%s\n",fun);
1464               printf("URL=\t%s\n",url);
1465               printf("CODE=\t%s\n",code);
1466               printf("LEN=\t%s\n",tam);
1467            }
1468         }
1469       }
1470       if( bool_ShowReadStatistics )
1471         printf("SARG: Records in file: " OFFSET_STRING ", reading: %3.2f%%\n",recs1, (float) 100 );
1472    }
1473
1474    if ( fp_Download_Unsort )
1475      fclose (fp_Download_Unsort);
1476
1477    if (fp_Write_User)
1478      fclose (fp_Write_User);
1479
1480    if(debug) {
1481       sprintf(msg, "   %s: %ld, %s: %ld, %s: %ld",text[10],totregsl,text[11],totregsg,text[68],totregsx);
1482       debuga(msg);
1483
1484       if((common_log) && (squid_log))
1485          debuga(text[12]);
1486
1487       if((common_log) && (!squid_log))
1488          debuga(text[13]);
1489
1490       if((!common_log) && (squid_log))
1491          debuga(text[14]);
1492
1493       if(sarglog)
1494          debuga(text[124]);
1495
1496       if((!common_log) && (!squid_log) && (!sarglog) && (!isalog)) {
1497          if(!totregsg) {
1498             fprintf(stderr, "SARG: %s\n",text[16]);
1499             fprintf(stderr, "SARG: %s\n",text[21]);
1500          } else fprintf(stderr, "SARG: %s\n",text[15]);
1501          bzero(msg,sizeof(msg));
1502          fclose(fp_in);
1503 //         fclose(fp_ou);
1504          if(fp_denied)
1505             fclose(fp_denied);
1506          if(fp_authfail)
1507             fclose(fp_authfail);
1508          if(tmp4)
1509             unlink(tmp4);
1510          if(tmp6)
1511             unlink(tmp6);
1512          unlink(tmp3);
1513          exit(0);
1514       }
1515    }
1516
1517    if(!totregsg){
1518       fprintf(stderr, "SARG: %s\n",text[16]);
1519       fprintf(stderr, "SARG: %s\n",text[21]);
1520       fclose(fp_in);
1521 //      fclose(fp_ou);
1522       if(fp_denied)
1523          fclose(fp_denied);
1524       if(fp_authfail)
1525          fclose(fp_authfail);
1526       exit(0);
1527    }
1528
1529    if(date[0] == '\0' && !sarglog) {
1530       strcat(period,tbuf);
1531       strcat(per_hour,tbuf2);
1532    }
1533
1534    if(debugz){
1535       debugaz("data",dia);
1536       debugaz("tbuf",tbuf);
1537       debugaz("period",period);
1538    }
1539
1540    if(debug){
1541      sprintf(msg, "%s: %s",text[17],period);
1542      debuga(msg);
1543    }
1544
1545    fclose(fp_in);
1546 //   fclose(fp_ou);
1547    if(fp_denied)
1548       fclose(fp_denied);
1549    if(fp_authfail)
1550       fclose(fp_authfail);
1551
1552    if(strcmp(ParsedOutputLog, "no") != 0 && !sarglog) {
1553       fclose(fp_log);
1554       strcpy(val1,period);
1555       if (getword_multisep(val2,sizeof(val2),val1,'-')<0){
1556          printf("SARG: Maybe you have a broken date range definition.\n");
1557          exit(1);
1558       }
1559       if (getword_multisep(val3,sizeof(val3),per_hour,'-')<0){
1560          printf("SARG: Maybe you have a broken date range definition.\n");
1561          exit(1);
1562       }
1563       sprintf(val4,"%s/sarg-%s_%s-%s_%s.log",ParsedOutputLog,val2,val3,val1,per_hour);
1564       rename(arq_log,val4);
1565       strcpy(arq_log,val4);
1566
1567       if(strcmp(ParsedOutputLogCompress,"nocompress") != 0) {
1568          sprintf(val1,"%s %s",ParsedOutputLogCompress,arq_log);
1569          system(val1);
1570       }
1571
1572       if(debug) {
1573          sprintf(msg,"%s %s",text[123],arq_log);
1574          debuga(msg);
1575       }
1576    }
1577
1578    if(strstr(ReportType,"denied") != 0) {
1579       sprintf(csort,"sort -T %s -k 3,3 -k 5,5 -o '%s' '%s'",tmp,tmp5,tmp4);
1580       system(csort);
1581       unlink(tmp4);
1582    }
1583
1584    sort_users_log(tmp, debug);
1585
1586    report_gen:
1587    if(strlen(DataFile) > 0)
1588       data_file(tmp);
1589    else
1590       gerarel();
1591
1592    unlink(tmp2);
1593    if(strstr(ReportType,"denied") != 0)
1594       unlink(tmp5);
1595
1596    if((strlen(zip) > 0 && strcmp(zip,"zcat") !=0)) {
1597       recomp(arq, zip); }
1598 //   else  unlink(arq);
1599
1600    if(strcmp(tmp,"/tmp") != 0) {
1601       sprintf(tmp4,"rm -rf %s",tmp);
1602       system(tmp4);
1603    }
1604
1605    if(excludecode)
1606       free(excludecode);
1607    if(userfile)
1608       free(userfile);
1609    if(excludefile)
1610       free(excludefile);
1611    if(excludeuser)
1612       free(excludeuser);
1613
1614    if(debug)
1615       debuga(text[21]);
1616
1617    exit(0);
1618
1619 }
1620
1621
1622 void getusers(char *pwdfile, int debug)
1623 {
1624
1625    FILE *fp_usr;
1626    char buf[255];
1627    char Msg[255];
1628    char user[255];
1629    unsigned long int nreg=0;
1630
1631    if(debug) {
1632       sprintf(Msg,"%s: %s",text[60],pwdfile);
1633       debuga(Msg);
1634    }
1635
1636    if ((fp_usr = fopen(pwdfile, "r")) == NULL) {
1637       fprintf(stderr, "SARG: (getusers) %s: %s - %s\n",text[45],pwdfile,strerror(errno));
1638       exit(1);
1639    }
1640
1641    nreg = lseek(fileno(fp_usr), (off_t)0, SEEK_END);
1642    nreg = nreg+5000;
1643    lseek(fileno(fp_usr), (off_t)0, 0);
1644
1645    if((userfile=(char *) malloc(nreg))==NULL){
1646       fprintf(stderr, "SARG: %s (%ld):\n",text[59],nreg);
1647       exit(1);
1648    }
1649
1650    bzero(userfile,nreg);
1651    sprintf(userfile,":");
1652
1653    while(fgets(buf,255,fp_usr)!=NULL) {
1654      if (getword_multisep(user,sizeof(user),buf,':')<0){
1655         printf("SARG: Maybe you have a broken record or garbage in your %s file.\n",pwdfile);
1656         exit(1);
1657      }
1658      strncat(userfile,user,strlen(user));
1659      strncat(userfile,":",1);
1660    }
1661
1662    fclose(fp_usr);
1663
1664    return;
1665 }
1666
1667
1668 void gethexclude(char *hexfile, int debug)
1669 {
1670
1671    FILE *fp_ex;
1672    char buf[255];
1673    char Msg[255];
1674    unsigned long int nreg=0;
1675
1676    if(debug) {
1677       sprintf(Msg,"%s: %s",text[67],hexfile);
1678       debuga(Msg);
1679    }
1680
1681    if ((fp_ex = fopen(hexfile, "r")) == NULL) {
1682       fprintf(stderr, "SARG: (gethexclude) %s: %s - %s\n",text[45],hexfile,strerror(errno));
1683       exit(1);
1684    }
1685
1686    nreg = lseek(fileno(fp_ex), (off_t)0, SEEK_END);
1687    lseek(fileno(fp_ex), (off_t)0, 0);
1688
1689    if((excludefile=(char *) malloc(nreg+11))==NULL){
1690       fprintf(stderr, "SARG: %s (%ld):\n",text[59],nreg);
1691       exit(1);
1692    }
1693
1694    bzero(excludefile,nreg+11);
1695
1696    while(fgets(buf,255,fp_ex)!=NULL){
1697      if(strstr(buf,"#") != 0)
1698         continue;
1699      buf[strlen(buf)-1]='\0';
1700      strcat(excludefile,buf);
1701      strcat(excludefile," ");
1702    }
1703
1704    strcat(excludefile,"*END* ");
1705
1706    fclose(fp_ex);
1707
1708    return;
1709 }
1710
1711
1712 void getuexclude(char *uexfile, int debug)
1713 {
1714
1715    FILE *fp_ex;
1716    char buf[255];
1717    char Msg[255];
1718    unsigned long int nreg=0;
1719
1720    if(debug) {
1721       sprintf(Msg,"%s: %s",text[67],uexfile);
1722       debuga(Msg);
1723    }
1724
1725    if ((fp_ex = fopen(uexfile, "r")) == NULL) {
1726       fprintf(stderr, "SARG: (gethexclude) %s: %s - %s\n",text[45],uexfile,strerror(errno));
1727       exit(1);
1728    }
1729
1730    nreg = lseek(fileno(fp_ex), (off_t)0, SEEK_END);
1731    lseek(fileno(fp_ex), (off_t)0, 0);
1732
1733    if((excludeuser=(char *) malloc(nreg+11))==NULL){
1734       fprintf(stderr, "SARG: %s (%ld):\n",text[59],nreg);
1735       exit(1);
1736    }
1737
1738    bzero(excludeuser,nreg+11);
1739
1740    while(fgets(buf,255,fp_ex)!=NULL){
1741      if(strstr(buf,"#") != 0)
1742         continue;
1743      buf[strlen(buf)-1]='\0';
1744      strcat(excludeuser,buf);
1745      strcat(excludeuser," ");
1746    }
1747
1748    strcat(excludeuser,"*END* ");
1749
1750    fclose(fp_ex);
1751
1752    return;
1753 }