log.c

   1 /*
   2  * SARG Squid Analysis Report Generator      http://sarg.sourceforge.net
   3  *                                                            1998, 2010
   4  *
   5  * SARG donations:
   6  *      please look at http://sarg.sourceforge.net/donations.php
   7  * Support:
   8  *     http://sourceforge.net/projects/sarg/forums/forum/363374
   9  * ---------------------------------------------------------------------
  10  *
  11  *  This program is free software; you can redistribute it and/or modify
  12  *  it under the terms of the GNU General Public License as published by
  13  *  the Free Software Foundation; either version 2 of the License, or
  14  *  (at your option) any later version.
  15  *
  16  *  This program is distributed in the hope that it will be useful,
  17  *  but WITHOUT ANY WARRANTY; without even the implied warranty of
  18  *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  19  *  GNU General Public License for more details.
  20  *
  21  *  You should have received a copy of the GNU General Public License
  22  *  along with this program; if not, write to the Free Software
  23  *  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111, USA.
  24  *
  25  */
  26
  27 #include "include/conf.h"
  28 #include "include/defs.h"
  29
  30 #define REPORT_EVERY_X_LINES 5000
  31
  32
  33 char *userfile;
  34
  35 numlist weekdays = { { 0, 1, 2, 3, 4, 5, 6 }, 7 };
  36 numlist hours = { { 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12,
  37              13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23 }, 24 };
  38
  39 static void getusers(const char *pwdfile, int debug);
  40
  41 int main(int argc,char *argv[])
  42 {
  43    enum isa_col_id {
  44       ISACOL_Ip,
  45       ISACOL_UserName,
  46       ISACOL_Date,
  47       ISACOL_Time,
  48       ISACOL_TimeTaken,
  49       ISACOL_Bytes,
  50       ISACOL_Uri,
  51       ISACOL_Status,
  52       ISACOL_Last //last entry of the list !
  53    };
  54    enum InputLogFormat {
  55       ILF_Unknown,
  56       ILF_Squid,
  57       ILF_Common,
  58       ILF_Sarg,
  59       ILF_Isa,
  60       ILF_Last //last entry of the list !
  61    };
  62
  63    FILE *fp_in = NULL, *fp_denied=NULL, *fp_authfail=NULL, *fp_log=NULL;
  64
  65    char sz_Download_Unsort[ 20000 ] ;
  66    FILE * fp_Download_Unsort = NULL ;
  67    FILE * fp_Write_User = NULL ;
  68
  69    extern int optind;
  70    extern int optopt;
  71    extern char *optarg;
  72
  73    char data[255];
  74    char elap[255];
  75    char ip[MAXLEN];
  76    char tam[255];
  77    char fun[MAXLEN];
  78    char wuser[MAXLEN];
  79    char smartfilter[MAXLEN];
  80    char dia[128];
  81    char wdata[128];
  82    char mes[30];
  83    char ano[30];
  84    char hora[30];
  85    char wtemp[MAXLEN];
  86    char wtemp2[255];
  87    char date[255];
  88    char arq[255];
  89    char arq_log[255];
  90    char hm[15], hmf[15], hmr[15];
  91    int  chm=0;
  92    char uagent[MAXLEN];
  93    char hexclude[MAXLEN];
  94    char csort[MAXLEN];
  95    int cstatus;
  96    char tbuf[128];
  97    char tbuf2[128];
  98    char zip[20];
  99    char *str;
 100    char bufz[MAXLEN];
 101    char bufy[MAXLEN];
 102    char tmp2[MAXLEN];
 103    char start_hour[128];
 104    char end_hour[128];
 105    enum InputLogFormat ilf;
 106    int ilf_count[ILF_Last];
 107    int  ch;
 108    int  x, l;
 109    int  errflg=0;
 110    int  puser=0;
 111    int  fhost=0;
 112    int  dns=0;
 113    int  fuser=0;
 114    int  idata=0;
 115    int  mindate=0;
 116    int  iarq=0;
 117    int  exstring=0;
 118    int isa_ncols=0,isa_cols[ISACOL_Last];
 119    int from_stdin;
 120    long totregsl=0;
 121    long totregsg=0;
 122    long totregsx=0;
 123    long totper=0;
 124    long int max_elapsed=0;
 125    time_t tt;
 126    struct tm *t;
 127    unsigned long recs1=0UL;
 128    unsigned long recs2=0UL;
 129    int OutputNonZero = REPORT_EVERY_X_LINES ;
 130    int download_flag=0;
 131    char download_url[MAXLEN];
 132    char sz_Last_User[MAXLEN]="";
 133    struct getwordstruct gwarea;
 134
 135    BgImage[0]='\0';
 136    LogoImage[0]='\0';
 137    LogoText[0]='\0';
 138    PasswdFile[0]='\0';
 139    OutputEmail[0]='\0';
 140    UserAgentLog[0]='\0';
 141    ExcludeHosts[0]='\0';
 142    ExcludeUsers[0]='\0';
 143    ConfigFile[0]='\0';
 144    code[0]='\0';
 145    LastLog=0;
 146    ReportType=0UL;
 147    UserTabFile[0]='\0';
 148    BlockIt[0]='\0';
 149    ExternalCSSFile[0]='\0';
 150    SquidGuardLogFormat[0]='\0';
 151    SquidGuardLogAlternate[0]='\0';
 152    for (ilf=0 ; ilf<ILF_Last ; ilf++) ilf_count[ilf]=0;
 153
 154    sprintf(ExcludeCodes,"%s/exclude_codes",SYSCONFDIR);
 155    strcpy(GraphDaysBytesBarColor,"orange");
 156    strcpy(BgColor,"#ffffff");
 157    strcpy(TxColor,"#000000");
 158    strcpy(TxBgColor,"lavender");
 159    strcpy(TiColor,"darkblue");
 160    strcpy(Width,"80");
 161    strcpy(Height,"45");
 162    strcpy(LogoTextColor,"#000000");
 163    strcpy(HeaderColor,"darkblue");
 164    strcpy(HeaderBgColor,"#dddddd");
 165    strcpy(LogoTextColor,"#006699");
 166    strcpy(FontSize,"9px");
 167    strcpy(TempDir,"/tmp");
 168    strcpy(OutputDir,"/var/www/html/squid-reports");
 169    Ip2Name=0;
 170    strcpy(DateFormat,"u");
 171    OverwriteReport=0;
 172    RemoveTempFiles=1;
 173    strcpy(ReplaceIndex,"index.html");
 174    Index=INDEX_YES;
 175    strcpy(RecordsWithoutUser,"ip");
 176    UseComma=0;
 177    strcpy(MailUtility,"mailx");
 178    strcpy(TopSitesNum,"100");
 179    UserIp=0;
 180    strcpy(TopuserSortField,"BYTES");
 181    strcpy(UserSortField,"BYTES");
 182    strcpy(TopuserSortOrder,"reverse");
 183    strcpy(UserSortOrder,"reverse");
 184    strcpy(TopsitesSortField,"CONNECT");
 185    strcpy(TopsitesSortType,"D");
 186    LongUrl=0;
 187    strcpy(language,"English");
 188    strcpy(FontFace,"Verdana,Tahoma,Arial");
 189    strcpy(datetimeby,"elap");
 190    strcpy(CharSet,"ISO-8859-1");
 191    Privacy=0;
 192    strcpy(PrivacyString,"***.***.***.***");
 193    strcpy(PrivacyStringColor,"blue");
 194    SuccessfulMsg=1;
 195    TopUserFields=TOPUSERFIELDS_NUM | TOPUSERFIELDS_DATE_TIME | TOPUSERFIELDS_USERID | TOPUSERFIELDS_CONNECT |
 196                  TOPUSERFIELDS_BYTES | TOPUSERFIELDS_SETYB | TOPUSERFIELDS_IN_CACHE_OUT |
 197                  TOPUSERFIELDS_USED_TIME | TOPUSERFIELDS_MILISEC | TOPUSERFIELDS_PTIME |
 198                  TOPUSERFIELDS_TOTAL | TOPUSERFIELDS_AVERAGE;
 199    UserReportFields=USERREPORTFIELDS_CONNECT | USERREPORTFIELDS_BYTES | USERREPORTFIELDS_SETYB |
 200                     USERREPORTFIELDS_IN_CACHE_OUT | USERREPORTFIELDS_USED_TIME | USERREPORTFIELDS_MILISEC |
 201                     USERREPORTFIELDS_PTIME | USERREPORTFIELDS_TOTAL | USERREPORTFIELDS_AVERAGE;
 202    strcpy(DataFileDelimiter,";");
 203    DataFileFields=DATA_FIELD_USER | DATA_FIELD_DATE | DATA_FIELD_TIME | DATA_FIELD_URL | DATA_FIELD_CONNECT |
 204                   DATA_FIELD_BYTES | DATA_FIELD_IN_CACHE | DATA_FIELD_OUT_CACHE | DATA_FIELD_ELAPSED;
 205    strcpy(SiteUserTimeDateType,"table");
 206    ShowReadStatistics=1;
 207    strcpy(IndexSortOrder,"D");
 208    ShowSargInfo=1;
 209    ShowSargLogo=1;
 210    strcpy(ParsedOutputLog,"no");
 211    strcpy(ParsedOutputLogCompress,"/bin/gzip -f");
 212    strcpy(DisplayedValues,"abbreviation");
 213    strcpy(HeaderFontSize,"9px");
 214    strcpy(TitleFontSize,"11px");
 215    strcpy(AuthUserFile,"/usr/local/sarg/passwd");
 216    strcpy(AuthName,"SARG, Restricted Access");
 217    strcpy(AuthType,"basic");
 218    strcpy(Require,"require user admin %u");
 219    set_download_suffix("7z,ace,arj,avi,bat,bin,bz2,bzip,cab,com,cpio,dll,doc,dot,exe,gz,iso,lha,lzh,mdb,mov,mp3,mpeg,mpg,mso,nrg,ogg,ppt,rar,rtf,shs,src,sys,tar,tgz,vcd,vob,wma,wmv,zip");
 220    Graphs=1;
 221    strcpy(Ulimit,"20000");
 222    strcpy(NtlmUserFormat,"domainname+username");
 223    IndexTree=INDEX_TREE_FILE;
 224    strcpy(RealtimeTypes,"GET,PUT,CONNECT");
 225    strcpy(RealtimeUnauthRec,"show");
 226    SquidguardIgnoreDate=0;
 227    DansguardianIgnoreDate=0;
 228    strcpy(DataFileUrl,"ip");
 229    strcpy(MaxElapsed,"28800000");
 230    BytesInSitesUsersReport=0;
 231    UserAuthentication=0;
 232    strcpy(LDAPHost,"127.0.0.1");
 233    LDAPPort=389;
 234    LDAPProtocolVersion=3;
 235    LDAPBindDN[0]='\0';
 236    LDAPBindPW[0]='\0';
 237    LDAPBaseSearch[0]='\0';
 238    strcpy(LDAPFilterSearch, "uid=%s");
 239    strcpy(LDAPTargetAttr, "cn");
 240
 241    dia[0]='\0';
 242    mes[0]='\0';
 243    ano[0]='\0';
 244    hora[0]='\0';
 245    tmp[0]='\0';
 246    tmp2[0]='\0';
 247    tmp3[0]='\0';
 248    wtemp[0]='\0';
 249    wtemp2[0]='\0';
 250    us[0]='\0';
 251    date[0]='\0';
 252    df[0]='\0';
 253    uagent[0]='\0';
 254    hexclude[0]='\0';
 255    addr[0]='\0';
 256    hm[0]='\0';
 257    hmf[0]='\0';
 258    site[0]='\0';
 259    outdir[0]='\0';
 260    elap[0]='\0';
 261    email[0]='\0';
 262    zip[0]='\0';
 263    UserInvalidChar[0]='\0';
 264    DataFile[0]='\0';
 265    SquidGuardConf[0]='\0';
 266    DansGuardianConf[0]='\0';
 267    start_hour[0]='\0';
 268    end_hour[0]='\0';
 269
 270    denied_count=0;
 271    download_count=0;
 272    authfail_count=0;
 273    dansguardian_count=0;
 274    squidguard_count=0;
 275    DeniedReportLimit=10;
 276    AuthfailReportLimit=10;
 277    DansGuardianReportLimit=10;
 278    SquidGuardReportLimit=10;
 279    DownloadReportLimit=50;
 280    UserReportLimit=0;
 281    debug=0;
 282    debugz=0;
 283    debugm=0;
 284    iprel=0;
 285    userip=0;
 286    color1=0;
 287    color2=0;
 288    color3=0;
 289    dotinuser=0;
 290    realt=0;
 291    realtime_refresh=3;
 292    realtime_access_log_lines=1000;
 293    cost=0.01;
 294    nocost=50000000;
 295    ndownload=0;
 296    squid24=0;
 297
 298    bzero(IncludeUsers, MAXLEN);
 299    bzero(ExcludeString, MAXLEN);
 300
 301 #ifdef HAVE_LOCALE_H
 302    setlocale(LC_TIME,"");
 303 #endif
 304
 305    NAccessLog=0;
 306    for(x=0; x<=MAXLOGS; x++)
 307       AccessLog[x][0]='\0';
 308    AccessLogFromCmdLine=0;
 309
 310    language_load(language);
 311    strcpy(Title,text[88]);
 312
 313    while((ch = getopt(argc, argv, "a:b:c:d:e:f:g:u:l:L:o:s:t:w:hijmnprvxyz")) != -1){
 314       switch(ch)
 315       {
 316          case 'a':
 317             strcpy(addr,optarg);
 318             break;
 319          case 'b':
 320             strcpy(uagent,optarg);
 321             break;
 322          case 'c':
 323             strcpy(hexclude,optarg);
 324             break;
 325          case 'd':
 326             strncpy(date,optarg,sizeof(date)-1);
 327             date[sizeof(date)-1]='\0';
 328             getword_start(&gwarea,optarg);
 329             if (getword(cdfrom,sizeof(cdfrom),&gwarea,'-')<0 || getword(cduntil,sizeof(cduntil),&gwarea,0)<0) {
 330                printf("SARG: Maybe you have a broken record or garbage in your date range.\n");
 331                exit(1);
 332             }
 333             date_from(date, cdfrom, cduntil);
 334             dfrom=atoi(cdfrom);
 335             duntil=atoi(cduntil);
 336             break;
 337          case 'e':
 338             strcpy(email,optarg);
 339             break;
 340          case 'f':
 341             strcpy(ConfigFile,optarg);
 342             break;
 343          case 'g':
 344             strcpy(df,optarg);
 345             break;
 346          case 'h':
 347             usage(argv[0]);
 348             exit(0);
 349             break;
 350          case 'i':
 351             iprel++;
 352             break;
 353          case 'l':
 354             if (NAccessLog>=MAXLOGS) {
 355                printf("SARG: Too many log files.\n");
 356                exit(1);
 357             }
 358             strcpy(AccessLog[NAccessLog],optarg);
 359             NAccessLog++;
 360             AccessLogFromCmdLine++;
 361             break;
 362          case 'L':
 363             strcpy(SquidGuardLogAlternate,optarg);
 364             break;
 365          case 'm':
 366             debugm++;
 367             break;
 368          case 'n':
 369             dns++;
 370             break;
 371          case 'o':
 372             strcpy(outdir,optarg);
 373             break;
 374          case 'p':
 375             userip++;
 376             break;
 377          case 'r':
 378             realt++;
 379             break;
 380          case 's':
 381             strcpy(site,optarg);
 382             break;
 383          case 't':
 384          {
 385             int h,m;
 386
 387             if(strstr(optarg,"-") == 0) {
 388               strcpy(hm,optarg);
 389               strcpy(hmf,optarg);
 390             } else {
 391                getword_start(&gwarea,optarg);
 392                if (getword(hm,sizeof(hm),&gwarea,'-')<0 || getword(hmf,sizeof(hmf),&gwarea,0)<0) {
 393                   fprintf(stderr,"SARG: Maybe you have a broken record or garbage in your time range.\n");
 394                   exit(1);
 395                }
 396             }
 397             if(sscanf(hm,"%d:%d",&h,&m)!=2) {
 398                fprintf(stderr,"SARG: time period must be MM or MM:SS. Exit.\n");
 399                exit(1);
 400             }
 401             sprintf(hm,"%02d%02d",h,m);
 402             if(sscanf(hmf,"%d:%d",&h,&m)!=2) {
 403                fprintf(stderr,"SARG: time period must be MM or MM:SS. Exit.\n");
 404                exit(1);
 405             }
 406             sprintf(hmf,"%02d%02d",h,m);
 407             break;
 408          }
 409          case 'u':
 410             strcpy(us,optarg);
 411             break;
 412          case 'v':
 413             version();
 414             break;
 415          case 'w':
 416             strcpy(tmp,optarg);
 417             break;
 418          case 'x':
 419             debug++;
 420             break;
 421          case 'y':
 422             langcode++;
 423             break;
 424          case 'z':
 425             debugz++;
 426             break;
 427          case ':':
 428             fprintf(stderr, "Option -%c require an argument\n",optopt);
 429             errflg++;
 430             break;
 431          case '?':
 432             usage(argv[0]);
 433             exit(1);
 434             break;
 435       }
 436
 437    }
 438
 439    if (errflg) {
 440       usage(argv[0]);
 441       exit(2);
 442    }
 443
 444    if(debug) debuga("Init");
 445
 446    if(ConfigFile[0] == '\0') sprintf(ConfigFile,"%s/sarg.conf",SYSCONFDIR);
 447    if(access(ConfigFile, R_OK) != 0) {
 448       debuga("Cannot open config file: %s - %s",ConfigFile,strerror(errno));
 449       exit(1);
 450    }
 451
 452    if(access(ConfigFile, R_OK) == 0)
 453       getconf();
 454
 455    if(UserIp) userip++;
 456
 457    if(dns) Ip2Name=1;
 458
 459    if(realt) {
 460       realtime();
 461       exit(0);
 462    }
 463
 464    if(IndexTree == INDEX_TREE_FILE)
 465       strcpy(ImageFile,"../images");
 466    else
 467       strcpy(ImageFile,"../../../images");
 468
 469    dataonly=0;
 470    if(DataFile[0] != '\0')
 471       dataonly++;
 472
 473    if(!NAccessLog) {
 474       strcpy(AccessLog[0],"/var/log/squid/access.log");
 475       NAccessLog++;
 476    }
 477
 478    if(strcmp(hexclude,"onvert") == 0 && strcmp(site,"plit") != 0) {
 479       convlog(AccessLog[0], df, dfrom, duntil);
 480       exit(0);
 481    }
 482
 483    if(strcmp(site,"plit") == 0) {
 484       splitlog(AccessLog[0], df, dfrom, duntil, hexclude);
 485       exit(0);
 486    }
 487
 488    load_excludecodes(ExcludeCodes);
 489
 490    if(access(PasswdFile, R_OK) == 0) {
 491       getusers(PasswdFile,debug);
 492       puser++;
 493    }
 494
 495    if(hexclude[0] == '\0')
 496       strcpy(hexclude,ExcludeHosts);
 497    if(hexclude[0] != '\0') {
 498       gethexclude(hexclude,debug);
 499       fhost++;
 500    }
 501
 502    if(ReportType == 0) {
 503       ReportType=REPORT_TYPE_TOPUSERS | REPORT_TYPE_TOPSITES | REPORT_TYPE_USERS_SITES |
 504                  REPORT_TYPE_SITES_USERS | REPORT_TYPE_DATE_TIME | REPORT_TYPE_DENIED |
 505                  REPORT_TYPE_AUTH_FAILURES | REPORT_TYPE_SITE_USER_TIME_DATE | REPORT_TYPE_DOWNLOADS;
 506    }
 507
 508    if(access(ExcludeUsers, R_OK) == 0) {
 509       getuexclude(ExcludeUsers,debug);
 510       fuser++;
 511    }
 512
 513    indexonly=0;
 514    if(fuser) {
 515       if(is_indexonly())
 516          indexonly++;
 517    }
 518    if(strcmp(ExcludeUsers,"indexonly") == 0) indexonly++;
 519    if(Index == INDEX_ONLY) indexonly++;
 520
 521    if(MaxElapsed[0] != '\0') max_elapsed=atol(MaxElapsed);
 522
 523    if(outdir[0] == '\0') strcpy(outdir,OutputDir);
 524    strcat(outdir,"/");
 525
 526    if(uagent[0] == '\0') strcpy(uagent,UserAgentLog);
 527
 528    if(tmp[0] == '\0') strcpy(tmp,TempDir);
 529    else strcpy(TempDir,tmp);
 530
 531    if(df[0] == '\0') strcpy(df,DateFormat);
 532    else strcpy(DateFormat,df);
 533
 534    if(df[0] == '\0') {
 535       strcpy(df,"u");
 536       strcpy(DateFormat,"u");
 537    }
 538
 539    if(email[0] == '\0' && OutputEmail[0] != '\0') strcpy(email,OutputEmail);
 540
 541    strcpy(tmp2,tmp);
 542
 543    if(email[0] != '\0') {
 544       sprintf(wtemp2,"%s/sarg",tmp2);
 545       my_mkdir(wtemp2);
 546       strcat(tmp2,"/sarg");
 547       strcpy(outdir,tmp2);
 548       strcat(outdir,"/");
 549    }
 550
 551    strcat(tmp2,"/sarg.log");
 552
 553    sprintf(tmp3,"%s/sarg",tmp);
 554    if(access(tmp3, R_OK) == 0) {
 555       unlinkdir(tmp3,1);
 556    }
 557    my_mkdir(tmp3);
 558    strcpy(tmp4,tmp3);
 559    strcpy(tmp5,tmp3);
 560    strcpy(tmp6,tmp3);
 561    strcat(tmp4,"/denied.log.unsort");
 562    strcat(tmp5,"/denied.log");
 563    strcat(tmp6,"/authfail.log.unsort");
 564
 565    if(debug) {
 566       fprintf(stderr, "SARG: %s:\nSARG:\n",text[22]);
 567       fprintf(stderr, "SARG: %35s (-a) = %s\n",text[23],addr);
 568       fprintf(stderr, "SARG: %35s (-b) = %s\n",text[71],uagent);
 569       fprintf(stderr, "SARG: %35s (-c) = %s\n",text[69],hexclude);
 570       fprintf(stderr, "SARG: %35s (-d) = %s\n",text[24],date);
 571       fprintf(stderr, "SARG: %35s (-e) = %s\n",text[41],email);
 572       fprintf(stderr, "SARG: %35s (-f) = %s\n",text[70],ConfigFile);
 573       if(strcmp(df,"e") == 0)
 574          fprintf(stderr, "SARG: %35s (-g) = %s (dd/mm/yyyy)\n",text[25],text[26]);
 575       if(strcmp(df,"u") == 0)
 576          fprintf(stderr, "SARG: %35s (-g) = %s (mm/dd/yyyy)\n",text[25],text[27]);
 577       if(strcmp(df,"w") == 0)
 578          fprintf(stderr, "SARG: %35s (-g) = %s (yyyy/ww)\n",text[25],text[85]);
 579       if(iprel)
 580          fprintf(stderr, "SARG: %35s (-i) = %s\n",text[28],text[1]);
 581       else
 582          fprintf(stderr, "SARG: %35s (-i) = %s\n",text[28],text[2]);
 583       for (iarq=0 ; iarq<NAccessLog ; iarq++)
 584          fprintf(stderr, "SARG: %35s (-l) = %s\n",text[37],AccessLog[iarq]);
 585       if(Ip2Name)
 586          fprintf(stderr, "SARG: %35s (-n) = %s\n",text[65],text[1]);
 587        else
 588          fprintf(stderr, "SARG: %35s (-n) = %s\n",text[65],text[2]);
 589       fprintf(stderr, "SARG: %35s (-o) = %s\n",text[38],outdir);
 590       if(UserIp)
 591          fprintf(stderr, "SARG: %35s (-p) = %s\n",text[29],text[1]);
 592        else
 593          fprintf(stderr, "SARG: %35s (-p) = %s\n",text[29],text[2]);
 594       fprintf(stderr, "SARG: %35s (-s) = %s\n",text[30],site);
 595       fprintf(stderr, "SARG: %35s (-t) = %s\n",text[31],hm);
 596       fprintf(stderr, "SARG: %35s (-u) = %s\n",text[32],us);
 597       fprintf(stderr, "SARG: %35s (-w) = %s\n",text[34],tmp);
 598       if(debug)
 599          fprintf(stderr, "SARG: %35s (-x) = %s\n",text[35],text[1]);
 600        else
 601          fprintf(stderr, "SARG: %35s (-x) = %s\n",text[35],text[2]);
 602       if(debugz)
 603          fprintf(stderr, "SARG: %35s (-z) = %s\n",text[36],text[1]);
 604        else
 605          fprintf(stderr, "SARG: %35s (-z) = %s\n",text[36],text[2]);
 606       fprintf(stderr, "SARG:\n");
 607    }
 608
 609    if(debugm) {
 610       printf("%s:\nSARG:\n",text[22]);
 611       printf("%35s (-a) = %s\n",text[23],addr);
 612       printf("%35s (-b) = %s\n",text[71],uagent);
 613       printf("%35s (-c) = %s\n",text[69],hexclude);
 614       printf("%35s (-d) = %s\n",text[24],date);
 615       printf("%35s (-e) = %s\n",text[41],email);
 616       printf("%35s (-f) = %s\n",text[70],ConfigFile);
 617       if(strcmp(df,"e") == 0)
 618          printf("%35s (-g) = %s (dd/mm/yyyy)\n",text[25],text[26]);
 619       if(strcmp(df,"u") == 0)
 620          printf("%35s (-g) = %s (mm/dd/yyyy)\n",text[25],text[27]);
 621       if(strcmp(df,"w") == 0)
 622          printf("%35s (-g) = %s (yyyy/ww)\n",text[25],text[85]);
 623       if(iprel)
 624          printf("%35s (-i) = %s\n",text[28],text[1]);
 625       else
 626          printf("%35s (-i) = %s\n",text[28],text[2]);
 627       for (iarq=0 ; iarq<NAccessLog ; iarq++)
 628          printf("%35s (-l) = %s\n",text[37],AccessLog[iarq]);
 629       if(Ip2Name)
 630          printf("%35s (-n) = %s\n",text[65],text[1]);
 631        else
 632          printf("%35s (-n) = %s\n",text[65],text[2]);
 633       printf("%35s (-o) = %s\n",text[38],outdir);
 634       if(UserIp)
 635          printf("%35s (-p) = %s\n",text[29],text[1]);
 636        else
 637          printf("%35s (-p) = %s\n",text[29],text[2]);
 638       printf("%35s (-s) = %s\n",text[30],site);
 639       printf("%35s (-t) = %s\n",text[31],hm);
 640       printf("%35s (-u) = %s\n",text[32],us);
 641       printf("%35s (-w) = %s\n",text[34],tmp);
 642       if(debug)
 643          printf("%35s (-x) = %s\n",text[35],text[1]);
 644        else
 645          printf("%35s (-x) = %s\n",text[35],text[2]);
 646       if(debugz)
 647          printf("%35s (-z) = %s\n",text[36],text[1]);
 648        else
 649          printf("%35s (-z) = %s\n",text[36],text[2]);
 650       printf("sarg %s: %s\n",text[73],VERSION);
 651       printf("Language=%s\n\n",text[3]);
 652    }
 653
 654    if(debug)
 655       debuga("sarg %s: %s",text[73],VERSION);
 656
 657 #ifdef HAVE_RLIM_T
 658    if (Ulimit[0] != '\0') {
 659       struct rlimit rl;
 660       long l1, l2;
 661       int rc=0;
 662
 663 #if defined(RLIMIT_NOFILE)
 664       getrlimit (RLIMIT_NOFILE, &rl);
 665 #elif defined(RLIMIT_OFILE)
 666       getrlimit (RLIMIT_OFILE, &rl);
 667 #else
 668 #warning "No rlimit resource for the number of open files"
 669 #endif
 670       l1 = rl.rlim_cur;
 671       l2 = rl.rlim_max;
 672
 673       rl.rlim_cur = atol(Ulimit);
 674       rl.rlim_max = atol(Ulimit);
 675 #if defined(RLIMIT_NOFILE)
 676       rc=setrlimit (RLIMIT_NOFILE, &rl);
 677 #elif defined(RLIMIT_OFILE)
 678       rc=setrlimit (RLIMIT_OFILE, &rl);
 679 #else
 680 #warning "No rlimit resource for the number of open files"
 681 #endif
 682       if(rc == -1) {
 683             debuga("setrlimit error - %s\n",strerror(errno));
 684       }
 685
 686       if(debug)
 687          debuga("Maximum file descriptor: cur=%ld max=%ld, changed to cur="RLIM_STRING" max="RLIM_STRING,l1,l2,rl.rlim_cur,rl.rlim_max);
 688    }
 689 #endif
 690
 691    init_usertab(UserTabFile);
 692
 693    sprintf ( sz_Download_Unsort , "%s/sarg/download.unsort", tmp);
 694
 695    if((ReportType & REPORT_TYPE_DENIED) != 0) {
 696       if((fp_denied=MY_FOPEN(tmp4,"w"))==NULL) {
 697          fprintf(stderr, "%s: (log) %s: %s - %s\n",argv[0],text[45],tmp4,strerror(errno));
 698          exit(1);
 699       }
 700    }
 701
 702    if(DataFile[0]=='\0') {
 703       if((ReportType & REPORT_TYPE_DENIED) != 0 || (ReportType & REPORT_TYPE_AUTH_FAILURES) != 0) {
 704          if((fp_authfail=MY_FOPEN(tmp6,"w"))==NULL) {
 705             fprintf(stderr, "%s: (log) %s: %s - %s\n",argv[0],text[45],tmp6,strerror(errno));
 706             exit(1);
 707          }
 708       }
 709    }
 710
 711    for (iarq=0 ; iarq<NAccessLog ; iarq++) {
 712       strcpy(arq,AccessLog[iarq]);
 713
 714       strcpy(arqtt,arq);
 715
 716       if(strcmp(arq,"-")==0) {
 717          if(debug)
 718             debuga("%s: %s",text[7],"stdin");
 719          fp_in=stdin;
 720          from_stdin=1;
 721       } else {
 722          decomp(arq,zip,tmp);
 723          if(debug)
 724             debuga("%s: %s",text[7],arq);
 725          if((fp_in=MY_FOPEN(arq,"r"))==NULL) {
 726             fprintf(stderr, "%s: (log) %s: %s - %s\n",argv[0],text[8],arq,strerror(errno));
 727             exit(1);
 728          }
 729          from_stdin=0;
 730       }
 731       ilf=ILF_Unknown;
 732       download_flag=0;
 733       // pre-Read the file only if I have to show stats
 734       if(ShowReadStatistics && !from_stdin) {
 735          rewind(fp_in);
 736          recs1=0UL;
 737          recs2=0UL;
 738
 739          while( fgets(bufz,sizeof(bufz),fp_in) != NULL ) recs1++;
 740          rewind(fp_in);
 741          printf("SARG: Records in file: %lu, reading: %3.2f%%\r",recs1,(float) 0);
 742          fflush( stdout ) ;
 743       }
 744
 745       while(fgets(bufz,sizeof(bufz),fp_in)!=NULL) {
 746
 747          if (ilf==ILF_Unknown) {
 748             if(strncmp(bufz,"#Software: Mic",14) == 0) {
 749                fixendofline(bufz);
 750                if (debug)
 751                   debuga("%s: %s",text[143],bufz);
 752                ilf=ILF_Isa;
 753                ilf_count[ilf]++;
 754                continue;
 755             }
 756
 757             if(strncmp(bufz,"*** SARG Log ***",16) == 0) {
 758                getword_start(&gwarea,arqtt);
 759                if (getword_skip(2000,&gwarea,'-')<0 || getword(val2,sizeof(val2),&gwarea,'_')<0 ||
 760                    getword_skip(10,&gwarea,'-')<0 || getword(val3,sizeof(val3),&gwarea,'_')<0) {
 761                   printf("SARG: The name of the file is invalid: %s\n",arq);
 762                   exit(1);
 763                }
 764                sprintf(period,"%s-%s",val2,val3);
 765                ilf=ILF_Sarg;
 766                ilf_count[ilf]++;
 767                continue;
 768             }
 769          }
 770
 771          if(!fp_log && strcmp(ParsedOutputLog, "no") != 0 && ilf!=ILF_Sarg) {
 772             if(access(ParsedOutputLog,R_OK) != 0) {
 773                my_mkdir(ParsedOutputLog);
 774             }
 775             sprintf(arq_log,"%s/sarg_temp.log",ParsedOutputLog);
 776             if((fp_log=MY_FOPEN(arq_log,"w"))==NULL) {
 777                fprintf(stderr, "%s: (log) %s: %s - %s\n",argv[0],text[8],arq_log,strerror(errno));
 778                exit(1);
 779             }
 780             fputs("*** SARG Log ***\n",fp_log);
 781          }
 782
 783          recs2++;
 784          if( ShowReadStatistics && !from_stdin && ! --OutputNonZero) {
 785            perc = recs2 * 100 ;
 786            perc = perc / recs1 ;
 787            printf("SARG: Records in file: %lu, reading: %3.2f%%\r",recs1,perc);
 788            fflush (stdout);
 789            OutputNonZero = REPORT_EVERY_X_LINES ;
 790          }
 791          if(strlen(bufz) > MAXLEN-1) continue;
 792          if(!bufz[0]) continue;
 793          if(strstr(bufz,"HTTP/0.0") != 0) continue;
 794          if(strstr(bufz,"logfile turned over") != 0) continue;
 795          if(bufz[0] == ' ') continue;
 796          if(strlen(bufz) < 58) continue;
 797
 798          // Record only hours usage which is required
 799          tt = (time_t) strtoul( bufz, NULL, 10 );
 800          t = localtime( &tt );
 801
 802          if( bsearch( &( t -> tm_wday ), weekdays.list, weekdays.len,
 803                                           sizeof( int ), compar ) == NULL )
 804             continue;
 805
 806          if( bsearch( &( t -> tm_hour ), hours.list, hours.len,
 807                                           sizeof( int ), compar ) == NULL )
 808             continue;
 809
 810          // exclude_string
 811          exstring=0;
 812          if(ExcludeString[0] != '\0') {
 813             strcpy(warea,bufz);
 814             getword_start(&gwarea,ExcludeString);
 815             while(strchr(gwarea.current,':') != 0) {
 816                if (getword_multisep(val1,sizeof(val1),&gwarea,':')<0) {
 817                   printf("SARG: Maybe you have a broken record or garbage in your exclusion string.\n");
 818                   exit(1);
 819                }
 820                if((str=(char *) strstr(warea,val1)) != (char *) NULL )
 821                   exstring++;
 822             }
 823             if((str=(char *) strstr(warea,gwarea.current)) != (char *) NULL )
 824                   exstring++;
 825          }
 826          if(exstring) continue;
 827
 828          strcpy(bufy,bufz);
 829          if ((str = strchr(bufz, '\n')) != NULL)
 830             *str = '\0';          /* strip \n */
 831
 832          totregsl++;
 833          if(debugm)
 834             printf("BUF=%s\n",bufz);
 835
 836          if (ilf==ILF_Squid || ilf==ILF_Common || ilf==ILF_Unknown) {
 837             getword_start(&gwarea,bufz);
 838             if (getword(data,sizeof(data),&gwarea,' ')<0) {
 839                printf("SARG: Maybe you have a broken record or garbage in your access.log file.\n");
 840                exit(1);
 841             }
 842             if((str=(char *) strchr(data, '.')) != (char *) NULL ) {
 843                if((str=(char *) strchr(str+1, '.')) != (char *) NULL ) {
 844                   strcpy(ip,data);
 845                   strcpy(elap,"0");
 846                   if(squid24) {
 847                      if (getword(user,sizeof(user),&gwarea,' ')<0 || getword_skip(255,&gwarea,' ')<0) {
 848                         printf("SARG: Maybe you have a broken record or garbage in your %s file.\n",arq);
 849                         exit(1);
 850                      }
 851                   } else {
 852                      if (getword_skip(255,&gwarea,' ')<0 || getword(user,sizeof(user),&gwarea,' ')<0) {
 853                         printf("SARG: Maybe you have a broken record or garbage in your %s file.\n",arq);
 854                         exit(1);
 855                      }
 856                   }
 857                   if (getword(data,sizeof(data),&gwarea,']')<0 || getword_skip(MAXLEN,&gwarea,'"')<0 ||
 858                       getword(fun,sizeof(fun),&gwarea,' ')<0 || getword(url,sizeof(url),&gwarea,' ')<0 ||
 859                       getword_skip(MAXLEN,&gwarea,' ')<0 || getword(code2,sizeof(code2),&gwarea,' ')<0 ||
 860                       getword(tam,sizeof(tam),&gwarea,' ')<0) {
 861                      printf("SARG: Maybe you have a broken record or garbage in your %s file.\n",arq);
 862                      exit(1);
 863                   }
 864                   if((str=(char *) strchr(gwarea.current, ' ')) != (char *) NULL ) {
 865                      if (getword(code,sizeof(code),&gwarea,' ')<0) {
 866                         printf("SARG: Maybe you have a broken record or garbage in your %s file.\n",arq);
 867                         exit(1);
 868                      }
 869                   } else {
 870                      if (getword(code,sizeof(code),&gwarea,'\0')<0) {
 871                         printf("SARG: Maybe you have a broken record or garbage in your %s file.\n",arq);
 872                         exit(1);
 873                      }
 874                   }
 875
 876                   if ((str = strchr(code, ':')) != NULL)
 877                      *str = '/';
 878
 879                   if(strcmp(tam,"\0") == 0)
 880                      strcpy(tam,"0");
 881
 882                   ilf=ILF_Common;
 883                   ilf_count[ilf]++;
 884                }
 885             }
 886
 887             if(ilf==ILF_Unknown || ilf==ILF_Squid) {
 888                if (getword(elap,sizeof(elap),&gwarea,' ')<0) {
 889                   printf("SARG: Maybe you have a broken record or garbage in your %s file.\n",arq);
 890                   exit(1);
 891                }
 892                while(strcmp(elap,"") == 0 && gwarea.current[0] != '\0')
 893                   if (getword(elap,sizeof(elap),&gwarea,' ')<0) {
 894                      printf("SARG: Maybe you have a broken record or garbage in your %s file.\n",arq);
 895                      exit(1);
 896                   }
 897                if(strlen(elap) < 1) continue;
 898                if (getword(ip,sizeof(ip),&gwarea,' ')<0){
 899                   printf("SARG: Maybe you have a broken record or garbage in your %s file.\n",arq);
 900                   exit(1);
 901                }
 902                if (getword(code,sizeof(code),&gwarea,' ')<0){
 903                   printf("SARG: Maybe you have a broken record or garbage in your %s file.\n",arq);
 904                   exit(1);
 905                }
 906                if (getword(tam,sizeof(tam),&gwarea,' ')<0){
 907                   printf("SARG: Maybe you have a broken record or garbage in your %s file.\n",arq);
 908                   exit(1);
 909                }
 910                if (getword(fun,sizeof(fun),&gwarea,' ')<0){
 911                   printf("SARG: Maybe you have a broken record or garbage in your %s file.\n",arq);
 912                   exit(1);
 913                }
 914                if (getword(url,sizeof(url),&gwarea,' ')<0){
 915                   printf("SARG: Maybe you have a broken record or garbage in your %s file.\n",arq);
 916                   exit(1);
 917                }
 918 //              while (strstr(bufz,"%20") != 0) {
 919 //                 getword(warea,bufz,' ');
 920 //                 strcat(url,warea);
 921 //              }
 922                if (getword(user,sizeof(user),&gwarea,' ')<0){
 923                   printf("SARG: Maybe you have a broken record or garbage in your %s file.\n",arq);
 924                   exit(1);
 925                }
 926                ilf=ILF_Squid;
 927                ilf_count[ilf]++;
 928             }
 929          }
 930          if (ilf==ILF_Sarg) {
 931             getword_start(&gwarea,bufz);
 932             if (getword(data,sizeof(data),&gwarea,'\t')<0){
 933                printf("SARG: Maybe you have a broken record or garbage in your %s file.\n",arq);
 934                exit(1);
 935             }
 936             if (getword(hora,sizeof(hora),&gwarea,'\t')<0) {
 937                printf("SARG: Maybe you have a broken record or garbage in your %s file.\n",arq);
 938                exit(1);
 939             }
 940             if (getword(user,sizeof(user),&gwarea,'\t')<0) {
 941                printf("SARG: Maybe you have a broken record or garbage in your %s file.\n",arq);
 942                exit(1);
 943             }
 944             if (getword(ip,sizeof(ip),&gwarea,'\t')<0) {
 945                printf("SARG: Maybe you have a broken record or garbage in your %s file.\n",arq);
 946                exit(1);
 947             }
 948             if (getword(url,sizeof(url),&gwarea,'\t')<0){
 949                printf("SARG: Maybe you have a broken record or garbage in your %s file.\n",arq);
 950                exit(1);
 951             }
 952             if (getword(tam,sizeof(tam),&gwarea,'\t')<0){
 953                printf("SARG: Maybe you have a broken record or garbage in your %s file.\n",arq);
 954                exit(1);
 955             }
 956             if (getword(code,sizeof(code),&gwarea,'\t')<0){
 957                printf("SARG: Maybe you have a broken record or garbage in your %s file.\n",arq);
 958                exit(1);
 959             }
 960             if (getword(elap,sizeof(elap),&gwarea,'\t')<0){
 961                printf("SARG: Maybe you have a broken record or garbage in your %s file.\n",arq);
 962                exit(1);
 963             }
 964             if (getword(smartfilter,sizeof(smartfilter),&gwarea,'\0')<0){
 965                printf("SARG: Maybe you have a broken record or garbage in your %s file.\n",arq);
 966                exit(1);
 967             }
 968          }
 969          if (ilf==ILF_Isa) {
 970             if (bufz[0] == '#') {
 971                int ncols,cols[ISACOL_Last];
 972
 973                fixendofline(bufz);
 974                getword_start(&gwarea,bufz);
 975                // remove the #Fields: column at the beginning of the line
 976                if (getword_skip(1000,&gwarea,' ')<0){
 977                   printf("SARG: Maybe you have a broken record or garbage in your %s file.\n",arq);
 978                   exit(1);
 979                }
 980                for (ncols=0 ; ncols<ISACOL_Last ; ncols++) cols[ncols]=-1;
 981                ncols=0;
 982                while(gwarea.current[0] != '\0') {
 983                   if (getword(val1,sizeof(val1),&gwarea,'\t')<0){
 984                      printf("SARG: Maybe you have a broken record or garbage in your %s file.\n",arq);
 985                      exit(1);
 986                   }
 987                   if(strcmp(val1,"c-ip") == 0) cols[ISACOL_Ip]=ncols;
 988                   if(strcmp(val1,"cs-username") == 0) cols[ISACOL_UserName]=ncols;
 989                   if(strcmp(val1,"date") == 0) cols[ISACOL_Date]=ncols;
 990                   if(strcmp(val1,"time") == 0) cols[ISACOL_Time]=ncols;
 991                   if(strcmp(val1,"time-taken") == 0) cols[ISACOL_TimeTaken]=ncols;
 992                   if(strcmp(val1,"sc-bytes") == 0) cols[ISACOL_Bytes]=ncols;
 993                   if(strcmp(val1,"cs-uri") == 0) cols[ISACOL_Uri]=ncols;
 994                   if(strcmp(val1,"sc-status") == 0) cols[ISACOL_Status]=ncols;
 995                   ncols++;
 996                }
 997                if (cols[ISACOL_Ip]>=0) {
 998                   isa_ncols=ncols;
 999                   for (isa_ncols=0 ; isa_ncols<ncols ; isa_ncols++)
1000                      isa_cols[isa_ncols]=cols[isa_ncols];
1001                }
1002                continue;
1003             }
1004             if (!isa_ncols) continue;
1005             getword_start(&gwarea,bufz);
1006             for (x=0 ; x<isa_ncols ; x++) {
1007                if (getword(val1,sizeof(val1),&gwarea,'\t')<0) {
1008                   printf("SARG: Maybe you have a broken record or garbage in your %s file.\n",arq);
1009                   exit(1);
1010                }
1011                if (x==isa_cols[ISACOL_Ip]) {
1012                   if (strlen(val1)>=sizeof(ip)) {
1013                      printf("SARG: Maybe you have a broken IP in your %s file.\n",arq);
1014                      exit(1);
1015                   }
1016                   strcpy(ip,val1);
1017                } else if (x==isa_cols[ISACOL_UserName]) {
1018                   if (strlen(val1)>=sizeof(user)) {
1019                      printf("SARG: Maybe you have a broken user in your %s file.\n",arq);
1020                      exit(1);
1021                   }
1022                   strcpy(user,val1);
1023                } else if (x==isa_cols[ISACOL_Date]) {
1024                   if (strlen(val1)>=sizeof(data)) {
1025                      printf("SARG: Maybe you have a broken date in your %s file.\n",arq);
1026                      exit(1);
1027                   }
1028                   strcpy(data,val1);
1029                } else if (x==isa_cols[ISACOL_Time]) {
1030                   if (strlen(val1)>=sizeof(hora)) {
1031                      printf("SARG: Maybe you have a broken time in your %s file.\n",arq);
1032                      exit(1);
1033                   }
1034                   strcpy(hora,val1);
1035                } else if (x==isa_cols[ISACOL_TimeTaken]) {
1036                   if (strlen(val1)>=sizeof(elap)) {
1037                      printf("SARG: Maybe you have a broken download duration in your %s file.\n",arq);
1038                      exit(1);
1039                   }
1040                   strcpy(elap,val1);
1041                } else if (x==isa_cols[ISACOL_Bytes]) {
1042                   if (strlen(val1)>=sizeof(tam)) {
1043                      printf("SARG: Maybe you have a broken download size in your %s file.\n",arq);
1044                      exit(1);
1045                   }
1046                   strcpy(tam,val1);
1047                } else if (x==isa_cols[ISACOL_Uri]) {
1048                   if (strlen(val1)>=sizeof(url)) {
1049                      printf("SARG: Maybe you have a broken URL in your %s file.\n",arq);
1050                      exit(1);
1051                   }
1052                   strcpy(url,val1);
1053                } else if (x==isa_cols[ISACOL_Status]) {
1054                   if (strlen(val1)>=sizeof(code)) {
1055                      printf("SARG: Maybe you have a broken access code in your %s file.\n",arq);
1056                      exit(1);
1057                   }
1058                   strcpy(code,val1);
1059                }
1060             }
1061
1062             if(strcmp(code,"401") == 0 || strcmp(code,"403") == 0 || strcmp(code,"407") == 0) {
1063                sprintf(val1,"DENIED/%s",code);
1064                strcpy(code,val1);
1065             }
1066             getword_start(&gwarea,data);
1067             if (getword(ano,sizeof(ano),&gwarea,'-')<0){
1068                printf("SARG: Maybe you have a broken record or garbage in your %s file.\n",arq);
1069                exit(1);
1070             }
1071             if (getword(mes,sizeof(mes),&gwarea,'-')<0){
1072                printf("SARG: Maybe you have a broken record or garbage in your %s file.\n",arq);
1073                exit(1);
1074             }
1075             if (getword(dia,sizeof(dia),&gwarea,'\0')<0){
1076                printf("SARG: Maybe you have a broken record or garbage in your %s file.\n",arq);
1077                exit(1);
1078             }
1079             conv_month_name(mes);
1080             sprintf(data," %s/%s/%s:%s",dia,mes,ano,hora);
1081          }
1082
1083          if(strlen(user) > 150) {
1084             if (debugm) printf("User too long: %s\n",user);
1085             totregsx++;
1086             continue;
1087          }
1088
1089          // include_users
1090          if(IncludeUsers[0] != '\0') {
1091             sprintf(val1,":%s:",user);
1092             if((str=(char *) strstr(IncludeUsers,val1)) == (char *) NULL )
1093                continue;
1094          }
1095
1096          if(vercode(code)) {
1097             if (debugm) printf("Excluded code: %s\n",code);
1098             totregsx++;
1099             continue;
1100          }
1101
1102          if(testvaliduserchar(user))
1103             continue;
1104
1105 #if 0
1106          if((str = strstr(user,"%20")) != NULL) {
1107             /*
1108             This is a patch introduced to solve bug #1624251 reported at sourceforge but
1109             the side effect is to truncate the name at the first space and merge the reports
1110             of people whose name is identical up to the first space.
1111
1112             The old code used to truncate the user name at the first % if a %20 was
1113             found anywhere in the string. That means the string could be truncated
1114             at the wrong place if another % occured before the %20. This new code should
1115             avoid that problem and only truncate at the space. There is no bug
1116             report indicating that anybody noticed this.
1117             */
1118             *str='\0';
1119          }
1120
1121          /*
1122          Code prior to 2.2.7 used to replace any %xx by a dot as long as a %5c was
1123          found in the user name.
1124          */
1125          while((str = strstr(user,"%5c")) != NULL) {
1126             *str='.';
1127             for (x=3 ; str[x] ; x++) str[x-2]=str[x];
1128          }
1129 #endif
1130
1131          for(str=user; *str; str++) {
1132             if(*str=='.') dotinuser++;
1133             if(*str=='?' || *str=='.' || *str==':' || *str=='/' || *str=='\\' || *str=='\'' || *str=='$' || *str=='@' ||
1134                *str=='\"' || *str=='*')
1135                *str='_';
1136          }
1137
1138          strlow(user);
1139          if(strncmp(NtlmUserFormat,"user",4) == 0) {
1140             if((str = strchr(user,'_')) != 0) {
1141                strcpy(warea,str+1);
1142                strcpy(user,warea);
1143             }
1144             if((str = strchr(user,'+')) != 0) {
1145                strcpy(warea,str+1);
1146                strcpy(user,warea);
1147             }
1148          }
1149
1150          if((ReportType & REPORT_TYPE_DENIED) != 0)
1151             strcpy(urly,url);
1152
1153          if(ilf!=ILF_Sarg) {
1154             /*
1155             The full URL is not saved in sarg log. There is no point in testing the URL to detect
1156             a downloaded file.
1157             */
1158             download_flag=is_download_suffix(url);
1159             if (download_flag) {
1160                strcpy(download_url,url);
1161                download_count++;
1162             }
1163          }
1164
1165          // remove any protocol:// at the beginning of the URL
1166          if ((str = strchr(url,'/')) != NULL && str[1] == '/') {
1167             int i;
1168
1169             str+=2;
1170             for (i=0 ; str[i] ; i++)
1171                url[i]=str[i];
1172             url[i]='\0';
1173          }
1174
1175          if(!LongUrl) {
1176             char *endofhost=strchr(url,'/');
1177             if (endofhost)
1178                *endofhost='\0';
1179             if(strlen(url) > 512 && (endofhost=strchr(url,'%')) != NULL) {
1180                *endofhost='\0';
1181             }
1182          }
1183
1184          if(ilf==ILF_Squid) {
1185             tt=atoi(data);
1186             t=localtime(&tt);
1187
1188             strftime(tbuf2, sizeof(tbuf2), "%H%M", t);
1189             sprintf(mes,"%d",t->tm_mon+1);
1190             conv_month_name(mes);
1191             if(strncmp(df,"u",1) == 0)
1192                sprintf(tbuf, "%04d%s%02d", t->tm_year+1900, mes, t->tm_mday);
1193             if(strncmp(df,"e",1) == 0)
1194                sprintf(tbuf, "%02d%s%04d", t->tm_mday, mes, t->tm_year+1900);
1195             if(strncmp(df,"w",1) == 0) {
1196                IndexTree=INDEX_TREE_FILE;
1197                strftime(tbuf, sizeof(tbuf), "%Y.%U", t);
1198             }
1199
1200             strftime(wdata, sizeof(wdata), "%Y%m%d", t);
1201             idata=atoi(wdata);
1202
1203             if(strncmp(df,"u",1)==0)
1204                strftime(dia, sizeof(dia), "%m/%d/%Y", t);
1205             else
1206                strftime(dia, sizeof(dia), "%d/%m/%Y", t);
1207             sprintf(hora,"%02d:%02d:%02d",t->tm_hour,t->tm_min,t->tm_sec);
1208          } else if(ilf==ILF_Common || ilf==ILF_Isa) {
1209             getword_start(&gwarea,data+1);
1210             if (getword_multisep(data,sizeof(data),&gwarea,':')<0){
1211                printf("SARG: Maybe you have a broken date in your %s file.\n",arq);
1212                exit(1);
1213             }
1214             if (getword_multisep(hora,sizeof(hora),&gwarea,' ')<0){
1215                printf("SARG: Maybe you have a broken date in your %s file.\n",arq);
1216                exit(1);
1217             }
1218             getword_start(&gwarea,data);
1219             if (getword(dia,sizeof(dia),&gwarea,'/')<0){
1220                printf("SARG: Maybe you have a broken date in your %s file.\n",arq);
1221                exit(1);
1222             }
1223             if (getword(mes,sizeof(mes),&gwarea,'/')<0){
1224                printf("SARG: Maybe you have a broken date in your %s file.\n",arq);
1225                exit(1);
1226             }
1227             if (getword(ano,sizeof(ano),&gwarea,'/')<0){
1228                printf("SARG: Maybe you have a broken date in your %s file.\n",arq);
1229                exit(1);
1230             }
1231
1232             if(strcmp(df,"u") == 0)
1233                snprintf(tbuf,sizeof(tbuf),"%s%s%s",ano,mes,dia);
1234             if(strcmp(df,"e") == 0)
1235                snprintf(tbuf,sizeof(tbuf),"%s%s%s",dia,mes,ano);
1236             builddia(dia,mes,ano,df,wdata);
1237             idata=atoi(wdata);
1238          } else if (ilf==ILF_Sarg) {
1239             getword_start(&gwarea,data);
1240             if(strcmp(df,"u") == 0) {
1241                if (getword(mes,sizeof(mes),&gwarea,'/')<0){
1242                   printf("SARG: Maybe you have a broken date in your %s file.\n",arq);
1243                   exit(1);
1244                }
1245                if (getword(dia,sizeof(dia),&gwarea,'/')<0){
1246                   printf("SARG: Maybe you have a broken date in your %s file.\n",arq);
1247                   exit(1);
1248                }
1249             } else {
1250                if (getword(dia,sizeof(dia),&gwarea,'/')<0){
1251                   printf("SARG: Maybe you have a broken date in your %s file.\n",arq);
1252                   exit(1);
1253                }
1254                if (getword(mes,sizeof(mes),&gwarea,'/')<0){
1255                   printf("SARG: Maybe you have a broken date in your %s file.\n",arq);
1256                   exit(1);
1257                }
1258             }
1259             if (getword(ano,sizeof(ano),&gwarea,0)<0){
1260                printf("SARG: Maybe you have a broken date in your %s file.\n",arq);
1261                exit(1);
1262             }
1263             snprintf(wdata,9,"%s%s%s",ano,mes,dia);
1264             idata=atoi(wdata);
1265          }
1266
1267          if(debugm)
1268             printf("DATE=%s IDATA=%d DFROM=%d DUNTIL=%d\n",date,idata,dfrom,duntil);
1269
1270          l=1;
1271          if(l){
1272             if(addr[0] != '\0'){
1273                if(strcmp(addr,ip)==0)
1274                   l=1;else l=0;
1275             }
1276             if(fhost) {
1277 //              l=vhexclude(ip);
1278                l=vhexclude(url);
1279                if(!l) {
1280                   if (debugm) printf("Excluded site: %s\n",url);
1281                   totregsx++;
1282                }
1283             }
1284          }
1285
1286          if(l){
1287             if(date[0] != '\0'){
1288                if(idata >= dfrom && idata <= duntil)
1289                   l=1;else l=0;
1290             }
1291          }
1292          if(l){
1293             if(hm[0] != '\0') {
1294                bzero(hmr,sizeof(hmr));
1295                chm++;
1296                getword_start(&gwarea,hora);
1297                while(chm) {
1298                   if (getword_multisep(warea,sizeof(warea),&gwarea,':')<0){
1299                      printf("SARG: Maybe you have a broken time in your %s file.\n",arq);
1300                      exit(1);
1301                   }
1302                   strncat(hmr,warea,2);
1303                   chm--;
1304                }
1305                strncat(hmr,gwarea.current,2);
1306
1307                if(atoi(hmr) >= atoi(hm) && atoi(hmr) <= atoi(hmf))
1308                   l=1;else l=0;
1309             }
1310          }
1311          if(l){
1312             if(site[0] != '\0'){
1313                if(strstr(url,site)!=0)
1314                   l=1;else l=0;
1315             }
1316          }
1317
1318          if(userip)
1319             strcpy(user,ip);
1320
1321          if(strcmp(user,"-") == 0 || strcmp(user," ") == 0 || strcmp(user,"") == 0) {
1322             if(strcmp(RecordsWithoutUser,"ip") == 0)
1323                strcpy(user,ip);
1324             if(strcmp(RecordsWithoutUser,"ignore") == 0)
1325                continue;
1326             if(strcmp(RecordsWithoutUser,"everybody") == 0)
1327                strcpy(user,"everybody");
1328          }
1329          if(us[0] != '\0'){
1330             if(strcmp(user,us)==0)
1331                l=1;
1332             else
1333                l=0;
1334          }
1335
1336          if(dotinuser) {
1337             subs(user,sizeof(user),"_",".");
1338             dotinuser=0;
1339          }
1340
1341          if(puser) {
1342             sprintf(wuser,":%s:",user);
1343             if(strstr(userfile, wuser) == 0)
1344                continue;
1345          }
1346
1347          if(l) {
1348             if(fuser) {
1349                l=vuexclude(user);
1350                if(!l) {
1351                   if (debugm) printf("Excluded user: %s\n",user);
1352                   totregsx++;
1353                }
1354             }
1355          }
1356
1357          if(l) {
1358             if(userip)
1359                fixip(user);
1360          }
1361
1362          if(l && max_elapsed) {
1363             if(atol(elap)>max_elapsed) {
1364                elap[0]='0';
1365                elap[1]='\0';
1366             }
1367          }
1368
1369          if(l) {
1370             if(strcmp(user,"-") !=0 && url[0] != '\0' && strcmp(user," ") !=0 && strcmp(user,"") !=0 && strcmp(user,":") !=0){
1371                if((str=(char *) strstr(bufz, "[SmartFilter:")) != (char *) NULL ) {
1372                   fixendofline(str);
1373                   sprintf(smartfilter,"\"%s\"",str+1);
1374                } else sprintf(smartfilter,"\"\"");
1375
1376                sprintf(bufz, "%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\n",dia,hora,user,ip,url,tam,code,elap,smartfilter);
1377
1378                if ( strcmp ( user , sz_Last_User ) != 0 ) {
1379                   if ( fp_Write_User )
1380                      fclose( fp_Write_User ) ;
1381                   sprintf (tmp3, "%s/sarg/%s.unsort", tmp, user);
1382
1383                   if ((fp_Write_User = MY_FOPEN (tmp3, "a")) == NULL) {
1384                      fprintf (stderr, "%s: (log) %s: %s - %s\n", argv[0], text[9], tmp3, strerror(errno));
1385                      exit (1);
1386                   }
1387                   strcpy( sz_Last_User , user ) ;
1388                }
1389                fputs (bufz, fp_Write_User);
1390
1391                if(fp_log && ilf!=ILF_Sarg) fputs(bufz,fp_log);
1392
1393                totregsg++;
1394
1395                if(download_flag && strstr(code,"DENIED") == 0) {
1396                   ndownload = 1;
1397
1398                   if ( ! fp_Download_Unsort ) {
1399                      if ((fp_Download_Unsort = MY_FOPEN ( sz_Download_Unsort, "a")) == NULL) {
1400                         fprintf (stderr, "%s: (log) %s: %s - %s\n", argv[0], text[9], tmp3, strerror(errno));
1401                         exit (1);
1402                      }
1403                   }
1404                   fprintf(fp_Download_Unsort,"%s\t%s\t%s\t%s\t%s\n",dia,hora,user,ip,download_url);
1405                }
1406
1407                if((ReportType & REPORT_TYPE_DENIED) != 0 || (ReportType & REPORT_TYPE_AUTH_FAILURES) != 0) {
1408                   if(strstr(code,"DENIED/403") != 0) {
1409                      fprintf(fp_denied, "%s\t%s\t%s\t%s\t%s\n",dia,hora,user,ip,urly);
1410                      denied_count++;
1411                   }
1412                   if(strstr(code,"DENIED/401") != 0 || strstr(code,"DENIED/407") != 0) {
1413                      if(fp_authfail)
1414                         fprintf(fp_authfail, "%s\t%s\t%s\t%s\t%s\n",dia,hora,user,ip,urly);
1415                      authfail_count++;
1416                   }
1417                }
1418
1419                if((!totper || idata<mindate) && ilf!=ILF_Sarg){
1420                   totper++;
1421                   mindate=idata;
1422                   sprintf(period,"%s-",tbuf);
1423                   strcpy(start_hour,tbuf2);
1424                   if(date[0] != '\0')
1425                      fixper(tbuf, period, cduntil);
1426                   if(debugz){
1427                      debugaz("tbuf",tbuf);
1428                      debugaz("period",period);
1429                   }
1430                }
1431             }
1432
1433             if(debugm){
1434                printf("IP=\t%s\n",ip);
1435                printf("USER=\t%s\n",user);
1436                printf("ELAP=\t%s\n",elap);
1437                printf("DATE=\t%s\n",dia);
1438                printf("TIME=\t%s\n",hora);
1439                printf("FUNC=\t%s\n",fun);
1440                printf("URL=\t%s\n",url);
1441                printf("CODE=\t%s\n",code);
1442                printf("LEN=\t%s\n",tam);
1443             }
1444          }
1445       }
1446       if (!from_stdin) {
1447          fclose(fp_in);
1448          if( ShowReadStatistics )
1449             printf("SARG: Records in file: %lu, reading: %3.2f%%\n",recs1, (float) 100 );
1450       }
1451    }
1452
1453    if ( fp_Download_Unsort )
1454      fclose (fp_Download_Unsort);
1455
1456    if (fp_Write_User)
1457      fclose (fp_Write_User);
1458
1459    free_download();
1460    free_excludecodes();
1461    free_exclude();
1462
1463    if(debug) {
1464       int totalcount=0;
1465
1466       for (ilf=0 ; ilf<ILF_Last ; ilf++) totalcount+=ilf_count[ilf];
1467
1468       debuga("   %s: %ld, %s: %ld, %s: %ld",text[10],totregsl,text[11],totregsg,text[68],totregsx);
1469
1470       if(ilf_count[ILF_Common]>0 && ilf_count[ILF_Squid]>0)
1471          debuga("%s",text[12]);
1472
1473       if(ilf_count[ILF_Common]>0 && ilf_count[ILF_Squid]==0)
1474          debuga("%s",text[13]);
1475
1476       if(ilf_count[ILF_Common]==0 && ilf_count[ILF_Squid]>0)
1477          debuga("%s",text[14]);
1478
1479       if(ilf_count[ILF_Sarg]>0)
1480          debuga("%s",text[124]);
1481
1482       if(totalcount==0) {
1483          if(!totregsg) {
1484             fprintf(stderr, "SARG: %s\n",text[16]);
1485             fprintf(stderr, "SARG: %s\n",text[21]);
1486          } else fprintf(stderr, "SARG: %s\n",text[15]);
1487          if(fp_denied)
1488             fclose(fp_denied);
1489          if(fp_authfail)
1490             fclose(fp_authfail);
1491 //         if(userfile)
1492 ////            free(userfile);
1493          unlink(tmp4);
1494          unlink(tmp6);
1495          unlink(tmp3);
1496          exit(0);
1497       }
1498    }
1499
1500    if(!totregsg){
1501       fprintf(stderr, "SARG: %s\n",text[16]);
1502       fprintf(stderr, "SARG: %s\n",text[21]);
1503 //      fclose(fp_ou);
1504       if(fp_denied)
1505          fclose(fp_denied);
1506       if(fp_authfail)
1507          fclose(fp_authfail);
1508       if(userfile)
1509          free(userfile);
1510       exit(0);
1511    }
1512
1513    if(date[0] == '\0' && ilf_count[ILF_Sarg]==0) {
1514       strcat(period,tbuf);
1515    }
1516
1517    if(debugz){
1518       debugaz("data",dia);
1519       debugaz("tbuf",tbuf);
1520       debugaz("period",period);
1521    }
1522
1523    if(debug)
1524       debuga("%s: %s",text[17],period);
1525
1526 //   fclose(fp_ou);
1527    if(fp_denied)
1528       fclose(fp_denied);
1529    if(fp_authfail)
1530       fclose(fp_authfail);
1531
1532    if(fp_log != NULL) {
1533       fclose(fp_log);
1534       strcpy(end_hour,tbuf2);
1535       getword_start(&gwarea,period);
1536       if (getword(val2,sizeof(val2),&gwarea,'-')<0){
1537          printf("SARG: Maybe you have a broken date range definition.\n");
1538          exit(1);
1539       }
1540       if (getword(val1,sizeof(val1),&gwarea,'\0')<0){
1541          printf("SARG: Maybe you have a broken date range definition.\n");
1542          exit(1);
1543       }
1544       sprintf(val4,"%s/sarg-%s_%s-%s_%s.log",ParsedOutputLog,val2,start_hour,val1,end_hour);
1545       if (rename(arq_log,val4)) {
1546          fprintf(stderr,"SARG: failed to rename %s to %s - %s\n",arq_log,val4,strerror(errno));
1547       } else {
1548          strcpy(arq_log,val4);
1549
1550          if(strcmp(ParsedOutputLogCompress,"nocompress") != 0 && ParsedOutputLogCompress[0] != '\0') {
1551             /*
1552             No double quotes around ParsedOutputLogCompress because it may contain command line options. If double quotes are
1553             necessary around the command name, put them in the configuration file.
1554             */
1555             sprintf(val1,"%s \"%s\"",ParsedOutputLogCompress,arq_log);
1556             cstatus=system(val1);
1557             if (!WIFEXITED(cstatus) || WEXITSTATUS(cstatus)) {
1558                fprintf(stderr, "SARG: command return status %d\n",WEXITSTATUS(cstatus));
1559                fprintf(stderr, "SARG: command: %s\n",val1);
1560                exit(1);
1561             }
1562          }
1563       }
1564       if(debug)
1565          debuga("%s %s",text[123],arq_log);
1566    }
1567
1568    if((ReportType & REPORT_TYPE_DENIED) != 0) {
1569       sprintf(csort,"sort -T \"%s\" -k 3,3 -k 5,5 -o \"%s\" \"%s\"",tmp,tmp5,tmp4);
1570       cstatus=system(csort);
1571       if (!WIFEXITED(cstatus) || WEXITSTATUS(cstatus)) {
1572          fprintf(stderr, "SARG: sort command return status %d\n",WEXITSTATUS(cstatus));
1573          fprintf(stderr, "SARG: sort command: %s\n",csort);
1574          exit(1);
1575       }
1576       unlink(tmp4);
1577    }
1578
1579    sort_users_log(tmp, debug);
1580
1581    if(DataFile[0] != '\0')
1582       data_file(tmp);
1583    else
1584       gerarel();
1585
1586    unlink(tmp2);
1587    if((ReportType & REPORT_TYPE_DENIED) != 0)
1588       unlink(tmp5);
1589
1590    if(zip[0] != '\0' && strcmp(zip,"zcat") !=0) {
1591       recomp(arq, zip);
1592    }
1593 //   else  unlink(arq);
1594
1595    if(strcmp(tmp,"/tmp") != 0) {
1596       unlinkdir(tmp,0);
1597    }
1598
1599    if(userfile)
1600       free(userfile);
1601
1602    if(debug)
1603       debuga("%s",text[21]);
1604
1605    exit(0);
1606
1607 }
1608
1609
1610 static void getusers(const char *pwdfile, int debug)
1611 {
1612
1613    FILE *fp_usr;
1614    char buf[255];
1615    char *str;
1616    long int nreg=0;
1617
1618    if(debug)
1619       debuga("%s: %s",text[60],pwdfile);
1620
1621    if ((fp_usr = fopen(pwdfile, "r")) == NULL) {
1622       fprintf(stderr, "SARG: (getusers) %s: %s - %s\n",text[45],pwdfile,strerror(errno));
1623       exit(1);
1624    }
1625
1626    fseek(fp_usr, 0, SEEK_END);
1627    nreg = ftell(fp_usr);
1628    if (nreg<0) {
1629       printf("SARG: Cannot get the size of file %s",pwdfile);
1630       exit(1);
1631    }
1632    nreg = nreg+5000;
1633    fseek(fp_usr, 0, SEEK_SET);
1634
1635    if((userfile=(char *) malloc(nreg))==NULL){
1636       fprintf(stderr, "SARG: %s (%ld):\n",text[59],nreg);
1637       exit(1);
1638    }
1639
1640    bzero(userfile,nreg);
1641    strcpy(userfile,":");
1642
1643    while(fgets(buf,sizeof(buf),fp_usr)!=NULL) {
1644       str=strchr(buf,':');
1645       if (!str) {
1646          printf("SARG: You have an invalid user in your %s file.\n",pwdfile);
1647          exit(1);
1648       }
1649       str[1]=0;
1650       strcat(userfile,buf);
1651    }
1652
1653    fclose(fp_usr);
1654
1655    return;
1656 }