log.c

   1 /*
   2  * AUTHOR: Pedro Lineu Orso                         pedro.orso@gmail.com
   3  *                                                            1998, 2009
   4  * SARG Squid Analysis Report Generator      http://sarg.sourceforge.net
   5  *
   6  * SARG donations:
   7  *      please look at http://sarg.sourceforge.net/donations.php
   8  * ---------------------------------------------------------------------
   9  *
  10  *  This program is free software; you can redistribute it and/or modify
  11  *  it under the terms of the GNU General Public License as published by
  12  *  the Free Software Foundation; either version 2 of the License, or
  13  *  (at your option) any later version.
  14  *
  15  *  This program is distributed in the hope that it will be useful,
  16  *  but WITHOUT ANY WARRANTY; without even the implied warranty of
  17  *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  18  *  GNU General Public License for more details.
  19  *
  20  *  You should have received a copy of the GNU General Public License
  21  *  along with this program; if not, write to the Free Software
  22  *  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111, USA.
  23  *
  24  */
  25
  26 #include "include/conf.h"
  27 #include "include/defs.h"
  28
  29 #define REPORT_EVERY_X_LINES 5000
  30
  31
  32 char *userfile;
  33 char *excludefile;
  34 char *excludeuser;
  35
  36 int bool_ShowReadStatistics ;
  37
  38 numlist weekdays = { { 0, 1, 2, 3, 4, 5, 6 }, 7 };
  39 numlist hours = { { 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12,
  40              13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23 }, 24 };
  41
  42 static void getusers(const char *pwdfile, int debug);
  43 static void gethexclude(const char *hexfile, int debug);
  44 static void getuexclude(const char *uexfile, int debug);
  45
  46 int main(int argc,char *argv[])
  47 {
  48    enum isa_col_id {
  49       ISACOL_Ip,
  50       ISACOL_UserName,
  51       ISACOL_Date,
  52       ISACOL_Time,
  53       ISACOL_TimeTaken,
  54       ISACOL_Bytes,
  55       ISACOL_Uri,
  56       ISACOL_Status,
  57       ISACOL_Last //last entry of the list !
  58    };
  59    enum InputLogFormat {
  60       ILF_Unknown,
  61       ILF_Squid,
  62       ILF_Common,
  63       ILF_Sarg,
  64       ILF_Isa,
  65       ILF_Last //last entry of the list !
  66    };
  67
  68    FILE *fp_in = NULL, *fp_denied=NULL, *fp_authfail=NULL, *fp_log=NULL;
  69
  70    char sz_Download_Unsort[ 20000 ] ;
  71    FILE * fp_Download_Unsort = NULL ;
  72    FILE * fp_Write_User = NULL ;
  73
  74    extern int optind;
  75    extern int optopt;
  76    extern char *optarg;
  77
  78    char w[MAXLEN];
  79    char data[255];
  80    char elap[255];
  81    char none[255];
  82    char ip[MAXLEN];
  83    char msg[MAXLEN];
  84    char tam[255];
  85    char fun[MAXLEN];
  86    char wuser[MAXLEN];
  87    char smartfilter[MAXLEN];
  88    char dia[128];
  89    char wdata[128];
  90    char mes[30];
  91    char ano[30];
  92    char hora[30];
  93    char wtemp[MAXLEN];
  94    char wtemp2[255];
  95    char date[255];
  96    char arq[255];
  97    char arq_log[255];
  98    char warq[255][255];
  99    char hm[15], hmf[15], hmr[15], whm[15];
 100    int  chm=0;
 101    char uagent[MAXLEN];
 102    char hexclude[MAXLEN];
 103    char csort[MAXLEN];
 104    int cstatus;
 105    char tbuf[128];
 106    char tbuf2[128];
 107    char zip[20];
 108    char *str;
 109    char bufz[MAXLEN];
 110    char bufy[MAXLEN];
 111    enum InputLogFormat ilf;
 112    int ilf_count[ILF_Last];
 113    int  ch;
 114    int  x, l;
 115    int  errflg=0;
 116    int  puser=0;
 117    int  fhost=0;
 118    int  dns=0;
 119    int  fuser=0;
 120    int  idata=0;
 121    int  mindate=0;
 122    int  narq=0;
 123    int  iarq=0;
 124    int  exstring=0;
 125    int isa_ncols=0,isa_cols[ISACOL_Last];
 126    int from_stdin;
 127    long totregsl=0;
 128    long totregsg=0;
 129    long totregsx=0;
 130    long totper=0;
 131    long int max_elapsed=0;
 132    time_t tt;
 133    struct tm *t;
 134    off_t recs1=0;
 135    unsigned long recs2=0;
 136    int OutputNonZero = REPORT_EVERY_X_LINES ;
 137    int download_flag;
 138    char download_url[MAXLEN];
 139    char sz_Last_User[MAXLEN]="";
 140
 141    BgImage[0]='\0';
 142    LogoImage[0]='\0';
 143    LogoText[0]='\0';
 144    PasswdFile[0]='\0';
 145    OutputEmail[0]='\0';
 146    Ip2Name[0]='\0';
 147    UserAgentLog[0]='\0';
 148    ExcludeHosts[0]='\0';
 149    ExcludeUsers[0]='\0';
 150    ConfigFile[0]='\0';
 151    code[0]='\0';
 152    LastLog[0]='\0';
 153    UserIp[0]='\0';
 154    ReportType[0]='\0';
 155    UserTabFile[0]='\0';
 156    BlockIt[0]='\0';
 157    ExternalCSSFile[0]='\0';
 158    SquidGuardLogFormat[0]='\0';
 159    SquidGuardLogAlternate[0]='\0';
 160    arq[0]='\0';
 161    for (ilf=0 ; ilf<ILF_Last ; ilf++) ilf_count[ilf]=0;
 162
 163    strcpy(AccessLog,"/var/log/squid/access.log");
 164    sprintf(ExcludeCodes,"%s/exclude_codes",SYSCONFDIR);
 165    strcpy(GraphDaysBytesBarColor,"orange");
 166    strcpy(BgColor,"#ffffff");
 167    strcpy(TxColor,"#000000");
 168    strcpy(TxBgColor,"lavender");
 169    strcpy(TiColor,"darkblue");
 170    strcpy(Width,"80");
 171    strcpy(Height,"45");
 172    strcpy(LogoTextColor,"#000000");
 173    strcpy(HeaderColor,"darkblue");
 174    strcpy(HeaderBgColor,"#dddddd");
 175    strcpy(LogoTextColor,"#006699");
 176    strcpy(FontSize,"9px");
 177    strcpy(TempDir,"/tmp");
 178    strcpy(OutputDir,"/var/www/html/squid-reports");
 179    strcpy(Ip2Name,"no");
 180    strcpy(DateFormat,"u");
 181    strcpy(OverwriteReport,"no");
 182    strcpy(RemoveTempFiles,"yes");
 183    strcpy(ReplaceIndex,"index.html");
 184    strcpy(Index,"yes");
 185    strcpy(RecordsWithoutUser,"ip");
 186    strcpy(UseComma,"no");
 187    strcpy(MailUtility,"mailx");
 188    strcpy(TopSitesNum,"100");
 189    strcpy(UserIp,"no");
 190    strcpy(TopuserSortField,"BYTES");
 191    strcpy(UserSortField,"BYTES");
 192    strcpy(TopuserSortOrder,"reverse");
 193    strcpy(UserSortOrder,"reverse");
 194    strcpy(TopsitesSortField,"CONNECT");
 195    strcpy(TopsitesSortType,"D");
 196    strcpy(LongUrl,"no");
 197    strcpy(language,"English");
 198    strcpy(FontFace,"Verdana,Tahoma,Arial");
 199    strcpy(datetimeby,"elap");
 200    strcpy(CharSet,"ISO-8859-1");
 201    strcpy(Privacy,"no");
 202    strcpy(PrivacyString,"***.***.***.***");
 203    strcpy(PrivacyStringColor,"blue");
 204    strcpy(SuccessfulMsg,"yes");
 205    strcpy(TopUserFields,"NUM DATE_TIME USERID CONNECT BYTES %BYTES IN-CACHE-OUT USED_TIME MILISEC %TIME TOTAL AVERAGE");
 206    strcpy(UserReportFields,"CONNECT BYTES %BYTES IN-CACHE-OUT USED_TIME MILISEC %TIME TOTAL AVERAGE");
 207    strcpy(DataFileDelimiter,";");
 208    strcpy(DataFileFields,"user;date;time;url;connect;bytes;in_cache;out_cache;elapsed");
 209    strcpy(SiteUserTimeDateType,"table");
 210    strcpy(ShowReadStatistics,"yes");
 211    strcpy(IndexSortOrder,"D");
 212    strcpy(ShowSargInfo,"yes");
 213    strcpy(ShowSargLogo,"yes");
 214    strcpy(ParsedOutputLog,"no");
 215    strcpy(ParsedOutputLogCompress,"/bin/gzip");
 216    strcpy(DisplayedValues,"abbreviation");
 217    strcpy(HeaderFontSize,"9px");
 218    strcpy(TitleFontSize,"11px");
 219    strcpy(AuthUserFile,"/usr/local/sarg/passwd");
 220    strcpy(AuthName,"SARG, Restricted Access");
 221    strcpy(AuthType,"basic");
 222    strcpy(Require,"require user admin %u");
 223    strcpy(DownloadSuffix,"7z,ace,arj,avi,bat,bin,bz2,bzip,cab,com,cpio,dll,doc,dot,exe,gz,iso,lha,lzh,mdb,mov,mp3,mpeg,mpg,mso,nrg,ogg,ppt,rar,rtf,shs,src,sys,tar,tgz,vcd,vob,wma,wmv,zip");
 224    strcpy(Graphs,"yes");
 225    strcpy(Ulimit,"20000");
 226    strcpy(NtlmUserFormat,"domainname+username");
 227    strcpy(IndexTree,"file");
 228    strcpy(RealtimeTypes,"GET,PUT,CONNECT");
 229    strcpy(RealtimeUnauthRec,"show");
 230    strcpy(SquidguardIgnoreDate,"off");
 231    strcpy(DansguardianIgnoreDate,"off");
 232    strcpy(DataFileUrl,"ip");
 233    strcpy(MaxElapsed,"28800000");
 234    strcpy(BytesInSitesUsersReport,"no");
 235
 236    dia[0]='\0';
 237    mes[0]='\0';
 238    ano[0]='\0';
 239    hora[0]='\0';
 240    tmp[0]='\0';
 241    tmp2[0]='\0';
 242    tmp3[0]='\0';
 243    wtemp[0]='\0';
 244    wtemp2[0]='\0';
 245    us[0]='\0';
 246    date[0]='\0';
 247    df[0]='\0';
 248    uagent[0]='\0';
 249    hexclude[0]='\0';
 250    addr[0]='\0';
 251    hm[0]='\0';
 252    hmf[0]='\0';
 253    site[0]='\0';
 254    outdir[0]='\0';
 255    elap[0]='\0';
 256    email[0]='\0';
 257    zip[0]='\0';
 258    UserInvalidChar[0]='\0';
 259    DataFile[0]='\0';
 260    SquidGuardConf[0]='\0';
 261    DansGuardianConf[0]='\0';
 262
 263    denied_count=0;
 264    download_count=0;
 265    authfail_count=0;
 266    dansguardian_count=0;
 267    squidguard_count=0;
 268    DeniedReportLimit=10;
 269    AuthfailReportLimit=10;
 270    DansGuardianReportLimit=10;
 271    SquidGuardReportLimit=10;
 272    DownloadReportLimit=50;
 273    UserReportLimit=0;
 274    debug=0;
 275    debugz=0;
 276    debugm=0;
 277    iprel=0;
 278    userip=0;
 279    color1=0;
 280    color2=0;
 281    color3=0;
 282    dotinuser=0;
 283    realt=0;
 284    realtime_refresh=3;
 285    realtime_access_log_lines=1000;
 286    cost=0.01;
 287    nocost=50000000;
 288    ndownload=0;
 289    squid24=0;
 290
 291    bzero(IncludeUsers, MAXLEN);
 292    bzero(ExcludeString, MAXLEN);
 293
 294    for(x=0; x<=254; x++)
 295       warq[x][0]='\0';
 296
 297    language_load(language);
 298    strcpy(Title,text[88]);
 299
 300    while((ch = getopt(argc, argv, "a:b:c:d:e:f:g:u:l:L:o:s:t:w:hijmnprvxyz")) != -1){
 301       switch(ch)
 302       {
 303          case 'a':
 304             strcpy(addr,optarg);
 305             break;
 306          case 'b':
 307             strcpy(uagent,optarg);
 308             break;
 309          case 'c':
 310             strcpy(hexclude,optarg);
 311             break;
 312          case 'd':
 313             strcpy(date,optarg);
 314             strcpy(cduntil,optarg);
 315             if (getword(cdfrom,sizeof(cdfrom),cduntil,'-')<0) {
 316                printf("SARG: Maybe you have a broken record or garbage in your date range.\n");
 317                exit(1);
 318             }
 319             date_from(date, cdfrom, cduntil);
 320             dfrom=atoi(cdfrom);
 321             duntil=atoi(cduntil);
 322             break;
 323          case 'e':
 324             strcpy(email,optarg);
 325             break;
 326          case 'f':
 327             strcpy(ConfigFile,optarg);
 328             break;
 329          case 'g':
 330             strcpy(df,optarg);
 331             break;
 332          case 'h':
 333             usage(argv[0]);
 334             exit(0);
 335             break;
 336          case 'i':
 337             iprel++;
 338             break;
 339          case 'l':
 340             if (narq>=sizeof(warq)/sizeof(warq[0])) {
 341                printf("SARG: Too many log files.\n");
 342                exit(1);
 343             }
 344             strcpy(warq[narq],optarg);
 345             narq++;
 346             break;
 347          case 'L':
 348             strcpy(SquidGuardLogAlternate,optarg);
 349             break;
 350          case 'm':
 351             debugm++;
 352             break;
 353          case 'n':
 354             dns++;
 355             break;
 356          case 'o':
 357             strcpy(outdir,optarg);
 358             break;
 359          case 'p':
 360             userip++;
 361             break;
 362          case 'r':
 363             realt++;
 364             break;
 365          case 's':
 366             strcpy(site,optarg);
 367             break;
 368          case 't':
 369             if(strstr(optarg,"-") == 0) {
 370               strcpy(hm,optarg);
 371               strcpy(hmf,optarg);
 372             } else {
 373                if (getword(hm,sizeof(hm),optarg,'-')<0) {
 374                   printf("SARG: Maybe you have a broken record or garbage in your time range.\n");
 375                   exit(1);
 376                }
 377                strcpy(hmf,optarg);
 378             }
 379             if(strlen(hm) > 5) {
 380                printf("SARG: time period must be MM or MM:SS. Exit.\n");
 381                exit(1);
 382             }
 383             bzero(whm,15);
 384             if(strstr(hm,":") != 0) {
 385                if (getword(warea,sizeof(warea),hm,':')<0) {
 386                   printf("SARG: Maybe you have a broken record or garbage in your time range.\n");
 387                   exit(1);
 388                }
 389                sprintf(whm,"%s%s",warea,hm);
 390                strcpy(hm,whm);
 391             }
 392             bzero(whm,15);
 393             if(strstr(hmf,":") != 0) {
 394                if (getword(warea,sizeof(warea),hmf,':')<0) {
 395                   printf("SARG: Maybe you have a broken record or garbage in your time range.\n");
 396                   exit(1);
 397                }
 398                sprintf(whm,"%s%s",warea,hmf);
 399                strcpy(hmf,whm);
 400             }
 401             break;
 402          case 'u':
 403             strcpy(us,optarg);
 404             break;
 405          case 'v':
 406             version();
 407             break;
 408          case 'w':
 409             strcpy(tmp,optarg);
 410             break;
 411          case 'x':
 412             debug++;
 413             break;
 414          case 'y':
 415             langcode++;
 416             break;
 417          case 'z':
 418             debugz++;
 419             break;
 420          case ':':
 421             fprintf(stderr, "Option -%c require an argument\n",optopt);
 422             errflg++;
 423             break;
 424          case '?':
 425             usage(argv[0]);
 426             exit(1);
 427             break;
 428       }
 429
 430    }
 431
 432    if (errflg) {
 433       usage(argv[0]);
 434       exit(2);
 435    }
 436
 437    if(debug) debuga("Init");
 438
 439    if(ConfigFile[0] == '\0') sprintf(ConfigFile,"%s/sarg.conf",SYSCONFDIR);
 440    if(access(ConfigFile, R_OK) != 0) {
 441       debuga("Cannot open config file: %s - %s",ConfigFile,strerror(errno));
 442       exit(1);
 443    }
 444
 445    if(access(ConfigFile, R_OK) == 0)
 446       getconf();
 447
 448    if(realt) {
 449       realtime();
 450       exit(0);
 451    }
 452
 453    if(strcmp(IndexTree,"file") == 0)
 454       strcpy(ImageFile,"../images");
 455    else
 456       strcpy(ImageFile,"../../../images");
 457
 458    dataonly=0;
 459    if(DataFile[0] != '\0')
 460       dataonly++;
 461
 462    subs(TopUserFields,sizeof(TopUserFields),"%BYTES","SETYB");
 463
 464    subs(UserReportFields,sizeof(UserReportFields),"%BYTES","SETYB");
 465
 466    if(!narq) {
 467       strcpy(warq[0],AccessLog);
 468       narq++;
 469    }
 470
 471    if(strcmp(hexclude,"onvert") == 0 && strcmp(site,"plit") != 0) {
 472       convlog(warq[0], df, dfrom, duntil);
 473       exit(0);
 474    }
 475
 476    if(strcmp(site,"plit") == 0) {
 477       splitlog(warq[0], df, dfrom, duntil, hexclude);
 478       exit(0);
 479    }
 480
 481    load_excludecodes(ExcludeCodes);
 482
 483    if(access(PasswdFile, R_OK) == 0) {
 484       getusers(PasswdFile,debug);
 485       puser++;
 486    }
 487
 488    if(hexclude[0] == '\0')
 489       strcpy(hexclude,ExcludeHosts);
 490    if(strlen(hexclude) > 0) {
 491       if(access(hexclude, R_OK) != 0) {
 492          debuga("Cannot open exclude_hosts file: %s - %s",hexclude,strerror(errno));
 493          exit(1);
 494       }
 495       gethexclude(hexclude,debug);
 496       fhost++;
 497    }
 498
 499    if(ReportType[0] == '\0')
 500       strcpy(ReportType,"topusers topsites users_sites sites_users date_time denied auth_failures site_user_time_date downloads");
 501
 502    if(access(ExcludeUsers, R_OK) == 0) {
 503       getuexclude(ExcludeUsers,debug);
 504       fuser++;
 505    }
 506
 507    indexonly=0;
 508    if(fuser) {
 509       if(strstr(excludeuser,"indexonly") != 0)
 510          indexonly++;
 511    }
 512    if(strcmp(ExcludeUsers,"indexonly") == 0) indexonly++;
 513    if(strcmp(Index,"only") == 0) indexonly++;
 514
 515    if(dns) strcpy(Ip2Name,"yes");
 516
 517    if(strcmp(UserIp,"yes") == 0) userip++;
 518
 519    if(strlen(MaxElapsed)>1) max_elapsed=atol(MaxElapsed);
 520
 521    if(strlen(outdir)<1) strcpy(outdir,OutputDir);
 522    strcat(outdir,"/");
 523
 524
 525    if(arq[0] == '\0') strcpy(arq,AccessLog);
 526
 527    if(uagent[0] == '\0') strcpy(uagent,UserAgentLog);
 528
 529    if(tmp[0] == '\0') strcpy(tmp,TempDir);
 530    else strcpy(TempDir,tmp);
 531
 532    if(df[0] == '\0') strcpy(df,DateFormat);
 533    else strcpy(DateFormat,df);
 534
 535    if(df[0] == '\0') {
 536       strcpy(df,"u");
 537       strcpy(DateFormat,"u");
 538    }
 539
 540    if(email[0] == '\0' && OutputEmail[0] != '\0') strcpy(email,OutputEmail);
 541
 542    strcpy(tmp2,tmp);
 543
 544    if(strlen(email) > 0) {
 545       sprintf(wtemp2,"%s/sarg",tmp2);
 546       my_mkdir(wtemp2);
 547       strcat(tmp2,"/sarg");
 548       strcpy(outdir,tmp2);
 549       strcat(outdir,"/");
 550    }
 551
 552    strcat(tmp2,"/sarg.log");
 553
 554    sprintf(warea,"%s/sarg",tmp);
 555    if(access(warea, R_OK) == 0) {
 556       if (snprintf(tmp3,sizeof(tmp3),"rm -rf \"%s\"",warea)>=sizeof(tmp3)) {
 557          fprintf(stderr,"SARG: Directory name too long: %s\n",warea);
 558          exit(1);
 559       }
 560       cstatus=system(tmp3);
 561       if (!WIFEXITED(cstatus) || WEXITSTATUS(cstatus)) {
 562          fprintf(stderr, "SARG: command return status %d\n",WEXITSTATUS(cstatus));
 563          fprintf(stderr, "SARG: command: %s\n",tmp3);
 564          exit(1);
 565       }
 566    }
 567
 568    sprintf(tmp3,"%s/sarg",tmp);
 569    my_mkdir(tmp3);
 570    strcpy(tmp4,tmp3);
 571    strcpy(tmp5,tmp3);
 572    strcpy(tmp6,tmp3);
 573    strcat(tmp4,"/denied.log.unsort");
 574    strcat(tmp5,"/denied.log");
 575    strcat(tmp6,"/authfail.log.unsort");
 576
 577    if(debug) {
 578       fprintf(stderr, "SARG: %s:\nSARG:\n",text[22]);
 579       fprintf(stderr, "SARG: %35s (-a) = %s\n",text[23],addr);
 580       fprintf(stderr, "SARG: %35s (-b) = %s\n",text[71],uagent);
 581       fprintf(stderr, "SARG: %35s (-c) = %s\n",text[69],hexclude);
 582       fprintf(stderr, "SARG: %35s (-d) = %s\n",text[24],date);
 583       fprintf(stderr, "SARG: %35s (-e) = %s\n",text[41],email);
 584       fprintf(stderr, "SARG: %35s (-f) = %s\n",text[70],ConfigFile);
 585       if(strcmp(df,"e") == 0)
 586          fprintf(stderr, "SARG: %35s (-g) = %s (dd/mm/yyyy)\n",text[25],text[26]);
 587       if(strcmp(df,"u") == 0)
 588          fprintf(stderr, "SARG: %35s (-g) = %s (mm/dd/yyyy)\n",text[25],text[27]);
 589       if(strcmp(df,"w") == 0)
 590          fprintf(stderr, "SARG: %35s (-g) = %s (yyyy/ww)\n",text[25],text[85]);
 591       if(iprel)
 592          fprintf(stderr, "SARG: %35s (-i) = %s\n",text[28],text[1]);
 593        else
 594          fprintf(stderr, "SARG: %35s (-i) = %s\n",text[28],text[2]);
 595       fprintf(stderr, "SARG: %35s (-l) = %s\n",text[37],arq);
 596       if(strcmp(Ip2Name,"yes") == 0)
 597          fprintf(stderr, "SARG: %35s (-n) = %s\n",text[65],text[1]);
 598        else
 599          fprintf(stderr, "SARG: %35s (-n) = %s\n",text[65],text[2]);
 600       fprintf(stderr, "SARG: %35s (-o) = %s\n",text[38],outdir);
 601       if(strcmp(UserIp,"yes") == 0)
 602          fprintf(stderr, "SARG: %35s (-p) = %s\n",text[29],text[1]);
 603        else
 604          fprintf(stderr, "SARG: %35s (-p) = %s\n",text[29],text[2]);
 605       fprintf(stderr, "SARG: %35s (-s) = %s\n",text[30],site);
 606       fprintf(stderr, "SARG: %35s (-t) = %s\n",text[31],hm);
 607       fprintf(stderr, "SARG: %35s (-u) = %s\n",text[32],us);
 608       fprintf(stderr, "SARG: %35s (-w) = %s\n",text[34],tmp);
 609       if(debug)
 610          fprintf(stderr, "SARG: %35s (-x) = %s\n",text[35],text[1]);
 611        else
 612          fprintf(stderr, "SARG: %35s (-x) = %s\n",text[35],text[2]);
 613       if(debugz)
 614          fprintf(stderr, "SARG: %35s (-z) = %s\n",text[36],text[1]);
 615        else
 616          fprintf(stderr, "SARG: %35s (-z) = %s\n",text[36],text[2]);
 617       fprintf(stderr, "SARG:\n");
 618    }
 619
 620    if(debugm) {
 621       printf("%s:\nSARG:\n",text[22]);
 622       printf("%35s (-a) = %s\n",text[23],addr);
 623       printf("%35s (-b) = %s\n",text[71],uagent);
 624       printf("%35s (-c) = %s\n",text[69],hexclude);
 625       printf("%35s (-d) = %s\n",text[24],date);
 626       printf("%35s (-e) = %s\n",text[41],email);
 627       printf("%35s (-f) = %s\n",text[70],ConfigFile);
 628       if(strcmp(df,"e") == 0)
 629          printf("%35s (-g) = %s (dd/mm/yyyy)\n",text[25],text[26]);
 630       if(strcmp(df,"u") == 0)
 631          printf("%35s (-g) = %s (mm/dd/yyyy)\n",text[25],text[27]);
 632       if(strcmp(df,"w") == 0)
 633          printf("%35s (-g) = %s (yyyy/ww)\n",text[25],text[85]);
 634       if(iprel)
 635          printf("%35s (-i) = %s\n",text[28],text[1]);
 636        else
 637          printf("%35s (-i) = %s\n",text[28],text[2]);
 638       printf("%35s (-l) = %s\n",text[37],arq);
 639       if(strcmp(Ip2Name,"yes") == 0)
 640          printf("%35s (-n) = %s\n",text[65],text[1]);
 641        else
 642          printf("%35s (-n) = %s\n",text[65],text[2]);
 643       printf("%35s (-o) = %s\n",text[38],outdir);
 644       if(strcmp(UserIp,"yes") == 0)
 645          printf("%35s (-p) = %s\n",text[29],text[1]);
 646        else
 647          printf("%35s (-p) = %s\n",text[29],text[2]);
 648       printf("%35s (-s) = %s\n",text[30],site);
 649       printf("%35s (-t) = %s\n",text[31],hm);
 650       printf("%35s (-u) = %s\n",text[32],us);
 651       printf("%35s (-w) = %s\n",text[34],tmp);
 652       if(debug)
 653          printf("%35s (-x) = %s\n",text[35],text[1]);
 654        else
 655          printf("%35s (-x) = %s\n",text[35],text[2]);
 656       if(debugz)
 657          printf("%35s (-z) = %s\n",text[36],text[1]);
 658        else
 659          printf("%35s (-z) = %s\n",text[36],text[2]);
 660       printf("sarg %s: %s\n",text[73],VERSION);
 661       printf("Language=%s\n\n",text[3]);
 662    }
 663
 664    if(debug)
 665       debuga("sarg %s: %s",text[73],VERSION);
 666
 667 #ifdef HAVE_RLIM_T
 668    if (Ulimit[0] != '\0') {
 669       struct rlimit rl;
 670       long l1, l2;
 671       int rc=0;
 672
 673 #if defined(RLIMIT_NOFILE)
 674       getrlimit (RLIMIT_NOFILE, &rl);
 675 #elif defined(RLIMIT_OFILE)
 676       getrlimit (RLIMIT_OFILE, &rl);
 677 #else
 678 #warning "No rlimit resource for the number of open files"
 679 #endif
 680       l1 = rl.rlim_cur;
 681       l2 = rl.rlim_max;
 682
 683       rl.rlim_cur = atol(Ulimit);
 684       rl.rlim_max = atol(Ulimit);
 685 #if defined(RLIMIT_NOFILE)
 686       rc=setrlimit (RLIMIT_NOFILE, &rl);
 687 #elif defined(RLIMIT_OFILE)
 688       rc=setrlimit (RLIMIT_OFILE, &rl);
 689 #else
 690 #warning "No rlimit resource for the number of open files"
 691 #endif
 692       if(rc == -1) {
 693             debuga("setrlimit error - %s\n",strerror(errno));
 694       }
 695
 696       if(debug)
 697          debuga("Maximum file descriptor: cur=%ld max=%ld, changed to cur="RLIM_STRING" max="RLIM_STRING,l1,l2,rl.rlim_cur,rl.rlim_max);
 698    }
 699 #endif
 700
 701    read_usertab(UserTabFile);
 702
 703    sprintf ( sz_Download_Unsort , "%s/sarg/download.unsort", tmp);
 704    bool_ShowReadStatistics = ( strcmp(ShowReadStatistics,"yes") == 0 ) ;
 705
 706    if(strstr(ReportType,"denied") != 0) {
 707       if((fp_denied=MY_FOPEN(tmp4,"w"))==NULL) {
 708          fprintf(stderr, "%s: (log) %s: %s - %s\n",argv[0],text[45],tmp4,strerror(errno));
 709          exit(1);
 710       }
 711    }
 712
 713    if(DataFile[0]=='\0') {
 714       if(strstr(ReportType,"denied") != 0 || strstr(ReportType,"auth_failures") != 0) {
 715          if((fp_authfail=MY_FOPEN(tmp6,"w"))==NULL) {
 716             fprintf(stderr, "%s: (log) %s: %s - %s\n",argv[0],text[45],tmp6,strerror(errno));
 717             exit(1);
 718          }
 719       }
 720    }
 721
 722    while(narq--) {
 723       strcpy(arq,warq[iarq]);
 724       iarq++;
 725
 726       strcpy(arqtt,arq);
 727
 728       if(strcmp(arq,"-")==0) {
 729          if(debug)
 730             debuga("%s: %s",text[7],"stdin");
 731          fp_in=stdin;
 732          from_stdin=1;
 733       } else {
 734          decomp(arq,zip,tmp);
 735          if(debug)
 736             debuga("%s: %s",text[7],arq);
 737          if((fp_in=MY_FOPEN(arq,"r"))==NULL) {
 738             fprintf(stderr, "%s: (log) %s: %s - %s\n",argv[0],text[8],arq,strerror(errno));
 739             exit(1);
 740          }
 741          from_stdin=0;
 742       }
 743       ilf=ILF_Unknown;
 744       // pre-Read the file only if I have to show stats
 745       if(bool_ShowReadStatistics && !from_stdin) {
 746          rewind(fp_in);
 747          recs1=0;
 748          recs2=0;
 749
 750          while( fgets(bufz,sizeof(bufz),fp_in) != NULL ) recs1++;
 751          rewind(fp_in);
 752          printf("SARG: Records in file: " OFFSET_STRING ", reading: %3.2f%%\r",recs1,(float) 0);
 753          fflush( stdout ) ;
 754       }
 755
 756       while(fgets(bufz,sizeof(bufz),fp_in)!=NULL) {
 757
 758          if (ilf==ILF_Unknown) {
 759             if(strncmp(bufz,"#Software: Mic",14) == 0) {
 760                fixendofline(bufz);
 761                debuga("%s: %s",text[143],bufz);
 762                ilf=ILF_Isa;
 763                ilf_count[ilf]++;
 764                continue;
 765             }
 766
 767             if(strncmp(bufz,"*** SARG Log ***",16) == 0) {
 768                if (getword(val2,sizeof(val2),arqtt,'-')<0 || getword(val2,sizeof(val2),arqtt,'_')<0 ||
 769                    getword(val3,sizeof(val3),arqtt,'-')<0 || getword(val3,sizeof(val3),arqtt,'_')<0) {
 770                   printf("SARG: Maybe you have a broken record or garbage in your %s file.\n",arq);
 771                   exit(1);
 772                }
 773                sprintf(period,"%s-%s",val2,val3);
 774                ilf=ILF_Sarg;
 775                ilf_count[ilf]++;
 776                continue;
 777             }
 778          }
 779
 780          if(strcmp(ParsedOutputLog, "no") != 0 && ilf!=ILF_Sarg) {
 781             if(access(ParsedOutputLog,R_OK) != 0) {
 782                sprintf(csort,"%s",ParsedOutputLog);
 783                my_mkdir(csort);
 784             }
 785             sprintf(arq_log,"%s/sarg_temp.log",ParsedOutputLog);
 786             if((fp_log=MY_FOPEN(arq_log,"w"))==NULL) {
 787                fprintf(stderr, "%s: (log) %s: %s - %s\n",argv[0],text[8],arq_log,strerror(errno));
 788                exit(1);
 789             }
 790             fputs("*** SARG Log ***\n",fp_log);
 791          }
 792
 793          recs2++;
 794          if( bool_ShowReadStatistics && !from_stdin && ! --OutputNonZero) {
 795            perc = recs2 * 100 ;
 796            perc = perc / recs1 ;
 797            printf("SARG: Records in file: " OFFSET_STRING", reading: %3.2f%%\r",recs1,perc);
 798            fflush (stdout);
 799            OutputNonZero = REPORT_EVERY_X_LINES ;
 800          }
 801          if(strlen(bufz) > MAXLEN-1) continue;
 802          if(!bufz[0]) continue;
 803          if(strstr(bufz,"HTTP/0.0") != 0) continue;
 804          if(strstr(bufz,"logfile turned over") != 0) continue;
 805          if(bufz[0] == ' ') continue;
 806          if(strlen(bufz) < 58) continue;
 807
 808          // Record only hours usage which is required
 809          tt = (time_t) strtoul( bufz, NULL, 10 );
 810          t = localtime( &tt );
 811
 812          if( bsearch( &( t -> tm_wday ), weekdays.list, weekdays.len,
 813                                           sizeof( int ), compar ) == NULL )
 814             continue;
 815
 816          if( bsearch( &( t -> tm_hour ), hours.list, hours.len,
 817                                           sizeof( int ), compar ) == NULL )
 818             continue;
 819
 820          // exclude_string
 821          exstring=0;
 822          if(strlen(ExcludeString) > 0) {
 823             strcpy(warea,bufz);
 824             strcpy(html,ExcludeString);
 825             while(strstr(html,":") != 0) {
 826                if (getword_multisep(val1,sizeof(val1),html,':')<0) {
 827                   printf("SARG: Maybe you have a broken record or garbage in your exclusion string.\n");
 828                   exit(1);
 829                }
 830                if((str=(char *) strstr(warea,val1)) != (char *) NULL )
 831                   exstring++;
 832             }
 833             if((str=(char *) strstr(warea,html)) != (char *) NULL )
 834                   exstring++;
 835          }
 836          if(exstring) continue;
 837
 838          strcpy(bufy,bufz);
 839          if ((str = strchr(bufz, '\n')) != NULL)
 840             *str = '\0';          /* strip \n */
 841
 842          totregsl++;
 843          if(debugm)
 844             printf("BUF=%s\n",bufz);
 845
 846          if (ilf==ILF_Squid || ilf==ILF_Common || ilf==ILF_Unknown) {
 847             if (getword(data,sizeof(data),bufz,' ')<0) {
 848                printf("SARG: Maybe you have a broken record or garbage in your access.log file.\n");
 849                exit(1);
 850             }
 851             if((str=(char *) strchr(data, '.')) != (char *) NULL ) {
 852                if((str=(char *) strchr(str+1, '.')) != (char *) NULL ) {
 853                   strcpy(ip,data);
 854                   strcpy(elap,"0");
 855                   if(squid24) {
 856                      if (getword(user,sizeof(user),bufz,' ')<0 || getword(none,sizeof(none),bufz,' ')<0) {
 857                         printf("SARG: Maybe you have a broken record or garbage in your %s file.\n",arq);
 858                         exit(1);
 859                      }
 860                   } else {
 861                      if (getword(none,sizeof(none),bufz,' ')<0 || getword(user,sizeof(user),bufz,' ')<0) {
 862                         printf("SARG: Maybe you have a broken record or garbage in your %s file.\n",arq);
 863                         exit(1);
 864                      }
 865                   }
 866                   if (getword(data,sizeof(data),bufz,']')<0 || getword(fun,sizeof(fun),bufz,'"')<0 ||
 867                       getword(fun,sizeof(fun),bufz,' ')<0 || getword(url,sizeof(url),bufz,' ')<0 ||
 868                       getword(code2,sizeof(code2),bufz,' ')<0 || getword(code2,sizeof(code2),bufz,' ')<0 ||
 869                       getword(tam,sizeof(tam),bufz,' ')<0) {
 870                      printf("SARG: Maybe you have a broken record or garbage in your %s file.\n",arq);
 871                      exit(1);
 872                   }
 873                   if((str=(char *) strchr(bufz, ' ')) != (char *) NULL ) {
 874                      if (getword(code,sizeof(code),bufz,' ')<0) {
 875                         printf("SARG: Maybe you have a broken record or garbage in your %s file.\n",arq);
 876                         exit(1);
 877                      }
 878                   }
 879                   else strcpy(code,bufz);
 880
 881                   if ((str = strchr(code, ':')) != NULL)
 882                      *str = '/';
 883
 884                   if(strcmp(tam,"\0") == 0)
 885                      strcpy(tam,"0");
 886
 887                   ilf=ILF_Common;
 888                   ilf_count[ilf]++;
 889                }
 890             }
 891
 892             if(ilf==ILF_Unknown || ilf==ILF_Squid) {
 893                if (getword(elap,sizeof(elap),bufz,' ')<0) {
 894                   printf("SARG: Maybe you have a broken record or garbage in your %s file.\n",arq);
 895                   exit(1);
 896                }
 897                while(strcmp(elap,"") == 0 && strlen(bufz) > 0)
 898                   if (getword(elap,sizeof(elap),bufz,' ')<0) {
 899                      printf("SARG: Maybe you have a broken record or garbage in your %s file.\n",arq);
 900                      exit(1);
 901                   }
 902                if(strlen(elap) < 1) continue;
 903                if (getword(ip,sizeof(ip),bufz,' ')<0){
 904                   printf("SARG: Maybe you have a broken record or garbage in your %s file.\n",arq);
 905                   exit(1);
 906                }
 907                if (getword(code,sizeof(code),bufz,' ')<0){
 908                   printf("SARG: Maybe you have a broken record or garbage in your %s file.\n",arq);
 909                   exit(1);
 910                }
 911                if (getword(tam,sizeof(tam),bufz,' ')<0){
 912                   printf("SARG: Maybe you have a broken record or garbage in your %s file.\n",arq);
 913                   exit(1);
 914                }
 915                if (getword(fun,sizeof(fun),bufz,' ')<0){
 916                   printf("SARG: Maybe you have a broken record or garbage in your %s file.\n",arq);
 917                   exit(1);
 918                }
 919                if (getword(url,sizeof(url),bufz,' ')<0){
 920                   printf("SARG: Maybe you have a broken record or garbage in your %s file.\n",arq);
 921                   exit(1);
 922                }
 923 //              while (strstr(bufz,"%20") != 0) {
 924 //                 getword(warea,bufz,' ');
 925 //                 strcat(url,warea);
 926 //              }
 927                if (getword(user,sizeof(user),bufz,' ')<0){
 928                   printf("SARG: Maybe you have a broken record or garbage in your %s file.\n",arq);
 929                   exit(1);
 930                }
 931                ilf=ILF_Squid;
 932                ilf_count[ilf]++;
 933             }
 934          }
 935          if (ilf==ILF_Sarg) {
 936             if (getword(data,sizeof(data),bufz,' ')<0){
 937                printf("SARG: Maybe you have a broken record or garbage in your %s file.\n",arq);
 938                exit(1);
 939             }
 940             if (getword(hora,sizeof(hora),bufz,' ')<0) {
 941                printf("SARG: Maybe you have a broken record or garbage in your %s file.\n",arq);
 942                exit(1);
 943             }
 944             if (getword(user,sizeof(user),bufz,' ')<0) {
 945                printf("SARG: Maybe you have a broken record or garbage in your %s file.\n",arq);
 946                exit(1);
 947             }
 948             if (getword(ip,sizeof(ip),bufz,' ')<0) {
 949                printf("SARG: Maybe you have a broken record or garbage in your %s file.\n",arq);
 950                exit(1);
 951             }
 952             if (getword(url,sizeof(url),bufz,' ')<0){
 953                printf("SARG: Maybe you have a broken record or garbage in your %s file.\n",arq);
 954                exit(1);
 955             }
 956             if (getword(tam,sizeof(tam),bufz,' ')<0){
 957                printf("SARG: Maybe you have a broken record or garbage in your %s file.\n",arq);
 958                exit(1);
 959             }
 960             if (getword(code,sizeof(code),bufz,' ')<0){
 961                printf("SARG: Maybe you have a broken record or garbage in your %s file.\n",arq);
 962                exit(1);
 963             }
 964             if (getword(elap,sizeof(elap),bufz,' ')<0){
 965                printf("SARG: Maybe you have a broken record or garbage in your %s file.\n",arq);
 966                exit(1);
 967             }
 968             if (getword(smartfilter,sizeof(smartfilter),bufz,' ')<0){
 969                printf("SARG: Maybe you have a broken record or garbage in your %s file.\n",arq);
 970                exit(1);
 971             }
 972          }
 973          if (ilf==ILF_Isa) {
 974             if (bufz[0] == '#') {
 975                int ncols,cols[ISACOL_Last];
 976
 977                fixendofline(bufz);
 978                // remove the #Fields: column at the beginning of the line
 979                if (getword_skip(1000,bufz,' ')<0){
 980                   printf("SARG: Maybe you have a broken record or garbage in your %s file.\n",arq);
 981                   exit(1);
 982                }
 983                for (ncols=0 ; ncols<ISACOL_Last ; ncols++) cols[ncols]=-1;
 984                ncols=0;
 985                while(bufz[0] != '\0') {
 986                   if (getword(val1,sizeof(val1),bufz,'\t')<0){
 987                      printf("SARG: Maybe you have a broken record or garbage in your %s file.\n",arq);
 988                      exit(1);
 989                   }
 990                   if(strcmp(val1,"c-ip") == 0) cols[ISACOL_Ip]=ncols;
 991                   if(strcmp(val1,"cs-username") == 0) cols[ISACOL_UserName]=ncols;
 992                   if(strcmp(val1,"date") == 0) cols[ISACOL_Date]=ncols;
 993                   if(strcmp(val1,"time") == 0) cols[ISACOL_Time]=ncols;
 994                   if(strcmp(val1,"time-taken") == 0) cols[ISACOL_TimeTaken]=ncols;
 995                   if(strcmp(val1,"sc-bytes") == 0) cols[ISACOL_Bytes]=ncols;
 996                   if(strcmp(val1,"cs-uri") == 0) cols[ISACOL_Uri]=ncols;
 997                   if(strcmp(val1,"sc-status") == 0) cols[ISACOL_Status]=ncols;
 998                   ncols++;
 999                }
1000                if (cols[ISACOL_Ip]>=0) {
1001                   isa_ncols=ncols;
1002                   for (isa_ncols=0 ; isa_ncols<ncols ; isa_ncols++)
1003                      isa_cols[isa_ncols]=cols[isa_ncols];
1004                }
1005                continue;
1006             }
1007             if (!isa_ncols) continue;
1008             for (x=0 ; x<isa_ncols ; x++) {
1009                if (getword(val1,sizeof(val1),bufz,'\t')<0) {
1010                   printf("SARG: Maybe you have a broken record or garbage in your %s file.\n",arq);
1011                   exit(1);
1012                }
1013                if (x==isa_cols[ISACOL_Ip]) {
1014                   if (strlen(val1)>=sizeof(ip)) {
1015                      printf("SARG: Maybe you have a broken IP in your %s file.\n",arq);
1016                      exit(1);
1017                   }
1018                   strcpy(ip,val1);
1019                } else if (x==isa_cols[ISACOL_UserName]) {
1020                   if (strlen(val1)>=sizeof(user)) {
1021                      printf("SARG: Maybe you have a broken user in your %s file.\n",arq);
1022                      exit(1);
1023                   }
1024                   strcpy(user,val1);
1025                } else if (x==isa_cols[ISACOL_Date]) {
1026                   if (strlen(val1)>=sizeof(data)) {
1027                      printf("SARG: Maybe you have a broken date in your %s file.\n",arq);
1028                      exit(1);
1029                   }
1030                   strcpy(data,val1);
1031                } else if (x==isa_cols[ISACOL_Time]) {
1032                   if (strlen(val1)>=sizeof(hora)) {
1033                      printf("SARG: Maybe you have a broken time in your %s file.\n",arq);
1034                      exit(1);
1035                   }
1036                   strcpy(hora,val1);
1037                } else if (x==isa_cols[ISACOL_TimeTaken]) {
1038                   if (strlen(val1)>=sizeof(elap)) {
1039                      printf("SARG: Maybe you have a broken download duration in your %s file.\n",arq);
1040                      exit(1);
1041                   }
1042                   strcpy(elap,val1);
1043                } else if (x==isa_cols[ISACOL_Bytes]) {
1044                   if (strlen(val1)>=sizeof(tam)) {
1045                      printf("SARG: Maybe you have a broken download size in your %s file.\n",arq);
1046                      exit(1);
1047                   }
1048                   strcpy(tam,val1);
1049                } else if (x==isa_cols[ISACOL_Uri]) {
1050                   if (strlen(val1)>=sizeof(url)) {
1051                      printf("SARG: Maybe you have a broken URL in your %s file.\n",arq);
1052                      exit(1);
1053                   }
1054                   strcpy(url,val1);
1055                } else if (x==isa_cols[ISACOL_Status]) {
1056                   if (strlen(val1)>=sizeof(code)) {
1057                      printf("SARG: Maybe you have a broken access code in your %s file.\n",arq);
1058                      exit(1);
1059                   }
1060                   strcpy(code,val1);
1061                }
1062             }
1063
1064             if(strcmp(code,"401") == 0 || strcmp(code,"403") == 0 || strcmp(code,"407") == 0) {
1065                sprintf(val1,"DENIED/%s",code);
1066                strcpy(code,val1);
1067             }
1068             if (getword(ano,sizeof(ano),data,'-')<0){
1069                printf("SARG: Maybe you have a broken record or garbage in your %s file.\n",arq);
1070                exit(1);
1071             }
1072             if (getword(mes,sizeof(mes),data,'-')<0){
1073                printf("SARG: Maybe you have a broken record or garbage in your %s file.\n",arq);
1074                exit(1);
1075             }
1076             strcpy(dia,data);
1077             conv_month_name(mes);
1078             sprintf(data," %s/%s/%s:%s",dia,mes,ano,hora);
1079          }
1080
1081          if(strlen(user) > 150) {
1082             if (debugm) printf("User too long: %s\n",user);
1083             totregsx++;
1084             continue;
1085          }
1086
1087          // include_users
1088          if(IncludeUsers[0] != '\0') {
1089             sprintf(val1,":%s:",user);
1090             if((str=(char *) strstr(IncludeUsers,val1)) == (char *) NULL )
1091                continue;
1092          }
1093
1094          if(vercode(code)) {
1095             if (debugm) printf("Excluded code: %s\n",code);
1096             totregsx++;
1097             continue;
1098          }
1099
1100          if(testvaliduserchar(user))
1101             continue;
1102
1103          if((str = strstr(user,"%20")) != NULL) {
1104             /*
1105             Why is it necessary to truncate the user name at the first space ?
1106
1107             The old code used to truncate the user name at the first % if a %20 was
1108             found anywhere in the string. That means the string could be truncated
1109             at the wrong place if another % occured before the %20. This new code should
1110             avoid that problem and only truncate at the space. There is no bug
1111             report indicating that anybody noticed this.
1112             */
1113             *str='\0';
1114          }
1115
1116          while(strstr(user,"%5c") != 0) {
1117             if (getword_multisep(w,sizeof(w),user,'%')<0){
1118                printf("SARG: Maybe you have a broken user in your %s file.\n",arq);
1119                exit(1);
1120             }
1121             strcpy(wuser,user+2);
1122             sprintf(user,"%s.%s",w,wuser);
1123          }
1124
1125          for(str=user; *str; str++) {
1126             if(*str=='.') dotinuser++;
1127             if(*str=='?' || *str=='.' || *str==':' || *str=='/' || *str=='\\')
1128                *str='_';
1129          }
1130
1131          strlow(user);
1132          if(strncmp(NtlmUserFormat,"user",4) == 0) {
1133             if(strchr(user,'_') != 0)
1134                if (getword_multisep(warea,sizeof(warea),user,'_')<0){
1135                   printf("SARG: Maybe you have a broken user in your %s file.\n",arq);
1136                   exit(1);
1137                }
1138             if(strchr(user,'+') != 0)
1139                if (getword_multisep(warea,sizeof(warea),user,'+')<0){
1140                   printf("SARG: Maybe you have a broken user in your %s file.\n",arq);
1141                   exit(1);
1142                }
1143          }
1144
1145          if(strstr(ReportType,"denied") != 0)
1146             strcpy(urly,url);
1147
1148          download_flag=is_download_suffix(url);
1149          if (download_flag) {
1150             strcpy(download_url,url);
1151             download_count++;
1152          }
1153
1154          if (strchr(url,'/')) {
1155             // remove any protocol:// at the beginning of the URL
1156             if (getword(w,sizeof(w),url,'/')<0){
1157                printf("SARG: Maybe you have a broken url in your %s file.\n",arq);
1158                exit(1);
1159             }
1160             if (getword(w,sizeof(w),url,'/')<0){
1161                printf("SARG: Maybe you have a broken url in your %s file.\n",arq);
1162                exit(1);
1163             }
1164             /*
1165             This code rejects any host without a directory. Now it
1166             doesn't look necessary so it is commented out.
1167             if (!strchr(url,'/')) {
1168                if (debugm) printf("URL without directory: %s\n",url);
1169                totregsx++;
1170                continue;
1171             }*/
1172          }
1173
1174          if(strcmp(LongUrl,"no") == 0) {
1175             char *endofhost=strchr(url,'/');
1176             if (endofhost)
1177                *endofhost='\0';
1178             if(strlen(url) > 512 && (endofhost=strchr(url,'%')) != NULL) {
1179                *endofhost='\0';
1180             }
1181          }
1182
1183          if(ilf==ILF_Squid) {
1184             tt=atoi(data);
1185             t=localtime(&tt);
1186
1187             strftime(tbuf2, sizeof(tbuf2), "%H%M", t);
1188             if(strncmp(df,"u",1) == 0)
1189                strftime(tbuf, sizeof(tbuf), "%Y%b%d", t);
1190             if(strncmp(df,"e",1) == 0)
1191                strftime(tbuf, sizeof(tbuf), "%d%b%Y", t);
1192             if(strncmp(df,"w",1) == 0) {
1193                strcpy(IndexTree,"file");
1194                strftime(tbuf, sizeof(tbuf), "%Y.%U", t);
1195             }
1196
1197             strftime(wdata, sizeof(wdata), "%Y%m%d", t);
1198             idata=atoi(wdata);
1199
1200             if(strncmp(df,"u",1)==0)
1201                strftime(dia, sizeof(dia), "%m/%d/%Y", t);
1202             else
1203                strftime(dia, sizeof(dia), "%d/%m/%Y", t);
1204             sprintf(hora,"%02d:%02d:%02d",t->tm_hour,t->tm_min,t->tm_sec);
1205          } else if(ilf==ILF_Common || ilf==ILF_Isa) {
1206             strcpy(wtemp,data+1);
1207             if (getword_multisep(data,sizeof(data),wtemp,':')<0){
1208                printf("SARG: Maybe you have a broken date in your %s file.\n",arq);
1209                exit(1);
1210             }
1211             if (getword_multisep(hora,sizeof(hora),wtemp,' ')<0){
1212                printf("SARG: Maybe you have a broken date in your %s file.\n",arq);
1213                exit(1);
1214             }
1215             if (getword_multisep(dia,sizeof(dia),data,'/')<0){
1216                printf("SARG: Maybe you have a broken date in your %s file.\n",arq);
1217                exit(1);
1218             }
1219             if (getword_multisep(mes,sizeof(mes),data,'/')<0){
1220                printf("SARG: Maybe you have a broken date in your %s file.\n",arq);
1221                exit(1);
1222             }
1223             if (getword_multisep(ano,sizeof(ano),data,'/')<0){
1224                printf("SARG: Maybe you have a broken date in your %s file.\n",arq);
1225                exit(1);
1226             }
1227
1228             if(strcmp(df,"u") == 0)
1229                snprintf(tbuf,sizeof(tbuf),"%s%s%s",ano,mes,dia);
1230             if(strcmp(df,"e") == 0)
1231                snprintf(tbuf,sizeof(tbuf),"%s%s%s",dia,mes,ano);
1232             builddia(dia,mes,ano,df,wdata);
1233             idata=atoi(wdata);
1234          } else if (ilf==ILF_Sarg) {
1235             if (getword_multisep(mes,sizeof(mes),data,'/')<0){
1236                printf("SARG: Maybe you have a broken date in your %s file.\n",arq);
1237                exit(1);
1238             }
1239             if (getword_multisep(dia,sizeof(dia),data,'/')<0){
1240                printf("SARG: Maybe you have a broken date in your %s file.\n",arq);
1241                exit(1);
1242             }
1243             if (getword_multisep(ano,sizeof(ano),data,0)<0){
1244                printf("SARG: Maybe you have a broken date in your %s file.\n",arq);
1245                exit(1);
1246             }
1247          }
1248
1249          if(debugm)
1250             printf("DATE=%s IDATA=%d DFROM=%d DUNTIL=%d\n",date,idata,dfrom,duntil);
1251
1252          l=1;
1253          if(l){
1254             if(addr[0] != '\0'){
1255                if(strcmp(addr,ip)==0)
1256                   l=1;else l=0;
1257             }
1258             if(fhost) {
1259 //              l=vhexclude(excludefile,ip);
1260                l=vhexclude(excludefile,url);
1261                if(!l) {
1262                   if (debugm) printf("Excluded site: %s\n",url);
1263                   totregsx++;
1264                }
1265             }
1266          }
1267
1268          if(l){
1269             if(date[0] != '\0'){
1270                if(idata >= dfrom && idata <= duntil)
1271                   l=1;else l=0;
1272             }
1273          }
1274          if(l){
1275             if(hm[0] != '\0') {
1276                strcpy(whm,hora);
1277                bzero(hmr,15);
1278                chm++;
1279                while(chm) {
1280                   if (getword_multisep(warea,sizeof(warea),whm,':')<0){
1281                      printf("SARG: Maybe you have a broken time in your %s file.\n",arq);
1282                      exit(1);
1283                   }
1284                   strncat(hmr,warea,2);
1285                   chm--;
1286                }
1287                strncat(hmr,whm,2);
1288
1289                if(atoi(hmr) >= atoi(hm) && atoi(hmr) <= atoi(hmf))
1290                   l=1;else l=0;
1291             }
1292          }
1293          if(l){
1294             if(site[0] != '\0'){
1295                if(strstr(url,site)!=0)
1296                   l=1;else l=0;
1297             }
1298          }
1299
1300          if(userip)
1301             strcpy(user,ip);
1302
1303          if(strcmp(user,"-") == 0 || strcmp(user," ") == 0 || strcmp(user,"") == 0) {
1304             if(strcmp(RecordsWithoutUser,"ip") == 0)
1305                strcpy(user,ip);
1306             if(strcmp(RecordsWithoutUser,"ignore") == 0)
1307                continue;
1308             if(strcmp(RecordsWithoutUser,"everybody") == 0)
1309                strcpy(user,"everybody");
1310          }
1311          if(us[0] != '\0'){
1312             if(strcmp(user,us)==0)
1313                l=1;
1314             else
1315                l=0;
1316          }
1317
1318          if(dotinuser) {
1319             subs(user,sizeof(user),"_",".");
1320             dotinuser=0;
1321          }
1322
1323          if(puser) {
1324             sprintf(wuser,":%s:",user);
1325             if(strstr(userfile, wuser) == 0)
1326                continue;
1327          }
1328
1329          if(l) {
1330             if(fuser) {
1331                l=vuexclude(excludeuser,user);
1332                if(!l) {
1333                   if (debugm) printf("Excluded user: %s\n",user);
1334                   totregsx++;
1335                }
1336             }
1337          }
1338
1339          if(l) {
1340             if(userip)
1341                fixip(user);
1342          }
1343
1344          if(l && max_elapsed) {
1345             if(atol(elap)>max_elapsed) {
1346                elap[0]='0';
1347                elap[1]='\0';
1348             }
1349          }
1350
1351          if(l) {
1352             if(strcmp(user,"-") !=0 && url[0] != '\0' && strcmp(user," ") !=0 && strcmp(user,"") !=0 && strcmp(user,":") !=0){
1353                if((str=(char *) strstr(bufz, "[SmartFilter:")) != (char *) NULL ) {
1354                   fixendofline(str);
1355                   sprintf(smartfilter,"\"%s\"",str+1);
1356                } else sprintf(smartfilter,"\"\"");
1357
1358                sprintf(bufz, "%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\n",dia,hora,user,ip,url,tam,code,elap,smartfilter);
1359
1360                if ( strcmp ( user , sz_Last_User ) != 0 ) {
1361                   if ( fp_Write_User )
1362                      fclose( fp_Write_User ) ;
1363                   sprintf (tmp3, "%s/sarg/%s.unsort", tmp, user);
1364
1365                   if ((fp_Write_User = MY_FOPEN (tmp3, "a")) == NULL) {
1366                      fprintf (stderr, "%s: (log) %s: %s - %s\n", argv[0], text[9], tmp3, strerror(errno));
1367                      exit (1);
1368                   }
1369                   strcpy( sz_Last_User , user ) ;
1370                }
1371                fputs (bufz, fp_Write_User);
1372
1373                if(strcmp(ParsedOutputLog, "no") != 0 && ilf!=ILF_Sarg)
1374                   fputs(bufz,fp_log);
1375
1376                totregsg++;
1377
1378                if(download_flag && strstr(code,"DENIED") == 0) {
1379                   ndownload = 1;
1380
1381                   if ( ! fp_Download_Unsort ) {
1382                      if ((fp_Download_Unsort = MY_FOPEN ( sz_Download_Unsort, "a")) == NULL) {
1383                         fprintf (stderr, "%s: (log) %s: %s - %s\n", argv[0], text[9], tmp3, strerror(errno));
1384                         exit (1);
1385                      }
1386                   }
1387                   fprintf(fp_Download_Unsort,"%s\t%s\t%s\t%s\t%s\n",dia,hora,user,ip,download_url);
1388                }
1389
1390                if(strstr(ReportType,"denied") != 0 || strstr(ReportType,"auth_failures") != 0) {
1391                   if(strstr(code,"DENIED/403") != 0) {
1392                      fprintf(fp_denied, "%s\t%s\t%s\t%s\t%s\n",dia,hora,user,ip,urly);
1393                      denied_count++;
1394                   }
1395                   if(strstr(code,"DENIED/401") != 0 || strstr(code,"DENIED/407") != 0) {
1396                      if(fp_authfail)
1397                         fprintf(fp_authfail, "%s\t%s\t%s\t%s\t%s\n",dia,hora,user,ip,urly);
1398                      authfail_count++;
1399                   }
1400                }
1401
1402                if((!totper || idata<mindate) && ilf!=ILF_Sarg){
1403                   totper++;
1404                   mindate=idata;
1405                   sprintf(period,"%s-",tbuf);
1406                   sprintf(per_hour,"%s-",tbuf2);
1407                   if(date[0] != '\0')
1408                      fixper(tbuf, period, cduntil);
1409                   if(debugz){
1410                      debugaz("tbuf",tbuf);
1411                      debugaz("period",period);
1412                   }
1413                }
1414             }
1415
1416             if(debugm){
1417                printf("IP=\t%s\n",ip);
1418                printf("USER=\t%s\n",user);
1419                printf("ELAP=\t%s\n",elap);
1420                printf("DATE=\t%s\n",dia);
1421                printf("TIME=\t%s\n",hora);
1422                printf("FUNC=\t%s\n",fun);
1423                printf("URL=\t%s\n",url);
1424                printf("CODE=\t%s\n",code);
1425                printf("LEN=\t%s\n",tam);
1426             }
1427          }
1428       }
1429       if (!from_stdin) {
1430          fclose(fp_in);
1431          if( bool_ShowReadStatistics )
1432             printf("SARG: Records in file: " OFFSET_STRING ", reading: %3.2f%%\n",recs1, (float) 100 );
1433       }
1434    }
1435
1436    if ( fp_Download_Unsort )
1437      fclose (fp_Download_Unsort);
1438
1439    if (fp_Write_User)
1440      fclose (fp_Write_User);
1441
1442    if(debug) {
1443       int totalcount=0;
1444
1445       for (ilf=0 ; ilf<ILF_Last ; ilf++) totalcount+=ilf_count[ilf];
1446
1447       debuga("   %s: %ld, %s: %ld, %s: %ld",text[10],totregsl,text[11],totregsg,text[68],totregsx);
1448
1449       if(ilf_count[ILF_Common]>0 && ilf_count[ILF_Squid]>0)
1450          debuga("%s",text[12]);
1451
1452       if(ilf_count[ILF_Common]>0 && ilf_count[ILF_Squid]==0)
1453          debuga("%s",text[13]);
1454
1455       if(ilf_count[ILF_Common]==0 && ilf_count[ILF_Squid]>0)
1456          debuga("%s",text[14]);
1457
1458       if(ilf_count[ILF_Sarg]>0)
1459          debuga("%s",text[124]);
1460
1461       if(totalcount==0) {
1462          if(!totregsg) {
1463             fprintf(stderr, "SARG: %s\n",text[16]);
1464             fprintf(stderr, "SARG: %s\n",text[21]);
1465          } else fprintf(stderr, "SARG: %s\n",text[15]);
1466          bzero(msg,sizeof(msg));
1467          if(fp_denied)
1468             fclose(fp_denied);
1469          if(fp_authfail)
1470             fclose(fp_authfail);
1471          free_excludecodes();
1472          if(userfile)
1473             free(userfile);
1474          if(excludefile)
1475             free(excludefile);
1476          if(excludeuser)
1477             free(excludeuser);
1478          unlink(tmp4);
1479          unlink(tmp6);
1480          unlink(tmp3);
1481          exit(0);
1482       }
1483    }
1484
1485    if(!totregsg){
1486       fprintf(stderr, "SARG: %s\n",text[16]);
1487       fprintf(stderr, "SARG: %s\n",text[21]);
1488 //      fclose(fp_ou);
1489       if(fp_denied)
1490          fclose(fp_denied);
1491       if(fp_authfail)
1492          fclose(fp_authfail);
1493       free_excludecodes();
1494       if(userfile)
1495          free(userfile);
1496       if(excludefile)
1497          free(excludefile);
1498       if(excludeuser)
1499          free(excludeuser);
1500       exit(0);
1501    }
1502
1503    if(date[0] == '\0' && ilf_count[ILF_Sarg]==0) {
1504       strcat(period,tbuf);
1505       strcat(per_hour,tbuf2);
1506    }
1507
1508    if(debugz){
1509       debugaz("data",dia);
1510       debugaz("tbuf",tbuf);
1511       debugaz("period",period);
1512    }
1513
1514    if(debug)
1515       debuga("%s: %s",text[17],period);
1516
1517 //   fclose(fp_ou);
1518    if(fp_denied)
1519       fclose(fp_denied);
1520    if(fp_authfail)
1521       fclose(fp_authfail);
1522
1523    if(fp_log != NULL) {
1524       fclose(fp_log);
1525       strcpy(val1,period);
1526       if (getword_multisep(val2,sizeof(val2),val1,'-')<0){
1527          printf("SARG: Maybe you have a broken date range definition.\n");
1528          exit(1);
1529       }
1530       if (getword_multisep(val3,sizeof(val3),per_hour,'-')<0){
1531          printf("SARG: Maybe you have a broken date range definition.\n");
1532          exit(1);
1533       }
1534       sprintf(val4,"%s/sarg-%s_%s-%s_%s.log",ParsedOutputLog,val2,val3,val1,per_hour);
1535       rename(arq_log,val4);
1536       strcpy(arq_log,val4);
1537
1538       if(strcmp(ParsedOutputLogCompress,"nocompress") != 0) {
1539          sprintf(val1,"\"%s\" \"%s\"",ParsedOutputLogCompress,arq_log);
1540          cstatus=system(val1);
1541          if (!WIFEXITED(cstatus) || WEXITSTATUS(cstatus)) {
1542             fprintf(stderr, "SARG: command return status %d\n",WEXITSTATUS(cstatus));
1543             fprintf(stderr, "SARG: command: %s\n",val1);
1544             exit(1);
1545          }
1546       }
1547
1548       if(debug)
1549          debuga("%s %s",text[123],arq_log);
1550    }
1551
1552    if(strstr(ReportType,"denied") != 0) {
1553       sprintf(csort,"sort -T \"%s\" -k 3,3 -k 5,5 -o \"%s\" \"%s\"",tmp,tmp5,tmp4);
1554       cstatus=system(csort);
1555       if (!WIFEXITED(cstatus) || WEXITSTATUS(cstatus)) {
1556          fprintf(stderr, "SARG: sort command return status %d\n",WEXITSTATUS(cstatus));
1557          fprintf(stderr, "SARG: sort command: %s\n",csort);
1558          exit(1);
1559       }
1560       unlink(tmp4);
1561    }
1562
1563    sort_users_log(tmp, debug);
1564
1565    if(DataFile[0] != '\0')
1566       data_file(tmp);
1567    else
1568       gerarel();
1569
1570    unlink(tmp2);
1571    if(strstr(ReportType,"denied") != 0)
1572       unlink(tmp5);
1573
1574    if(zip[0] != '\0' && strcmp(zip,"zcat") !=0) {
1575       recomp(arq, zip);
1576    }
1577 //   else  unlink(arq);
1578
1579    if(strcmp(tmp,"/tmp") != 0) {
1580       if (snprintf(tmp4,sizeof(tmp4),"rm -rf \"%s\"",tmp)>=sizeof(tmp4)) {
1581          fprintf(stderr,"SARG: Directory name too long: %s\n",tmp);
1582          exit(1);
1583       }
1584       cstatus=system(tmp4);
1585       if (!WIFEXITED(cstatus) || WEXITSTATUS(cstatus)) {
1586          fprintf(stderr, "SARG: command return status %d\n",WEXITSTATUS(cstatus));
1587          fprintf(stderr, "SARG: command: %s\n",tmp4);
1588          exit(1);
1589       }
1590    }
1591
1592    free_excludecodes();
1593    if(userfile)
1594       free(userfile);
1595    if(excludefile)
1596       free(excludefile);
1597    if(excludeuser)
1598       free(excludeuser);
1599
1600    if(debug)
1601       debuga("%s",text[21]);
1602
1603    exit(0);
1604
1605 }
1606
1607
1608 static void getusers(const char *pwdfile, int debug)
1609 {
1610
1611    FILE *fp_usr;
1612    char buf[255];
1613    char *str;
1614    long int nreg=0;
1615
1616    if(debug)
1617       debuga("%s: %s",text[60],pwdfile);
1618
1619    if ((fp_usr = fopen(pwdfile, "r")) == NULL) {
1620       fprintf(stderr, "SARG: (getusers) %s: %s - %s\n",text[45],pwdfile,strerror(errno));
1621       exit(1);
1622    }
1623
1624    fseek(fp_usr, 0, SEEK_END);
1625    nreg = ftell(fp_usr);
1626    if (nreg<0) {
1627       printf("SARG: Cannot get the size of file %s",pwdfile);
1628       exit(1);
1629    }
1630    nreg = nreg+5000;
1631    fseek(fp_usr, 0, SEEK_SET);
1632
1633    if((userfile=(char *) malloc(nreg))==NULL){
1634       fprintf(stderr, "SARG: %s (%ld):\n",text[59],nreg);
1635       exit(1);
1636    }
1637
1638    bzero(userfile,nreg);
1639    strcpy(userfile,":");
1640
1641    while(fgets(buf,sizeof(buf),fp_usr)!=NULL) {
1642       str=strchr(buf,':');
1643       if (!str) {
1644          printf("SARG: You have an invalid user in your %s file.\n",pwdfile);
1645          exit(1);
1646       }
1647       str[1]=0;
1648       strcat(userfile,buf);
1649    }
1650
1651    fclose(fp_usr);
1652
1653    return;
1654 }
1655
1656
1657 static void gethexclude(const char *hexfile, int debug)
1658 {
1659
1660    FILE *fp_ex;
1661    char buf[255];
1662    long int nreg=0;
1663
1664    if(debug)
1665       debuga("%s: %s",text[67],hexfile);
1666
1667    if ((fp_ex = fopen(hexfile, "r")) == NULL) {
1668       fprintf(stderr, "SARG: (gethexclude) %s: %s - %s\n",text[45],hexfile,strerror(errno));
1669       exit(1);
1670    }
1671
1672    fseek(fp_ex, 0, SEEK_END);
1673    nreg = ftell(fp_ex);
1674    if (nreg<0) {
1675       printf("SARG: Cannot get the size of file %s",hexfile);
1676       exit(1);
1677    }
1678    nreg += 11;
1679    fseek(fp_ex, 0, SEEK_SET);
1680
1681    if((excludefile=(char *) malloc(nreg))==NULL){
1682       fprintf(stderr, "SARG: %s (%ld):\n",text[59],nreg);
1683       exit(1);
1684    }
1685
1686    bzero(excludefile,nreg);
1687
1688    while(fgets(buf,sizeof(buf),fp_ex)!=NULL){
1689      if(strchr(buf,'#') != NULL)
1690         continue;
1691      fixendofline(buf);
1692      strcat(excludefile,buf);
1693      strcat(excludefile," ");
1694    }
1695
1696    strcat(excludefile,"*END* ");
1697
1698    fclose(fp_ex);
1699
1700    return;
1701 }
1702
1703
1704 static void getuexclude(const char *uexfile, int debug)
1705 {
1706
1707    FILE *fp_ex;
1708    char buf[255];
1709    long int nreg=0;
1710
1711    if(debug)
1712       debuga("%s: %s",text[67],uexfile);
1713
1714    if ((fp_ex = fopen(uexfile, "r")) == NULL) {
1715       fprintf(stderr, "SARG: (gethexclude) %s: %s - %s\n",text[45],uexfile,strerror(errno));
1716       exit(1);
1717    }
1718
1719    fseek(fp_ex, 0, SEEK_END);
1720    nreg = ftell(fp_ex);
1721    if (nreg<0) {
1722       printf("SARG: Cannot get the size of file %s",uexfile);
1723       exit(1);
1724    }
1725    nreg += 11;
1726    fseek(fp_ex, 0, SEEK_SET);
1727
1728    if((excludeuser=(char *) malloc(nreg))==NULL){
1729       fprintf(stderr, "SARG: %s (%ld):\n",text[59],nreg);
1730       exit(1);
1731    }
1732
1733    bzero(excludeuser,nreg);
1734
1735    while(fgets(buf,sizeof(buf),fp_ex)!=NULL){
1736      if(strchr(buf,'#') != NULL)
1737         continue;
1738      fixendofline(buf);
1739      strcat(excludeuser,buf);
1740      strcat(excludeuser," ");
1741    }
1742
1743    strcat(excludeuser,"*END* ");
1744
1745    fclose(fp_ex);
1746
1747    return;
1748 }