log.c

   1 /*
   2  * SARG Squid Analysis Report Generator      http://sarg.sourceforge.net
   3  *                                                            1998, 2010
   4  *
   5  * SARG donations:
   6  *      please look at http://sarg.sourceforge.net/donations.php
   7  * Support:
   8  *     http://sourceforge.net/projects/sarg/forums/forum/363374
   9  * ---------------------------------------------------------------------
  10  *
  11  *  This program is free software; you can redistribute it and/or modify
  12  *  it under the terms of the GNU General Public License as published by
  13  *  the Free Software Foundation; either version 2 of the License, or
  14  *  (at your option) any later version.
  15  *
  16  *  This program is distributed in the hope that it will be useful,
  17  *  but WITHOUT ANY WARRANTY; without even the implied warranty of
  18  *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  19  *  GNU General Public License for more details.
  20  *
  21  *  You should have received a copy of the GNU General Public License
  22  *  along with this program; if not, write to the Free Software
  23  *  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111, USA.
  24  *
  25  */
  26
  27 #include "include/conf.h"
  28 #include "include/defs.h"
  29
  30 #define REPORT_EVERY_X_LINES 5000
  31
  32 static char *userfile=NULL;
  33
  34 numlist weekdays = { { 0, 1, 2, 3, 4, 5, 6 }, 7 };
  35 numlist hours = { { 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12,
  36              13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23 }, 24 };
  37
  38 static void getusers(const char *pwdfile, int debug);
  39
  40 int main(int argc,char *argv[])
  41 {
  42    enum isa_col_id {
  43       ISACOL_Ip,
  44       ISACOL_UserName,
  45       ISACOL_Date,
  46       ISACOL_Time,
  47       ISACOL_TimeTaken,
  48       ISACOL_Bytes,
  49       ISACOL_Uri,
  50       ISACOL_Status,
  51       ISACOL_Last //last entry of the list !
  52    };
  53    enum InputLogFormat {
  54       ILF_Unknown,
  55       ILF_Squid,
  56       ILF_Common,
  57       ILF_Sarg,
  58       ILF_Isa,
  59       ILF_Last //last entry of the list !
  60    };
  61
  62    FILE *fp_in = NULL, *fp_denied=NULL, *fp_authfail=NULL, *fp_log=NULL;
  63
  64    char sz_Download_Unsort[ 20000 ] ;
  65    FILE * fp_Download_Unsort = NULL ;
  66    FILE * fp_Write_User = NULL ;
  67
  68    extern int optind;
  69    extern int optopt;
  70    extern char *optarg;
  71
  72    char data[255];
  73    char elap[255];
  74    char ip[MAXLEN];
  75    char tam[255];
  76    char fun[MAXLEN];
  77    char wuser[MAXLEN];
  78    char smartfilter[MAXLEN];
  79    char dia[128];
  80    char wdata[128];
  81    char mes[30];
  82    char ano[30];
  83    char hora[30];
  84    char wtemp[MAXLEN];
  85    char wtemp2[255];
  86    char date[255];
  87    char arq[255];
  88    char arq_log[255];
  89    char hm[15], hmf[15], hmr[15];
  90    int  chm=0;
  91    char uagent[MAXLEN];
  92    char hexclude[MAXLEN];
  93    char csort[MAXLEN];
  94    int cstatus;
  95    char tbuf[128];
  96    char tbuf2[128];
  97    char zip[20];
  98    char *str;
  99    char tmp2[MAXLEN];
 100    char start_hour[128];
 101    char end_hour[128];
 102    char *linebuf;
 103    char hostname[512];
 104    char *url;
 105    char *urly;
 106    enum InputLogFormat ilf;
 107    int ilf_count[ILF_Last];
 108    int  ch;
 109    int  x, l;
 110    int  errflg=0;
 111    int  puser=0;
 112    int  fhost=0;
 113    int  dns=0;
 114    int  fuser=0;
 115    int  idata=0;
 116    int  mindate=0;
 117    int  iarq=0;
 118    int  exstring=0;
 119    int isa_ncols=0,isa_cols[ISACOL_Last];
 120    int from_stdin;
 121    int blen;
 122    long totregsl=0;
 123    long totregsg=0;
 124    long totregsx=0;
 125    long totper=0;
 126    long int max_elapsed=0;
 127    time_t tt;
 128    struct tm *t;
 129    unsigned long recs1=0UL;
 130    unsigned long recs2=0UL;
 131    int OutputNonZero = REPORT_EVERY_X_LINES ;
 132    int download_flag=0;
 133    char *download_url;
 134    char sz_Last_User[MAXLEN]="";
 135    struct getwordstruct gwarea;
 136    struct longlinestruct line;
 137
 138    BgImage[0]='\0';
 139    LogoImage[0]='\0';
 140    LogoText[0]='\0';
 141    PasswdFile[0]='\0';
 142    OutputEmail[0]='\0';
 143    UserAgentLog[0]='\0';
 144    ExcludeHosts[0]='\0';
 145    ExcludeUsers[0]='\0';
 146    ConfigFile[0]='\0';
 147    code[0]='\0';
 148    LastLog=0;
 149    ReportType=0UL;
 150    UserTabFile[0]='\0';
 151    BlockIt[0]='\0';
 152    ExternalCSSFile[0]='\0';
 153    SquidGuardLogFormat[0]='\0';
 154    SquidGuardLogAlternate[0]='\0';
 155    for (ilf=0 ; ilf<ILF_Last ; ilf++) ilf_count[ilf]=0;
 156
 157    sprintf(ExcludeCodes,"%s/exclude_codes",SYSCONFDIR);
 158    strcpy(GraphDaysBytesBarColor,"orange");
 159    strcpy(BgColor,"#ffffff");
 160    strcpy(TxColor,"#000000");
 161    strcpy(TxBgColor,"lavender");
 162    strcpy(TiColor,"darkblue");
 163    strcpy(Width,"80");
 164    strcpy(Height,"45");
 165    strcpy(LogoTextColor,"#000000");
 166    strcpy(HeaderColor,"darkblue");
 167    strcpy(HeaderBgColor,"#dddddd");
 168    strcpy(LogoTextColor,"#006699");
 169    strcpy(FontSize,"9px");
 170    strcpy(TempDir,"/tmp");
 171    strcpy(OutputDir,"/var/www/html/squid-reports");
 172    Ip2Name=0;
 173    strcpy(DateFormat,"u");
 174    OverwriteReport=0;
 175    RemoveTempFiles=1;
 176    strcpy(ReplaceIndex,"index.html");
 177    Index=INDEX_YES;
 178    strcpy(RecordsWithoutUser,"ip");
 179    UseComma=0;
 180    strcpy(MailUtility,"mailx");
 181    TopSitesNum=100;
 182    TopUsersNum=0;
 183    UserIp=0;
 184    strcpy(TopuserSortField,"BYTES");
 185    strcpy(UserSortField,"BYTES");
 186    strcpy(TopuserSortOrder,"reverse");
 187    strcpy(UserSortOrder,"reverse");
 188    strcpy(TopsitesSortField,"CONNECT");
 189    strcpy(TopsitesSortType,"D");
 190    LongUrl=0;
 191    strcpy(language,"English");
 192    strcpy(FontFace,"Verdana,Tahoma,Arial");
 193    strcpy(datetimeby,"elap");
 194    strcpy(CharSet,"ISO-8859-1");
 195    Privacy=0;
 196    strcpy(PrivacyString,"***.***.***.***");
 197    strcpy(PrivacyStringColor,"blue");
 198    SuccessfulMsg=1;
 199    TopUserFields=TOPUSERFIELDS_NUM | TOPUSERFIELDS_DATE_TIME | TOPUSERFIELDS_USERID | TOPUSERFIELDS_CONNECT |
 200                  TOPUSERFIELDS_BYTES | TOPUSERFIELDS_SETYB | TOPUSERFIELDS_IN_CACHE_OUT |
 201                  TOPUSERFIELDS_USED_TIME | TOPUSERFIELDS_MILISEC | TOPUSERFIELDS_PTIME |
 202                  TOPUSERFIELDS_TOTAL | TOPUSERFIELDS_AVERAGE;
 203    UserReportFields=USERREPORTFIELDS_CONNECT | USERREPORTFIELDS_BYTES | USERREPORTFIELDS_SETYB |
 204                     USERREPORTFIELDS_IN_CACHE_OUT | USERREPORTFIELDS_USED_TIME | USERREPORTFIELDS_MILISEC |
 205                     USERREPORTFIELDS_PTIME | USERREPORTFIELDS_TOTAL | USERREPORTFIELDS_AVERAGE;
 206    strcpy(DataFileDelimiter,";");
 207    DataFileFields=DATA_FIELD_USER | DATA_FIELD_DATE | DATA_FIELD_TIME | DATA_FIELD_URL | DATA_FIELD_CONNECT |
 208                   DATA_FIELD_BYTES | DATA_FIELD_IN_CACHE | DATA_FIELD_OUT_CACHE | DATA_FIELD_ELAPSED;
 209    ShowReadStatistics=1;
 210    strcpy(IndexSortOrder,"D");
 211    ShowSargInfo=1;
 212    ShowSargLogo=1;
 213    strcpy(ParsedOutputLog,"no");
 214    strcpy(ParsedOutputLogCompress,"/bin/gzip -f");
 215    strcpy(DisplayedValues,"abbreviation");
 216    strcpy(HeaderFontSize,"9px");
 217    strcpy(TitleFontSize,"11px");
 218    strcpy(AuthUserFile,"/usr/local/sarg/passwd");
 219    strcpy(AuthName,"SARG, Restricted Access");
 220    strcpy(AuthType,"basic");
 221    strcpy(Require,"require user admin %u");
 222    set_download_suffix("7z,ace,arj,avi,bat,bin,bz2,bzip,cab,com,cpio,dll,doc,dot,exe,gz,iso,lha,lzh,mdb,mov,mp3,mpeg,mpg,mso,nrg,ogg,ppt,rar,rtf,shs,src,sys,tar,tgz,vcd,vob,wma,wmv,zip");
 223    Graphs=1;
 224 #if defined(FONTDIR)
 225    strcpy(GraphFont,FONTDIR"/DejaVuSans.ttf");
 226 #else
 227    GraphFont[0]='\0';
 228 #endif
 229    strcpy(Ulimit,"20000");
 230    strcpy(NtlmUserFormat,"domainname+username");
 231    IndexTree=INDEX_TREE_FILE;
 232    strcpy(RealtimeTypes,"GET,PUT,CONNECT");
 233    strcpy(RealtimeUnauthRec,"show");
 234    SquidguardIgnoreDate=0;
 235    DansguardianIgnoreDate=0;
 236    strcpy(DataFileUrl,"ip");
 237    strcpy(MaxElapsed,"28800000");
 238    BytesInSitesUsersReport=0;
 239    UserAuthentication=0;
 240    strcpy(LDAPHost,"127.0.0.1");
 241    LDAPPort=389;
 242    LDAPProtocolVersion=3;
 243    LDAPBindDN[0]='\0';
 244    LDAPBindPW[0]='\0';
 245    LDAPBaseSearch[0]='\0';
 246    strcpy(LDAPFilterSearch, "uid=%s");
 247    strcpy(LDAPTargetAttr, "cn");
 248
 249    dia[0]='\0';
 250    mes[0]='\0';
 251    ano[0]='\0';
 252    hora[0]='\0';
 253    tmp[0]='\0';
 254    tmp2[0]='\0';
 255    tmp3[0]='\0';
 256    wtemp[0]='\0';
 257    wtemp2[0]='\0';
 258    us[0]='\0';
 259    date[0]='\0';
 260    df[0]='\0';
 261    uagent[0]='\0';
 262    hexclude[0]='\0';
 263    addr[0]='\0';
 264    hm[0]='\0';
 265    hmf[0]='\0';
 266    site[0]='\0';
 267    outdir[0]='\0';
 268    elap[0]='\0';
 269    email[0]='\0';
 270    zip[0]='\0';
 271    UserInvalidChar[0]='\0';
 272    DataFile[0]='\0';
 273    SquidGuardConf[0]='\0';
 274    DansGuardianConf[0]='\0';
 275    start_hour[0]='\0';
 276    end_hour[0]='\0';
 277
 278    denied_count=0;
 279    download_count=0;
 280    authfail_count=0;
 281    dansguardian_count=0;
 282    squidguard_count=0;
 283    useragent_count=0;
 284    DeniedReportLimit=10;
 285    AuthfailReportLimit=10;
 286    DansGuardianReportLimit=10;
 287    SquidGuardReportLimit=10;
 288    DownloadReportLimit=50;
 289    UserReportLimit=0;
 290    debug=0;
 291    debugz=0;
 292    debugm=0;
 293    iprel=0;
 294    userip=0;
 295    color1=0;
 296    color2=0;
 297    color3=0;
 298    dotinuser=0;
 299    realt=0;
 300    realtime_refresh=3;
 301    realtime_access_log_lines=1000;
 302    cost=0.01;
 303    nocost=50000000;
 304    ndownload=0;
 305    squid24=0;
 306
 307    bzero(IncludeUsers, MAXLEN);
 308    bzero(ExcludeString, MAXLEN);
 309
 310 #ifdef HAVE_LOCALE_H
 311    setlocale(LC_TIME,"");
 312    setlocale (LC_MESSAGES, "");
 313 #endif
 314    bindtextdomain (PACKAGE_NAME, LOCALEDIR);
 315    textdomain (PACKAGE_NAME);
 316
 317    NAccessLog=0;
 318    for(x=0; x<MAXLOGS; x++)
 319       AccessLog[x][0]='\0';
 320    AccessLogFromCmdLine=0;
 321
 322    language_load(language);
 323    strcpy(Title,text[88]);
 324
 325    while((ch = getopt(argc, argv, "a:b:c:d:e:f:g:u:l:L:o:s:t:w:hijmnprvxyz")) != -1){
 326       switch(ch)
 327       {
 328          case 'a':
 329             strcpy(addr,optarg);
 330             break;
 331          case 'b':
 332             strcpy(uagent,optarg);
 333             break;
 334          case 'c':
 335             strcpy(hexclude,optarg);
 336             break;
 337          case 'd':
 338             strncpy(date,optarg,sizeof(date)-1);
 339             date[sizeof(date)-1]='\0';
 340             getword_start(&gwarea,optarg);
 341             if (getword(cdfrom,sizeof(cdfrom),&gwarea,'-')<0 || getword(cduntil,sizeof(cduntil),&gwarea,0)<0) {
 342                printf("SARG: Maybe you have a broken record or garbage in your date range.\n");
 343                exit(1);
 344             }
 345             date_from(date, cdfrom, cduntil);
 346             dfrom=atoi(cdfrom);
 347             duntil=atoi(cduntil);
 348             break;
 349          case 'e':
 350             strcpy(email,optarg);
 351             break;
 352          case 'f':
 353             strcpy(ConfigFile,optarg);
 354             break;
 355          case 'g':
 356             strcpy(df,optarg);
 357             break;
 358          case 'h':
 359             usage(argv[0]);
 360             exit(0);
 361             break;
 362          case 'i':
 363             iprel++;
 364             break;
 365          case 'l':
 366             if (NAccessLog>=MAXLOGS) {
 367                printf("SARG: Too many log files.\n");
 368                exit(1);
 369             }
 370             strcpy(AccessLog[NAccessLog],optarg);
 371             NAccessLog++;
 372             AccessLogFromCmdLine++;
 373             break;
 374          case 'L':
 375             strcpy(SquidGuardLogAlternate,optarg);
 376             break;
 377          case 'm':
 378             debugm++;
 379             break;
 380          case 'n':
 381             dns++;
 382             break;
 383          case 'o':
 384             strcpy(outdir,optarg);
 385             break;
 386          case 'p':
 387             userip++;
 388             break;
 389          case 'r':
 390             realt++;
 391             break;
 392          case 's':
 393             strcpy(site,optarg);
 394             break;
 395          case 't':
 396          {
 397             int h,m;
 398
 399             if(strstr(optarg,"-") == 0) {
 400               strcpy(hm,optarg);
 401               strcpy(hmf,optarg);
 402             } else {
 403                getword_start(&gwarea,optarg);
 404                if (getword(hm,sizeof(hm),&gwarea,'-')<0 || getword(hmf,sizeof(hmf),&gwarea,0)<0) {
 405                   fprintf(stderr,"SARG: Maybe you have a broken record or garbage in your time range.\n");
 406                   exit(1);
 407                }
 408             }
 409             if(sscanf(hm,"%d:%d",&h,&m)!=2) {
 410                fprintf(stderr,"SARG: time period must be MM or MM:SS. Exit.\n");
 411                exit(1);
 412             }
 413             sprintf(hm,"%02d%02d",h,m);
 414             if(sscanf(hmf,"%d:%d",&h,&m)!=2) {
 415                fprintf(stderr,"SARG: time period must be MM or MM:SS. Exit.\n");
 416                exit(1);
 417             }
 418             sprintf(hmf,"%02d%02d",h,m);
 419             break;
 420          }
 421          case 'u':
 422             strcpy(us,optarg);
 423             break;
 424          case 'v':
 425             version();
 426             break;
 427          case 'w':
 428             strcpy(tmp,optarg);
 429             break;
 430          case 'x':
 431             debug++;
 432             break;
 433          case 'y':
 434             langcode++;
 435             break;
 436          case 'z':
 437             debugz++;
 438             break;
 439          case ':':
 440             fprintf(stderr, "Option -%c require an argument\n",optopt);
 441             errflg++;
 442             break;
 443          case '?':
 444             usage(argv[0]);
 445             exit(1);
 446             break;
 447       }
 448
 449    }
 450
 451    if (errflg) {
 452       usage(argv[0]);
 453       exit(2);
 454    }
 455
 456    if(debug) debuga("Init");
 457
 458    if(ConfigFile[0] == '\0') sprintf(ConfigFile,"%s/sarg.conf",SYSCONFDIR);
 459    if(access(ConfigFile, R_OK) != 0) {
 460       debuga("Cannot open config file: %s - %s",ConfigFile,strerror(errno));
 461       exit(1);
 462    }
 463
 464    if(access(ConfigFile, R_OK) == 0)
 465       getconf();
 466
 467    if(UserIp) userip++;
 468
 469    if(dns) Ip2Name=1;
 470
 471    if(realt) {
 472       realtime();
 473       exit(0);
 474    }
 475
 476    if(IndexTree == INDEX_TREE_FILE)
 477       strcpy(ImageFile,"../images");
 478    else
 479       strcpy(ImageFile,"../../../images");
 480
 481    dataonly=0;
 482    if(DataFile[0] != '\0')
 483       dataonly++;
 484
 485    if(!NAccessLog) {
 486       strcpy(AccessLog[0],"/var/log/squid/access.log");
 487       NAccessLog++;
 488    }
 489
 490    if(strcmp(hexclude,"onvert") == 0 && strcmp(site,"plit") != 0) {
 491       convlog(AccessLog[0], df, dfrom, duntil);
 492       exit(0);
 493    }
 494
 495    if(strcmp(site,"plit") == 0) {
 496       splitlog(AccessLog[0], df, dfrom, duntil, hexclude);
 497       exit(0);
 498    }
 499
 500    load_excludecodes(ExcludeCodes);
 501
 502    if(access(PasswdFile, R_OK) == 0) {
 503       getusers(PasswdFile,debug);
 504       puser++;
 505    }
 506
 507    if(hexclude[0] == '\0')
 508       strcpy(hexclude,ExcludeHosts);
 509    if(hexclude[0] != '\0') {
 510       gethexclude(hexclude,debug);
 511       fhost++;
 512    }
 513
 514    if(ReportType == 0) {
 515       ReportType=REPORT_TYPE_TOPUSERS | REPORT_TYPE_TOPSITES | REPORT_TYPE_USERS_SITES |
 516                  REPORT_TYPE_SITES_USERS | REPORT_TYPE_DATE_TIME | REPORT_TYPE_DENIED |
 517                  REPORT_TYPE_AUTH_FAILURES | REPORT_TYPE_SITE_USER_TIME_DATE | REPORT_TYPE_DOWNLOADS;
 518    }
 519
 520    if(access(ExcludeUsers, R_OK) == 0) {
 521       getuexclude(ExcludeUsers,debug);
 522       fuser++;
 523    }
 524
 525    indexonly=0;
 526    if(fuser) {
 527       if(is_indexonly())
 528          indexonly++;
 529    }
 530    if(strcmp(ExcludeUsers,"indexonly") == 0) indexonly++;
 531    if(Index == INDEX_ONLY) indexonly++;
 532
 533    if(MaxElapsed[0] != '\0') max_elapsed=atol(MaxElapsed);
 534
 535    if(outdir[0] == '\0') strcpy(outdir,OutputDir);
 536    strcat(outdir,"/");
 537
 538    if(uagent[0] == '\0') strcpy(uagent,UserAgentLog);
 539
 540    if(tmp[0] == '\0') strcpy(tmp,TempDir);
 541    else strcpy(TempDir,tmp);
 542
 543    if(df[0] == '\0') strcpy(df,DateFormat);
 544    else strcpy(DateFormat,df);
 545
 546    if(df[0] == '\0') {
 547       strcpy(df,"u");
 548       strcpy(DateFormat,"u");
 549    }
 550
 551    if(email[0] == '\0' && OutputEmail[0] != '\0') strcpy(email,OutputEmail);
 552
 553    strcpy(tmp2,tmp);
 554
 555    if(email[0] != '\0') {
 556       sprintf(wtemp2,"%s/sarg",tmp2);
 557       my_mkdir(wtemp2);
 558       strcat(tmp2,"/sarg");
 559       strcpy(outdir,tmp2);
 560       strcat(outdir,"/");
 561    }
 562
 563    strcat(tmp2,"/sarg.log");
 564
 565    sprintf(tmp3,"%s/sarg",tmp);
 566    if(access(tmp3, R_OK) == 0) {
 567       unlinkdir(tmp3,1);
 568    }
 569    my_mkdir(tmp3);
 570    strcpy(tmp4,tmp3);
 571    strcpy(tmp5,tmp3);
 572    strcpy(tmp6,tmp3);
 573    strcat(tmp4,"/denied.log.unsort");
 574    strcat(tmp5,"/denied.log");
 575    strcat(tmp6,"/authfail.log.unsort");
 576
 577    if(debug) {
 578       fprintf(stderr, "SARG: %s:\nSARG:\n",text[22]);
 579       fprintf(stderr, "SARG: %35s (-a) = %s\n",text[23],addr);
 580       fprintf(stderr, "SARG: %35s (-b) = %s\n",text[71],uagent);
 581       fprintf(stderr, "SARG: %35s (-c) = %s\n",text[69],hexclude);
 582       fprintf(stderr, "SARG: %35s (-d) = %s\n",text[24],date);
 583       fprintf(stderr, "SARG: %35s (-e) = %s\n",text[41],email);
 584       fprintf(stderr, "SARG: %35s (-f) = %s\n",text[70],ConfigFile);
 585       if(strcmp(df,"e") == 0)
 586          fprintf(stderr, "SARG: %35s (-g) = %s (dd/mm/yyyy)\n",text[25],text[26]);
 587       if(strcmp(df,"u") == 0)
 588          fprintf(stderr, "SARG: %35s (-g) = %s (mm/dd/yyyy)\n",text[25],text[27]);
 589       if(strcmp(df,"w") == 0)
 590          fprintf(stderr, "SARG: %35s (-g) = %s (yyyy/ww)\n",text[25],text[85]);
 591       if(iprel)
 592          fprintf(stderr, "SARG: %35s (-i) = %s\n",text[28],text[1]);
 593       else
 594          fprintf(stderr, "SARG: %35s (-i) = %s\n",text[28],text[2]);
 595       for (iarq=0 ; iarq<NAccessLog ; iarq++)
 596          fprintf(stderr, "SARG: %35s (-l) = %s\n",text[37],AccessLog[iarq]);
 597       if(Ip2Name)
 598          fprintf(stderr, "SARG: %35s (-n) = %s\n",text[65],text[1]);
 599        else
 600          fprintf(stderr, "SARG: %35s (-n) = %s\n",text[65],text[2]);
 601       fprintf(stderr, "SARG: %35s (-o) = %s\n",text[38],outdir);
 602       if(UserIp)
 603          fprintf(stderr, "SARG: %35s (-p) = %s\n",text[29],text[1]);
 604        else
 605          fprintf(stderr, "SARG: %35s (-p) = %s\n",text[29],text[2]);
 606       fprintf(stderr, "SARG: %35s (-s) = %s\n",text[30],site);
 607       fprintf(stderr, "SARG: %35s (-t) = %s\n",text[31],hm);
 608       fprintf(stderr, "SARG: %35s (-u) = %s\n",text[32],us);
 609       fprintf(stderr, "SARG: %35s (-w) = %s\n",text[34],tmp);
 610       if(debug)
 611          fprintf(stderr, "SARG: %35s (-x) = %s\n",text[35],text[1]);
 612        else
 613          fprintf(stderr, "SARG: %35s (-x) = %s\n",text[35],text[2]);
 614       if(debugz)
 615          fprintf(stderr, "SARG: %35s (-z) = %s\n",text[36],text[1]);
 616        else
 617          fprintf(stderr, "SARG: %35s (-z) = %s\n",text[36],text[2]);
 618       fprintf(stderr, "SARG:\n");
 619    }
 620
 621    if(debugm) {
 622       printf("%s:\nSARG:\n",text[22]);
 623       printf("%35s (-a) = %s\n",text[23],addr);
 624       printf("%35s (-b) = %s\n",text[71],uagent);
 625       printf("%35s (-c) = %s\n",text[69],hexclude);
 626       printf("%35s (-d) = %s\n",text[24],date);
 627       printf("%35s (-e) = %s\n",text[41],email);
 628       printf("%35s (-f) = %s\n",text[70],ConfigFile);
 629       if(strcmp(df,"e") == 0)
 630          printf("%35s (-g) = %s (dd/mm/yyyy)\n",text[25],text[26]);
 631       if(strcmp(df,"u") == 0)
 632          printf("%35s (-g) = %s (mm/dd/yyyy)\n",text[25],text[27]);
 633       if(strcmp(df,"w") == 0)
 634          printf("%35s (-g) = %s (yyyy/ww)\n",text[25],text[85]);
 635       if(iprel)
 636          printf("%35s (-i) = %s\n",text[28],text[1]);
 637       else
 638          printf("%35s (-i) = %s\n",text[28],text[2]);
 639       for (iarq=0 ; iarq<NAccessLog ; iarq++)
 640          printf("%35s (-l) = %s\n",text[37],AccessLog[iarq]);
 641       if(Ip2Name)
 642          printf("%35s (-n) = %s\n",text[65],text[1]);
 643        else
 644          printf("%35s (-n) = %s\n",text[65],text[2]);
 645       printf("%35s (-o) = %s\n",text[38],outdir);
 646       if(UserIp)
 647          printf("%35s (-p) = %s\n",text[29],text[1]);
 648        else
 649          printf("%35s (-p) = %s\n",text[29],text[2]);
 650       printf("%35s (-s) = %s\n",text[30],site);
 651       printf("%35s (-t) = %s\n",text[31],hm);
 652       printf("%35s (-u) = %s\n",text[32],us);
 653       printf("%35s (-w) = %s\n",text[34],tmp);
 654       if(debug)
 655          printf("%35s (-x) = %s\n",text[35],text[1]);
 656        else
 657          printf("%35s (-x) = %s\n",text[35],text[2]);
 658       if(debugz)
 659          printf("%35s (-z) = %s\n",text[36],text[1]);
 660        else
 661          printf("%35s (-z) = %s\n",text[36],text[2]);
 662       printf("sarg %s: %s\n",text[73],VERSION);
 663       printf("Language=%s\n\n",text[3]);
 664    }
 665
 666    if(debug)
 667       debuga("sarg %s: %s",text[73],VERSION);
 668
 669 #ifdef HAVE_RLIM_T
 670    if (Ulimit[0] != '\0') {
 671       struct rlimit rl;
 672       long l1, l2;
 673       int rc=0;
 674
 675 #if defined(RLIMIT_NOFILE)
 676       getrlimit (RLIMIT_NOFILE, &rl);
 677 #elif defined(RLIMIT_OFILE)
 678       getrlimit (RLIMIT_OFILE, &rl);
 679 #else
 680 #warning "No rlimit resource for the number of open files"
 681 #endif
 682       l1 = rl.rlim_cur;
 683       l2 = rl.rlim_max;
 684
 685       rl.rlim_cur = atol(Ulimit);
 686       rl.rlim_max = atol(Ulimit);
 687 #if defined(RLIMIT_NOFILE)
 688       rc=setrlimit (RLIMIT_NOFILE, &rl);
 689 #elif defined(RLIMIT_OFILE)
 690       rc=setrlimit (RLIMIT_OFILE, &rl);
 691 #else
 692 #warning "No rlimit resource for the number of open files"
 693 #endif
 694       if(rc == -1) {
 695             debuga("setrlimit error - %s\n",strerror(errno));
 696       }
 697
 698       if(debug)
 699          debuga("Maximum file descriptor: cur=%ld max=%ld, changed to cur="RLIM_STRING" max="RLIM_STRING,l1,l2,rl.rlim_cur,rl.rlim_max);
 700    }
 701 #endif
 702
 703    init_usertab(UserTabFile);
 704
 705    if (longline_prepare(&line)<0) {
 706       debuga(_("Not enough memory to read a log file"));
 707       exit(1);
 708    }
 709
 710    sprintf ( sz_Download_Unsort , "%s/sarg/download.unsort", tmp);
 711
 712    if((ReportType & REPORT_TYPE_DENIED) != 0) {
 713       if((fp_denied=MY_FOPEN(tmp4,"w"))==NULL) {
 714          fprintf(stderr, "%s: (log) %s: %s - %s\n",argv[0],text[45],tmp4,strerror(errno));
 715          exit(1);
 716       }
 717    }
 718
 719    if(DataFile[0]=='\0') {
 720       if((ReportType & REPORT_TYPE_DENIED) != 0 || (ReportType & REPORT_TYPE_AUTH_FAILURES) != 0) {
 721          if((fp_authfail=MY_FOPEN(tmp6,"w"))==NULL) {
 722             fprintf(stderr, "%s: (log) %s: %s - %s\n",argv[0],text[45],tmp6,strerror(errno));
 723             exit(1);
 724          }
 725       }
 726    }
 727
 728    for (iarq=0 ; iarq<NAccessLog ; iarq++) {
 729       strcpy(arq,AccessLog[iarq]);
 730
 731       strcpy(arqtt,arq);
 732
 733       if(strcmp(arq,"-")==0) {
 734          if(debug)
 735             debuga("%s: %s",text[7],"stdin");
 736          fp_in=stdin;
 737          from_stdin=1;
 738       } else {
 739          decomp(arq,zip,tmp);
 740          if(debug)
 741             debuga("%s: %s",text[7],arq);
 742          if((fp_in=MY_FOPEN(arq,"r"))==NULL) {
 743             fprintf(stderr, "%s: (log) %s: %s - %s\n",argv[0],text[8],arq,strerror(errno));
 744             exit(1);
 745          }
 746          from_stdin=0;
 747       }
 748       ilf=ILF_Unknown;
 749       download_flag=0;
 750       // pre-read the file only if we have to show stats
 751       if(ShowReadStatistics && !from_stdin) {
 752          size_t nread,i;
 753          int skipcr=0;
 754
 755          recs1=0UL;
 756          recs2=0UL;
 757
 758          while ((nread=fread(line.buffer,1,line.size,fp_in))>0) {
 759             for (i=0 ; i<nread ; i++)
 760                if (skipcr) {
 761                   if (line.buffer[i]!='\n' && line.buffer[i]!='\r') {
 762                      skipcr=0;
 763                   }
 764                } else {
 765                   if (line.buffer[i]=='\n' || line.buffer[i]=='\r') {
 766                      skipcr=1;
 767                      recs1++;
 768                   }
 769                }
 770          }
 771          rewind(fp_in);
 772          printf("SARG: Records in file: %lu, reading: %3.2f%%\r",recs1,(float) 0);
 773          fflush( stdout ) ;
 774       }
 775
 776       line.start=0;
 777       line.end=0;
 778       line.length=0;
 779
 780       while ((linebuf=longline_read(fp_in,&line))!=NULL) {
 781          blen=strlen(linebuf);
 782
 783          if (ilf==ILF_Unknown) {
 784             if(strncmp(linebuf,"#Software: Mic",14) == 0) {
 785                fixendofline(linebuf);
 786                if (debug)
 787                   debuga("%s: %s",text[143],linebuf);
 788                ilf=ILF_Isa;
 789                ilf_count[ilf]++;
 790                continue;
 791             }
 792
 793             if(strncmp(linebuf,"*** SARG Log ***",16) == 0) {
 794                getword_start(&gwarea,arqtt);
 795                if (getword_skip(2000,&gwarea,'-')<0 || getword(val2,sizeof(val2),&gwarea,'_')<0 ||
 796                    getword_skip(10,&gwarea,'-')<0 || getword(val3,sizeof(val3),&gwarea,'_')<0) {
 797                   printf("SARG: The name of the file is invalid: %s\n",arq);
 798                   exit(1);
 799                }
 800                sprintf(period,"%s-%s",val2,val3);
 801                ilf=ILF_Sarg;
 802                ilf_count[ilf]++;
 803                continue;
 804             }
 805          }
 806
 807          if(!fp_log && strcmp(ParsedOutputLog, "no") != 0 && ilf!=ILF_Sarg) {
 808             if(access(ParsedOutputLog,R_OK) != 0) {
 809                my_mkdir(ParsedOutputLog);
 810             }
 811             sprintf(arq_log,"%s/sarg_temp.log",ParsedOutputLog);
 812             if((fp_log=MY_FOPEN(arq_log,"w"))==NULL) {
 813                fprintf(stderr, "%s: (log) %s: %s - %s\n",argv[0],text[8],arq_log,strerror(errno));
 814                exit(1);
 815             }
 816             fputs("*** SARG Log ***\n",fp_log);
 817          }
 818
 819          recs2++;
 820          if( ShowReadStatistics && !from_stdin && ! --OutputNonZero) {
 821            perc = recs2 * 100 ;
 822            perc = perc / recs1 ;
 823            printf("SARG: Records in file: %lu, reading: %3.2f%%\r",recs1,perc);
 824            fflush (stdout);
 825            OutputNonZero = REPORT_EVERY_X_LINES ;
 826          }
 827          if(blen < 58) continue;
 828          if(strstr(linebuf,"HTTP/0.0") != 0) continue;
 829          if(strstr(linebuf,"logfile turned over") != 0) continue;
 830          if(linebuf[0] == ' ') continue;
 831
 832          // Record only hours usage which is required
 833          tt = (time_t) strtoul( linebuf, NULL, 10 );
 834          t = localtime( &tt );
 835
 836          if( bsearch( &( t -> tm_wday ), weekdays.list, weekdays.len,
 837                                           sizeof( int ), compar ) == NULL )
 838             continue;
 839
 840          if( bsearch( &( t -> tm_hour ), hours.list, hours.len,
 841                                           sizeof( int ), compar ) == NULL )
 842             continue;
 843
 844          // exclude_string
 845          exstring=0;
 846          if(ExcludeString[0] != '\0') {
 847             getword_start(&gwarea,ExcludeString);
 848             while(strchr(gwarea.current,':') != 0) {
 849                if (getword_multisep(val1,sizeof(val1),&gwarea,':')<0) {
 850                   printf("SARG: Maybe you have a broken record or garbage in your exclusion string.\n");
 851                   exit(1);
 852                }
 853                if((str=(char *) strstr(linebuf,val1)) != (char *) NULL )
 854                   exstring++;
 855             }
 856             if((str=(char *) strstr(linebuf,gwarea.current)) != (char *) NULL )
 857                   exstring++;
 858          }
 859          if(exstring) continue;
 860
 861          if ((str = strchr(linebuf, '\n')) != NULL)
 862             *str = '\0';          /* strip \n */
 863
 864          totregsl++;
 865          if(debugm)
 866             printf("BUF=%s\n",linebuf);
 867
 868          if (ilf==ILF_Squid || ilf==ILF_Common || ilf==ILF_Unknown) {
 869             getword_start(&gwarea,linebuf);
 870             if (getword(data,sizeof(data),&gwarea,' ')<0) {
 871                printf("SARG: Maybe you have a broken time in your access.log file.\n");
 872                exit(1);
 873             }
 874             if((str=(char *) strchr(data, '.')) != (char *) NULL ) {
 875                if((str=(char *) strchr(str+1, '.')) != (char *) NULL ) {
 876                   strcpy(ip,data);
 877                   strcpy(elap,"0");
 878                   if(squid24) {
 879                      if (getword(user,sizeof(user),&gwarea,' ')<0 || getword_skip(255,&gwarea,' ')<0) {
 880                         printf("SARG: Maybe you have a broken record or garbage in your %s file.\n",arq);
 881                         exit(1);
 882                      }
 883                   } else {
 884                      if (getword_skip(255,&gwarea,' ')<0 || getword(user,sizeof(user),&gwarea,' ')<0) {
 885                         printf("SARG: Maybe you have a broken record or garbage in your %s file.\n",arq);
 886                         exit(1);
 887                      }
 888                   }
 889                   if (getword(data,sizeof(data),&gwarea,']')<0 || getword_skip(MAXLEN,&gwarea,'"')<0 ||
 890                       getword(fun,sizeof(fun),&gwarea,' ')<0) {
 891                      printf("SARG: Maybe you have a broken record or garbage in your %s file.\n",arq);
 892                      exit(1);
 893                   }
 894                   if (getword_ptr(linebuf,&url,&gwarea,' ')<0) {
 895                      debuga(_("Maybe you have a broken url in your %s file"),arq);
 896                      exit(1);
 897                   }
 898                   if (getword_skip(MAXLEN,&gwarea,' ')<0 || getword(code2,sizeof(code2),&gwarea,' ')<0 ||
 899                       getword(tam,sizeof(tam),&gwarea,' ')<0) {
 900                      printf("SARG: Maybe you have a broken record or garbage in your %s file.\n",arq);
 901                      exit(1);
 902                   }
 903                   if((str=(char *) strchr(gwarea.current, ' ')) != (char *) NULL ) {
 904                      if (getword(code,sizeof(code),&gwarea,' ')<0) {
 905                         printf("SARG: Maybe you have a broken record or garbage in your %s file.\n",arq);
 906                         exit(1);
 907                      }
 908                   } else {
 909                      if (getword(code,sizeof(code),&gwarea,'\0')<0) {
 910                         printf("SARG: Maybe you have a broken record or garbage in your %s file.\n",arq);
 911                         exit(1);
 912                      }
 913                   }
 914
 915                   if ((str = strchr(code, ':')) != NULL)
 916                      *str = '/';
 917
 918                   if(strcmp(tam,"\0") == 0)
 919                      strcpy(tam,"0");
 920
 921                   ilf=ILF_Common;
 922                   ilf_count[ilf]++;
 923                }
 924             }
 925
 926             if(ilf==ILF_Unknown || ilf==ILF_Squid) {
 927                if (getword(elap,sizeof(elap),&gwarea,' ')<0) {
 928                   printf("SARG: Maybe you have a broken elapsed time in your %s file.\n",arq);
 929                   exit(1);
 930                }
 931                while(strcmp(elap,"") == 0 && gwarea.current[0] != '\0')
 932                   if (getword(elap,sizeof(elap),&gwarea,' ')<0) {
 933                      printf("SARG: Maybe you have a broken elapsed time in your %s file.\n",arq);
 934                      exit(1);
 935                   }
 936                if(strlen(elap) < 1) continue;
 937                if (getword(ip,sizeof(ip),&gwarea,' ')<0){
 938                   printf("SARG: Maybe you have a broken client IP address in your %s file.\n",arq);
 939                   exit(1);
 940                }
 941                if (getword(code,sizeof(code),&gwarea,' ')<0){
 942                   printf("SARG: Maybe you have a broken result code in your %s file.\n",arq);
 943                   exit(1);
 944                }
 945                if (getword(tam,sizeof(tam),&gwarea,' ')<0){
 946                   printf("SARG: Maybe you have a broken amount of data in your %s file.\n",arq);
 947                   exit(1);
 948                }
 949                if (getword(fun,sizeof(fun),&gwarea,' ')<0){
 950                   printf("SARG: Maybe you have a broken request method in your %s file.\n",arq);
 951                   exit(1);
 952                }
 953                if (getword_ptr(linebuf,&url,&gwarea,' ')<0){
 954                   debuga(_("Maybe you have a broken url in your %s file"),arq);
 955                   exit(1);
 956                }
 957                if (getword(user,sizeof(user),&gwarea,' ')<0){
 958                   printf("SARG: Maybe you have a broken user ID in your %s file.\n",arq);
 959                   exit(1);
 960                }
 961                ilf=ILF_Squid;
 962                ilf_count[ilf]++;
 963             }
 964          }
 965          if (ilf==ILF_Sarg) {
 966             getword_start(&gwarea,linebuf);
 967             if (getword(data,sizeof(data),&gwarea,'\t')<0){
 968                printf("SARG: Maybe you have a broken record or garbage in your %s file.\n",arq);
 969                exit(1);
 970             }
 971             if (getword(hora,sizeof(hora),&gwarea,'\t')<0) {
 972                printf("SARG: Maybe you have a broken record or garbage in your %s file.\n",arq);
 973                exit(1);
 974             }
 975             if (getword(user,sizeof(user),&gwarea,'\t')<0) {
 976                printf("SARG: Maybe you have a broken record or garbage in your %s file.\n",arq);
 977                exit(1);
 978             }
 979             if (getword(ip,sizeof(ip),&gwarea,'\t')<0) {
 980                printf("SARG: Maybe you have a broken record or garbage in your %s file.\n",arq);
 981                exit(1);
 982             }
 983             if (getword_ptr(linebuf,&url,&gwarea,'\t')<0){
 984                debuga(_("Maybe you have a broken record or garbage in your %s file"),arq);
 985                exit(1);
 986             }
 987             if (getword(tam,sizeof(tam),&gwarea,'\t')<0){
 988                printf("SARG: Maybe you have a broken record or garbage in your %s file.\n",arq);
 989                exit(1);
 990             }
 991             if (getword(code,sizeof(code),&gwarea,'\t')<0){
 992                printf("SARG: Maybe you have a broken record or garbage in your %s file.\n",arq);
 993                exit(1);
 994             }
 995             if (getword(elap,sizeof(elap),&gwarea,'\t')<0){
 996                printf("SARG: Maybe you have a broken record or garbage in your %s file.\n",arq);
 997                exit(1);
 998             }
 999             if (getword(smartfilter,sizeof(smartfilter),&gwarea,'\0')<0){
1000                printf("SARG: Maybe you have a broken record or garbage in your %s file.\n",arq);
1001                exit(1);
1002             }
1003          }
1004          if (ilf==ILF_Isa) {
1005             if (linebuf[0] == '#') {
1006                int ncols,cols[ISACOL_Last];
1007
1008                fixendofline(linebuf);
1009                getword_start(&gwarea,linebuf);
1010                // remove the #Fields: column at the beginning of the line
1011                if (getword_skip(1000,&gwarea,' ')<0){
1012                   printf("SARG: Maybe you have a broken record or garbage in your %s file.\n",arq);
1013                   exit(1);
1014                }
1015                for (ncols=0 ; ncols<ISACOL_Last ; ncols++) cols[ncols]=-1;
1016                ncols=0;
1017                while(gwarea.current[0] != '\0') {
1018                   if (getword(val1,sizeof(val1),&gwarea,'\t')<0){
1019                      printf("SARG: Maybe you have a broken record or garbage in your %s file.\n",arq);
1020                      exit(1);
1021                   }
1022                   if(strcmp(val1,"c-ip") == 0) cols[ISACOL_Ip]=ncols;
1023                   if(strcmp(val1,"cs-username") == 0) cols[ISACOL_UserName]=ncols;
1024                   if(strcmp(val1,"date") == 0) cols[ISACOL_Date]=ncols;
1025                   if(strcmp(val1,"time") == 0) cols[ISACOL_Time]=ncols;
1026                   if(strcmp(val1,"time-taken") == 0) cols[ISACOL_TimeTaken]=ncols;
1027                   if(strcmp(val1,"sc-bytes") == 0) cols[ISACOL_Bytes]=ncols;
1028                   if(strcmp(val1,"cs-uri") == 0) cols[ISACOL_Uri]=ncols;
1029                   if(strcmp(val1,"sc-status") == 0) cols[ISACOL_Status]=ncols;
1030                   ncols++;
1031                }
1032                if (cols[ISACOL_Ip]>=0) {
1033                   isa_ncols=ncols;
1034                   for (isa_ncols=0 ; isa_ncols<ncols ; isa_ncols++)
1035                      isa_cols[isa_ncols]=cols[isa_ncols];
1036                }
1037                continue;
1038             }
1039             if (!isa_ncols) continue;
1040             getword_start(&gwarea,linebuf);
1041             for (x=0 ; x<isa_ncols ; x++) {
1042                if (getword_ptr(linebuf,&str,&gwarea,'\t')<0) {
1043                   debuga(_("Maybe you have a broken record or garbage in your %s file"),arq);
1044                   exit(1);
1045                }
1046                if (x==isa_cols[ISACOL_Ip]) {
1047                   if (strlen(str)>=sizeof(ip)) {
1048                      printf("SARG: Maybe you have a broken IP in your %s file.\n",arq);
1049                      exit(1);
1050                   }
1051                   strcpy(ip,str);
1052                } else if (x==isa_cols[ISACOL_UserName]) {
1053                   if (strlen(str)>=sizeof(user)) {
1054                      printf("SARG: Maybe you have a broken user in your %s file.\n",arq);
1055                      exit(1);
1056                   }
1057                   strcpy(user,str);
1058                } else if (x==isa_cols[ISACOL_Date]) {
1059                   if (strlen(str)>=sizeof(data)) {
1060                      printf("SARG: Maybe you have a broken date in your %s file.\n",arq);
1061                      exit(1);
1062                   }
1063                   strcpy(data,str);
1064                } else if (x==isa_cols[ISACOL_Time]) {
1065                   if (strlen(str)>=sizeof(hora)) {
1066                      printf("SARG: Maybe you have a broken time in your %s file.\n",arq);
1067                      exit(1);
1068                   }
1069                   strcpy(hora,str);
1070                } else if (x==isa_cols[ISACOL_TimeTaken]) {
1071                   if (strlen(str)>=sizeof(elap)) {
1072                      printf("SARG: Maybe you have a broken download duration in your %s file.\n",arq);
1073                      exit(1);
1074                   }
1075                   strcpy(elap,str);
1076                } else if (x==isa_cols[ISACOL_Bytes]) {
1077                   if (strlen(str)>=sizeof(tam)) {
1078                      printf("SARG: Maybe you have a broken download size in your %s file.\n",arq);
1079                      exit(1);
1080                   }
1081                   strcpy(tam,str);
1082                } else if (x==isa_cols[ISACOL_Uri]) {
1083                   url=str;
1084                } else if (x==isa_cols[ISACOL_Status]) {
1085                   if (strlen(str)>=sizeof(code)) {
1086                      printf("SARG: Maybe you have a broken access code in your %s file.\n",arq);
1087                      exit(1);
1088                   }
1089                   strcpy(code,str);
1090                }
1091             }
1092
1093             if(strcmp(code,"401") == 0 || strcmp(code,"403") == 0 || strcmp(code,"407") == 0) {
1094                sprintf(val1,"DENIED/%s",code);
1095                strcpy(code,val1);
1096             }
1097             getword_start(&gwarea,data);
1098             if (getword(ano,sizeof(ano),&gwarea,'-')<0){
1099                printf("SARG: Maybe you have a broken record or garbage in your %s file.\n",arq);
1100                exit(1);
1101             }
1102             if (getword(mes,sizeof(mes),&gwarea,'-')<0){
1103                printf("SARG: Maybe you have a broken record or garbage in your %s file.\n",arq);
1104                exit(1);
1105             }
1106             if (getword(dia,sizeof(dia),&gwarea,'\0')<0){
1107                printf("SARG: Maybe you have a broken record or garbage in your %s file.\n",arq);
1108                exit(1);
1109             }
1110             conv_month_name(mes);
1111             sprintf(data," %s/%s/%s:%s",dia,mes,ano,hora);
1112          }
1113
1114          if(strlen(user) > 150) {
1115             if (debugm) printf("User too long: %s\n",user);
1116             totregsx++;
1117             continue;
1118          }
1119
1120          // include_users
1121          if(IncludeUsers[0] != '\0') {
1122             sprintf(val1,":%s:",user);
1123             if((str=(char *) strstr(IncludeUsers,val1)) == (char *) NULL )
1124                continue;
1125          }
1126
1127          if(vercode(code)) {
1128             if (debugm) printf("Excluded code: %s\n",code);
1129             totregsx++;
1130             continue;
1131          }
1132
1133          if(testvaliduserchar(user))
1134             continue;
1135
1136 #if 0
1137          if((str = strstr(user,"%20")) != NULL) {
1138             /*
1139             This is a patch introduced to solve bug #1624251 reported at sourceforge but
1140             the side effect is to truncate the name at the first space and merge the reports
1141             of people whose name is identical up to the first space.
1142
1143             The old code used to truncate the user name at the first % if a %20 was
1144             found anywhere in the string. That means the string could be truncated
1145             at the wrong place if another % occured before the %20. This new code should
1146             avoid that problem and only truncate at the space. There is no bug
1147             report indicating that anybody noticed this.
1148             */
1149             *str='\0';
1150          }
1151
1152          /*
1153          Code prior to 2.2.7 used to replace any %xx by a dot as long as a %5c was
1154          found in the user name.
1155          */
1156          while((str = strstr(user,"%5c")) != NULL) {
1157             *str='.';
1158             for (x=3 ; str[x] ; x++) str[x-2]=str[x];
1159          }
1160 #endif
1161
1162          for(str=user; *str; str++) {
1163             if(*str=='.') dotinuser++;
1164             if(*str=='?' || *str=='.' || *str==':' || *str=='/' || *str=='\\' || *str=='\'' || *str=='$' || *str=='@' ||
1165                *str=='\"' || *str=='*')
1166                *str='_';
1167          }
1168
1169          strlow(user);
1170          if(strncmp(NtlmUserFormat,"user",4) == 0) {
1171             if((str = strchr(user,'_')) != 0) {
1172                strcpy(warea,str+1);
1173                strcpy(user,warea);
1174             }
1175             if((str = strchr(user,'+')) != 0) {
1176                strcpy(warea,str+1);
1177                strcpy(user,warea);
1178             }
1179          }
1180
1181          urly=url;
1182
1183          if(ilf!=ILF_Sarg) {
1184             /*
1185             The full URL is not saved in sarg log. There is no point in testing the URL to detect
1186             a downloaded file.
1187             */
1188             download_flag=is_download_suffix(url);
1189             if (download_flag) {
1190                download_url=url;
1191                download_count++;
1192             }
1193          }
1194
1195          // remove any protocol:// at the beginning of the URL
1196          if ((str = strchr(url,'/')) != NULL && str[1] == '/') {
1197             int i;
1198
1199             str+=2;
1200             for (i=0 ; str[i] ; i++)
1201                url[i]=str[i];
1202             url[i]='\0';
1203          }
1204
1205          if(!LongUrl) {
1206             int i;
1207
1208             for (i=0 ; i<sizeof(hostname)-1 && url[i] && url[i]!='/' ; i++)
1209                hostname[i]=url[i];
1210             hostname[i]='\0';
1211             url=hostname;
1212          }
1213
1214          if(ilf==ILF_Squid) {
1215             tt=atoi(data);
1216             t=localtime(&tt);
1217
1218             strftime(tbuf2, sizeof(tbuf2), "%H%M", t);
1219             sprintf(mes,"%d",t->tm_mon+1);
1220             conv_month_name(mes);
1221             if(strncmp(df,"u",1) == 0)
1222                sprintf(tbuf, "%04d%s%02d", t->tm_year+1900, mes, t->tm_mday);
1223             if(strncmp(df,"e",1) == 0)
1224                sprintf(tbuf, "%02d%s%04d", t->tm_mday, mes, t->tm_year+1900);
1225             if(strncmp(df,"w",1) == 0) {
1226                IndexTree=INDEX_TREE_FILE;
1227                strftime(tbuf, sizeof(tbuf), "%Y.%U", t);
1228             }
1229
1230             strftime(wdata, sizeof(wdata), "%Y%m%d", t);
1231             idata=atoi(wdata);
1232
1233             if(strncmp(df,"u",1)==0)
1234                strftime(dia, sizeof(dia), "%m/%d/%Y", t);
1235             else
1236                strftime(dia, sizeof(dia), "%d/%m/%Y", t);
1237             sprintf(hora,"%02d:%02d:%02d",t->tm_hour,t->tm_min,t->tm_sec);
1238          } else if(ilf==ILF_Common || ilf==ILF_Isa) {
1239             getword_start(&gwarea,data+1);
1240             if (getword_multisep(data,sizeof(data),&gwarea,':')<0){
1241                printf("SARG: Maybe you have a broken date in your %s file.\n",arq);
1242                exit(1);
1243             }
1244             if (getword_multisep(hora,sizeof(hora),&gwarea,' ')<0){
1245                printf("SARG: Maybe you have a broken date in your %s file.\n",arq);
1246                exit(1);
1247             }
1248             getword_start(&gwarea,data);
1249             if (getword(dia,sizeof(dia),&gwarea,'/')<0){
1250                printf("SARG: Maybe you have a broken date in your %s file.\n",arq);
1251                exit(1);
1252             }
1253             if (getword(mes,sizeof(mes),&gwarea,'/')<0){
1254                printf("SARG: Maybe you have a broken date in your %s file.\n",arq);
1255                exit(1);
1256             }
1257             if (getword(ano,sizeof(ano),&gwarea,'/')<0){
1258                printf("SARG: Maybe you have a broken date in your %s file.\n",arq);
1259                exit(1);
1260             }
1261
1262             if(strcmp(df,"u") == 0)
1263                snprintf(tbuf,sizeof(tbuf),"%s%s%s",ano,mes,dia);
1264             if(strcmp(df,"e") == 0)
1265                snprintf(tbuf,sizeof(tbuf),"%s%s%s",dia,mes,ano);
1266             builddia(dia,mes,ano,df,wdata);
1267             idata=atoi(wdata);
1268          } else if (ilf==ILF_Sarg) {
1269             getword_start(&gwarea,data);
1270             if(strcmp(df,"u") == 0) {
1271                if (getword(mes,sizeof(mes),&gwarea,'/')<0){
1272                   printf("SARG: Maybe you have a broken date in your %s file.\n",arq);
1273                   exit(1);
1274                }
1275                if (getword(dia,sizeof(dia),&gwarea,'/')<0){
1276                   printf("SARG: Maybe you have a broken date in your %s file.\n",arq);
1277                   exit(1);
1278                }
1279             } else {
1280                if (getword(dia,sizeof(dia),&gwarea,'/')<0){
1281                   printf("SARG: Maybe you have a broken date in your %s file.\n",arq);
1282                   exit(1);
1283                }
1284                if (getword(mes,sizeof(mes),&gwarea,'/')<0){
1285                   printf("SARG: Maybe you have a broken date in your %s file.\n",arq);
1286                   exit(1);
1287                }
1288             }
1289             if (getword(ano,sizeof(ano),&gwarea,0)<0){
1290                printf("SARG: Maybe you have a broken date in your %s file.\n",arq);
1291                exit(1);
1292             }
1293             snprintf(wdata,9,"%s%s%s",ano,mes,dia);
1294             idata=atoi(wdata);
1295          }
1296
1297          if(debugm)
1298             printf("DATE=%s IDATA=%d DFROM=%d DUNTIL=%d\n",date,idata,dfrom,duntil);
1299
1300          l=1;
1301          if(l){
1302             if(addr[0] != '\0'){
1303                if(strcmp(addr,ip)==0)
1304                   l=1;else l=0;
1305             }
1306             if(fhost) {
1307 //              l=vhexclude(ip);
1308                l=vhexclude(url);
1309                if(!l) {
1310                   if (debugm) printf("Excluded site: %s\n",url);
1311                   totregsx++;
1312                }
1313             }
1314          }
1315
1316          if(l){
1317             if(date[0] != '\0'){
1318                if(idata >= dfrom && idata <= duntil)
1319                   l=1;else l=0;
1320             }
1321          }
1322          if(l){
1323             if(hm[0] != '\0') {
1324                bzero(hmr,sizeof(hmr));
1325                chm++;
1326                getword_start(&gwarea,hora);
1327                while(chm) {
1328                   if (getword_multisep(warea,sizeof(warea),&gwarea,':')<0){
1329                      printf("SARG: Maybe you have a broken time in your %s file.\n",arq);
1330                      exit(1);
1331                   }
1332                   strncat(hmr,warea,2);
1333                   chm--;
1334                }
1335                strncat(hmr,gwarea.current,2);
1336
1337                if(atoi(hmr) >= atoi(hm) && atoi(hmr) <= atoi(hmf))
1338                   l=1;else l=0;
1339             }
1340          }
1341          if(l){
1342             if(site[0] != '\0'){
1343                if(strstr(url,site)!=0)
1344                   l=1;else l=0;
1345             }
1346          }
1347
1348          if(userip)
1349             strcpy(user,ip);
1350
1351          if(strcmp(user,"-") == 0 || strcmp(user," ") == 0 || strcmp(user,"") == 0) {
1352             if(strcmp(RecordsWithoutUser,"ip") == 0)
1353                strcpy(user,ip);
1354             if(strcmp(RecordsWithoutUser,"ignore") == 0)
1355                continue;
1356             if(strcmp(RecordsWithoutUser,"everybody") == 0)
1357                strcpy(user,"everybody");
1358          }
1359          if(us[0] != '\0'){
1360             if(strcmp(user,us)==0)
1361                l=1;
1362             else
1363                l=0;
1364          }
1365
1366          if(dotinuser) {
1367             subs(user,sizeof(user),"_",".");
1368             dotinuser=0;
1369          }
1370
1371          if(puser) {
1372             sprintf(wuser,":%s:",user);
1373             if(strstr(userfile, wuser) == 0)
1374                continue;
1375          }
1376
1377          if(l) {
1378             if(fuser) {
1379                l=vuexclude(user);
1380                if(!l) {
1381                   if (debugm) printf("Excluded user: %s\n",user);
1382                   totregsx++;
1383                }
1384             }
1385          }
1386
1387          if(l) {
1388             if(userip)
1389                fixip(user);
1390          }
1391
1392          if(l && max_elapsed) {
1393             if(atol(elap)>max_elapsed) {
1394                elap[0]='0';
1395                elap[1]='\0';
1396             }
1397          }
1398
1399          if(l) {
1400             if(strcmp(user,"-") !=0 && url[0] != '\0' && strcmp(user," ") !=0 && strcmp(user,"") !=0 && strcmp(user,":") !=0){
1401                if((str=(char *) strstr(linebuf, "[SmartFilter:")) != (char *) NULL ) {
1402                   fixendofline(str);
1403                   sprintf(smartfilter,"\"%s\"",str+1);
1404                } else sprintf(smartfilter,"\"\"");
1405
1406                if ( strcmp ( user , sz_Last_User ) != 0 ) {
1407                   if ( fp_Write_User )
1408                      fclose( fp_Write_User ) ;
1409                   sprintf (tmp3, "%s/sarg/%s.unsort", tmp, user);
1410
1411                   if ((fp_Write_User = MY_FOPEN (tmp3, "a")) == NULL) {
1412                      fprintf (stderr, "%s: (log) %s: %s - %s\n", argv[0], text[9], tmp3, strerror(errno));
1413                      exit (1);
1414                   }
1415                   strcpy( sz_Last_User , user ) ;
1416                }
1417                fprintf(fp_Write_User, "%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\n",dia,hora,user,ip,url,tam,code,elap,smartfilter);
1418
1419                if(fp_log && ilf!=ILF_Sarg)
1420                   fprintf(fp_log, "%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\n",dia,hora,user,ip,url,tam,code,elap,smartfilter);
1421
1422                totregsg++;
1423
1424                if(download_flag && strstr(code,"DENIED") == 0) {
1425                   ndownload = 1;
1426
1427                   if ( ! fp_Download_Unsort ) {
1428                      if ((fp_Download_Unsort = MY_FOPEN ( sz_Download_Unsort, "a")) == NULL) {
1429                         fprintf (stderr, "%s: (log) %s: %s - %s\n", argv[0], text[9], tmp3, strerror(errno));
1430                         exit (1);
1431                      }
1432                   }
1433                   fprintf(fp_Download_Unsort,"%s\t%s\t%s\t%s\t%s\n",dia,hora,user,ip,download_url);
1434                }
1435
1436                if((ReportType & REPORT_TYPE_DENIED) != 0 || (ReportType & REPORT_TYPE_AUTH_FAILURES) != 0) {
1437                   if(strstr(code,"DENIED/403") != 0) {
1438                      fprintf(fp_denied, "%s\t%s\t%s\t%s\t%s\n",dia,hora,user,ip,urly);
1439                      denied_count++;
1440                   }
1441                   if(strstr(code,"DENIED/401") != 0 || strstr(code,"DENIED/407") != 0) {
1442                      if(fp_authfail)
1443                         fprintf(fp_authfail, "%s\t%s\t%s\t%s\t%s\n",dia,hora,user,ip,urly);
1444                      authfail_count++;
1445                   }
1446                }
1447
1448                if((!totper || idata<mindate) && ilf!=ILF_Sarg){
1449                   totper++;
1450                   mindate=idata;
1451                   sprintf(period,"%s-",tbuf);
1452                   strcpy(start_hour,tbuf2);
1453                   if(date[0] != '\0')
1454                      fixper(tbuf, period, cduntil);
1455                   if(debugz){
1456                      debugaz("tbuf",tbuf);
1457                      debugaz("period",period);
1458                   }
1459                }
1460             }
1461
1462             if(debugm){
1463                printf("IP=\t%s\n",ip);
1464                printf("USER=\t%s\n",user);
1465                printf("ELAP=\t%s\n",elap);
1466                printf("DATE=\t%s\n",dia);
1467                printf("TIME=\t%s\n",hora);
1468                printf("FUNC=\t%s\n",fun);
1469                printf("URL=\t%s\n",url);
1470                printf("CODE=\t%s\n",code);
1471                printf("LEN=\t%s\n",tam);
1472             }
1473          }
1474       }
1475       if (!from_stdin) {
1476          fclose(fp_in);
1477          if( ShowReadStatistics )
1478             printf("SARG: Records in file: %lu, reading: %3.2f%%\n",recs1, (float) 100 );
1479       }
1480    }
1481
1482    longline_free(&line);
1483    if ( fp_Download_Unsort )
1484      fclose (fp_Download_Unsort);
1485
1486    if (fp_Write_User)
1487      fclose (fp_Write_User);
1488
1489    free_download();
1490    free_excludecodes();
1491    free_exclude();
1492
1493    if(debug) {
1494       int totalcount=0;
1495
1496       for (ilf=0 ; ilf<ILF_Last ; ilf++) totalcount+=ilf_count[ilf];
1497
1498       debuga("   %s: %ld, %s: %ld, %s: %ld",text[10],totregsl,text[11],totregsg,text[68],totregsx);
1499
1500       if(ilf_count[ILF_Common]>0 && ilf_count[ILF_Squid]>0)
1501          debuga("%s",text[12]);
1502
1503       if(ilf_count[ILF_Common]>0 && ilf_count[ILF_Squid]==0)
1504          debuga("%s",text[13]);
1505
1506       if(ilf_count[ILF_Common]==0 && ilf_count[ILF_Squid]>0)
1507          debuga("%s",text[14]);
1508
1509       if(ilf_count[ILF_Sarg]>0)
1510          debuga("%s",text[124]);
1511
1512       if(totalcount==0) {
1513          if(!totregsg) {
1514             fprintf(stderr, "SARG: %s\n",text[16]);
1515             fprintf(stderr, "SARG: %s\n",text[21]);
1516          } else fprintf(stderr, "SARG: %s\n",text[15]);
1517          if(fp_denied)
1518             fclose(fp_denied);
1519          if(fp_authfail)
1520             fclose(fp_authfail);
1521          if(userfile)
1522             free(userfile);
1523          close_usertab();
1524          unlink(tmp4);
1525          unlink(tmp6);
1526          unlink(tmp3);
1527          exit(0);
1528       }
1529    }
1530
1531    if(!totregsg){
1532       fprintf(stderr, "SARG: %s\n",text[16]);
1533       fprintf(stderr, "SARG: %s\n",text[21]);
1534 //      fclose(fp_ou);
1535       if(fp_denied)
1536          fclose(fp_denied);
1537       if(fp_authfail)
1538          fclose(fp_authfail);
1539       if(userfile)
1540          free(userfile);
1541       close_usertab();
1542       exit(0);
1543    }
1544
1545    if(date[0] == '\0' && ilf_count[ILF_Sarg]==0) {
1546       strcat(period,tbuf);
1547    }
1548
1549    if(debugz){
1550       debugaz("data",dia);
1551       debugaz("tbuf",tbuf);
1552       debugaz("period",period);
1553    }
1554
1555    if(debug)
1556       debuga("%s: %s",text[17],period);
1557
1558 //   fclose(fp_ou);
1559    if(fp_denied)
1560       fclose(fp_denied);
1561    if(fp_authfail)
1562       fclose(fp_authfail);
1563
1564    if(fp_log != NULL) {
1565       fclose(fp_log);
1566       strcpy(end_hour,tbuf2);
1567       getword_start(&gwarea,period);
1568       if (getword(val2,sizeof(val2),&gwarea,'-')<0){
1569          printf("SARG: Maybe you have a broken date range definition.\n");
1570          exit(1);
1571       }
1572       if (getword(val1,sizeof(val1),&gwarea,'\0')<0){
1573          printf("SARG: Maybe you have a broken date range definition.\n");
1574          exit(1);
1575       }
1576       sprintf(val4,"%s/sarg-%s_%s-%s_%s.log",ParsedOutputLog,val2,start_hour,val1,end_hour);
1577       if (rename(arq_log,val4)) {
1578          fprintf(stderr,"SARG: failed to rename %s to %s - %s\n",arq_log,val4,strerror(errno));
1579       } else {
1580          strcpy(arq_log,val4);
1581
1582          if(strcmp(ParsedOutputLogCompress,"nocompress") != 0 && ParsedOutputLogCompress[0] != '\0') {
1583             /*
1584             No double quotes around ParsedOutputLogCompress because it may contain command line options. If double quotes are
1585             necessary around the command name, put them in the configuration file.
1586             */
1587             sprintf(val1,"%s \"%s\"",ParsedOutputLogCompress,arq_log);
1588             cstatus=system(val1);
1589             if (!WIFEXITED(cstatus) || WEXITSTATUS(cstatus)) {
1590                fprintf(stderr, "SARG: command return status %d\n",WEXITSTATUS(cstatus));
1591                fprintf(stderr, "SARG: command: %s\n",val1);
1592                exit(1);
1593             }
1594          }
1595       }
1596       if(debug)
1597          debuga("%s %s",text[123],arq_log);
1598    }
1599
1600    if((ReportType & REPORT_TYPE_DENIED) != 0) {
1601       sprintf(csort,"sort -T \"%s\" -k 3,3 -k 5,5 -o \"%s\" \"%s\"",tmp,tmp5,tmp4);
1602       cstatus=system(csort);
1603       if (!WIFEXITED(cstatus) || WEXITSTATUS(cstatus)) {
1604          fprintf(stderr, "SARG: sort command return status %d\n",WEXITSTATUS(cstatus));
1605          fprintf(stderr, "SARG: sort command: %s\n",csort);
1606          exit(1);
1607       }
1608       unlink(tmp4);
1609    }
1610
1611    sort_users_log(tmp, debug);
1612
1613    if(DataFile[0] != '\0')
1614       data_file(tmp);
1615    else
1616       gerarel();
1617
1618    unlink(tmp2);
1619    if((ReportType & REPORT_TYPE_DENIED) != 0)
1620       unlink(tmp5);
1621
1622    if(zip[0] != '\0' && strcmp(zip,"zcat") !=0) {
1623       recomp(arq, zip);
1624    }
1625 //   else  unlink(arq);
1626
1627    if(strcmp(tmp,"/tmp") != 0) {
1628       unlinkdir(tmp,0);
1629    }
1630
1631    if(userfile)
1632       free(userfile);
1633    close_usertab();
1634
1635    if(debug)
1636       debuga("%s",text[21]);
1637
1638    exit(0);
1639
1640 }
1641
1642
1643 static void getusers(const char *pwdfile, int debug)
1644 {
1645
1646    FILE *fp_usr;
1647    char buf[255];
1648    char *str;
1649    long int nreg=0;
1650
1651    if(debug)
1652       debuga("%s: %s",text[60],pwdfile);
1653
1654    if ((fp_usr = fopen(pwdfile, "r")) == NULL) {
1655       fprintf(stderr, "SARG: (getusers) %s: %s - %s\n",text[45],pwdfile,strerror(errno));
1656       exit(1);
1657    }
1658
1659    fseek(fp_usr, 0, SEEK_END);
1660    nreg = ftell(fp_usr);
1661    if (nreg<0) {
1662       printf("SARG: Cannot get the size of file %s",pwdfile);
1663       exit(1);
1664    }
1665    nreg = nreg+5000;
1666    fseek(fp_usr, 0, SEEK_SET);
1667
1668    if((userfile=(char *) malloc(nreg))==NULL){
1669       fprintf(stderr, "SARG: %s (%ld):\n",text[59],nreg);
1670       exit(1);
1671    }
1672
1673    bzero(userfile,nreg);
1674    strcpy(userfile,":");
1675
1676    while(fgets(buf,sizeof(buf),fp_usr)!=NULL) {
1677       str=strchr(buf,':');
1678       if (!str) {
1679          printf("SARG: You have an invalid user in your %s file.\n",pwdfile);
1680          exit(1);
1681       }
1682       str[1]=0;
1683       strcat(userfile,buf);
1684    }
1685
1686    fclose(fp_usr);
1687
1688    return;
1689 }