[thirdparty/sarg.git] / log.c

/*
 * AUTHOR: Pedro Lineu Orso                         pedro.orso@gmail.com
 *                                                            1998, 2009
 * SARG Squid Analysis Report Generator      http://sarg.sourceforge.net
 *
 * SARG donations:
 *      please look at http://sarg.sourceforge.net/donations.php
 * ---------------------------------------------------------------------
 *
 *  This program is free software; you can redistribute it and/or modify
 *  it under the terms of the GNU General Public License as published by
 *  the Free Software Foundation; either version 2 of the License, or
 *  (at your option) any later version.
 *
 *  This program is distributed in the hope that it will be useful,
 *  but WITHOUT ANY WARRANTY; without even the implied warranty of
 *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 *  GNU General Public License for more details.
 *
 *  You should have received a copy of the GNU General Public License
 *  along with this program; if not, write to the Free Software
 *  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111, USA.
 *
 */

#include "include/conf.h"
#include "include/defs.h"

#define REPORT_EVERY_X_LINES 5000


char *userfile;
char *excludefile;
char *excludeuser;

int bool_ShowReadStatistics ;

numlist weekdays = { { 0, 1, 2, 3, 4, 5, 6 }, 7 };
numlist hours = { { 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12,
             13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23 }, 24 };

static void getusers(const char *pwdfile, int debug);
static void gethexclude(const char *hexfile, int debug);
static void getuexclude(const char *uexfile, int debug);

int main(int argc,char *argv[])
{
   enum isa_col_id {
      ISACOL_Ip,
      ISACOL_UserName,
      ISACOL_Date,
      ISACOL_Time,
      ISACOL_TimeTaken,
      ISACOL_Bytes,
      ISACOL_Uri,
      ISACOL_Status,
      ISACOL_Last //last entry of the list !
   };
   enum InputLogFormat {
      ILF_Unknown,
      ILF_Squid,
      ILF_Common,
      ILF_Sarg,
      ILF_Isa,
      ILF_Last //last entry of the list !
   };

   FILE *fp_in = NULL, *fp_denied=NULL, *fp_authfail=NULL, *fp_log=NULL;

   char sz_Download_Unsort[ 20000 ] ;
   FILE * fp_Download_Unsort = NULL ;
   FILE * fp_Write_User = NULL ;

   extern int optind;
   extern int optopt;
   extern char *optarg;

   char data[255];
   char elap[255];
   char ip[MAXLEN];
   char msg[MAXLEN];
   char tam[255];
   char fun[MAXLEN];
   char wuser[MAXLEN];
   char smartfilter[MAXLEN];
   char dia[128];
   char wdata[128];
   char mes[30];
   char ano[30];
   char hora[30];
   char wtemp[MAXLEN];
   char wtemp2[255];
   char date[255];
   char arq[255];
   char arq_log[255];
   char hm[15], hmf[15], hmr[15];
   int  chm=0;
   char uagent[MAXLEN];
   char hexclude[MAXLEN];
   char csort[MAXLEN];
   int cstatus;
   char tbuf[128];
   char tbuf2[128];
   char zip[20];
   char *str;
   char bufz[MAXLEN];
   char bufy[MAXLEN];
   enum InputLogFormat ilf;
   int ilf_count[ILF_Last];
   int  ch;
   int  x, l;
   int  errflg=0;
   int  puser=0;
   int  fhost=0;
   int  dns=0;
   int  fuser=0;
   int  idata=0;
   int  mindate=0;
   int  iarq=0;
   int  exstring=0;
   int isa_ncols=0,isa_cols[ISACOL_Last];
   int from_stdin;
   long totregsl=0;
   long totregsg=0;
   long totregsx=0;
   long totper=0;
   long int max_elapsed=0;
   time_t tt;
   struct tm *t;
   unsigned long recs1=0UL;
   unsigned long recs2=0UL;
   int OutputNonZero = REPORT_EVERY_X_LINES ;
   int download_flag;
   char download_url[MAXLEN];
   char sz_Last_User[MAXLEN]="";
   struct getwordstruct gwarea;

   BgImage[0]='\0';
   LogoImage[0]='\0';
   LogoText[0]='\0';
   PasswdFile[0]='\0';
   OutputEmail[0]='\0';
   UserAgentLog[0]='\0';
   ExcludeHosts[0]='\0';
   ExcludeUsers[0]='\0';
   ConfigFile[0]='\0';
   code[0]='\0';
   LastLog[0]='\0';
   ReportType[0]='\0';
   UserTabFile[0]='\0';
   BlockIt[0]='\0';
   ExternalCSSFile[0]='\0';
   SquidGuardLogFormat[0]='\0';
   SquidGuardLogAlternate[0]='\0';
   for (ilf=0 ; ilf<ILF_Last ; ilf++) ilf_count[ilf]=0;

   sprintf(ExcludeCodes,"%s/exclude_codes",SYSCONFDIR);
   strcpy(GraphDaysBytesBarColor,"orange");
   strcpy(BgColor,"#ffffff");
   strcpy(TxColor,"#000000");
   strcpy(TxBgColor,"lavender");
   strcpy(TiColor,"darkblue");
   strcpy(Width,"80");
   strcpy(Height,"45");
   strcpy(LogoTextColor,"#000000");
   strcpy(HeaderColor,"darkblue");
   strcpy(HeaderBgColor,"#dddddd");
   strcpy(LogoTextColor,"#006699");
   strcpy(FontSize,"9px");
   strcpy(TempDir,"/tmp");
   strcpy(OutputDir,"/var/www/html/squid-reports");
   Ip2Name=0;
   strcpy(DateFormat,"u");
   strcpy(OverwriteReport,"no");
   strcpy(RemoveTempFiles,"yes");
   strcpy(ReplaceIndex,"index.html");
   strcpy(Index,"yes");
   strcpy(RecordsWithoutUser,"ip");
   strcpy(UseComma,"no");
   strcpy(MailUtility,"mailx");
   strcpy(TopSitesNum,"100");
   UserIp=0;
   strcpy(TopuserSortField,"BYTES");
   strcpy(UserSortField,"BYTES");
   strcpy(TopuserSortOrder,"reverse");
   strcpy(UserSortOrder,"reverse");
   strcpy(TopsitesSortField,"CONNECT");
   strcpy(TopsitesSortType,"D");
   LongUrl=0;
   strcpy(language,"English");
   strcpy(FontFace,"Verdana,Tahoma,Arial");
   strcpy(datetimeby,"elap");
   strcpy(CharSet,"ISO-8859-1");
   strcpy(Privacy,"no");
   strcpy(PrivacyString,"***.***.***.***");
   strcpy(PrivacyStringColor,"blue");
   strcpy(SuccessfulMsg,"yes");
   strcpy(TopUserFields,"NUM DATE_TIME USERID CONNECT BYTES %BYTES IN-CACHE-OUT USED_TIME MILISEC %TIME TOTAL AVERAGE");
   strcpy(UserReportFields,"CONNECT BYTES %BYTES IN-CACHE-OUT USED_TIME MILISEC %TIME TOTAL AVERAGE");
   strcpy(DataFileDelimiter,";");
   strcpy(DataFileFields,"user;date;time;url;connect;bytes;in_cache;out_cache;elapsed");
   strcpy(SiteUserTimeDateType,"table");
   strcpy(ShowReadStatistics,"yes");
   strcpy(IndexSortOrder,"D");
   strcpy(ShowSargInfo,"yes");
   strcpy(ShowSargLogo,"yes");
   strcpy(ParsedOutputLog,"no");
   strcpy(ParsedOutputLogCompress,"/bin/gzip");
   strcpy(DisplayedValues,"abbreviation");
   strcpy(HeaderFontSize,"9px");
   strcpy(TitleFontSize,"11px");
   strcpy(AuthUserFile,"/usr/local/sarg/passwd");
   strcpy(AuthName,"SARG, Restricted Access");
   strcpy(AuthType,"basic");
   strcpy(Require,"require user admin %u");
   set_download_suffix("7z,ace,arj,avi,bat,bin,bz2,bzip,cab,com,cpio,dll,doc,dot,exe,gz,iso,lha,lzh,mdb,mov,mp3,mpeg,mpg,mso,nrg,ogg,ppt,rar,rtf,shs,src,sys,tar,tgz,vcd,vob,wma,wmv,zip");
   strcpy(Graphs,"yes");
   strcpy(Ulimit,"20000");
   strcpy(NtlmUserFormat,"domainname+username");
   strcpy(IndexTree,"file");
   strcpy(RealtimeTypes,"GET,PUT,CONNECT");
   strcpy(RealtimeUnauthRec,"show");
   strcpy(SquidguardIgnoreDate,"off");
   strcpy(DansguardianIgnoreDate,"off");
   strcpy(DataFileUrl,"ip");
   strcpy(MaxElapsed,"28800000");
   strcpy(BytesInSitesUsersReport,"no");

   dia[0]='\0';
   mes[0]='\0';
   ano[0]='\0';
   hora[0]='\0';
   tmp[0]='\0';
   tmp2[0]='\0';
   tmp3[0]='\0';
   wtemp[0]='\0';
   wtemp2[0]='\0';
   us[0]='\0';
   date[0]='\0';
   df[0]='\0';
   uagent[0]='\0';
   hexclude[0]='\0';
   addr[0]='\0';
   hm[0]='\0';
   hmf[0]='\0';
   site[0]='\0';
   outdir[0]='\0';
   elap[0]='\0';
   email[0]='\0';
   zip[0]='\0';
   UserInvalidChar[0]='\0';
   DataFile[0]='\0';
   SquidGuardConf[0]='\0';
   DansGuardianConf[0]='\0';

   denied_count=0;
   download_count=0;
   authfail_count=0;
   dansguardian_count=0;
   squidguard_count=0;
   DeniedReportLimit=10;
   AuthfailReportLimit=10;
   DansGuardianReportLimit=10;
   SquidGuardReportLimit=10;
   DownloadReportLimit=50;
   UserReportLimit=0;
   debug=0;
   debugz=0;
   debugm=0;
   iprel=0;
   userip=0;
   color1=0;
   color2=0;
   color3=0;
   dotinuser=0;
   realt=0;
   realtime_refresh=3;
   realtime_access_log_lines=1000;
   cost=0.01;
   nocost=50000000;
   ndownload=0;
   squid24=0;

   bzero(IncludeUsers, MAXLEN);
   bzero(ExcludeString, MAXLEN);

#ifdef HAVE_LOCALE_H
   setlocale(LC_TIME,"");
#endif

   NAccessLog=0;
   for(x=0; x<=MAXLOGS; x++)
      AccessLog[x][0]='\0';
   AccessLogFromCmdLine=0;

   language_load(language);
   strcpy(Title,text[88]);

   while((ch = getopt(argc, argv, "a:b:c:d:e:f:g:u:l:L:o:s:t:w:hijmnprvxyz")) != -1){
      switch(ch)
      {
         case 'a':
            strcpy(addr,optarg);
            break;
         case 'b':
            strcpy(uagent,optarg);
            break;
         case 'c':
            strcpy(hexclude,optarg);
            break;
         case 'd':
            strncpy(date,optarg,sizeof(date)-1);
            date[sizeof(date)-1]='\0';
            getword_start(&gwarea,optarg);
            if (getword(cdfrom,sizeof(cdfrom),&gwarea,'-')<0 || getword(cduntil,sizeof(cduntil),&gwarea,0)<0) {
               printf("SARG: Maybe you have a broken record or garbage in your date range.\n");
               exit(1);
            }
            date_from(date, cdfrom, cduntil);
            dfrom=atoi(cdfrom);
            duntil=atoi(cduntil);
            break;
         case 'e':
            strcpy(email,optarg);
            break;
         case 'f':
            strcpy(ConfigFile,optarg);
            break;
         case 'g':
            strcpy(df,optarg);
            break;
         case 'h':
            usage(argv[0]);
            exit(0);
            break;
         case 'i':
            iprel++;
            break;
         case 'l':
            if (NAccessLog>=MAXLOGS) {
               printf("SARG: Too many log files.\n");
               exit(1);
            }
            strcpy(AccessLog[NAccessLog],optarg);
            NAccessLog++;
            AccessLogFromCmdLine++;
            break;
         case 'L':
            strcpy(SquidGuardLogAlternate,optarg);
            break;
         case 'm':
            debugm++;
            break;
         case 'n':
            dns++;
            break;
         case 'o':
            strcpy(outdir,optarg);
            break;
         case 'p':
            userip++;
            break;
         case 'r':
            realt++;
            break;
         case 's':
            strcpy(site,optarg);
            break;
         case 't':
         {
            int h,m;

            if(strstr(optarg,"-") == 0) {
              strcpy(hm,optarg);
              strcpy(hmf,optarg);
            } else {
               getword_start(&gwarea,optarg);
               if (getword(hm,sizeof(hm),&gwarea,'-')<0 || getword(hmf,sizeof(hmf),&gwarea,0)<0) {
                  fprintf(stderr,"SARG: Maybe you have a broken record or garbage in your time range.\n");
                  exit(1);
               }
            }
            if(sscanf(hm,"%d:%d",&h,&m)!=2) {
               fprintf(stderr,"SARG: time period must be MM or MM:SS. Exit.\n");
               exit(1);
            }
            sprintf(hm,"%02d%02d",h,m);
            if(sscanf(hmf,"%d:%d",&h,&m)!=2) {
               fprintf(stderr,"SARG: time period must be MM or MM:SS. Exit.\n");
               exit(1);
            }
            sprintf(hmf,"%02d%02d",h,m);
            break;
         }
         case 'u':
            strcpy(us,optarg);
            break;
         case 'v':
            version();
            break;
         case 'w':
            strcpy(tmp,optarg);
            break;
         case 'x':
            debug++;
            break;
         case 'y':
            langcode++;
            break;
         case 'z':
            debugz++;
            break;
         case ':':
            fprintf(stderr, "Option -%c require an argument\n",optopt);
            errflg++;
            break;
         case '?':
            usage(argv[0]);
            exit(1);
            break;
      }

   }

   if (errflg) {
      usage(argv[0]);
      exit(2);
   }

   if(debug) debuga("Init");

   if(ConfigFile[0] == '\0') sprintf(ConfigFile,"%s/sarg.conf",SYSCONFDIR);
   if(access(ConfigFile, R_OK) != 0) {
      debuga("Cannot open config file: %s - %s",ConfigFile,strerror(errno));
      exit(1);
   }

   if(access(ConfigFile, R_OK) == 0)
      getconf();

   if(realt) {
      realtime();
      exit(0);
   }

   if(strcmp(IndexTree,"file") == 0)
      strcpy(ImageFile,"../images");
   else
      strcpy(ImageFile,"../../../images");

   dataonly=0;
   if(DataFile[0] != '\0')
      dataonly++;

   subs(TopUserFields,sizeof(TopUserFields),"%BYTES","SETYB");

   subs(UserReportFields,sizeof(UserReportFields),"%BYTES","SETYB");

   if(!NAccessLog) {
      strcpy(AccessLog[0],"/var/log/squid/access.log");
      NAccessLog++;
   }

   if(strcmp(hexclude,"onvert") == 0 && strcmp(site,"plit") != 0) {
      convlog(AccessLog[0], df, dfrom, duntil);
      exit(0);
   }

   if(strcmp(site,"plit") == 0) {
      splitlog(AccessLog[0], df, dfrom, duntil, hexclude);
      exit(0);
   }

   load_excludecodes(ExcludeCodes);

   if(access(PasswdFile, R_OK) == 0) {
      getusers(PasswdFile,debug);
      puser++;
   }

   if(hexclude[0] == '\0')
      strcpy(hexclude,ExcludeHosts);
   if(strlen(hexclude) > 0) {
      if(access(hexclude, R_OK) != 0) {
         debuga("Cannot open exclude_hosts file: %s - %s",hexclude,strerror(errno));
         exit(1);
      }
      gethexclude(hexclude,debug);
      fhost++;
   }

   if(ReportType[0] == '\0')
      strcpy(ReportType,"topusers topsites users_sites sites_users date_time denied auth_failures site_user_time_date downloads");

   if(access(ExcludeUsers, R_OK) == 0) {
      getuexclude(ExcludeUsers,debug);
      fuser++;
   }

   indexonly=0;
   if(fuser) {
      if(strstr(excludeuser,"indexonly") != 0)
         indexonly++;
   }
   if(strcmp(ExcludeUsers,"indexonly") == 0) indexonly++;
   if(strcmp(Index,"only") == 0) indexonly++;

   if(dns) Ip2Name=1;

   if(UserIp) userip++;

   if(strlen(MaxElapsed)>1) max_elapsed=atol(MaxElapsed);

   if(strlen(outdir)<1) strcpy(outdir,OutputDir);
   strcat(outdir,"/");

   if(uagent[0] == '\0') strcpy(uagent,UserAgentLog);

   if(tmp[0] == '\0') strcpy(tmp,TempDir);
   else strcpy(TempDir,tmp);

   if(df[0] == '\0') strcpy(df,DateFormat);
   else strcpy(DateFormat,df);

   if(df[0] == '\0') {
      strcpy(df,"u");
      strcpy(DateFormat,"u");
   }

   if(email[0] == '\0' && OutputEmail[0] != '\0') strcpy(email,OutputEmail);

   strcpy(tmp2,tmp);

   if(strlen(email) > 0) {
      sprintf(wtemp2,"%s/sarg",tmp2);
      my_mkdir(wtemp2);
      strcat(tmp2,"/sarg");
      strcpy(outdir,tmp2);
      strcat(outdir,"/");
   }

   strcat(tmp2,"/sarg.log");

   sprintf(warea,"%s/sarg",tmp);
   if(access(warea, R_OK) == 0) {
      if (snprintf(tmp3,sizeof(tmp3),"rm -rf \"%s\"",warea)>=sizeof(tmp3)) {
         fprintf(stderr,"SARG: Directory name too long: %s\n",warea);
         exit(1);
      }
      cstatus=system(tmp3);
      if (!WIFEXITED(cstatus) || WEXITSTATUS(cstatus)) {
         fprintf(stderr, "SARG: command return status %d\n",WEXITSTATUS(cstatus));
         fprintf(stderr, "SARG: command: %s\n",tmp3);
         exit(1);
      }
   }

   sprintf(tmp3,"%s/sarg",tmp);
   my_mkdir(tmp3);
   strcpy(tmp4,tmp3);
   strcpy(tmp5,tmp3);
   strcpy(tmp6,tmp3);
   strcat(tmp4,"/denied.log.unsort");
   strcat(tmp5,"/denied.log");
   strcat(tmp6,"/authfail.log.unsort");

   if(debug) {
      fprintf(stderr, "SARG: %s:\nSARG:\n",text[22]);
      fprintf(stderr, "SARG: %35s (-a) = %s\n",text[23],addr);
      fprintf(stderr, "SARG: %35s (-b) = %s\n",text[71],uagent);
      fprintf(stderr, "SARG: %35s (-c) = %s\n",text[69],hexclude);
      fprintf(stderr, "SARG: %35s (-d) = %s\n",text[24],date);
      fprintf(stderr, "SARG: %35s (-e) = %s\n",text[41],email);
      fprintf(stderr, "SARG: %35s (-f) = %s\n",text[70],ConfigFile);
      if(strcmp(df,"e") == 0)
         fprintf(stderr, "SARG: %35s (-g) = %s (dd/mm/yyyy)\n",text[25],text[26]);
      if(strcmp(df,"u") == 0)
         fprintf(stderr, "SARG: %35s (-g) = %s (mm/dd/yyyy)\n",text[25],text[27]);
      if(strcmp(df,"w") == 0)
         fprintf(stderr, "SARG: %35s (-g) = %s (yyyy/ww)\n",text[25],text[85]);
      if(iprel)
         fprintf(stderr, "SARG: %35s (-i) = %s\n",text[28],text[1]);
      else
         fprintf(stderr, "SARG: %35s (-i) = %s\n",text[28],text[2]);
      for (iarq=0 ; iarq<NAccessLog ; iarq++)
         fprintf(stderr, "SARG: %35s (-l) = %s\n",text[37],AccessLog[iarq]);
      if(Ip2Name)
         fprintf(stderr, "SARG: %35s (-n) = %s\n",text[65],text[1]);
       else
         fprintf(stderr, "SARG: %35s (-n) = %s\n",text[65],text[2]);
      fprintf(stderr, "SARG: %35s (-o) = %s\n",text[38],outdir);
      if(UserIp)
         fprintf(stderr, "SARG: %35s (-p) = %s\n",text[29],text[1]);
       else
         fprintf(stderr, "SARG: %35s (-p) = %s\n",text[29],text[2]);
      fprintf(stderr, "SARG: %35s (-s) = %s\n",text[30],site);
      fprintf(stderr, "SARG: %35s (-t) = %s\n",text[31],hm);
      fprintf(stderr, "SARG: %35s (-u) = %s\n",text[32],us);
      fprintf(stderr, "SARG: %35s (-w) = %s\n",text[34],tmp);
      if(debug)
         fprintf(stderr, "SARG: %35s (-x) = %s\n",text[35],text[1]);
       else
         fprintf(stderr, "SARG: %35s (-x) = %s\n",text[35],text[2]);
      if(debugz)
         fprintf(stderr, "SARG: %35s (-z) = %s\n",text[36],text[1]);
       else
         fprintf(stderr, "SARG: %35s (-z) = %s\n",text[36],text[2]);
      fprintf(stderr, "SARG:\n");
   }

   if(debugm) {
      printf("%s:\nSARG:\n",text[22]);
      printf("%35s (-a) = %s\n",text[23],addr);
      printf("%35s (-b) = %s\n",text[71],uagent);
      printf("%35s (-c) = %s\n",text[69],hexclude);
      printf("%35s (-d) = %s\n",text[24],date);
      printf("%35s (-e) = %s\n",text[41],email);
      printf("%35s (-f) = %s\n",text[70],ConfigFile);
      if(strcmp(df,"e") == 0)
         printf("%35s (-g) = %s (dd/mm/yyyy)\n",text[25],text[26]);
      if(strcmp(df,"u") == 0)
         printf("%35s (-g) = %s (mm/dd/yyyy)\n",text[25],text[27]);
      if(strcmp(df,"w") == 0)
         printf("%35s (-g) = %s (yyyy/ww)\n",text[25],text[85]);
      if(iprel)
         printf("%35s (-i) = %s\n",text[28],text[1]);
      else
         printf("%35s (-i) = %s\n",text[28],text[2]);
      for (iarq=0 ; iarq<NAccessLog ; iarq++)
         printf("%35s (-l) = %s\n",text[37],AccessLog[iarq]);
      if(Ip2Name)
         printf("%35s (-n) = %s\n",text[65],text[1]);
       else
         printf("%35s (-n) = %s\n",text[65],text[2]);
      printf("%35s (-o) = %s\n",text[38],outdir);
      if(UserIp)
         printf("%35s (-p) = %s\n",text[29],text[1]);
       else
         printf("%35s (-p) = %s\n",text[29],text[2]);
      printf("%35s (-s) = %s\n",text[30],site);
      printf("%35s (-t) = %s\n",text[31],hm);
      printf("%35s (-u) = %s\n",text[32],us);
      printf("%35s (-w) = %s\n",text[34],tmp);
      if(debug)
         printf("%35s (-x) = %s\n",text[35],text[1]);
       else
         printf("%35s (-x) = %s\n",text[35],text[2]);
      if(debugz)
         printf("%35s (-z) = %s\n",text[36],text[1]);
       else
         printf("%35s (-z) = %s\n",text[36],text[2]);
      printf("sarg %s: %s\n",text[73],VERSION);
      printf("Language=%s\n\n",text[3]);
   }

   if(debug)
      debuga("sarg %s: %s",text[73],VERSION);

#ifdef HAVE_RLIM_T
   if (Ulimit[0] != '\0') {
      struct rlimit rl;
      long l1, l2;
      int rc=0;

#if defined(RLIMIT_NOFILE)
      getrlimit (RLIMIT_NOFILE, &rl);
#elif defined(RLIMIT_OFILE)
      getrlimit (RLIMIT_OFILE, &rl);
#else
#warning "No rlimit resource for the number of open files"
#endif
      l1 = rl.rlim_cur;
      l2 = rl.rlim_max;

      rl.rlim_cur = atol(Ulimit);
      rl.rlim_max = atol(Ulimit);
#if defined(RLIMIT_NOFILE)
      rc=setrlimit (RLIMIT_NOFILE, &rl);
#elif defined(RLIMIT_OFILE)
      rc=setrlimit (RLIMIT_OFILE, &rl);
#else
#warning "No rlimit resource for the number of open files"
#endif
      if(rc == -1) {
            debuga("setrlimit error - %s\n",strerror(errno));
      }

      if(debug)
         debuga("Maximum file descriptor: cur=%ld max=%ld, changed to cur="RLIM_STRING" max="RLIM_STRING,l1,l2,rl.rlim_cur,rl.rlim_max);
   }
#endif

   read_usertab(UserTabFile);

   sprintf ( sz_Download_Unsort , "%s/sarg/download.unsort", tmp);
   bool_ShowReadStatistics = ( strcmp(ShowReadStatistics,"yes") == 0 ) ;

   if(strstr(ReportType,"denied") != 0) {
      if((fp_denied=MY_FOPEN(tmp4,"w"))==NULL) {
         fprintf(stderr, "%s: (log) %s: %s - %s\n",argv[0],text[45],tmp4,strerror(errno));
         exit(1);
      }
   }

   if(DataFile[0]=='\0') {
      if(strstr(ReportType,"denied") != 0 || strstr(ReportType,"auth_failures") != 0) {
         if((fp_authfail=MY_FOPEN(tmp6,"w"))==NULL) {
            fprintf(stderr, "%s: (log) %s: %s - %s\n",argv[0],text[45],tmp6,strerror(errno));
            exit(1);
         }
      }
   }

   for (iarq=0 ; iarq<NAccessLog ; iarq++) {
      strcpy(arq,AccessLog[iarq]);

      strcpy(arqtt,arq);

      if(strcmp(arq,"-")==0) {
         if(debug)
            debuga("%s: %s",text[7],"stdin");
         fp_in=stdin;
         from_stdin=1;
      } else {
         decomp(arq,zip,tmp);
         if(debug)
            debuga("%s: %s",text[7],arq);
         if((fp_in=MY_FOPEN(arq,"r"))==NULL) {
            fprintf(stderr, "%s: (log) %s: %s - %s\n",argv[0],text[8],arq,strerror(errno));
            exit(1);
         }
         from_stdin=0;
      }
      ilf=ILF_Unknown;
      // pre-Read the file only if I have to show stats
      if(bool_ShowReadStatistics && !from_stdin) {
         rewind(fp_in);
         recs1=0UL;
         recs2=0UL;

         while( fgets(bufz,sizeof(bufz),fp_in) != NULL ) recs1++;
         rewind(fp_in);
         printf("SARG: Records in file: %lu, reading: %3.2f%%\r",recs1,(float) 0);
         fflush( stdout ) ;
      }

      while(fgets(bufz,sizeof(bufz),fp_in)!=NULL) {

         if (ilf==ILF_Unknown) {
            if(strncmp(bufz,"#Software: Mic",14) == 0) {
               fixendofline(bufz);
               debuga("%s: %s",text[143],bufz);
               ilf=ILF_Isa;
               ilf_count[ilf]++;
               continue;
            }

            if(strncmp(bufz,"*** SARG Log ***",16) == 0) {
               getword_start(&gwarea,arqtt);
               if (getword(val2,sizeof(val2),&gwarea,'-')<0 || getword(val2,sizeof(val2),&gwarea,'_')<0 ||
                   getword(val3,sizeof(val3),&gwarea,'-')<0 || getword(val3,sizeof(val3),&gwarea,'_')<0) {
                  printf("SARG: Maybe you have a broken record or garbage in your %s file.\n",arq);
                  exit(1);
               }
               sprintf(period,"%s-%s",val2,val3);
               ilf=ILF_Sarg;
               ilf_count[ilf]++;
               continue;
            }
         }

         if(strcmp(ParsedOutputLog, "no") != 0 && ilf!=ILF_Sarg) {
            if(access(ParsedOutputLog,R_OK) != 0) {
               sprintf(csort,"%s",ParsedOutputLog);
               my_mkdir(csort);
            }
            sprintf(arq_log,"%s/sarg_temp.log",ParsedOutputLog);
            if((fp_log=MY_FOPEN(arq_log,"w"))==NULL) {
               fprintf(stderr, "%s: (log) %s: %s - %s\n",argv[0],text[8],arq_log,strerror(errno));
               exit(1);
            }
            fputs("*** SARG Log ***\n",fp_log);
         }

         recs2++;
         if( bool_ShowReadStatistics && !from_stdin && ! --OutputNonZero) {
           perc = recs2 * 100 ;
           perc = perc / recs1 ;
           printf("SARG: Records in file: %lu, reading: %3.2f%%\r",recs1,perc);
           fflush (stdout);
           OutputNonZero = REPORT_EVERY_X_LINES ;
         }
         if(strlen(bufz) > MAXLEN-1) continue;
         if(!bufz[0]) continue;
         if(strstr(bufz,"HTTP/0.0") != 0) continue;
         if(strstr(bufz,"logfile turned over") != 0) continue;
         if(bufz[0] == ' ') continue;
         if(strlen(bufz) < 58) continue;

         // Record only hours usage which is required
         tt = (time_t) strtoul( bufz, NULL, 10 );
         t = localtime( &tt );

         if( bsearch( &( t -> tm_wday ), weekdays.list, weekdays.len,
                                          sizeof( int ), compar ) == NULL )
            continue;

         if( bsearch( &( t -> tm_hour ), hours.list, hours.len,
                                          sizeof( int ), compar ) == NULL )
            continue;

         // exclude_string
         exstring=0;
         if(ExcludeString[0] != '\0') {
            strcpy(warea,bufz);
            getword_start(&gwarea,ExcludeString);
            while(strchr(gwarea.current,':') != 0) {
               if (getword_multisep(val1,sizeof(val1),&gwarea,':')<0) {
                  printf("SARG: Maybe you have a broken record or garbage in your exclusion string.\n");
                  exit(1);
               }
               if((str=(char *) strstr(warea,val1)) != (char *) NULL )
                  exstring++;
            }
            if((str=(char *) strstr(warea,gwarea.current)) != (char *) NULL )
                  exstring++;
         }
         if(exstring) continue;

         strcpy(bufy,bufz);
         if ((str = strchr(bufz, '\n')) != NULL)
            *str = '\0';          /* strip \n */

         totregsl++;
         if(debugm)
            printf("BUF=%s\n",bufz);

         if (ilf==ILF_Squid || ilf==ILF_Common || ilf==ILF_Unknown) {
            getword_start(&gwarea,bufz);
            if (getword(data,sizeof(data),&gwarea,' ')<0) {
               printf("SARG: Maybe you have a broken record or garbage in your access.log file.\n");
               exit(1);
            }
            if((str=(char *) strchr(data, '.')) != (char *) NULL ) {
               if((str=(char *) strchr(str+1, '.')) != (char *) NULL ) {
                  strcpy(ip,data);
                  strcpy(elap,"0");
                  if(squid24) {
                     if (getword(user,sizeof(user),&gwarea,' ')<0 || getword_skip(255,&gwarea,' ')<0) {
                        printf("SARG: Maybe you have a broken record or garbage in your %s file.\n",arq);
                        exit(1);
                     }
                  } else {
                     if (getword_skip(255,&gwarea,' ')<0 || getword(user,sizeof(user),&gwarea,' ')<0) {
                        printf("SARG: Maybe you have a broken record or garbage in your %s file.\n",arq);
                        exit(1);
                     }
                  }
                  if (getword(data,sizeof(data),&gwarea,']')<0 || getword_skip(MAXLEN,&gwarea,'"')<0 ||
                      getword(fun,sizeof(fun),&gwarea,' ')<0 || getword(url,sizeof(url),&gwarea,' ')<0 ||
                      getword_skip(MAXLEN,&gwarea,' ')<0 || getword(code2,sizeof(code2),&gwarea,' ')<0 ||
                      getword(tam,sizeof(tam),&gwarea,' ')<0) {
                     printf("SARG: Maybe you have a broken record or garbage in your %s file.\n",arq);
                     exit(1);
                  }
                  if((str=(char *) strchr(gwarea.current, ' ')) != (char *) NULL ) {
                     if (getword(code,sizeof(code),&gwarea,' ')<0) {
                        printf("SARG: Maybe you have a broken record or garbage in your %s file.\n",arq);
                        exit(1);
                     }
                  } else {
                     if (getword(code,sizeof(code),&gwarea,'\0')<0) {
                        printf("SARG: Maybe you have a broken record or garbage in your %s file.\n",arq);
                        exit(1);
                     }
                  }

                  if ((str = strchr(code, ':')) != NULL)
                     *str = '/';

                  if(strcmp(tam,"\0") == 0)
                     strcpy(tam,"0");

                  ilf=ILF_Common;
                  ilf_count[ilf]++;
               }
            }

            if(ilf==ILF_Unknown || ilf==ILF_Squid) {
               if (getword(elap,sizeof(elap),&gwarea,' ')<0) {
                  printf("SARG: Maybe you have a broken record or garbage in your %s file.\n",arq);
                  exit(1);
               }
               while(strcmp(elap,"") == 0 && gwarea.current[0] != '\0')
                  if (getword(elap,sizeof(elap),&gwarea,' ')<0) {
                     printf("SARG: Maybe you have a broken record or garbage in your %s file.\n",arq);
                     exit(1);
                  }
               if(strlen(elap) < 1) continue;
               if (getword(ip,sizeof(ip),&gwarea,' ')<0){
                  printf("SARG: Maybe you have a broken record or garbage in your %s file.\n",arq);
                  exit(1);
               }
               if (getword(code,sizeof(code),&gwarea,' ')<0){
                  printf("SARG: Maybe you have a broken record or garbage in your %s file.\n",arq);
                  exit(1);
               }
               if (getword(tam,sizeof(tam),&gwarea,' ')<0){
                  printf("SARG: Maybe you have a broken record or garbage in your %s file.\n",arq);
                  exit(1);
               }
               if (getword(fun,sizeof(fun),&gwarea,' ')<0){
                  printf("SARG: Maybe you have a broken record or garbage in your %s file.\n",arq);
                  exit(1);
               }
               if (getword(url,sizeof(url),&gwarea,' ')<0){
                  printf("SARG: Maybe you have a broken record or garbage in your %s file.\n",arq);
                  exit(1);
               }
//              while (strstr(bufz,"%20") != 0) {
//                 getword(warea,bufz,' ');
//                 strcat(url,warea);
//              }
               if (getword(user,sizeof(user),&gwarea,' ')<0){
                  printf("SARG: Maybe you have a broken record or garbage in your %s file.\n",arq);
                  exit(1);
               }
               ilf=ILF_Squid;
               ilf_count[ilf]++;
            }
         }
         if (ilf==ILF_Sarg) {
            getword_start(&gwarea,bufz);
            if (getword(data,sizeof(data),&gwarea,' ')<0){
               printf("SARG: Maybe you have a broken record or garbage in your %s file.\n",arq);
               exit(1);
            }
            if (getword(hora,sizeof(hora),&gwarea,' ')<0) {
               printf("SARG: Maybe you have a broken record or garbage in your %s file.\n",arq);
               exit(1);
            }
            if (getword(user,sizeof(user),&gwarea,' ')<0) {
               printf("SARG: Maybe you have a broken record or garbage in your %s file.\n",arq);
               exit(1);
            }
            if (getword(ip,sizeof(ip),&gwarea,' ')<0) {
               printf("SARG: Maybe you have a broken record or garbage in your %s file.\n",arq);
               exit(1);
            }
            if (getword(url,sizeof(url),&gwarea,' ')<0){
               printf("SARG: Maybe you have a broken record or garbage in your %s file.\n",arq);
               exit(1);
            }
            if (getword(tam,sizeof(tam),&gwarea,' ')<0){
               printf("SARG: Maybe you have a broken record or garbage in your %s file.\n",arq);
               exit(1);
            }
            if (getword(code,sizeof(code),&gwarea,' ')<0){
               printf("SARG: Maybe you have a broken record or garbage in your %s file.\n",arq);
               exit(1);
            }
            if (getword(elap,sizeof(elap),&gwarea,' ')<0){
               printf("SARG: Maybe you have a broken record or garbage in your %s file.\n",arq);
               exit(1);
            }
            if (getword(smartfilter,sizeof(smartfilter),&gwarea,' ')<0){
               printf("SARG: Maybe you have a broken record or garbage in your %s file.\n",arq);
               exit(1);
            }
         }
         if (ilf==ILF_Isa) {
            if (bufz[0] == '#') {
               int ncols,cols[ISACOL_Last];

               fixendofline(bufz);
               getword_start(&gwarea,bufz);
               // remove the #Fields: column at the beginning of the line
               if (getword_skip(1000,&gwarea,' ')<0){
                  printf("SARG: Maybe you have a broken record or garbage in your %s file.\n",arq);
                  exit(1);
               }
               for (ncols=0 ; ncols<ISACOL_Last ; ncols++) cols[ncols]=-1;
               ncols=0;
               while(gwarea.current[0] != '\0') {
                  if (getword(val1,sizeof(val1),&gwarea,'\t')<0){
                     printf("SARG: Maybe you have a broken record or garbage in your %s file.\n",arq);
                     exit(1);
                  }
                  if(strcmp(val1,"c-ip") == 0) cols[ISACOL_Ip]=ncols;
                  if(strcmp(val1,"cs-username") == 0) cols[ISACOL_UserName]=ncols;
                  if(strcmp(val1,"date") == 0) cols[ISACOL_Date]=ncols;
                  if(strcmp(val1,"time") == 0) cols[ISACOL_Time]=ncols;
                  if(strcmp(val1,"time-taken") == 0) cols[ISACOL_TimeTaken]=ncols;
                  if(strcmp(val1,"sc-bytes") == 0) cols[ISACOL_Bytes]=ncols;
                  if(strcmp(val1,"cs-uri") == 0) cols[ISACOL_Uri]=ncols;
                  if(strcmp(val1,"sc-status") == 0) cols[ISACOL_Status]=ncols;
                  ncols++;
               }
               if (cols[ISACOL_Ip]>=0) {
                  isa_ncols=ncols;
                  for (isa_ncols=0 ; isa_ncols<ncols ; isa_ncols++)
                     isa_cols[isa_ncols]=cols[isa_ncols];
               }
               continue;
            }
            if (!isa_ncols) continue;
            getword_start(&gwarea,bufz);
            for (x=0 ; x<isa_ncols ; x++) {
               if (getword(val1,sizeof(val1),&gwarea,'\t')<0) {
                  printf("SARG: Maybe you have a broken record or garbage in your %s file.\n",arq);
                  exit(1);
               }
               if (x==isa_cols[ISACOL_Ip]) {
                  if (strlen(val1)>=sizeof(ip)) {
                     printf("SARG: Maybe you have a broken IP in your %s file.\n",arq);
                     exit(1);
                  }
                  strcpy(ip,val1);
               } else if (x==isa_cols[ISACOL_UserName]) {
                  if (strlen(val1)>=sizeof(user)) {
                     printf("SARG: Maybe you have a broken user in your %s file.\n",arq);
                     exit(1);
                  }
                  strcpy(user,val1);
               } else if (x==isa_cols[ISACOL_Date]) {
                  if (strlen(val1)>=sizeof(data)) {
                     printf("SARG: Maybe you have a broken date in your %s file.\n",arq);
                     exit(1);
                  }
                  strcpy(data,val1);
               } else if (x==isa_cols[ISACOL_Time]) {
                  if (strlen(val1)>=sizeof(hora)) {
                     printf("SARG: Maybe you have a broken time in your %s file.\n",arq);
                     exit(1);
                  }
                  strcpy(hora,val1);
               } else if (x==isa_cols[ISACOL_TimeTaken]) {
                  if (strlen(val1)>=sizeof(elap)) {
                     printf("SARG: Maybe you have a broken download duration in your %s file.\n",arq);
                     exit(1);
                  }
                  strcpy(elap,val1);
               } else if (x==isa_cols[ISACOL_Bytes]) {
                  if (strlen(val1)>=sizeof(tam)) {
                     printf("SARG: Maybe you have a broken download size in your %s file.\n",arq);
                     exit(1);
                  }
                  strcpy(tam,val1);
               } else if (x==isa_cols[ISACOL_Uri]) {
                  if (strlen(val1)>=sizeof(url)) {
                     printf("SARG: Maybe you have a broken URL in your %s file.\n",arq);
                     exit(1);
                  }
                  strcpy(url,val1);
               } else if (x==isa_cols[ISACOL_Status]) {
                  if (strlen(val1)>=sizeof(code)) {
                     printf("SARG: Maybe you have a broken access code in your %s file.\n",arq);
                     exit(1);
                  }
                  strcpy(code,val1);
               }
            }

            if(strcmp(code,"401") == 0 || strcmp(code,"403") == 0 || strcmp(code,"407") == 0) {
               sprintf(val1,"DENIED/%s",code);
               strcpy(code,val1);
            }
            getword_start(&gwarea,data);
            if (getword(ano,sizeof(ano),&gwarea,'-')<0){
               printf("SARG: Maybe you have a broken record or garbage in your %s file.\n",arq);
               exit(1);
            }
            if (getword(mes,sizeof(mes),&gwarea,'-')<0){
               printf("SARG: Maybe you have a broken record or garbage in your %s file.\n",arq);
               exit(1);
            }
            if (getword(dia,sizeof(dia),&gwarea,'\0')<0){
               printf("SARG: Maybe you have a broken record or garbage in your %s file.\n",arq);
               exit(1);
            }
            conv_month_name(mes);
            sprintf(data," %s/%s/%s:%s",dia,mes,ano,hora);
         }

         if(strlen(user) > 150) {
            if (debugm) printf("User too long: %s\n",user);
            totregsx++;
            continue;
         }

         // include_users
         if(IncludeUsers[0] != '\0') {
            sprintf(val1,":%s:",user);
            if((str=(char *) strstr(IncludeUsers,val1)) == (char *) NULL )
               continue;
         }

         if(vercode(code)) {
            if (debugm) printf("Excluded code: %s\n",code);
            totregsx++;
            continue;
         }

         if(testvaliduserchar(user))
            continue;

         if((str = strstr(user,"%20")) != NULL) {
            /*
            Why is it necessary to truncate the user name at the first space ?

            The old code used to truncate the user name at the first % if a %20 was
            found anywhere in the string. That means the string could be truncated
            at the wrong place if another % occured before the %20. This new code should
            avoid that problem and only truncate at the space. There is no bug
            report indicating that anybody noticed this.
            */
            *str='\0';
         }

         /*
         Code prior to 2.2.7 used to replace any %xx by a dot as long as a %5c was
         found in the user name.
         */
         while((str = strstr(user,"%5c")) != NULL) {
            *str='.';
            for (x=3 ; str[x] ; x++) str[x-2]=str[x];
         }

         for(str=user; *str; str++) {
            if(*str=='.') dotinuser++;
            if(*str=='?' || *str=='.' || *str==':' || *str=='/' || *str=='\\')
               *str='_';
         }

         strlow(user);
         if(strncmp(NtlmUserFormat,"user",4) == 0) {
            if((str = strchr(user,'_')) != 0) {
               strcpy(warea,str+1);
               strcpy(user,warea);
            }
            if((str = strchr(user,'+')) != 0) {
               strcpy(warea,str+1);
               strcpy(user,warea);
            }
         }

         if(strstr(ReportType,"denied") != 0)
            strcpy(urly,url);

         download_flag=is_download_suffix(url);
         if (download_flag) {
            strcpy(download_url,url);
            download_count++;
         }

         // remove any protocol:// at the beginning of the URL
         if ((str = strchr(url,'/')) != NULL && str[1] == '/') {
            int i;

            str+=2;
            for (i=0 ; str[i] ; i++)
               url[i]=str[i];
            url[i]='\0';
         }

         if(!LongUrl) {
            char *endofhost=strchr(url,'/');
            if (endofhost)
               *endofhost='\0';
            if(strlen(url) > 512 && (endofhost=strchr(url,'%')) != NULL) {
               *endofhost='\0';
            }
         }

         if(ilf==ILF_Squid) {
            tt=atoi(data);
            t=localtime(&tt);

            strftime(tbuf2, sizeof(tbuf2), "%H%M", t);
            if(strncmp(df,"u",1) == 0)
               strftime(tbuf, sizeof(tbuf), "%Y%b%d", t);
            if(strncmp(df,"e",1) == 0)
               strftime(tbuf, sizeof(tbuf), "%d%b%Y", t);
            if(strncmp(df,"w",1) == 0) {
               strcpy(IndexTree,"file");
               strftime(tbuf, sizeof(tbuf), "%Y.%U", t);
            }

            strftime(wdata, sizeof(wdata), "%Y%m%d", t);
            idata=atoi(wdata);

            if(strncmp(df,"u",1)==0)
               strftime(dia, sizeof(dia), "%m/%d/%Y", t);
            else
               strftime(dia, sizeof(dia), "%d/%m/%Y", t);
            sprintf(hora,"%02d:%02d:%02d",t->tm_hour,t->tm_min,t->tm_sec);
         } else if(ilf==ILF_Common || ilf==ILF_Isa) {
            getword_start(&gwarea,data+1);
            if (getword_multisep(data,sizeof(data),&gwarea,':')<0){
               printf("SARG: Maybe you have a broken date in your %s file.\n",arq);
               exit(1);
            }
            if (getword_multisep(hora,sizeof(hora),&gwarea,' ')<0){
               printf("SARG: Maybe you have a broken date in your %s file.\n",arq);
               exit(1);
            }
            getword_start(&gwarea,data);
            if (getword_multisep(dia,sizeof(dia),&gwarea,'/')<0){
               printf("SARG: Maybe you have a broken date in your %s file.\n",arq);
               exit(1);
            }
            if (getword_multisep(mes,sizeof(mes),&gwarea,'/')<0){
               printf("SARG: Maybe you have a broken date in your %s file.\n",arq);
               exit(1);
            }
            if (getword_multisep(ano,sizeof(ano),&gwarea,'/')<0){
               printf("SARG: Maybe you have a broken date in your %s file.\n",arq);
               exit(1);
            }

            if(strcmp(df,"u") == 0)
               snprintf(tbuf,sizeof(tbuf),"%s%s%s",ano,mes,dia);
            if(strcmp(df,"e") == 0)
               snprintf(tbuf,sizeof(tbuf),"%s%s%s",dia,mes,ano);
            builddia(dia,mes,ano,df,wdata);
            idata=atoi(wdata);
         } else if (ilf==ILF_Sarg) {
            getword_start(&gwarea,data);
            if (getword_multisep(mes,sizeof(mes),&gwarea,'/')<0){
               printf("SARG: Maybe you have a broken date in your %s file.\n",arq);
               exit(1);
            }
            if (getword_multisep(dia,sizeof(dia),&gwarea,'/')<0){
               printf("SARG: Maybe you have a broken date in your %s file.\n",arq);
               exit(1);
            }
            if (getword_multisep(ano,sizeof(ano),&gwarea,0)<0){
               printf("SARG: Maybe you have a broken date in your %s file.\n",arq);
               exit(1);
            }
         }

         if(debugm)
            printf("DATE=%s IDATA=%d DFROM=%d DUNTIL=%d\n",date,idata,dfrom,duntil);

         l=1;
         if(l){
            if(addr[0] != '\0'){
               if(strcmp(addr,ip)==0)
                  l=1;else l=0;
            }
            if(fhost) {
//              l=vhexclude(excludefile,ip);
               l=vhexclude(excludefile,url);
               if(!l) {
                  if (debugm) printf("Excluded site: %s\n",url);
                  totregsx++;
               }
            }
         }

         if(l){
            if(date[0] != '\0'){
               if(idata >= dfrom && idata <= duntil)
                  l=1;else l=0;
            }
         }
         if(l){
            if(hm[0] != '\0') {
               bzero(hmr,sizeof(hmr));
               chm++;
               getword_start(&gwarea,hora);
               while(chm) {
                  if (getword_multisep(warea,sizeof(warea),&gwarea,':')<0){
                     printf("SARG: Maybe you have a broken time in your %s file.\n",arq);
                     exit(1);
                  }
                  strncat(hmr,warea,2);
                  chm--;
               }
               strncat(hmr,gwarea.current,2);

               if(atoi(hmr) >= atoi(hm) && atoi(hmr) <= atoi(hmf))
                  l=1;else l=0;
            }
         }
         if(l){
            if(site[0] != '\0'){
               if(strstr(url,site)!=0)
                  l=1;else l=0;
            }
         }

         if(userip)
            strcpy(user,ip);

         if(strcmp(user,"-") == 0 || strcmp(user," ") == 0 || strcmp(user,"") == 0) {
            if(strcmp(RecordsWithoutUser,"ip") == 0)
               strcpy(user,ip);
            if(strcmp(RecordsWithoutUser,"ignore") == 0)
               continue;
            if(strcmp(RecordsWithoutUser,"everybody") == 0)
               strcpy(user,"everybody");
         }
         if(us[0] != '\0'){
            if(strcmp(user,us)==0)
               l=1;
            else
               l=0;
         }

         if(dotinuser) {
            subs(user,sizeof(user),"_",".");
            dotinuser=0;
         }

         if(puser) {
            sprintf(wuser,":%s:",user);
            if(strstr(userfile, wuser) == 0)
               continue;
         }

         if(l) {
            if(fuser) {
               l=vuexclude(excludeuser,user);
               if(!l) {
                  if (debugm) printf("Excluded user: %s\n",user);
                  totregsx++;
               }
            }
         }

         if(l) {
            if(userip)
               fixip(user);
         }

         if(l && max_elapsed) {
            if(atol(elap)>max_elapsed) {
               elap[0]='0';
               elap[1]='\0';
            }
         }

         if(l) {
            if(strcmp(user,"-") !=0 && url[0] != '\0' && strcmp(user," ") !=0 && strcmp(user,"") !=0 && strcmp(user,":") !=0){
               if((str=(char *) strstr(bufz, "[SmartFilter:")) != (char *) NULL ) {
                  fixendofline(str);
                  sprintf(smartfilter,"\"%s\"",str+1);
               } else sprintf(smartfilter,"\"\"");

               sprintf(bufz, "%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\n",dia,hora,user,ip,url,tam,code,elap,smartfilter);

               if ( strcmp ( user , sz_Last_User ) != 0 ) {
                  if ( fp_Write_User )
                     fclose( fp_Write_User ) ;
                  sprintf (tmp3, "%s/sarg/%s.unsort", tmp, user);

                  if ((fp_Write_User = MY_FOPEN (tmp3, "a")) == NULL) {
                     fprintf (stderr, "%s: (log) %s: %s - %s\n", argv[0], text[9], tmp3, strerror(errno));
                     exit (1);
                  }
                  strcpy( sz_Last_User , user ) ;
               }
               fputs (bufz, fp_Write_User);

               if(strcmp(ParsedOutputLog, "no") != 0 && ilf!=ILF_Sarg)
                  fputs(bufz,fp_log);

               totregsg++;

               if(download_flag && strstr(code,"DENIED") == 0) {
                  ndownload = 1;

                  if ( ! fp_Download_Unsort ) {
                     if ((fp_Download_Unsort = MY_FOPEN ( sz_Download_Unsort, "a")) == NULL) {
                        fprintf (stderr, "%s: (log) %s: %s - %s\n", argv[0], text[9], tmp3, strerror(errno));
                        exit (1);
                     }
                  }
                  fprintf(fp_Download_Unsort,"%s\t%s\t%s\t%s\t%s\n",dia,hora,user,ip,download_url);
               }

               if(strstr(ReportType,"denied") != 0 || strstr(ReportType,"auth_failures") != 0) {
                  if(strstr(code,"DENIED/403") != 0) {
                     fprintf(fp_denied, "%s\t%s\t%s\t%s\t%s\n",dia,hora,user,ip,urly);
                     denied_count++;
                  }
                  if(strstr(code,"DENIED/401") != 0 || strstr(code,"DENIED/407") != 0) {
                     if(fp_authfail)
                        fprintf(fp_authfail, "%s\t%s\t%s\t%s\t%s\n",dia,hora,user,ip,urly);
                     authfail_count++;
                  }
               }

               if((!totper || idata<mindate) && ilf!=ILF_Sarg){
                  totper++;
                  mindate=idata;
                  sprintf(period,"%s-",tbuf);
                  sprintf(per_hour,"%s-",tbuf2);
                  if(date[0] != '\0')
                     fixper(tbuf, period, cduntil);
                  if(debugz){
                     debugaz("tbuf",tbuf);
                     debugaz("period",period);
                  }
               }
            }

            if(debugm){
               printf("IP=\t%s\n",ip);
               printf("USER=\t%s\n",user);
               printf("ELAP=\t%s\n",elap);
               printf("DATE=\t%s\n",dia);
               printf("TIME=\t%s\n",hora);
               printf("FUNC=\t%s\n",fun);
               printf("URL=\t%s\n",url);
               printf("CODE=\t%s\n",code);
               printf("LEN=\t%s\n",tam);
            }
         }
      }
      if (!from_stdin) {
         fclose(fp_in);
         if( bool_ShowReadStatistics )
            printf("SARG: Records in file: %lu, reading: %3.2f%%\n",recs1, (float) 100 );
      }
   }

   if ( fp_Download_Unsort )
     fclose (fp_Download_Unsort);

   if (fp_Write_User)
     fclose (fp_Write_User);

   if(debug) {
      int totalcount=0;

      for (ilf=0 ; ilf<ILF_Last ; ilf++) totalcount+=ilf_count[ilf];

      debuga("   %s: %ld, %s: %ld, %s: %ld",text[10],totregsl,text[11],totregsg,text[68],totregsx);

      if(ilf_count[ILF_Common]>0 && ilf_count[ILF_Squid]>0)
         debuga("%s",text[12]);

      if(ilf_count[ILF_Common]>0 && ilf_count[ILF_Squid]==0)
         debuga("%s",text[13]);

      if(ilf_count[ILF_Common]==0 && ilf_count[ILF_Squid]>0)
         debuga("%s",text[14]);

      if(ilf_count[ILF_Sarg]>0)
         debuga("%s",text[124]);

      if(totalcount==0) {
         if(!totregsg) {
            fprintf(stderr, "SARG: %s\n",text[16]);
            fprintf(stderr, "SARG: %s\n",text[21]);
         } else fprintf(stderr, "SARG: %s\n",text[15]);
         bzero(msg,sizeof(msg));
         if(fp_denied)
            fclose(fp_denied);
         if(fp_authfail)
            fclose(fp_authfail);
         free_excludecodes();
         if(userfile)
            free(userfile);
         if(excludefile)
            free(excludefile);
         if(excludeuser)
            free(excludeuser);
         free_download();
         unlink(tmp4);
         unlink(tmp6);
         unlink(tmp3);
         exit(0);
      }
   }

   if(!totregsg){
      fprintf(stderr, "SARG: %s\n",text[16]);
      fprintf(stderr, "SARG: %s\n",text[21]);
//      fclose(fp_ou);
      if(fp_denied)
         fclose(fp_denied);
      if(fp_authfail)
         fclose(fp_authfail);
      free_excludecodes();
      if(userfile)
         free(userfile);
      if(excludefile)
         free(excludefile);
      if(excludeuser)
         free(excludeuser);
      free_download();
      exit(0);
   }

   if(date[0] == '\0' && ilf_count[ILF_Sarg]==0) {
      strcat(period,tbuf);
      strcat(per_hour,tbuf2);
   }

   if(debugz){
      debugaz("data",dia);
      debugaz("tbuf",tbuf);
      debugaz("period",period);
   }

   if(debug)
      debuga("%s: %s",text[17],period);

//   fclose(fp_ou);
   if(fp_denied)
      fclose(fp_denied);
   if(fp_authfail)
      fclose(fp_authfail);

   if(fp_log != NULL) {
      fclose(fp_log);
      getword_start(&gwarea,period);
      if (getword_multisep(val2,sizeof(val2),&gwarea,'-')<0){
         printf("SARG: Maybe you have a broken date range definition.\n");
         exit(1);
      }
      if (getword_multisep(val1,sizeof(val1),&gwarea,'\0')<0){
         printf("SARG: Maybe you have a broken date range definition.\n");
         exit(1);
      }
      getword_start(&gwarea,per_hour);
      if (getword_multisep(val3,sizeof(val3),&gwarea,'-')<0){
         printf("SARG: Maybe you have a broken date range definition.\n");
         exit(1);
      }
      sprintf(val4,"%s/sarg-%s_%s-%s_%s.log",ParsedOutputLog,val2,val3,val1,gwarea.current);
      rename(arq_log,val4);
      strcpy(arq_log,val4);

      if(strcmp(ParsedOutputLogCompress,"nocompress") != 0) {
         sprintf(val1,"\"%s\" \"%s\"",ParsedOutputLogCompress,arq_log);
         cstatus=system(val1);
         if (!WIFEXITED(cstatus) || WEXITSTATUS(cstatus)) {
            fprintf(stderr, "SARG: command return status %d\n",WEXITSTATUS(cstatus));
            fprintf(stderr, "SARG: command: %s\n",val1);
            exit(1);
         }
      }

      if(debug)
         debuga("%s %s",text[123],arq_log);
   }

   if(strstr(ReportType,"denied") != 0) {
      sprintf(csort,"sort -T \"%s\" -k 3,3 -k 5,5 -o \"%s\" \"%s\"",tmp,tmp5,tmp4);
      cstatus=system(csort);
      if (!WIFEXITED(cstatus) || WEXITSTATUS(cstatus)) {
         fprintf(stderr, "SARG: sort command return status %d\n",WEXITSTATUS(cstatus));
         fprintf(stderr, "SARG: sort command: %s\n",csort);
         exit(1);
      }
      unlink(tmp4);
   }

   sort_users_log(tmp, debug);

   if(DataFile[0] != '\0')
      data_file(tmp);
   else
      gerarel();

   unlink(tmp2);
   if(strstr(ReportType,"denied") != 0)
      unlink(tmp5);

   if(zip[0] != '\0' && strcmp(zip,"zcat") !=0) {
      recomp(arq, zip);
   }
//   else  unlink(arq);

   if(strcmp(tmp,"/tmp") != 0) {
      if (snprintf(tmp4,sizeof(tmp4),"rm -rf \"%s\"",tmp)>=sizeof(tmp4)) {
         fprintf(stderr,"SARG: Directory name too long: %s\n",tmp);
         exit(1);
      }
      cstatus=system(tmp4);
      if (!WIFEXITED(cstatus) || WEXITSTATUS(cstatus)) {
         fprintf(stderr, "SARG: command return status %d\n",WEXITSTATUS(cstatus));
         fprintf(stderr, "SARG: command: %s\n",tmp4);
         exit(1);
      }
   }

   free_excludecodes();
   if(userfile)
      free(userfile);
   if(excludefile)
      free(excludefile);
   if(excludeuser)
      free(excludeuser);
   free_download();

   if(debug)
      debuga("%s",text[21]);

   exit(0);

}


static void getusers(const char *pwdfile, int debug)
{

   FILE *fp_usr;
   char buf[255];
   char *str;
   long int nreg=0;

   if(debug)
      debuga("%s: %s",text[60],pwdfile);

   if ((fp_usr = fopen(pwdfile, "r")) == NULL) {
      fprintf(stderr, "SARG: (getusers) %s: %s - %s\n",text[45],pwdfile,strerror(errno));
      exit(1);
   }

   fseek(fp_usr, 0, SEEK_END);
   nreg = ftell(fp_usr);
   if (nreg<0) {
      printf("SARG: Cannot get the size of file %s",pwdfile);
      exit(1);
   }
   nreg = nreg+5000;
   fseek(fp_usr, 0, SEEK_SET);

   if((userfile=(char *) malloc(nreg))==NULL){
      fprintf(stderr, "SARG: %s (%ld):\n",text[59],nreg);
      exit(1);
   }

   bzero(userfile,nreg);
   strcpy(userfile,":");

   while(fgets(buf,sizeof(buf),fp_usr)!=NULL) {
      str=strchr(buf,':');
      if (!str) {
         printf("SARG: You have an invalid user in your %s file.\n",pwdfile);
         exit(1);
      }
      str[1]=0;
      strcat(userfile,buf);
   }

   fclose(fp_usr);

   return;
}


static void gethexclude(const char *hexfile, int debug)
{

   FILE *fp_ex;
   char buf[255];
   long int nreg=0;

   if(debug)
      debuga("%s: %s",text[67],hexfile);

   if ((fp_ex = fopen(hexfile, "r")) == NULL) {
      fprintf(stderr, "SARG: (gethexclude) %s: %s - %s\n",text[45],hexfile,strerror(errno));
      exit(1);
   }

   fseek(fp_ex, 0, SEEK_END);
   nreg = ftell(fp_ex);
   if (nreg<0) {
      printf("SARG: Cannot get the size of file %s",hexfile);
      exit(1);
   }
   nreg += 11;
   fseek(fp_ex, 0, SEEK_SET);

   if((excludefile=(char *) malloc(nreg))==NULL){
      fprintf(stderr, "SARG: %s (%ld):\n",text[59],nreg);
      exit(1);
   }

   bzero(excludefile,nreg);

   while(fgets(buf,sizeof(buf),fp_ex)!=NULL){
     if(strchr(buf,'#') != NULL)
        continue;
     fixendofline(buf);
     strcat(excludefile,buf);
     strcat(excludefile," ");
   }

   strcat(excludefile,"*END* ");

   fclose(fp_ex);

   return;
}


static void getuexclude(const char *uexfile, int debug)
{

   FILE *fp_ex;
   char buf[255];
   long int nreg=0;

   if(debug)
      debuga("%s: %s",text[67],uexfile);

   if ((fp_ex = fopen(uexfile, "r")) == NULL) {
      fprintf(stderr, "SARG: (gethexclude) %s: %s - %s\n",text[45],uexfile,strerror(errno));
      exit(1);
   }

   fseek(fp_ex, 0, SEEK_END);
   nreg = ftell(fp_ex);
   if (nreg<0) {
      printf("SARG: Cannot get the size of file %s",uexfile);
      exit(1);
   }
   nreg += 11;
   fseek(fp_ex, 0, SEEK_SET);

   if((excludeuser=(char *) malloc(nreg))==NULL){
      fprintf(stderr, "SARG: %s (%ld):\n",text[59],nreg);
      exit(1);
   }

   bzero(excludeuser,nreg);

   while(fgets(buf,sizeof(buf),fp_ex)!=NULL){
     if(strchr(buf,'#') != NULL)
        continue;
     fixendofline(buf);
     strcat(excludeuser,buf);
     strcat(excludeuser," ");
   }

   strcat(excludeuser,"*END* ");

   fclose(fp_ex);

   return;
}