[thirdparty/sarg.git] / log.c

/*
 * SARG Squid Analysis Report Generator      http://sarg.sourceforge.net
 *                                                            1998, 2010
 *
 * SARG donations:
 *      please look at http://sarg.sourceforge.net/donations.php
 * Support:
 *     http://sourceforge.net/projects/sarg/forums/forum/363374
 * ---------------------------------------------------------------------
 *
 *  This program is free software; you can redistribute it and/or modify
 *  it under the terms of the GNU General Public License as published by
 *  the Free Software Foundation; either version 2 of the License, or
 *  (at your option) any later version.
 *
 *  This program is distributed in the hope that it will be useful,
 *  but WITHOUT ANY WARRANTY; without even the implied warranty of
 *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 *  GNU General Public License for more details.
 *
 *  You should have received a copy of the GNU General Public License
 *  along with this program; if not, write to the Free Software
 *  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111, USA.
 *
 */

#include "include/conf.h"
#include "include/defs.h"

#define REPORT_EVERY_X_LINES 5000

char *userfile;

numlist weekdays = { { 0, 1, 2, 3, 4, 5, 6 }, 7 };
numlist hours = { { 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12,
             13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23 }, 24 };

static void getusers(const char *pwdfile, int debug);

int main(int argc,char *argv[])
{
   enum isa_col_id {
      ISACOL_Ip,
      ISACOL_UserName,
      ISACOL_Date,
      ISACOL_Time,
      ISACOL_TimeTaken,
      ISACOL_Bytes,
      ISACOL_Uri,
      ISACOL_Status,
      ISACOL_Last //last entry of the list !
   };
   enum InputLogFormat {
      ILF_Unknown,
      ILF_Squid,
      ILF_Common,
      ILF_Sarg,
      ILF_Isa,
      ILF_Last //last entry of the list !
   };

   FILE *fp_in = NULL, *fp_denied=NULL, *fp_authfail=NULL, *fp_log=NULL;

   char sz_Download_Unsort[ 20000 ] ;
   FILE * fp_Download_Unsort = NULL ;
   FILE * fp_Write_User = NULL ;

   extern int optind;
   extern int optopt;
   extern char *optarg;

   char data[255];
   char elap[255];
   char ip[MAXLEN];
   char tam[255];
   char fun[MAXLEN];
   char wuser[MAXLEN];
   char smartfilter[MAXLEN];
   char dia[128];
   char wdata[128];
   char mes[30];
   char ano[30];
   char hora[30];
   char wtemp[MAXLEN];
   char wtemp2[255];
   char date[255];
   char arq[255];
   char arq_log[255];
   char hm[15], hmf[15], hmr[15];
   int  chm=0;
   char uagent[MAXLEN];
   char hexclude[MAXLEN];
   char csort[MAXLEN];
   int cstatus;
   char tbuf[128];
   char tbuf2[128];
   char zip[20];
   char *str;
   char tmp2[MAXLEN];
   char start_hour[128];
   char end_hour[128];
   char *linebuf;
   char hostname[512];
   char *url;
   char *urly;
   enum InputLogFormat ilf;
   int ilf_count[ILF_Last];
   int  ch;
   int  x, l;
   int  errflg=0;
   int  puser=0;
   int  fhost=0;
   int  dns=0;
   int  fuser=0;
   int  idata=0;
   int  mindate=0;
   int  iarq=0;
   int  exstring=0;
   int isa_ncols=0,isa_cols[ISACOL_Last];
   int from_stdin;
   int blen;
   long totregsl=0;
   long totregsg=0;
   long totregsx=0;
   long totper=0;
   long int max_elapsed=0;
   time_t tt;
   struct tm *t;
   unsigned long recs1=0UL;
   unsigned long recs2=0UL;
   int OutputNonZero = REPORT_EVERY_X_LINES ;
   int download_flag=0;
   char *download_url;
   char sz_Last_User[MAXLEN]="";
   struct getwordstruct gwarea;
   struct longlinestruct line;

   BgImage[0]='\0';
   LogoImage[0]='\0';
   LogoText[0]='\0';
   PasswdFile[0]='\0';
   OutputEmail[0]='\0';
   UserAgentLog[0]='\0';
   ExcludeHosts[0]='\0';
   ExcludeUsers[0]='\0';
   ConfigFile[0]='\0';
   code[0]='\0';
   LastLog=0;
   ReportType=0UL;
   UserTabFile[0]='\0';
   BlockIt[0]='\0';
   ExternalCSSFile[0]='\0';
   SquidGuardLogFormat[0]='\0';
   SquidGuardLogAlternate[0]='\0';
   for (ilf=0 ; ilf<ILF_Last ; ilf++) ilf_count[ilf]=0;

   sprintf(ExcludeCodes,"%s/exclude_codes",SYSCONFDIR);
   strcpy(GraphDaysBytesBarColor,"orange");
   strcpy(BgColor,"#ffffff");
   strcpy(TxColor,"#000000");
   strcpy(TxBgColor,"lavender");
   strcpy(TiColor,"darkblue");
   strcpy(Width,"80");
   strcpy(Height,"45");
   strcpy(LogoTextColor,"#000000");
   strcpy(HeaderColor,"darkblue");
   strcpy(HeaderBgColor,"#dddddd");
   strcpy(LogoTextColor,"#006699");
   strcpy(FontSize,"9px");
   strcpy(TempDir,"/tmp");
   strcpy(OutputDir,"/var/www/html/squid-reports");
   Ip2Name=0;
   strcpy(DateFormat,"u");
   OverwriteReport=0;
   RemoveTempFiles=1;
   strcpy(ReplaceIndex,"index.html");
   Index=INDEX_YES;
   strcpy(RecordsWithoutUser,"ip");
   UseComma=0;
   strcpy(MailUtility,"mailx");
   TopSitesNum=100;
   TopUsersNum=0;
   UserIp=0;
   strcpy(TopuserSortField,"BYTES");
   strcpy(UserSortField,"BYTES");
   strcpy(TopuserSortOrder,"reverse");
   strcpy(UserSortOrder,"reverse");
   strcpy(TopsitesSortField,"CONNECT");
   strcpy(TopsitesSortType,"D");
   LongUrl=0;
   strcpy(language,"English");
   strcpy(FontFace,"Verdana,Tahoma,Arial");
   strcpy(datetimeby,"elap");
   strcpy(CharSet,"ISO-8859-1");
   Privacy=0;
   strcpy(PrivacyString,"***.***.***.***");
   strcpy(PrivacyStringColor,"blue");
   SuccessfulMsg=1;
   TopUserFields=TOPUSERFIELDS_NUM | TOPUSERFIELDS_DATE_TIME | TOPUSERFIELDS_USERID | TOPUSERFIELDS_CONNECT |
                 TOPUSERFIELDS_BYTES | TOPUSERFIELDS_SETYB | TOPUSERFIELDS_IN_CACHE_OUT |
                 TOPUSERFIELDS_USED_TIME | TOPUSERFIELDS_MILISEC | TOPUSERFIELDS_PTIME |
                 TOPUSERFIELDS_TOTAL | TOPUSERFIELDS_AVERAGE;
   UserReportFields=USERREPORTFIELDS_CONNECT | USERREPORTFIELDS_BYTES | USERREPORTFIELDS_SETYB |
                    USERREPORTFIELDS_IN_CACHE_OUT | USERREPORTFIELDS_USED_TIME | USERREPORTFIELDS_MILISEC |
                    USERREPORTFIELDS_PTIME | USERREPORTFIELDS_TOTAL | USERREPORTFIELDS_AVERAGE;
   strcpy(DataFileDelimiter,";");
   DataFileFields=DATA_FIELD_USER | DATA_FIELD_DATE | DATA_FIELD_TIME | DATA_FIELD_URL | DATA_FIELD_CONNECT |
                  DATA_FIELD_BYTES | DATA_FIELD_IN_CACHE | DATA_FIELD_OUT_CACHE | DATA_FIELD_ELAPSED;
   ShowReadStatistics=1;
   strcpy(IndexSortOrder,"D");
   ShowSargInfo=1;
   ShowSargLogo=1;
   strcpy(ParsedOutputLog,"no");
   strcpy(ParsedOutputLogCompress,"/bin/gzip -f");
   strcpy(DisplayedValues,"abbreviation");
   strcpy(HeaderFontSize,"9px");
   strcpy(TitleFontSize,"11px");
   strcpy(AuthUserFile,"/usr/local/sarg/passwd");
   strcpy(AuthName,"SARG, Restricted Access");
   strcpy(AuthType,"basic");
   strcpy(Require,"require user admin %u");
   set_download_suffix("7z,ace,arj,avi,bat,bin,bz2,bzip,cab,com,cpio,dll,doc,dot,exe,gz,iso,lha,lzh,mdb,mov,mp3,mpeg,mpg,mso,nrg,ogg,ppt,rar,rtf,shs,src,sys,tar,tgz,vcd,vob,wma,wmv,zip");
   Graphs=1;
#if defined(FONTDIR)
   strcpy(GraphFont,FONTDIR"/DejaVuSans.ttf");
#else
   GraphFont[0]='\0';
#endif
   strcpy(Ulimit,"20000");
   strcpy(NtlmUserFormat,"domainname+username");
   IndexTree=INDEX_TREE_FILE;
   strcpy(RealtimeTypes,"GET,PUT,CONNECT");
   strcpy(RealtimeUnauthRec,"show");
   SquidguardIgnoreDate=0;
   DansguardianIgnoreDate=0;
   strcpy(DataFileUrl,"ip");
   strcpy(MaxElapsed,"28800000");
   BytesInSitesUsersReport=0;
   UserAuthentication=0;
   strcpy(LDAPHost,"127.0.0.1");
   LDAPPort=389;
   LDAPProtocolVersion=3;
   LDAPBindDN[0]='\0';
   LDAPBindPW[0]='\0';
   LDAPBaseSearch[0]='\0';
   strcpy(LDAPFilterSearch, "uid=%s");
   strcpy(LDAPTargetAttr, "cn");

   dia[0]='\0';
   mes[0]='\0';
   ano[0]='\0';
   hora[0]='\0';
   tmp[0]='\0';
   tmp2[0]='\0';
   tmp3[0]='\0';
   wtemp[0]='\0';
   wtemp2[0]='\0';
   us[0]='\0';
   date[0]='\0';
   df[0]='\0';
   uagent[0]='\0';
   hexclude[0]='\0';
   addr[0]='\0';
   hm[0]='\0';
   hmf[0]='\0';
   site[0]='\0';
   outdir[0]='\0';
   elap[0]='\0';
   email[0]='\0';
   zip[0]='\0';
   UserInvalidChar[0]='\0';
   DataFile[0]='\0';
   SquidGuardConf[0]='\0';
   DansGuardianConf[0]='\0';
   start_hour[0]='\0';
   end_hour[0]='\0';

   denied_count=0;
   download_count=0;
   authfail_count=0;
   dansguardian_count=0;
   squidguard_count=0;
   useragent_count=0;
   DeniedReportLimit=10;
   AuthfailReportLimit=10;
   DansGuardianReportLimit=10;
   SquidGuardReportLimit=10;
   DownloadReportLimit=50;
   UserReportLimit=0;
   debug=0;
   debugz=0;
   debugm=0;
   iprel=0;
   userip=0;
   color1=0;
   color2=0;
   color3=0;
   dotinuser=0;
   realt=0;
   realtime_refresh=3;
   realtime_access_log_lines=1000;
   cost=0.01;
   nocost=50000000;
   ndownload=0;
   squid24=0;

   bzero(IncludeUsers, MAXLEN);
   bzero(ExcludeString, MAXLEN);

#ifdef HAVE_LOCALE_H
   setlocale(LC_TIME,"");
   setlocale (LC_MESSAGES, "");
#endif
   bindtextdomain (PACKAGE_NAME, LOCALEDIR);
   textdomain (PACKAGE_NAME);

   NAccessLog=0;
   for(x=0; x<MAXLOGS; x++)
      AccessLog[x][0]='\0';
   AccessLogFromCmdLine=0;

   language_load(language);
   strcpy(Title,text[88]);

   while((ch = getopt(argc, argv, "a:b:c:d:e:f:g:u:l:L:o:s:t:w:hijmnprvxyz")) != -1){
      switch(ch)
      {
         case 'a':
            strcpy(addr,optarg);
            break;
         case 'b':
            strcpy(uagent,optarg);
            break;
         case 'c':
            strcpy(hexclude,optarg);
            break;
         case 'd':
            strncpy(date,optarg,sizeof(date)-1);
            date[sizeof(date)-1]='\0';
            getword_start(&gwarea,optarg);
            if (getword(cdfrom,sizeof(cdfrom),&gwarea,'-')<0 || getword(cduntil,sizeof(cduntil),&gwarea,0)<0) {
               printf("SARG: Maybe you have a broken record or garbage in your date range.\n");
               exit(1);
            }
            date_from(date, cdfrom, cduntil);
            dfrom=atoi(cdfrom);
            duntil=atoi(cduntil);
            break;
         case 'e':
            strcpy(email,optarg);
            break;
         case 'f':
            strcpy(ConfigFile,optarg);
            break;
         case 'g':
            strcpy(df,optarg);
            break;
         case 'h':
            usage(argv[0]);
            exit(0);
            break;
         case 'i':
            iprel++;
            break;
         case 'l':
            if (NAccessLog>=MAXLOGS) {
               printf("SARG: Too many log files.\n");
               exit(1);
            }
            strcpy(AccessLog[NAccessLog],optarg);
            NAccessLog++;
            AccessLogFromCmdLine++;
            break;
         case 'L':
            strcpy(SquidGuardLogAlternate,optarg);
            break;
         case 'm':
            debugm++;
            break;
         case 'n':
            dns++;
            break;
         case 'o':
            strcpy(outdir,optarg);
            break;
         case 'p':
            userip++;
            break;
         case 'r':
            realt++;
            break;
         case 's':
            strcpy(site,optarg);
            break;
         case 't':
         {
            int h,m;

            if(strstr(optarg,"-") == 0) {
              strcpy(hm,optarg);
              strcpy(hmf,optarg);
            } else {
               getword_start(&gwarea,optarg);
               if (getword(hm,sizeof(hm),&gwarea,'-')<0 || getword(hmf,sizeof(hmf),&gwarea,0)<0) {
                  fprintf(stderr,"SARG: Maybe you have a broken record or garbage in your time range.\n");
                  exit(1);
               }
            }
            if(sscanf(hm,"%d:%d",&h,&m)!=2) {
               fprintf(stderr,"SARG: time period must be MM or MM:SS. Exit.\n");
               exit(1);
            }
            sprintf(hm,"%02d%02d",h,m);
            if(sscanf(hmf,"%d:%d",&h,&m)!=2) {
               fprintf(stderr,"SARG: time period must be MM or MM:SS. Exit.\n");
               exit(1);
            }
            sprintf(hmf,"%02d%02d",h,m);
            break;
         }
         case 'u':
            strcpy(us,optarg);
            break;
         case 'v':
            version();
            break;
         case 'w':
            strcpy(tmp,optarg);
            break;
         case 'x':
            debug++;
            break;
         case 'y':
            langcode++;
            break;
         case 'z':
            debugz++;
            break;
         case ':':
            fprintf(stderr, "Option -%c require an argument\n",optopt);
            errflg++;
            break;
         case '?':
            usage(argv[0]);
            exit(1);
            break;
      }

   }

   if (errflg) {
      usage(argv[0]);
      exit(2);
   }

   if(debug) debuga("Init");

   if(ConfigFile[0] == '\0') sprintf(ConfigFile,"%s/sarg.conf",SYSCONFDIR);
   if(access(ConfigFile, R_OK) != 0) {
      debuga("Cannot open config file: %s - %s",ConfigFile,strerror(errno));
      exit(1);
   }

   if(access(ConfigFile, R_OK) == 0)
      getconf();

   if(UserIp) userip++;

   if(dns) Ip2Name=1;

   if(realt) {
      realtime();
      exit(0);
   }

   if(IndexTree == INDEX_TREE_FILE)
      strcpy(ImageFile,"../images");
   else
      strcpy(ImageFile,"../../../images");

   dataonly=0;
   if(DataFile[0] != '\0')
      dataonly++;

   if(!NAccessLog) {
      strcpy(AccessLog[0],"/var/log/squid/access.log");
      NAccessLog++;
   }

   if(strcmp(hexclude,"onvert") == 0 && strcmp(site,"plit") != 0) {
      convlog(AccessLog[0], df, dfrom, duntil);
      exit(0);
   }

   if(strcmp(site,"plit") == 0) {
      splitlog(AccessLog[0], df, dfrom, duntil, hexclude);
      exit(0);
   }

   load_excludecodes(ExcludeCodes);

   if(access(PasswdFile, R_OK) == 0) {
      getusers(PasswdFile,debug);
      puser++;
   }

   if(hexclude[0] == '\0')
      strcpy(hexclude,ExcludeHosts);
   if(hexclude[0] != '\0') {
      gethexclude(hexclude,debug);
      fhost++;
   }

   if(ReportType == 0) {
      ReportType=REPORT_TYPE_TOPUSERS | REPORT_TYPE_TOPSITES | REPORT_TYPE_USERS_SITES |
                 REPORT_TYPE_SITES_USERS | REPORT_TYPE_DATE_TIME | REPORT_TYPE_DENIED |
                 REPORT_TYPE_AUTH_FAILURES | REPORT_TYPE_SITE_USER_TIME_DATE | REPORT_TYPE_DOWNLOADS;
   }

   if(access(ExcludeUsers, R_OK) == 0) {
      getuexclude(ExcludeUsers,debug);
      fuser++;
   }

   indexonly=0;
   if(fuser) {
      if(is_indexonly())
         indexonly++;
   }
   if(strcmp(ExcludeUsers,"indexonly") == 0) indexonly++;
   if(Index == INDEX_ONLY) indexonly++;

   if(MaxElapsed[0] != '\0') max_elapsed=atol(MaxElapsed);

   if(outdir[0] == '\0') strcpy(outdir,OutputDir);
   strcat(outdir,"/");

   if(uagent[0] == '\0') strcpy(uagent,UserAgentLog);

   if(tmp[0] == '\0') strcpy(tmp,TempDir);
   else strcpy(TempDir,tmp);

   if(df[0] == '\0') strcpy(df,DateFormat);
   else strcpy(DateFormat,df);

   if(df[0] == '\0') {
      strcpy(df,"u");
      strcpy(DateFormat,"u");
   }

   if(email[0] == '\0' && OutputEmail[0] != '\0') strcpy(email,OutputEmail);

   strcpy(tmp2,tmp);

   if(email[0] != '\0') {
      sprintf(wtemp2,"%s/sarg",tmp2);
      my_mkdir(wtemp2);
      strcat(tmp2,"/sarg");
      strcpy(outdir,tmp2);
      strcat(outdir,"/");
   }

   strcat(tmp2,"/sarg.log");

   sprintf(tmp3,"%s/sarg",tmp);
   if(access(tmp3, R_OK) == 0) {
      unlinkdir(tmp3,1);
   }
   my_mkdir(tmp3);
   strcpy(tmp4,tmp3);
   strcpy(tmp5,tmp3);
   strcpy(tmp6,tmp3);
   strcat(tmp4,"/denied.log.unsort");
   strcat(tmp5,"/denied.log");
   strcat(tmp6,"/authfail.log.unsort");

   if(debug) {
      fprintf(stderr, "SARG: %s:\nSARG:\n",text[22]);
      fprintf(stderr, "SARG: %35s (-a) = %s\n",text[23],addr);
      fprintf(stderr, "SARG: %35s (-b) = %s\n",text[71],uagent);
      fprintf(stderr, "SARG: %35s (-c) = %s\n",text[69],hexclude);
      fprintf(stderr, "SARG: %35s (-d) = %s\n",text[24],date);
      fprintf(stderr, "SARG: %35s (-e) = %s\n",text[41],email);
      fprintf(stderr, "SARG: %35s (-f) = %s\n",text[70],ConfigFile);
      if(strcmp(df,"e") == 0)
         fprintf(stderr, "SARG: %35s (-g) = %s (dd/mm/yyyy)\n",text[25],text[26]);
      if(strcmp(df,"u") == 0)
         fprintf(stderr, "SARG: %35s (-g) = %s (mm/dd/yyyy)\n",text[25],text[27]);
      if(strcmp(df,"w") == 0)
         fprintf(stderr, "SARG: %35s (-g) = %s (yyyy/ww)\n",text[25],text[85]);
      if(iprel)
         fprintf(stderr, "SARG: %35s (-i) = %s\n",text[28],text[1]);
      else
         fprintf(stderr, "SARG: %35s (-i) = %s\n",text[28],text[2]);
      for (iarq=0 ; iarq<NAccessLog ; iarq++)
         fprintf(stderr, "SARG: %35s (-l) = %s\n",text[37],AccessLog[iarq]);
      if(Ip2Name)
         fprintf(stderr, "SARG: %35s (-n) = %s\n",text[65],text[1]);
       else
         fprintf(stderr, "SARG: %35s (-n) = %s\n",text[65],text[2]);
      fprintf(stderr, "SARG: %35s (-o) = %s\n",text[38],outdir);
      if(UserIp)
         fprintf(stderr, "SARG: %35s (-p) = %s\n",text[29],text[1]);
       else
         fprintf(stderr, "SARG: %35s (-p) = %s\n",text[29],text[2]);
      fprintf(stderr, "SARG: %35s (-s) = %s\n",text[30],site);
      fprintf(stderr, "SARG: %35s (-t) = %s\n",text[31],hm);
      fprintf(stderr, "SARG: %35s (-u) = %s\n",text[32],us);
      fprintf(stderr, "SARG: %35s (-w) = %s\n",text[34],tmp);
      if(debug)
         fprintf(stderr, "SARG: %35s (-x) = %s\n",text[35],text[1]);
       else
         fprintf(stderr, "SARG: %35s (-x) = %s\n",text[35],text[2]);
      if(debugz)
         fprintf(stderr, "SARG: %35s (-z) = %s\n",text[36],text[1]);
       else
         fprintf(stderr, "SARG: %35s (-z) = %s\n",text[36],text[2]);
      fprintf(stderr, "SARG:\n");
   }

   if(debugm) {
      printf("%s:\nSARG:\n",text[22]);
      printf("%35s (-a) = %s\n",text[23],addr);
      printf("%35s (-b) = %s\n",text[71],uagent);
      printf("%35s (-c) = %s\n",text[69],hexclude);
      printf("%35s (-d) = %s\n",text[24],date);
      printf("%35s (-e) = %s\n",text[41],email);
      printf("%35s (-f) = %s\n",text[70],ConfigFile);
      if(strcmp(df,"e") == 0)
         printf("%35s (-g) = %s (dd/mm/yyyy)\n",text[25],text[26]);
      if(strcmp(df,"u") == 0)
         printf("%35s (-g) = %s (mm/dd/yyyy)\n",text[25],text[27]);
      if(strcmp(df,"w") == 0)
         printf("%35s (-g) = %s (yyyy/ww)\n",text[25],text[85]);
      if(iprel)
         printf("%35s (-i) = %s\n",text[28],text[1]);
      else
         printf("%35s (-i) = %s\n",text[28],text[2]);
      for (iarq=0 ; iarq<NAccessLog ; iarq++)
         printf("%35s (-l) = %s\n",text[37],AccessLog[iarq]);
      if(Ip2Name)
         printf("%35s (-n) = %s\n",text[65],text[1]);
       else
         printf("%35s (-n) = %s\n",text[65],text[2]);
      printf("%35s (-o) = %s\n",text[38],outdir);
      if(UserIp)
         printf("%35s (-p) = %s\n",text[29],text[1]);
       else
         printf("%35s (-p) = %s\n",text[29],text[2]);
      printf("%35s (-s) = %s\n",text[30],site);
      printf("%35s (-t) = %s\n",text[31],hm);
      printf("%35s (-u) = %s\n",text[32],us);
      printf("%35s (-w) = %s\n",text[34],tmp);
      if(debug)
         printf("%35s (-x) = %s\n",text[35],text[1]);
       else
         printf("%35s (-x) = %s\n",text[35],text[2]);
      if(debugz)
         printf("%35s (-z) = %s\n",text[36],text[1]);
       else
         printf("%35s (-z) = %s\n",text[36],text[2]);
      printf("sarg %s: %s\n",text[73],VERSION);
      printf("Language=%s\n\n",text[3]);
   }

   if(debug)
      debuga("sarg %s: %s",text[73],VERSION);

#ifdef HAVE_RLIM_T
   if (Ulimit[0] != '\0') {
      struct rlimit rl;
      long l1, l2;
      int rc=0;

#if defined(RLIMIT_NOFILE)
      getrlimit (RLIMIT_NOFILE, &rl);
#elif defined(RLIMIT_OFILE)
      getrlimit (RLIMIT_OFILE, &rl);
#else
#warning "No rlimit resource for the number of open files"
#endif
      l1 = rl.rlim_cur;
      l2 = rl.rlim_max;

      rl.rlim_cur = atol(Ulimit);
      rl.rlim_max = atol(Ulimit);
#if defined(RLIMIT_NOFILE)
      rc=setrlimit (RLIMIT_NOFILE, &rl);
#elif defined(RLIMIT_OFILE)
      rc=setrlimit (RLIMIT_OFILE, &rl);
#else
#warning "No rlimit resource for the number of open files"
#endif
      if(rc == -1) {
            debuga("setrlimit error - %s\n",strerror(errno));
      }

      if(debug)
         debuga("Maximum file descriptor: cur=%ld max=%ld, changed to cur="RLIM_STRING" max="RLIM_STRING,l1,l2,rl.rlim_cur,rl.rlim_max);
   }
#endif

   init_usertab(UserTabFile);

   if (longline_prepare(&line)<0) {
      debuga(_("Not enough memory to read a log file"));
      exit(1);
   }

   sprintf ( sz_Download_Unsort , "%s/sarg/download.unsort", tmp);

   if((ReportType & REPORT_TYPE_DENIED) != 0) {
      if((fp_denied=MY_FOPEN(tmp4,"w"))==NULL) {
         fprintf(stderr, "%s: (log) %s: %s - %s\n",argv[0],text[45],tmp4,strerror(errno));
         exit(1);
      }
   }

   if(DataFile[0]=='\0') {
      if((ReportType & REPORT_TYPE_DENIED) != 0 || (ReportType & REPORT_TYPE_AUTH_FAILURES) != 0) {
         if((fp_authfail=MY_FOPEN(tmp6,"w"))==NULL) {
            fprintf(stderr, "%s: (log) %s: %s - %s\n",argv[0],text[45],tmp6,strerror(errno));
            exit(1);
         }
      }
   }

   for (iarq=0 ; iarq<NAccessLog ; iarq++) {
      strcpy(arq,AccessLog[iarq]);

      strcpy(arqtt,arq);

      if(strcmp(arq,"-")==0) {
         if(debug)
            debuga("%s: %s",text[7],"stdin");
         fp_in=stdin;
         from_stdin=1;
      } else {
         decomp(arq,zip,tmp);
         if(debug)
            debuga("%s: %s",text[7],arq);
         if((fp_in=MY_FOPEN(arq,"r"))==NULL) {
            fprintf(stderr, "%s: (log) %s: %s - %s\n",argv[0],text[8],arq,strerror(errno));
            exit(1);
         }
         from_stdin=0;
      }
      ilf=ILF_Unknown;
      download_flag=0;
      // pre-read the file only if we have to show stats
      if(ShowReadStatistics && !from_stdin) {
         size_t nread,i;
         int skipcr=0;

         recs1=0UL;
         recs2=0UL;

         while ((nread=fread(line.buffer,1,line.size,fp_in))>0) {
            for (i=0 ; i<nread ; i++)
               if (skipcr) {
                  if (line.buffer[i]!='\n' && line.buffer[i]!='\r') {
                     skipcr=0;
                  }
               } else {
                  if (line.buffer[i]=='\n' || line.buffer[i]=='\r') {
                     skipcr=1;
                     recs1++;
                  }
               }
         }
         rewind(fp_in);
         printf("SARG: Records in file: %lu, reading: %3.2f%%\r",recs1,(float) 0);
         fflush( stdout ) ;
      }

      line.start=0;
      line.end=0;
      line.length=0;

      while ((linebuf=longline_read(fp_in,&line))!=NULL) {
         blen=strlen(linebuf);

         if (ilf==ILF_Unknown) {
            if(strncmp(linebuf,"#Software: Mic",14) == 0) {
               fixendofline(linebuf);
               if (debug)
                  debuga("%s: %s",text[143],linebuf);
               ilf=ILF_Isa;
               ilf_count[ilf]++;
               continue;
            }

            if(strncmp(linebuf,"*** SARG Log ***",16) == 0) {
               getword_start(&gwarea,arqtt);
               if (getword_skip(2000,&gwarea,'-')<0 || getword(val2,sizeof(val2),&gwarea,'_')<0 ||
                   getword_skip(10,&gwarea,'-')<0 || getword(val3,sizeof(val3),&gwarea,'_')<0) {
                  printf("SARG: The name of the file is invalid: %s\n",arq);
                  exit(1);
               }
               sprintf(period,"%s-%s",val2,val3);
               ilf=ILF_Sarg;
               ilf_count[ilf]++;
               continue;
            }
         }

         if(!fp_log && strcmp(ParsedOutputLog, "no") != 0 && ilf!=ILF_Sarg) {
            if(access(ParsedOutputLog,R_OK) != 0) {
               my_mkdir(ParsedOutputLog);
            }
            sprintf(arq_log,"%s/sarg_temp.log",ParsedOutputLog);
            if((fp_log=MY_FOPEN(arq_log,"w"))==NULL) {
               fprintf(stderr, "%s: (log) %s: %s - %s\n",argv[0],text[8],arq_log,strerror(errno));
               exit(1);
            }
            fputs("*** SARG Log ***\n",fp_log);
         }

         recs2++;
         if( ShowReadStatistics && !from_stdin && ! --OutputNonZero) {
           perc = recs2 * 100 ;
           perc = perc / recs1 ;
           printf("SARG: Records in file: %lu, reading: %3.2f%%\r",recs1,perc);
           fflush (stdout);
           OutputNonZero = REPORT_EVERY_X_LINES ;
         }
         if(blen < 58) continue;
         if(strstr(linebuf,"HTTP/0.0") != 0) continue;
         if(strstr(linebuf,"logfile turned over") != 0) continue;
         if(linebuf[0] == ' ') continue;

         // Record only hours usage which is required
         tt = (time_t) strtoul( linebuf, NULL, 10 );
         t = localtime( &tt );

         if( bsearch( &( t -> tm_wday ), weekdays.list, weekdays.len,
                                          sizeof( int ), compar ) == NULL )
            continue;

         if( bsearch( &( t -> tm_hour ), hours.list, hours.len,
                                          sizeof( int ), compar ) == NULL )
            continue;

         // exclude_string
         exstring=0;
         if(ExcludeString[0] != '\0') {
            getword_start(&gwarea,ExcludeString);
            while(strchr(gwarea.current,':') != 0) {
               if (getword_multisep(val1,sizeof(val1),&gwarea,':')<0) {
                  printf("SARG: Maybe you have a broken record or garbage in your exclusion string.\n");
                  exit(1);
               }
               if((str=(char *) strstr(linebuf,val1)) != (char *) NULL )
                  exstring++;
            }
            if((str=(char *) strstr(linebuf,gwarea.current)) != (char *) NULL )
                  exstring++;
         }
         if(exstring) continue;

         if ((str = strchr(linebuf, '\n')) != NULL)
            *str = '\0';          /* strip \n */

         totregsl++;
         if(debugm)
            printf("BUF=%s\n",linebuf);

         if (ilf==ILF_Squid || ilf==ILF_Common || ilf==ILF_Unknown) {
            getword_start(&gwarea,linebuf);
            if (getword(data,sizeof(data),&gwarea,' ')<0) {
               printf("SARG: Maybe you have a broken time in your access.log file.\n");
               exit(1);
            }
            if((str=(char *) strchr(data, '.')) != (char *) NULL ) {
               if((str=(char *) strchr(str+1, '.')) != (char *) NULL ) {
                  strcpy(ip,data);
                  strcpy(elap,"0");
                  if(squid24) {
                     if (getword(user,sizeof(user),&gwarea,' ')<0 || getword_skip(255,&gwarea,' ')<0) {
                        printf("SARG: Maybe you have a broken record or garbage in your %s file.\n",arq);
                        exit(1);
                     }
                  } else {
                     if (getword_skip(255,&gwarea,' ')<0 || getword(user,sizeof(user),&gwarea,' ')<0) {
                        printf("SARG: Maybe you have a broken record or garbage in your %s file.\n",arq);
                        exit(1);
                     }
                  }
                  if (getword(data,sizeof(data),&gwarea,']')<0 || getword_skip(MAXLEN,&gwarea,'"')<0 ||
                      getword(fun,sizeof(fun),&gwarea,' ')<0) {
                     printf("SARG: Maybe you have a broken record or garbage in your %s file.\n",arq);
                     exit(1);
                  }
                  if (getword_ptr(linebuf,&url,&gwarea,' ')<0) {
                     debuga(_("Maybe you have a broken url in your %s file"),arq);
                     exit(1);
                  }
                  if (getword_skip(MAXLEN,&gwarea,' ')<0 || getword(code2,sizeof(code2),&gwarea,' ')<0 ||
                      getword(tam,sizeof(tam),&gwarea,' ')<0) {
                     printf("SARG: Maybe you have a broken record or garbage in your %s file.\n",arq);
                     exit(1);
                  }
                  if((str=(char *) strchr(gwarea.current, ' ')) != (char *) NULL ) {
                     if (getword(code,sizeof(code),&gwarea,' ')<0) {
                        printf("SARG: Maybe you have a broken record or garbage in your %s file.\n",arq);
                        exit(1);
                     }
                  } else {
                     if (getword(code,sizeof(code),&gwarea,'\0')<0) {
                        printf("SARG: Maybe you have a broken record or garbage in your %s file.\n",arq);
                        exit(1);
                     }
                  }

                  if ((str = strchr(code, ':')) != NULL)
                     *str = '/';

                  if(strcmp(tam,"\0") == 0)
                     strcpy(tam,"0");

                  ilf=ILF_Common;
                  ilf_count[ilf]++;
               }
            }

            if(ilf==ILF_Unknown || ilf==ILF_Squid) {
               if (getword(elap,sizeof(elap),&gwarea,' ')<0) {
                  printf("SARG: Maybe you have a broken elapsed time in your %s file.\n",arq);
                  exit(1);
               }
               while(strcmp(elap,"") == 0 && gwarea.current[0] != '\0')
                  if (getword(elap,sizeof(elap),&gwarea,' ')<0) {
                     printf("SARG: Maybe you have a broken elapsed time in your %s file.\n",arq);
                     exit(1);
                  }
               if(strlen(elap) < 1) continue;
               if (getword(ip,sizeof(ip),&gwarea,' ')<0){
                  printf("SARG: Maybe you have a broken client IP address in your %s file.\n",arq);
                  exit(1);
               }
               if (getword(code,sizeof(code),&gwarea,' ')<0){
                  printf("SARG: Maybe you have a broken result code in your %s file.\n",arq);
                  exit(1);
               }
               if (getword(tam,sizeof(tam),&gwarea,' ')<0){
                  printf("SARG: Maybe you have a broken amount of data in your %s file.\n",arq);
                  exit(1);
               }
               if (getword(fun,sizeof(fun),&gwarea,' ')<0){
                  printf("SARG: Maybe you have a broken request method in your %s file.\n",arq);
                  exit(1);
               }
               if (getword_ptr(linebuf,&url,&gwarea,' ')<0){
                  debuga(_("Maybe you have a broken url in your %s file"),arq);
                  exit(1);
               }
               if (getword(user,sizeof(user),&gwarea,' ')<0){
                  printf("SARG: Maybe you have a broken user ID in your %s file.\n",arq);
                  exit(1);
               }
               ilf=ILF_Squid;
               ilf_count[ilf]++;
            }
         }
         if (ilf==ILF_Sarg) {
            getword_start(&gwarea,linebuf);
            if (getword(data,sizeof(data),&gwarea,'\t')<0){
               printf("SARG: Maybe you have a broken record or garbage in your %s file.\n",arq);
               exit(1);
            }
            if (getword(hora,sizeof(hora),&gwarea,'\t')<0) {
               printf("SARG: Maybe you have a broken record or garbage in your %s file.\n",arq);
               exit(1);
            }
            if (getword(user,sizeof(user),&gwarea,'\t')<0) {
               printf("SARG: Maybe you have a broken record or garbage in your %s file.\n",arq);
               exit(1);
            }
            if (getword(ip,sizeof(ip),&gwarea,'\t')<0) {
               printf("SARG: Maybe you have a broken record or garbage in your %s file.\n",arq);
               exit(1);
            }
            if (getword_ptr(linebuf,&url,&gwarea,'\t')<0){
               debuga(_("Maybe you have a broken record or garbage in your %s file"),arq);
               exit(1);
            }
            if (getword(tam,sizeof(tam),&gwarea,'\t')<0){
               printf("SARG: Maybe you have a broken record or garbage in your %s file.\n",arq);
               exit(1);
            }
            if (getword(code,sizeof(code),&gwarea,'\t')<0){
               printf("SARG: Maybe you have a broken record or garbage in your %s file.\n",arq);
               exit(1);
            }
            if (getword(elap,sizeof(elap),&gwarea,'\t')<0){
               printf("SARG: Maybe you have a broken record or garbage in your %s file.\n",arq);
               exit(1);
            }
            if (getword(smartfilter,sizeof(smartfilter),&gwarea,'\0')<0){
               printf("SARG: Maybe you have a broken record or garbage in your %s file.\n",arq);
               exit(1);
            }
         }
         if (ilf==ILF_Isa) {
            if (linebuf[0] == '#') {
               int ncols,cols[ISACOL_Last];

               fixendofline(linebuf);
               getword_start(&gwarea,linebuf);
               // remove the #Fields: column at the beginning of the line
               if (getword_skip(1000,&gwarea,' ')<0){
                  printf("SARG: Maybe you have a broken record or garbage in your %s file.\n",arq);
                  exit(1);
               }
               for (ncols=0 ; ncols<ISACOL_Last ; ncols++) cols[ncols]=-1;
               ncols=0;
               while(gwarea.current[0] != '\0') {
                  if (getword(val1,sizeof(val1),&gwarea,'\t')<0){
                     printf("SARG: Maybe you have a broken record or garbage in your %s file.\n",arq);
                     exit(1);
                  }
                  if(strcmp(val1,"c-ip") == 0) cols[ISACOL_Ip]=ncols;
                  if(strcmp(val1,"cs-username") == 0) cols[ISACOL_UserName]=ncols;
                  if(strcmp(val1,"date") == 0) cols[ISACOL_Date]=ncols;
                  if(strcmp(val1,"time") == 0) cols[ISACOL_Time]=ncols;
                  if(strcmp(val1,"time-taken") == 0) cols[ISACOL_TimeTaken]=ncols;
                  if(strcmp(val1,"sc-bytes") == 0) cols[ISACOL_Bytes]=ncols;
                  if(strcmp(val1,"cs-uri") == 0) cols[ISACOL_Uri]=ncols;
                  if(strcmp(val1,"sc-status") == 0) cols[ISACOL_Status]=ncols;
                  ncols++;
               }
               if (cols[ISACOL_Ip]>=0) {
                  isa_ncols=ncols;
                  for (isa_ncols=0 ; isa_ncols<ncols ; isa_ncols++)
                     isa_cols[isa_ncols]=cols[isa_ncols];
               }
               continue;
            }
            if (!isa_ncols) continue;
            getword_start(&gwarea,linebuf);
            for (x=0 ; x<isa_ncols ; x++) {
               if (getword_ptr(linebuf,&str,&gwarea,'\t')<0) {
                  debuga(_("Maybe you have a broken record or garbage in your %s file"),arq);
                  exit(1);
               }
               if (x==isa_cols[ISACOL_Ip]) {
                  if (strlen(str)>=sizeof(ip)) {
                     printf("SARG: Maybe you have a broken IP in your %s file.\n",arq);
                     exit(1);
                  }
                  strcpy(ip,str);
               } else if (x==isa_cols[ISACOL_UserName]) {
                  if (strlen(str)>=sizeof(user)) {
                     printf("SARG: Maybe you have a broken user in your %s file.\n",arq);
                     exit(1);
                  }
                  strcpy(user,str);
               } else if (x==isa_cols[ISACOL_Date]) {
                  if (strlen(str)>=sizeof(data)) {
                     printf("SARG: Maybe you have a broken date in your %s file.\n",arq);
                     exit(1);
                  }
                  strcpy(data,str);
               } else if (x==isa_cols[ISACOL_Time]) {
                  if (strlen(str)>=sizeof(hora)) {
                     printf("SARG: Maybe you have a broken time in your %s file.\n",arq);
                     exit(1);
                  }
                  strcpy(hora,str);
               } else if (x==isa_cols[ISACOL_TimeTaken]) {
                  if (strlen(str)>=sizeof(elap)) {
                     printf("SARG: Maybe you have a broken download duration in your %s file.\n",arq);
                     exit(1);
                  }
                  strcpy(elap,str);
               } else if (x==isa_cols[ISACOL_Bytes]) {
                  if (strlen(str)>=sizeof(tam)) {
                     printf("SARG: Maybe you have a broken download size in your %s file.\n",arq);
                     exit(1);
                  }
                  strcpy(tam,str);
               } else if (x==isa_cols[ISACOL_Uri]) {
                  url=str;
               } else if (x==isa_cols[ISACOL_Status]) {
                  if (strlen(str)>=sizeof(code)) {
                     printf("SARG: Maybe you have a broken access code in your %s file.\n",arq);
                     exit(1);
                  }
                  strcpy(code,str);
               }
            }

            if(strcmp(code,"401") == 0 || strcmp(code,"403") == 0 || strcmp(code,"407") == 0) {
               sprintf(val1,"DENIED/%s",code);
               strcpy(code,val1);
            }
            getword_start(&gwarea,data);
            if (getword(ano,sizeof(ano),&gwarea,'-')<0){
               printf("SARG: Maybe you have a broken record or garbage in your %s file.\n",arq);
               exit(1);
            }
            if (getword(mes,sizeof(mes),&gwarea,'-')<0){
               printf("SARG: Maybe you have a broken record or garbage in your %s file.\n",arq);
               exit(1);
            }
            if (getword(dia,sizeof(dia),&gwarea,'\0')<0){
               printf("SARG: Maybe you have a broken record or garbage in your %s file.\n",arq);
               exit(1);
            }
            conv_month_name(mes);
            sprintf(data," %s/%s/%s:%s",dia,mes,ano,hora);
         }

         if(strlen(user) > 150) {
            if (debugm) printf("User too long: %s\n",user);
            totregsx++;
            continue;
         }

         // include_users
         if(IncludeUsers[0] != '\0') {
            sprintf(val1,":%s:",user);
            if((str=(char *) strstr(IncludeUsers,val1)) == (char *) NULL )
               continue;
         }

         if(vercode(code)) {
            if (debugm) printf("Excluded code: %s\n",code);
            totregsx++;
            continue;
         }

         if(testvaliduserchar(user))
            continue;

#if 0
         if((str = strstr(user,"%20")) != NULL) {
            /*
            This is a patch introduced to solve bug #1624251 reported at sourceforge but
            the side effect is to truncate the name at the first space and merge the reports
            of people whose name is identical up to the first space.

            The old code used to truncate the user name at the first % if a %20 was
            found anywhere in the string. That means the string could be truncated
            at the wrong place if another % occured before the %20. This new code should
            avoid that problem and only truncate at the space. There is no bug
            report indicating that anybody noticed this.
            */
            *str='\0';
         }

         /*
         Code prior to 2.2.7 used to replace any %xx by a dot as long as a %5c was
         found in the user name.
         */
         while((str = strstr(user,"%5c")) != NULL) {
            *str='.';
            for (x=3 ; str[x] ; x++) str[x-2]=str[x];
         }
#endif

         for(str=user; *str; str++) {
            if(*str=='.') dotinuser++;
            if(*str=='?' || *str=='.' || *str==':' || *str=='/' || *str=='\\' || *str=='\'' || *str=='$' || *str=='@' ||
               *str=='\"' || *str=='*')
               *str='_';
         }

         strlow(user);
         if(strncmp(NtlmUserFormat,"user",4) == 0) {
            if((str = strchr(user,'_')) != 0) {
               strcpy(warea,str+1);
               strcpy(user,warea);
            }
            if((str = strchr(user,'+')) != 0) {
               strcpy(warea,str+1);
               strcpy(user,warea);
            }
         }

         urly=url;

         if(ilf!=ILF_Sarg) {
            /*
            The full URL is not saved in sarg log. There is no point in testing the URL to detect
            a downloaded file.
            */
            download_flag=is_download_suffix(url);
            if (download_flag) {
               download_url=url;
               download_count++;
            }
         }

         // remove any protocol:// at the beginning of the URL
         if ((str = strchr(url,'/')) != NULL && str[1] == '/') {
            int i;

            str+=2;
            for (i=0 ; str[i] ; i++)
               url[i]=str[i];
            url[i]='\0';
         }

         if(!LongUrl) {
            int i;

            for (i=0 ; i<sizeof(hostname)-1 && url[i] && url[i]!='/' ; i++)
               hostname[i]=url[i];
            hostname[i]='\0';
            url=hostname;
         }

         if(ilf==ILF_Squid) {
            tt=atoi(data);
            t=localtime(&tt);

            strftime(tbuf2, sizeof(tbuf2), "%H%M", t);
            sprintf(mes,"%d",t->tm_mon+1);
            conv_month_name(mes);
            if(strncmp(df,"u",1) == 0)
               sprintf(tbuf, "%04d%s%02d", t->tm_year+1900, mes, t->tm_mday);
            if(strncmp(df,"e",1) == 0)
               sprintf(tbuf, "%02d%s%04d", t->tm_mday, mes, t->tm_year+1900);
            if(strncmp(df,"w",1) == 0) {
               IndexTree=INDEX_TREE_FILE;
               strftime(tbuf, sizeof(tbuf), "%Y.%U", t);
            }

            strftime(wdata, sizeof(wdata), "%Y%m%d", t);
            idata=atoi(wdata);

            if(strncmp(df,"u",1)==0)
               strftime(dia, sizeof(dia), "%m/%d/%Y", t);
            else
               strftime(dia, sizeof(dia), "%d/%m/%Y", t);
            sprintf(hora,"%02d:%02d:%02d",t->tm_hour,t->tm_min,t->tm_sec);
         } else if(ilf==ILF_Common || ilf==ILF_Isa) {
            getword_start(&gwarea,data+1);
            if (getword_multisep(data,sizeof(data),&gwarea,':')<0){
               printf("SARG: Maybe you have a broken date in your %s file.\n",arq);
               exit(1);
            }
            if (getword_multisep(hora,sizeof(hora),&gwarea,' ')<0){
               printf("SARG: Maybe you have a broken date in your %s file.\n",arq);
               exit(1);
            }
            getword_start(&gwarea,data);
            if (getword(dia,sizeof(dia),&gwarea,'/')<0){
               printf("SARG: Maybe you have a broken date in your %s file.\n",arq);
               exit(1);
            }
            if (getword(mes,sizeof(mes),&gwarea,'/')<0){
               printf("SARG: Maybe you have a broken date in your %s file.\n",arq);
               exit(1);
            }
            if (getword(ano,sizeof(ano),&gwarea,'/')<0){
               printf("SARG: Maybe you have a broken date in your %s file.\n",arq);
               exit(1);
            }

            if(strcmp(df,"u") == 0)
               snprintf(tbuf,sizeof(tbuf),"%s%s%s",ano,mes,dia);
            if(strcmp(df,"e") == 0)
               snprintf(tbuf,sizeof(tbuf),"%s%s%s",dia,mes,ano);
            builddia(dia,mes,ano,df,wdata);
            idata=atoi(wdata);
         } else if (ilf==ILF_Sarg) {
            getword_start(&gwarea,data);
            if(strcmp(df,"u") == 0) {
               if (getword(mes,sizeof(mes),&gwarea,'/')<0){
                  printf("SARG: Maybe you have a broken date in your %s file.\n",arq);
                  exit(1);
               }
               if (getword(dia,sizeof(dia),&gwarea,'/')<0){
                  printf("SARG: Maybe you have a broken date in your %s file.\n",arq);
                  exit(1);
               }
            } else {
               if (getword(dia,sizeof(dia),&gwarea,'/')<0){
                  printf("SARG: Maybe you have a broken date in your %s file.\n",arq);
                  exit(1);
               }
               if (getword(mes,sizeof(mes),&gwarea,'/')<0){
                  printf("SARG: Maybe you have a broken date in your %s file.\n",arq);
                  exit(1);
               }
            }
            if (getword(ano,sizeof(ano),&gwarea,0)<0){
               printf("SARG: Maybe you have a broken date in your %s file.\n",arq);
               exit(1);
            }
            snprintf(wdata,9,"%s%s%s",ano,mes,dia);
            idata=atoi(wdata);
         }

         if(debugm)
            printf("DATE=%s IDATA=%d DFROM=%d DUNTIL=%d\n",date,idata,dfrom,duntil);

         l=1;
         if(l){
            if(addr[0] != '\0'){
               if(strcmp(addr,ip)==0)
                  l=1;else l=0;
            }
            if(fhost) {
//              l=vhexclude(ip);
               l=vhexclude(url);
               if(!l) {
                  if (debugm) printf("Excluded site: %s\n",url);
                  totregsx++;
               }
            }
         }

         if(l){
            if(date[0] != '\0'){
               if(idata >= dfrom && idata <= duntil)
                  l=1;else l=0;
            }
         }
         if(l){
            if(hm[0] != '\0') {
               bzero(hmr,sizeof(hmr));
               chm++;
               getword_start(&gwarea,hora);
               while(chm) {
                  if (getword_multisep(warea,sizeof(warea),&gwarea,':')<0){
                     printf("SARG: Maybe you have a broken time in your %s file.\n",arq);
                     exit(1);
                  }
                  strncat(hmr,warea,2);
                  chm--;
               }
               strncat(hmr,gwarea.current,2);

               if(atoi(hmr) >= atoi(hm) && atoi(hmr) <= atoi(hmf))
                  l=1;else l=0;
            }
         }
         if(l){
            if(site[0] != '\0'){
               if(strstr(url,site)!=0)
                  l=1;else l=0;
            }
         }

         if(userip)
            strcpy(user,ip);

         if(strcmp(user,"-") == 0 || strcmp(user," ") == 0 || strcmp(user,"") == 0) {
            if(strcmp(RecordsWithoutUser,"ip") == 0)
               strcpy(user,ip);
            if(strcmp(RecordsWithoutUser,"ignore") == 0)
               continue;
            if(strcmp(RecordsWithoutUser,"everybody") == 0)
               strcpy(user,"everybody");
         }
         if(us[0] != '\0'){
            if(strcmp(user,us)==0)
               l=1;
            else
               l=0;
         }

         if(dotinuser) {
            subs(user,sizeof(user),"_",".");
            dotinuser=0;
         }

         if(puser) {
            sprintf(wuser,":%s:",user);
            if(strstr(userfile, wuser) == 0)
               continue;
         }

         if(l) {
            if(fuser) {
               l=vuexclude(user);
               if(!l) {
                  if (debugm) printf("Excluded user: %s\n",user);
                  totregsx++;
               }
            }
         }

         if(l) {
            if(userip)
               fixip(user);
         }

         if(l && max_elapsed) {
            if(atol(elap)>max_elapsed) {
               elap[0]='0';
               elap[1]='\0';
            }
         }

         if(l) {
            if(strcmp(user,"-") !=0 && url[0] != '\0' && strcmp(user," ") !=0 && strcmp(user,"") !=0 && strcmp(user,":") !=0){
               if((str=(char *) strstr(linebuf, "[SmartFilter:")) != (char *) NULL ) {
                  fixendofline(str);
                  sprintf(smartfilter,"\"%s\"",str+1);
               } else sprintf(smartfilter,"\"\"");

               if ( strcmp ( user , sz_Last_User ) != 0 ) {
                  if ( fp_Write_User )
                     fclose( fp_Write_User ) ;
                  sprintf (tmp3, "%s/sarg/%s.unsort", tmp, user);

                  if ((fp_Write_User = MY_FOPEN (tmp3, "a")) == NULL) {
                     fprintf (stderr, "%s: (log) %s: %s - %s\n", argv[0], text[9], tmp3, strerror(errno));
                     exit (1);
                  }
                  strcpy( sz_Last_User , user ) ;
               }
               fprintf(fp_Write_User, "%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\n",dia,hora,user,ip,url,tam,code,elap,smartfilter);

               if(fp_log && ilf!=ILF_Sarg)
                  fprintf(fp_log, "%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\n",dia,hora,user,ip,url,tam,code,elap,smartfilter);

               totregsg++;

               if(download_flag && strstr(code,"DENIED") == 0) {
                  ndownload = 1;

                  if ( ! fp_Download_Unsort ) {
                     if ((fp_Download_Unsort = MY_FOPEN ( sz_Download_Unsort, "a")) == NULL) {
                        fprintf (stderr, "%s: (log) %s: %s - %s\n", argv[0], text[9], tmp3, strerror(errno));
                        exit (1);
                     }
                  }
                  fprintf(fp_Download_Unsort,"%s\t%s\t%s\t%s\t%s\n",dia,hora,user,ip,download_url);
               }

               if((ReportType & REPORT_TYPE_DENIED) != 0 || (ReportType & REPORT_TYPE_AUTH_FAILURES) != 0) {
                  if(strstr(code,"DENIED/403") != 0) {
                     fprintf(fp_denied, "%s\t%s\t%s\t%s\t%s\n",dia,hora,user,ip,urly);
                     denied_count++;
                  }
                  if(strstr(code,"DENIED/401") != 0 || strstr(code,"DENIED/407") != 0) {
                     if(fp_authfail)
                        fprintf(fp_authfail, "%s\t%s\t%s\t%s\t%s\n",dia,hora,user,ip,urly);
                     authfail_count++;
                  }
               }

               if((!totper || idata<mindate) && ilf!=ILF_Sarg){
                  totper++;
                  mindate=idata;
                  sprintf(period,"%s-",tbuf);
                  strcpy(start_hour,tbuf2);
                  if(date[0] != '\0')
                     fixper(tbuf, period, cduntil);
                  if(debugz){
                     debugaz("tbuf",tbuf);
                     debugaz("period",period);
                  }
               }
            }

            if(debugm){
               printf("IP=\t%s\n",ip);
               printf("USER=\t%s\n",user);
               printf("ELAP=\t%s\n",elap);
               printf("DATE=\t%s\n",dia);
               printf("TIME=\t%s\n",hora);
               printf("FUNC=\t%s\n",fun);
               printf("URL=\t%s\n",url);
               printf("CODE=\t%s\n",code);
               printf("LEN=\t%s\n",tam);
            }
         }
      }
      if (!from_stdin) {
         fclose(fp_in);
         if( ShowReadStatistics )
            printf("SARG: Records in file: %lu, reading: %3.2f%%\n",recs1, (float) 100 );
      }
   }

   longline_free(&line);
   if ( fp_Download_Unsort )
     fclose (fp_Download_Unsort);

   if (fp_Write_User)
     fclose (fp_Write_User);

   free_download();
   free_excludecodes();
   free_exclude();

   if(debug) {
      int totalcount=0;

      for (ilf=0 ; ilf<ILF_Last ; ilf++) totalcount+=ilf_count[ilf];

      debuga("   %s: %ld, %s: %ld, %s: %ld",text[10],totregsl,text[11],totregsg,text[68],totregsx);

      if(ilf_count[ILF_Common]>0 && ilf_count[ILF_Squid]>0)
         debuga("%s",text[12]);

      if(ilf_count[ILF_Common]>0 && ilf_count[ILF_Squid]==0)
         debuga("%s",text[13]);

      if(ilf_count[ILF_Common]==0 && ilf_count[ILF_Squid]>0)
         debuga("%s",text[14]);

      if(ilf_count[ILF_Sarg]>0)
         debuga("%s",text[124]);

      if(totalcount==0) {
         if(!totregsg) {
            fprintf(stderr, "SARG: %s\n",text[16]);
            fprintf(stderr, "SARG: %s\n",text[21]);
         } else fprintf(stderr, "SARG: %s\n",text[15]);
         if(fp_denied)
            fclose(fp_denied);
         if(fp_authfail)
            fclose(fp_authfail);
//         if(userfile)
////            free(userfile);
         unlink(tmp4);
         unlink(tmp6);
         unlink(tmp3);
         exit(0);
      }
   }

   if(!totregsg){
      fprintf(stderr, "SARG: %s\n",text[16]);
      fprintf(stderr, "SARG: %s\n",text[21]);
//      fclose(fp_ou);
      if(fp_denied)
         fclose(fp_denied);
      if(fp_authfail)
         fclose(fp_authfail);
      if(userfile)
         free(userfile);
      exit(0);
   }

   if(date[0] == '\0' && ilf_count[ILF_Sarg]==0) {
      strcat(period,tbuf);
   }

   if(debugz){
      debugaz("data",dia);
      debugaz("tbuf",tbuf);
      debugaz("period",period);
   }

   if(debug)
      debuga("%s: %s",text[17],period);

//   fclose(fp_ou);
   if(fp_denied)
      fclose(fp_denied);
   if(fp_authfail)
      fclose(fp_authfail);

   if(fp_log != NULL) {
      fclose(fp_log);
      strcpy(end_hour,tbuf2);
      getword_start(&gwarea,period);
      if (getword(val2,sizeof(val2),&gwarea,'-')<0){
         printf("SARG: Maybe you have a broken date range definition.\n");
         exit(1);
      }
      if (getword(val1,sizeof(val1),&gwarea,'\0')<0){
         printf("SARG: Maybe you have a broken date range definition.\n");
         exit(1);
      }
      sprintf(val4,"%s/sarg-%s_%s-%s_%s.log",ParsedOutputLog,val2,start_hour,val1,end_hour);
      if (rename(arq_log,val4)) {
         fprintf(stderr,"SARG: failed to rename %s to %s - %s\n",arq_log,val4,strerror(errno));
      } else {
         strcpy(arq_log,val4);

         if(strcmp(ParsedOutputLogCompress,"nocompress") != 0 && ParsedOutputLogCompress[0] != '\0') {
            /*
            No double quotes around ParsedOutputLogCompress because it may contain command line options. If double quotes are
            necessary around the command name, put them in the configuration file.
            */
            sprintf(val1,"%s \"%s\"",ParsedOutputLogCompress,arq_log);
            cstatus=system(val1);
            if (!WIFEXITED(cstatus) || WEXITSTATUS(cstatus)) {
               fprintf(stderr, "SARG: command return status %d\n",WEXITSTATUS(cstatus));
               fprintf(stderr, "SARG: command: %s\n",val1);
               exit(1);
            }
         }
      }
      if(debug)
         debuga("%s %s",text[123],arq_log);
   }

   if((ReportType & REPORT_TYPE_DENIED) != 0) {
      sprintf(csort,"sort -T \"%s\" -k 3,3 -k 5,5 -o \"%s\" \"%s\"",tmp,tmp5,tmp4);
      cstatus=system(csort);
      if (!WIFEXITED(cstatus) || WEXITSTATUS(cstatus)) {
         fprintf(stderr, "SARG: sort command return status %d\n",WEXITSTATUS(cstatus));
         fprintf(stderr, "SARG: sort command: %s\n",csort);
         exit(1);
      }
      unlink(tmp4);
   }

   sort_users_log(tmp, debug);

   if(DataFile[0] != '\0')
      data_file(tmp);
   else
      gerarel();

   unlink(tmp2);
   if((ReportType & REPORT_TYPE_DENIED) != 0)
      unlink(tmp5);

   if(zip[0] != '\0' && strcmp(zip,"zcat") !=0) {
      recomp(arq, zip);
   }
//   else  unlink(arq);

   if(strcmp(tmp,"/tmp") != 0) {
      unlinkdir(tmp,0);
   }

   if(userfile)
      free(userfile);

   if(debug)
      debuga("%s",text[21]);

   exit(0);

}


static void getusers(const char *pwdfile, int debug)
{

   FILE *fp_usr;
   char buf[255];
   char *str;
   long int nreg=0;

   if(debug)
      debuga("%s: %s",text[60],pwdfile);

   if ((fp_usr = fopen(pwdfile, "r")) == NULL) {
      fprintf(stderr, "SARG: (getusers) %s: %s - %s\n",text[45],pwdfile,strerror(errno));
      exit(1);
   }

   fseek(fp_usr, 0, SEEK_END);
   nreg = ftell(fp_usr);
   if (nreg<0) {
      printf("SARG: Cannot get the size of file %s",pwdfile);
      exit(1);
   }
   nreg = nreg+5000;
   fseek(fp_usr, 0, SEEK_SET);

   if((userfile=(char *) malloc(nreg))==NULL){
      fprintf(stderr, "SARG: %s (%ld):\n",text[59],nreg);
      exit(1);
   }

   bzero(userfile,nreg);
   strcpy(userfile,":");

   while(fgets(buf,sizeof(buf),fp_usr)!=NULL) {
      str=strchr(buf,':');
      if (!str) {
         printf("SARG: You have an invalid user in your %s file.\n",pwdfile);
         exit(1);
      }
      str[1]=0;
      strcat(userfile,buf);
   }

   fclose(fp_usr);

   return;
}