[thirdparty/sarg.git] / log.c

/*
 * SARG Squid Analysis Report Generator      http://sarg.sourceforge.net
 *                                                            1998, 2010
 *
 * SARG donations:
 *      please look at http://sarg.sourceforge.net/donations.php
 * Support:
 *     http://sourceforge.net/projects/sarg/forums/forum/363374
 * ---------------------------------------------------------------------
 *
 *  This program is free software; you can redistribute it and/or modify
 *  it under the terms of the GNU General Public License as published by
 *  the Free Software Foundation; either version 2 of the License, or
 *  (at your option) any later version.
 *
 *  This program is distributed in the hope that it will be useful,
 *  but WITHOUT ANY WARRANTY; without even the implied warranty of
 *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 *  GNU General Public License for more details.
 *
 *  You should have received a copy of the GNU General Public License
 *  along with this program; if not, write to the Free Software
 *  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111, USA.
 *
 */

#include "include/conf.h"
#include "include/defs.h"

#define REPORT_EVERY_X_LINES 5000
#define MAX_OPEN_USER_FILES 10

struct userfilestruct
{
   struct userfilestruct *next;
   struct userinfostruct *user;
   FILE *file;
};

static char *userfile=NULL;

numlist weekdays = { { 0, 1, 2, 3, 4, 5, 6 }, 7 };
numlist hours = { { 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12,
             13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23 }, 24 };

static void getusers(const char *pwdfile, int debug);

int main(int argc,char *argv[])
{
   enum isa_col_id {
      ISACOL_Ip,
      ISACOL_UserName,
      ISACOL_Date,
      ISACOL_Time,
      ISACOL_TimeTaken,
      ISACOL_Bytes,
      ISACOL_Uri,
      ISACOL_Status,
      ISACOL_Last //last entry of the list !
   };
   enum InputLogFormat {
      ILF_Unknown,
      ILF_Squid,
      ILF_Common,
      ILF_Sarg,
      ILF_Isa,
      ILF_Last //last entry of the list !
   };

   FILE *fp_in = NULL, *fp_denied=NULL, *fp_authfail=NULL, *fp_log=NULL;

   char sz_Download_Unsort[ 20000 ] ;
   FILE * fp_Download_Unsort = NULL ;

   extern int optind;
   extern int optopt;
   extern char *optarg;

   char data[255];
   char elap[255];
   char ip[MAXLEN];
   char tam[255];
   char fun[MAXLEN];
   char wuser[MAXLEN];
   char smartfilter[MAXLEN];
   char dia[128];
   char wdata[128];
   char mes[30];
   char ano[30];
   char hora[30];
   char wtemp[MAXLEN];
   char wtemp2[255];
   char date[255];
   char arq[255];
   char arq_log[255];
   char hm[15], hmf[15], hmr[15];
   int  chm=0;
   char uagent[MAXLEN];
   char hexclude[MAXLEN];
   char csort[MAXLEN];
   int cstatus;
   char tbuf[128];
   char tbuf2[128];
   char zip[20];
   char *str;
   char tmp2[MAXLEN];
   char tmp3[MAXLEN];
   char denied_unsort[MAXLEN];
   char denied_sort[MAXLEN];
   char authfail_unsort[MAXLEN];
   char start_hour[128];
   char end_hour[128];
   char *linebuf;
   char hostname[512];
   char *url;
   char *urly;
   char user[MAX_USER_LEN];
   enum InputLogFormat ilf;
   int ilf_count[ILF_Last];
   int  ch;
   int  x;
   int  errflg=0;
   int  puser=0;
   int  fhost=0;
   int  dns=0;
   int  fuser=0;
   int  idata=0;
   int  mindate=0;
   int  iarq=0;
   int  exstring=0;
   int isa_ncols=0,isa_cols[ISACOL_Last];
   int from_stdin;
   int blen;
   int maxopenfiles;
   int nopen;
   int id_is_ip;
   long totregsl=0;
   long totregsg=0;
   long totregsx=0;
   long totper=0;
   long int max_elapsed=0;
   time_t tt;
   struct tm *t;
   unsigned long recs1=0UL;
   unsigned long recs2=0UL;
   int OutputNonZero = REPORT_EVERY_X_LINES ;
   int download_flag=0;
   char *download_url=NULL;
   struct getwordstruct gwarea;
   struct longlinestruct line;
   struct userinfostruct *uinfo;
   struct userfilestruct *first_user_file, *ufile, *ufile1, *prev_ufile;

#ifdef HAVE_LOCALE_H
   setlocale(LC_TIME,"");
#endif

#if defined(ENABLE_NLS) && defined(HAVE_LOCALE_H)
   setlocale (LC_ALL, "");
   bindtextdomain (PACKAGE_NAME, LOCALEDIR);
   textdomain (PACKAGE_NAME);
#endif //ENABLE_NLS

   BgImage[0]='\0';
   LogoImage[0]='\0';
   LogoText[0]='\0';
   PasswdFile[0]='\0';
   OutputEmail[0]='\0';
   UserAgentLog[0]='\0';
   ExcludeHosts[0]='\0';
   ExcludeUsers[0]='\0';
   ConfigFile[0]='\0';
   code[0]='\0';
   LastLog=0;
   ReportType=0UL;
   UserTabFile[0]='\0';
   BlockIt[0]='\0';
   ExternalCSSFile[0]='\0';
   SquidGuardLogFormat[0]='\0';
   SquidGuardLogAlternate[0]='\0';
   for (ilf=0 ; ilf<ILF_Last ; ilf++) ilf_count[ilf]=0;

   sprintf(ExcludeCodes,"%s/exclude_codes",SYSCONFDIR);
   strcpy(GraphDaysBytesBarColor,"orange");
   strcpy(BgColor,"#ffffff");
   strcpy(TxColor,"#000000");
   strcpy(TxBgColor,"lavender");
   strcpy(TiColor,"darkblue");
   strcpy(Width,"80");
   strcpy(Height,"45");
   strcpy(LogoTextColor,"#000000");
   strcpy(HeaderColor,"darkblue");
   strcpy(HeaderBgColor,"#dddddd");
   strcpy(LogoTextColor,"#006699");
   strcpy(FontSize,"9px");
   strcpy(TempDir,"/tmp");
   strcpy(OutputDir,"/var/www/html/squid-reports");
   Ip2Name=0;
   strcpy(DateFormat,"u");
   OverwriteReport=0;
   RemoveTempFiles=1;
   strcpy(ReplaceIndex,"index.html");
   Index=INDEX_YES;
   RecordsWithoutUser=RECORDWITHOUTUSER_IP;
   UseComma=0;
   strcpy(MailUtility,"mailx");
   TopSitesNum=100;
   TopUsersNum=0;
   UserIp=0;
   strcpy(TopuserSortField,"BYTES");
   strcpy(UserSortField,"BYTES");
   strcpy(TopuserSortOrder,"reverse");
   strcpy(UserSortOrder,"reverse");
   strcpy(TopsitesSortField,"CONNECT");
   strcpy(TopsitesSortType,"D");
   LongUrl=0;
   strcpy(FontFace,"Verdana,Tahoma,Arial");
   strcpy(datetimeby,"elap");
   strcpy(CharSet,"ISO-8859-1");
   Privacy=0;
   strcpy(PrivacyString,"***.***.***.***");
   strcpy(PrivacyStringColor,"blue");
   SuccessfulMsg=1;
   TopUserFields=TOPUSERFIELDS_NUM | TOPUSERFIELDS_DATE_TIME | TOPUSERFIELDS_USERID | TOPUSERFIELDS_CONNECT |
                 TOPUSERFIELDS_BYTES | TOPUSERFIELDS_SETYB | TOPUSERFIELDS_IN_CACHE_OUT |
                 TOPUSERFIELDS_USED_TIME | TOPUSERFIELDS_MILISEC | TOPUSERFIELDS_PTIME |
                 TOPUSERFIELDS_TOTAL | TOPUSERFIELDS_AVERAGE;
   UserReportFields=USERREPORTFIELDS_CONNECT | USERREPORTFIELDS_BYTES | USERREPORTFIELDS_SETYB |
                    USERREPORTFIELDS_IN_CACHE_OUT | USERREPORTFIELDS_USED_TIME | USERREPORTFIELDS_MILISEC |
                    USERREPORTFIELDS_PTIME | USERREPORTFIELDS_TOTAL | USERREPORTFIELDS_AVERAGE;
   strcpy(DataFileDelimiter,";");
   DataFileFields=DATA_FIELD_USER | DATA_FIELD_DATE | DATA_FIELD_TIME | DATA_FIELD_URL | DATA_FIELD_CONNECT |
                  DATA_FIELD_BYTES | DATA_FIELD_IN_CACHE | DATA_FIELD_OUT_CACHE | DATA_FIELD_ELAPSED;
   ShowReadStatistics=1;
   strcpy(IndexSortOrder,"D");
   ShowSargInfo=1;
   ShowSargLogo=1;
   strcpy(ParsedOutputLog,"no");
   strcpy(ParsedOutputLogCompress,"/bin/gzip -f");
   strcpy(DisplayedValues,"abbreviation");
   strcpy(HeaderFontSize,"9px");
   strcpy(TitleFontSize,"11px");
   strcpy(AuthUserTemplateFile,"sarg_htaccess");
   set_download_suffix("7z,ace,arj,avi,bat,bin,bz2,bzip,cab,com,cpio,dll,doc,dot,exe,gz,iso,lha,lzh,mdb,mov,mp3,mpeg,mpg,mso,nrg,ogg,ppt,rar,rtf,shs,src,sys,tar,tgz,vcd,vob,wma,wmv,zip");
   Graphs=1;
#if defined(FONTDIR)
   strcpy(GraphFont,FONTDIR"/DejaVuSans.ttf");
#else
   GraphFont[0]='\0';
#endif
   strcpy(Ulimit,"20000");
   NtlmUserFormat=NTLMUSERFORMAT_DOMAINUSER;
   IndexTree=INDEX_TREE_FILE;
   strcpy(RealtimeTypes,"GET,PUT,CONNECT");
   strcpy(RealtimeUnauthRec,"show");
   SquidguardIgnoreDate=0;
   DansguardianIgnoreDate=0;
   DataFileUrl=DATAFILEURL_IP;
   strcpy(MaxElapsed,"28800000");
   BytesInSitesUsersReport=0;
   UserAuthentication=0;
   strcpy(LDAPHost,"127.0.0.1");
   LDAPPort=389;
   LDAPProtocolVersion=3;
   LDAPBindDN[0]='\0';
   LDAPBindPW[0]='\0';
   LDAPBaseSearch[0]='\0';
   strcpy(LDAPFilterSearch, "uid=%s");
   strcpy(LDAPTargetAttr, "cn");

   dia[0]='\0';
   mes[0]='\0';
   ano[0]='\0';
   hora[0]='\0';
   tmp[0]='\0';
   tmp2[0]='\0';
   tmp3[0]='\0';
   wtemp[0]='\0';
   wtemp2[0]='\0';
   us[0]='\0';
   date[0]='\0';
   df[0]='\0';
   uagent[0]='\0';
   hexclude[0]='\0';
   addr[0]='\0';
   hm[0]='\0';
   hmf[0]='\0';
   site[0]='\0';
   outdir[0]='\0';
   elap[0]='\0';
   email[0]='\0';
   zip[0]='\0';
   UserInvalidChar[0]='\0';
   DataFile[0]='\0';
   SquidGuardConf[0]='\0';
   DansGuardianConf[0]='\0';
   start_hour[0]='\0';
   end_hour[0]='\0';

   denied_count=0;
   download_count=0;
   authfail_count=0;
   dansguardian_count=0;
   squidguard_count=0;
   useragent_count=0;
   DeniedReportLimit=10;
   AuthfailReportLimit=10;
   DansGuardianReportLimit=10;
   SquidGuardReportLimit=10;
   DownloadReportLimit=50;
   UserReportLimit=0;
   debug=0;
   debugz=0;
   debugm=0;
   iprel=0;
   userip=0;
   color1=0;
   color2=0;
   color3=0;
   realt=0;
   realtime_refresh=3;
   realtime_access_log_lines=1000;
   cost=0.01;
   nocost=50000000;
   ndownload=0;
   squid24=0;

   bzero(IncludeUsers, sizeof(IncludeUsers));
   bzero(ExcludeString, sizeof(ExcludeString));
   first_user_file=NULL;

   NAccessLog=0;
   for(x=0; x<MAXLOGS; x++)
      AccessLog[x][0]='\0';
   AccessLogFromCmdLine=0;

   strcpy(Title,_("Squid User Access Report"));

   while((ch = getopt(argc, argv, "a:b:c:d:e:f:g:u:l:L:o:s:t:w:hijmnprvxyz")) != -1){
      switch(ch)
      {
         case 'a':
            strcpy(addr,optarg);
            break;
         case 'b':
            strcpy(uagent,optarg);
            break;
         case 'c':
            strcpy(hexclude,optarg);
            break;
         case 'd':
            strncpy(date,optarg,sizeof(date)-1);
            date[sizeof(date)-1]='\0';
            getword_start(&gwarea,optarg);
            if (getword(cdfrom,sizeof(cdfrom),&gwarea,'-')<0 || getword(cduntil,sizeof(cduntil),&gwarea,0)<0) {
               printf(_("SARG: The date range requested on the command line by option -d is invalid.\n"));
               exit(EXIT_FAILURE);
            }
            date_from(date, cdfrom, cduntil);
            dfrom=atoi(cdfrom);
            duntil=atoi(cduntil);
            break;
         case 'e':
            strcpy(email,optarg);
            break;
         case 'f':
            strcpy(ConfigFile,optarg);
            break;
         case 'g':
            strcpy(df,optarg);
            break;
         case 'h':
            usage(argv[0]);
            exit(0);
            break;
         case 'i':
            iprel++;
            break;
         case 'l':
            if (NAccessLog>=MAXLOGS) {
               printf(_("SARG: Too many log files passed on command line with option -l.\n"));
               exit(EXIT_FAILURE);
            }
            strcpy(AccessLog[NAccessLog],optarg);
            NAccessLog++;
            AccessLogFromCmdLine++;
            break;
         case 'L':
            strcpy(SquidGuardLogAlternate,optarg);
            break;
         case 'm':
            debugm++;
            break;
         case 'n':
            dns++;
            break;
         case 'o':
            strcpy(outdir,optarg);
            break;
         case 'p':
            userip++;
            break;
         case 'r':
            realt++;
            break;
         case 's':
            strcpy(site,optarg);
            break;
         case 't':
         {
            int h,m;

            if(strstr(optarg,"-") == 0) {
              strcpy(hm,optarg);
              strcpy(hmf,optarg);
            } else {
               getword_start(&gwarea,optarg);
               if (getword(hm,sizeof(hm),&gwarea,'-')<0 || getword(hmf,sizeof(hmf),&gwarea,0)<0) {
                  debuga(_("The time range passed on the command line with option -t is invalid\n"));
                  exit(EXIT_FAILURE);
               }
            }
            if(sscanf(hm,"%d:%d",&h,&m)!=2) {
               debuga(_("Time period must be MM or MM:SS. Exit\n"));
               exit(EXIT_FAILURE);
            }
            sprintf(hm,"%02d%02d",h,m);
            if(sscanf(hmf,"%d:%d",&h,&m)!=2) {
               debuga(_("Time period must be MM or MM:SS. Exit\n"));
               exit(EXIT_FAILURE);
            }
            sprintf(hmf,"%02d%02d",h,m);
            break;
         }
         case 'u':
            strcpy(us,optarg);
            break;
         case 'v':
            version();
            break;
         case 'w':
            strcpy(tmp,optarg);
            break;
         case 'x':
            debug++;
            break;
         case 'y':
            langcode++;
            break;
         case 'z':
            debugz++;
            break;
         case ':':
            debuga(_("Option -%c require an argument\n"),optopt);
            errflg++;
            break;
         case '?':
            usage(argv[0]);
            exit(EXIT_FAILURE);
            break;
      }

   }

   if (errflg) {
      usage(argv[0]);
      exit(2);
   }

   if(debug) debuga(_("Init\n"));

   if(ConfigFile[0] == '\0') sprintf(ConfigFile,"%s/sarg.conf",SYSCONFDIR);
   if(access(ConfigFile, R_OK) != 0) {
      debuga(_("Cannot open config file: %s - %s\n"),ConfigFile,strerror(errno));
      exit(EXIT_FAILURE);
   }

   if(access(ConfigFile, R_OK) == 0)
      getconf();

   if(UserIp) userip++;

   if(dns) Ip2Name=1;

   if(realt) {
      realtime();
      exit(0);
   }

   if(IndexTree == INDEX_TREE_FILE)
      strcpy(ImageFile,"../images");
   else
      strcpy(ImageFile,"../../../images");

   dataonly=0;
   if(DataFile[0] != '\0')
      dataonly++;

   if(!NAccessLog) {
      strcpy(AccessLog[0],"/var/log/squid/access.log");
      NAccessLog++;
   }

   if(strcmp(hexclude,"onvert") == 0 && strcmp(site,"plit") != 0) {
      convlog(AccessLog[0], df, dfrom, duntil);
      exit(0);
   }

   if(strcmp(site,"plit") == 0) {
      splitlog(AccessLog[0], df, dfrom, duntil, hexclude);
      exit(0);
   }

   load_excludecodes(ExcludeCodes);

   if(access(PasswdFile, R_OK) == 0) {
      getusers(PasswdFile,debug);
      puser++;
   }

   if(hexclude[0] == '\0')
      strcpy(hexclude,ExcludeHosts);
   if(hexclude[0] != '\0') {
      gethexclude(hexclude,debug);
      fhost++;
   }

   if(ReportType == 0) {
      ReportType=REPORT_TYPE_TOPUSERS | REPORT_TYPE_TOPSITES | REPORT_TYPE_USERS_SITES |
                 REPORT_TYPE_SITES_USERS | REPORT_TYPE_DATE_TIME | REPORT_TYPE_DENIED |
                 REPORT_TYPE_AUTH_FAILURES | REPORT_TYPE_SITE_USER_TIME_DATE | REPORT_TYPE_DOWNLOADS;
   }

   if(access(ExcludeUsers, R_OK) == 0) {
      getuexclude(ExcludeUsers,debug);
      fuser++;
   }

   indexonly=0;
   if(fuser) {
      if(is_indexonly())
         indexonly++;
   }
   if(strcmp(ExcludeUsers,"indexonly") == 0) indexonly++;
   if(Index == INDEX_ONLY) indexonly++;

   if(MaxElapsed[0] != '\0') max_elapsed=atol(MaxElapsed);

   if(outdir[0] == '\0') strcpy(outdir,OutputDir);
   strcat(outdir,"/");

   if(uagent[0] == '\0') strcpy(uagent,UserAgentLog);

   if(tmp[0] == '\0') strcpy(tmp,TempDir);
   else strcpy(TempDir,tmp);

   if(df[0] == '\0') strcpy(df,DateFormat);
   else strcpy(DateFormat,df);

   if(df[0] == '\0') {
      strcpy(df,"u");
      strcpy(DateFormat,"u");
   }

   if(email[0] == '\0' && OutputEmail[0] != '\0') strcpy(email,OutputEmail);

   strcpy(tmp2,tmp);

   if(email[0] != '\0') {
      sprintf(wtemp2,"%s/sarg",tmp2);
      my_mkdir(wtemp2);
      strcat(tmp2,"/sarg");
      strcpy(outdir,tmp2);
      strcat(outdir,"/");
   }

   strcat(tmp2,"/sarg.log");

   sprintf(tmp3,"%s/sarg",tmp);
   if(access(tmp3, R_OK) == 0) {
      unlinkdir(tmp3,1);
   }
   my_mkdir(tmp3);
   strcpy(denied_unsort,tmp3);
   strcpy(denied_sort,tmp3);
   strcpy(authfail_unsort,tmp3);
   strcat(denied_unsort,"/denied.log.unsort");
   strcat(denied_sort,"/denied.log");
   strcat(authfail_unsort,"/authfail.log.unsort");

   if(debug) {
      debuga(_("Parameters:\n"));
      debuga(_("          Hostname or IP address (-a) = %s\n"),addr);
      debuga(_("                   Useragent log (-b) = %s\n"),uagent);
      debuga(_("                    Exclude file (-c) = %s\n"),hexclude);
      debuga(_("                 Date from-until (-d) = %s\n"),date);
      debuga(_("   Email address to send reports (-e) = %s\n"),email);
      debuga(_("                     Config file (-f) = %s\n"),ConfigFile);
      if(strcmp(df,"e") == 0)
         debuga(_("                     Date format (-g) = Europe (dd/mm/yyyy)\n"));
      if(strcmp(df,"u") == 0)
         debuga(_("                     Date format (-g) = USA (mm/dd/yyyy)\n"));
      if(strcmp(df,"w") == 0)
         debuga(_("                     Date format (-g) = Sites & Users (yyyy/ww)\n"));
      debuga(_("                       IP report (-i) = %s\n"),(iprel) ? _("Yes") : _("No"));
      for (iarq=0 ; iarq<NAccessLog ; iarq++)
         debuga(_("                       Input log (-l) = %s\n"),AccessLog[iarq]);
      debuga(_("              Resolve IP Address (-n) = %s\n"),(Ip2Name) ? _("Yes") : _("No"));
      debuga(_("                      Output dir (-o) = %s\n"),outdir);
      debuga(_("Use Ip Address instead of userid (-p) = %s\n"),(UserIp) ? _("Yes") : _("No"));
      debuga(_("                   Accessed site (-s) = %s\n"),site);
      debuga(_("                            Time (-t) = %s\n"),hm);
      debuga(_("                            User (-u) = %s\n"),us);
      debuga(_("                   Temporary dir (-w) = %s\n"),tmp);
      debuga(_("                  Debug messages (-x) = %s\n"),(debug) ? _("Yes") : _("No"));
      debuga(_("                Process messages (-z) = %s\n"),(debugz) ? _("Yes") : _("No"));
      debuga("\n");
   }

   if(debugm) {
      printf(_("Parameters:\n"));
      printf(_("          Hostname or IP address (-a) = %s\n"),addr);
      printf(_("                   Useragent log (-b) = %s\n"),uagent);
      printf(_("                    Exclude file (-c) = %s\n"),hexclude);
      printf(_("                 Date from-until (-d) = %s\n"),date);
      printf(_("   Email address to send reports (-e) = %s\n"),email);
      printf(_("                     Config file (-f) = %s\n"),ConfigFile);
      if(strcmp(df,"e") == 0)
         printf(_("                     Date format (-g) = Europe (dd/mm/yyyy)\n"));
      if(strcmp(df,"u") == 0)
         printf(_("                     Date format (-g) = USA (mm/dd/yyyy)\n"));
      if(strcmp(df,"w") == 0)
         printf(_("                     Date format (-g) = Sites & Users (yyyy/ww)\n"));
      printf(_("                       IP report (-i) = %s\n"),(iprel) ? _("Yes") : _("No"));
      for (iarq=0 ; iarq<NAccessLog ; iarq++)
         printf(_("                       Input log (-l) = %s\n"),AccessLog[iarq]);
      printf(_("              Resolve IP Address (-n) = %s\n"),(Ip2Name) ? _("Yes") : _("No"));
      printf(_("                      Output dir (-o) = %s\n"),outdir);
      printf(_("Use Ip Address instead of userid (-p) = %s\n"),(UserIp) ? _("Yes") : _("No"));
      printf(_("                   Accessed site (-s) = %s\n"),site);
      printf(_("                            Time (-t) = %s\n"),hm);
      printf(_("                            User (-u) = %s\n"),us);
      printf(_("                   Temporary dir (-w) = %s\n"),tmp);
      printf(_("                  Debug messages (-x) = %s\n"),(debug) ? _("Yes") : _("No"));
      printf(_("                Process messages (-z) = %s\n"),(debugz) ? _("Yes") : _("No"));
      printf(_("sarg version: %s\n"),VERSION);
   }

   if(debug)
      debuga(_("sarg version: %s\n"),VERSION);

   maxopenfiles=MAX_OPEN_USER_FILES;
#ifdef HAVE_RLIM_T
   if (Ulimit[0] != '\0') {
      struct rlimit rl;
      long l1, l2;
      int rc=0;

#if defined(RLIMIT_NOFILE)
      getrlimit (RLIMIT_NOFILE, &rl);
#elif defined(RLIMIT_OFILE)
      getrlimit (RLIMIT_OFILE, &rl);
#else
#warning "No rlimit resource for the number of open files"
#endif
      l1 = rl.rlim_cur;
      l2 = rl.rlim_max;

      rl.rlim_cur = atol(Ulimit);
      rl.rlim_max = atol(Ulimit);
#if defined(RLIMIT_NOFILE)
      rc=setrlimit (RLIMIT_NOFILE, &rl);
#elif defined(RLIMIT_OFILE)
      rc=setrlimit (RLIMIT_OFILE, &rl);
#else
#warning "No rlimit resource for the number of open files"
#endif
      if(rc == -1) {
            debuga(_("setrlimit error - %s\n"),strerror(errno));
      }

      if(debug)
         debuga("Maximum file descriptor: cur=%ld max=%ld, changed to cur="RLIM_STRING" max="RLIM_STRING,l1,l2,rl.rlim_cur,rl.rlim_max);
   }
#endif

   init_usertab(UserTabFile);

   if (longline_prepare(&line)<0) {
      debuga(_("Not enough memory to read a log file\n"));
      exit(EXIT_FAILURE);
   }

   sprintf ( sz_Download_Unsort , "%s/sarg/download.unsort", tmp);

   if(DataFile[0]=='\0') {
      if((ReportType & REPORT_TYPE_DENIED) != 0) {
         if((fp_denied=MY_FOPEN(denied_unsort,"w"))==NULL) {
            debuga(_("(log) Cannot open file: %s - %s\n"),denied_unsort,strerror(errno));
            exit(EXIT_FAILURE);
         }
      }

      if((ReportType & REPORT_TYPE_DENIED) != 0 || (ReportType & REPORT_TYPE_AUTH_FAILURES) != 0) {
         if((fp_authfail=MY_FOPEN(authfail_unsort,"w"))==NULL) {
            debuga(_("(log) Cannot open file: %s - %s\n"),authfail_unsort,strerror(errno));
            exit(EXIT_FAILURE);
         }
      }
   }

   for (iarq=0 ; iarq<NAccessLog ; iarq++) {
      strcpy(arq,AccessLog[iarq]);

      strcpy(arqtt,arq);

      if(strcmp(arq,"-")==0) {
         if(debug)
            debuga(_("Reading access log file: from stdin\n"));
         fp_in=stdin;
         from_stdin=1;
      } else {
         decomp(arq,zip,tmp);
         if(debug)
            debuga(_("Reading access log file: %s\n"),arq);
         if((fp_in=MY_FOPEN(arq,"r"))==NULL) {
            debuga(_("(log) Cannot open log file: %s - %s\n"),arq,strerror(errno));
            exit(EXIT_FAILURE);
         }
         from_stdin=0;
      }
      ilf=ILF_Unknown;
      download_flag=0;
      // pre-read the file only if we have to show stats
      if(ShowReadStatistics && !from_stdin) {
         size_t nread,i;
         int skipcr=0;

         recs1=0UL;
         recs2=0UL;

         while ((nread=fread(line.buffer,1,line.size,fp_in))>0) {
            for (i=0 ; i<nread ; i++)
               if (skipcr) {
                  if (line.buffer[i]!='\n' && line.buffer[i]!='\r') {
                     skipcr=0;
                  }
               } else {
                  if (line.buffer[i]=='\n' || line.buffer[i]=='\r') {
                     skipcr=1;
                     recs1++;
                  }
               }
         }
         rewind(fp_in);
         printf(_("SARG: Records in file: %lu, reading: %3.2f%%"),recs1,(float) 0);
         putchar('\r');
         fflush( stdout ) ;
      }

      line.start=0;
      line.end=0;
      line.length=0;

      while ((linebuf=longline_read(fp_in,&line))!=NULL) {
         blen=strlen(linebuf);

         if (ilf==ILF_Unknown) {
            if(strncmp(linebuf,"#Software: Mic",14) == 0) {
               fixendofline(linebuf);
               if (debug)
                  debuga(_("Log is from Microsoft ISA: %s\n"),linebuf);
               ilf=ILF_Isa;
               ilf_count[ilf]++;
               continue;
            }

            if(strncmp(linebuf,"*** SARG Log ***",16) == 0) {
               getword_start(&gwarea,arqtt);
               if (getword_skip(2000,&gwarea,'-')<0 || getword(val2,sizeof(val2),&gwarea,'_')<0 ||
                   getword_skip(10,&gwarea,'-')<0 || getword(val3,sizeof(val3),&gwarea,'_')<0) {
                  debuga(_("The name of the file is invalid: %s\n"),arq);
                  exit(EXIT_FAILURE);
               }
               sprintf(period,"%s-%s",val2,val3);
               ilf=ILF_Sarg;
               ilf_count[ilf]++;
               continue;
            }
         }

         if(!fp_log && strcmp(ParsedOutputLog, "no") != 0 && ilf!=ILF_Sarg) {
            if(access(ParsedOutputLog,R_OK) != 0) {
               my_mkdir(ParsedOutputLog);
            }
            sprintf(arq_log,"%s/sarg_temp.log",ParsedOutputLog);
            if((fp_log=MY_FOPEN(arq_log,"w"))==NULL) {
               debuga(_("(log) Cannot open log file: %s - %s\n"),arq_log,strerror(errno));
               exit(EXIT_FAILURE);
            }
            fputs("*** SARG Log ***\n",fp_log);
         }

         recs2++;
         if( ShowReadStatistics && !from_stdin && ! --OutputNonZero) {
           perc = recs2 * 100 ;
           perc = perc / recs1 ;
           printf(_("SARG: Records in file: %lu, reading: %3.2f%%"),recs1,perc);
           putchar('\r');
           fflush (stdout);
           OutputNonZero = REPORT_EVERY_X_LINES ;
         }
         if(blen < 58) continue;
         if(strstr(linebuf,"HTTP/0.0") != 0) continue;
         if(strstr(linebuf,"logfile turned over") != 0) continue;
         if(linebuf[0] == ' ') continue;

         // exclude_string
         if(ExcludeString[0] != '\0') {
            exstring=0;
            getword_start(&gwarea,ExcludeString);
            while(strchr(gwarea.current,':') != 0) {
               if (getword_multisep(val1,sizeof(val1),&gwarea,':')<0) {
                  debuga(_("Maybe you have a broken record or garbage in your exclusion string\n"));
                  exit(EXIT_FAILURE);
               }
               if((str=(char *) strstr(linebuf,val1)) != (char *) NULL )
                  exstring++;
            }
            if((str=(char *) strstr(linebuf,gwarea.current)) != (char *) NULL )
                  exstring++;
            if(exstring) continue;
         }

         totregsl++;
         if(debugm)
            printf("BUF=%s\n",linebuf);

         if (ilf==ILF_Squid || ilf==ILF_Common || ilf==ILF_Unknown) {
            getword_start(&gwarea,linebuf);
            if (getword(data,sizeof(data),&gwarea,' ')<0) {
               debuga(_("Maybe you have a broken time in your access.log file\n"));
               exit(EXIT_FAILURE);
            }
            if((str=(char *) strchr(data, '.')) != (char *) NULL ) {
               if((str=(char *) strchr(str+1, '.')) != (char *) NULL ) {
                  strcpy(ip,data);
                  strcpy(elap,"0");
                  if(squid24) {
                     if (getword(user,sizeof(user),&gwarea,' ')<0 || getword_skip(255,&gwarea,' ')<0) {
                        debuga(_("Maybe you have a broken record or garbage in your %s file\n"),arq);
                        exit(EXIT_FAILURE);
                     }
                  } else {
                     if (getword_skip(255,&gwarea,' ')<0 || getword(user,sizeof(user),&gwarea,' ')<0) {
                        debuga(_("Maybe you have a broken record or garbage in your %s file\n"),arq);
                        exit(EXIT_FAILURE);
                     }
                  }
                  if (getword(data,sizeof(data),&gwarea,']')<0 || getword_skip(MAXLEN,&gwarea,'"')<0 ||
                      getword(fun,sizeof(fun),&gwarea,' ')<0) {
                     debuga(_("Maybe you have a broken record or garbage in your %s file\n"),arq);
                     exit(EXIT_FAILURE);
                  }
                  if (getword_ptr(linebuf,&url,&gwarea,' ')<0) {
                     debuga(_("Maybe you have a broken url in your %s file\n"),arq);
                     exit(EXIT_FAILURE);
                  }
                  if (getword_skip(MAXLEN,&gwarea,' ')<0 || getword(code2,sizeof(code2),&gwarea,' ')<0 ||
                      getword(tam,sizeof(tam),&gwarea,' ')<0) {
                     debuga(_("Maybe you have a broken record or garbage in your %s file\n"),arq);
                     exit(EXIT_FAILURE);
                  }
                  if((str=(char *) strchr(gwarea.current, ' ')) != (char *) NULL ) {
                     if (getword(code,sizeof(code),&gwarea,' ')<0) {
                        debuga(_("Maybe you have a broken record or garbage in your %s file\n"),arq);
                        exit(EXIT_FAILURE);
                     }
                  } else {
                     if (getword(code,sizeof(code),&gwarea,'\0')<0) {
                        debuga(_("Maybe you have a broken record or garbage in your %s file\n"),arq);
                        exit(EXIT_FAILURE);
                     }
                  }

                  if ((str = strchr(code, ':')) != NULL)
                     *str = '/';

                  if(strcmp(tam,"\0") == 0)
                     strcpy(tam,"0");

                  ilf=ILF_Common;
                  ilf_count[ilf]++;
               }
            }

            if(ilf==ILF_Unknown || ilf==ILF_Squid) {
               if (getword(elap,sizeof(elap),&gwarea,' ')<0) {
                  debuga(_("Maybe you have a broken elapsed time in your %s file\n"),arq);
                  exit(EXIT_FAILURE);
               }
               while(strcmp(elap,"") == 0 && gwarea.current[0] != '\0')
                  if (getword(elap,sizeof(elap),&gwarea,' ')<0) {
                     debuga(_("Maybe you have a broken elapsed time in your %s file\n"),arq);
                     exit(EXIT_FAILURE);
                  }
               if(strlen(elap) < 1) continue;
               if (getword(ip,sizeof(ip),&gwarea,' ')<0){
                  debuga(_("Maybe you have a broken client IP address in your %s file\n"),arq);
                  exit(EXIT_FAILURE);
               }
               if (getword(code,sizeof(code),&gwarea,' ')<0){
                  debuga(_("Maybe you have a broken result code in your %s file\n"),arq);
                  exit(EXIT_FAILURE);
               }
               if (getword(tam,sizeof(tam),&gwarea,' ')<0){
                  debuga(_("Maybe you have a broken amount of data in your %s file\n"),arq);
                  exit(EXIT_FAILURE);
               }
               if (getword(fun,sizeof(fun),&gwarea,' ')<0){
                  debuga(_("Maybe you have a broken request method in your %s file\n"),arq);
                  exit(EXIT_FAILURE);
               }
               if (getword_ptr(linebuf,&url,&gwarea,' ')<0){
                  debuga(_("Maybe you have a broken url in your %s file\n"),arq);
                  exit(EXIT_FAILURE);
               }
               if (getword(user,sizeof(user),&gwarea,' ')<0){
                  debuga(_("Maybe you have a broken user ID in your %s file\n"),arq);
                  exit(EXIT_FAILURE);
               }
               ilf=ILF_Squid;
               ilf_count[ilf]++;
            }
         }
         if (ilf==ILF_Sarg) {
            getword_start(&gwarea,linebuf);
            if (getword(data,sizeof(data),&gwarea,'\t')<0){
               debuga(_("Maybe you have a broken record or garbage in your %s file\n"),arq);
               exit(EXIT_FAILURE);
            }
            if (getword(hora,sizeof(hora),&gwarea,'\t')<0) {
               debuga(_("Maybe you have a broken record or garbage in your %s file\n"),arq);
               exit(EXIT_FAILURE);
            }
            if (getword(user,sizeof(user),&gwarea,'\t')<0) {
               debuga(_("Maybe you have a broken record or garbage in your %s file\n"),arq);
               exit(EXIT_FAILURE);
            }
            if (getword(ip,sizeof(ip),&gwarea,'\t')<0) {
               debuga(_("Maybe you have a broken record or garbage in your %s file\n"),arq);
               exit(EXIT_FAILURE);
            }
            if (getword_ptr(linebuf,&url,&gwarea,'\t')<0){
               debuga(_("Maybe you have a broken record or garbage in your %s file\n"),arq);
               exit(EXIT_FAILURE);
            }
            if (getword(tam,sizeof(tam),&gwarea,'\t')<0){
               debuga(_("Maybe you have a broken record or garbage in your %s file\n"),arq);
               exit(EXIT_FAILURE);
            }
            if (getword(code,sizeof(code),&gwarea,'\t')<0){
               debuga(_("Maybe you have a broken record or garbage in your %s file\n"),arq);
               exit(EXIT_FAILURE);
            }
            if (getword(elap,sizeof(elap),&gwarea,'\t')<0){
               debuga(_("Maybe you have a broken record or garbage in your %s file\n"),arq);
               exit(EXIT_FAILURE);
            }
            if (getword(smartfilter,sizeof(smartfilter),&gwarea,'\0')<0){
               debuga(_("Maybe you have a broken record or garbage in your %s file\n"),arq);
               exit(EXIT_FAILURE);
            }
         }
         if (ilf==ILF_Isa) {
            if (linebuf[0] == '#') {
               int ncols,cols[ISACOL_Last];

               fixendofline(linebuf);
               getword_start(&gwarea,linebuf);
               // remove the #Fields: column at the beginning of the line
               if (getword_skip(1000,&gwarea,' ')<0){
                  debuga(_("Maybe you have a broken record or garbage in your %s file\n"),arq);
                  exit(EXIT_FAILURE);
               }
               for (ncols=0 ; ncols<ISACOL_Last ; ncols++) cols[ncols]=-1;
               ncols=0;
               while(gwarea.current[0] != '\0') {
                  if (getword(val1,sizeof(val1),&gwarea,'\t')<0){
                     debuga(_("Maybe you have a broken record or garbage in your %s file\n"),arq);
                     exit(EXIT_FAILURE);
                  }
                  if(strcmp(val1,"c-ip") == 0) cols[ISACOL_Ip]=ncols;
                  if(strcmp(val1,"cs-username") == 0) cols[ISACOL_UserName]=ncols;
                  if(strcmp(val1,"date") == 0) cols[ISACOL_Date]=ncols;
                  if(strcmp(val1,"time") == 0) cols[ISACOL_Time]=ncols;
                  if(strcmp(val1,"time-taken") == 0) cols[ISACOL_TimeTaken]=ncols;
                  if(strcmp(val1,"sc-bytes") == 0) cols[ISACOL_Bytes]=ncols;
                  if(strcmp(val1,"cs-uri") == 0) cols[ISACOL_Uri]=ncols;
                  if(strcmp(val1,"sc-status") == 0) cols[ISACOL_Status]=ncols;
                  ncols++;
               }
               if (cols[ISACOL_Ip]>=0) {
                  isa_ncols=ncols;
                  for (ncols=0 ; ncols<ISACOL_Last ; ncols++)
                     isa_cols[ncols]=cols[ncols];
               }
               continue;
            }
            if (!isa_ncols) continue;
            getword_start(&gwarea,linebuf);
            for (x=0 ; x<isa_ncols ; x++) {
               if (getword_ptr(linebuf,&str,&gwarea,'\t')<0) {
                  debuga(_("Maybe you have a broken record or garbage in your %s file\n"),arq);
                  exit(EXIT_FAILURE);
               }
               if (x==isa_cols[ISACOL_Ip]) {
                  if (strlen(str)>=sizeof(ip)) {
                     debuga(_("Maybe you have a broken IP in your %s file\n"),arq);
                     exit(EXIT_FAILURE);
                  }
                  strcpy(ip,str);
               } else if (x==isa_cols[ISACOL_UserName]) {
                  if (strlen(str)>=sizeof(user)) {
                     debuga(_("Maybe you have a broken user ID in your %s file\n"),arq);
                     exit(EXIT_FAILURE);
                  }
                  strcpy(user,str);
               } else if (x==isa_cols[ISACOL_Date]) {
                  if (strlen(str)>=sizeof(data)) {
                     debuga(_("Maybe you have a broken date in your %s file\n"),arq);
                     exit(EXIT_FAILURE);
                  }
                  strcpy(data,str);
               } else if (x==isa_cols[ISACOL_Time]) {
                  if (strlen(str)>=sizeof(hora)) {
                     debuga(_("Maybe you have a broken time in your %s file\n"),arq);
                     exit(EXIT_FAILURE);
                  }
                  strcpy(hora,str);
               } else if (x==isa_cols[ISACOL_TimeTaken]) {
                  if (strlen(str)>=sizeof(elap)) {
                     debuga(_("Maybe you have a broken download duration in your %s file\n"),arq);
                     exit(EXIT_FAILURE);
                  }
                  strcpy(elap,str);
               } else if (x==isa_cols[ISACOL_Bytes]) {
                  if (strlen(str)>=sizeof(tam)) {
                     debuga(_("Maybe you have a broken download size in your %s file\n"),arq);
                     exit(EXIT_FAILURE);
                  }
                  strcpy(tam,str);
               } else if (x==isa_cols[ISACOL_Uri]) {
                  url=str;
               } else if (x==isa_cols[ISACOL_Status]) {
                  if (strlen(str)>=sizeof(code)) {
                     debuga(_("Maybe you have a broken access code in your %s file\n"),arq);
                     exit(EXIT_FAILURE);
                  }
                  strcpy(code,str);
               }
            }

            if(strcmp(code,"401") == 0 || strcmp(code,"403") == 0 || strcmp(code,"407") == 0) {
               sprintf(val1,"DENIED/%s",code);
               strcpy(code,val1);
            }
            getword_start(&gwarea,data);
            if (getword(ano,sizeof(ano),&gwarea,'-')<0){
               debuga(_("Maybe you have a broken record or garbage in your %s file\n"),arq);
               exit(EXIT_FAILURE);
            }
            if (getword(mes,sizeof(mes),&gwarea,'-')<0){
               debuga(_("Maybe you have a broken record or garbage in your %s file\n"),arq);
               exit(EXIT_FAILURE);
            }
            if (getword(dia,sizeof(dia),&gwarea,'\0')<0){
               debuga(_("Maybe you have a broken record or garbage in your %s file\n"),arq);
               exit(EXIT_FAILURE);
            }
            conv_month_name(mes);
            sprintf(data," %s/%s/%s:%s",dia,mes,ano,hora);
         }

         if(ilf==ILF_Squid) {
            tt=atoi(data);
            t=localtime(&tt);

            strftime(tbuf2, sizeof(tbuf2), "%H%M", t);
            sprintf(mes,"%d",t->tm_mon+1);
            conv_month_name(mes);
            if(strncmp(df,"u",1) == 0)
               sprintf(tbuf, "%04d%s%02d", t->tm_year+1900, mes, t->tm_mday);
            if(strncmp(df,"e",1) == 0)
               sprintf(tbuf, "%02d%s%04d", t->tm_mday, mes, t->tm_year+1900);
            if(strncmp(df,"w",1) == 0) {
               IndexTree=INDEX_TREE_FILE;
               strftime(tbuf, sizeof(tbuf), "%Y.%U", t);
            }

            strftime(wdata, sizeof(wdata), "%Y%m%d", t);
            idata=atoi(wdata);

            if(strncmp(df,"u",1)==0)
               strftime(dia, sizeof(dia), "%m/%d/%Y", t);
            else
               strftime(dia, sizeof(dia), "%d/%m/%Y", t);
            sprintf(hora,"%02d:%02d:%02d",t->tm_hour,t->tm_min,t->tm_sec);
         } else if(ilf==ILF_Common || ilf==ILF_Isa) {
            getword_start(&gwarea,data+1);
            if (getword_multisep(data,sizeof(data),&gwarea,':')<0){
               debuga(_("Maybe you have a broken date in your %s file\n"),arq);
               exit(EXIT_FAILURE);
            }
            if (getword_multisep(hora,sizeof(hora),&gwarea,' ')<0){
               debuga(_("Maybe you have a broken date in your %s file\n"),arq);
               exit(EXIT_FAILURE);
            }
            getword_start(&gwarea,data);
            if (getword(dia,sizeof(dia),&gwarea,'/')<0){
               debuga(_("Maybe you have a broken date in your %s file\n"),arq);
               exit(EXIT_FAILURE);
            }
            if (getword(mes,sizeof(mes),&gwarea,'/')<0){
               debuga(_("Maybe you have a broken date in your %s file\n"),arq);
               exit(EXIT_FAILURE);
            }
            if (getword(ano,sizeof(ano),&gwarea,'/')<0){
               debuga(_("Maybe you have a broken date in your %s file\n"),arq);
               exit(EXIT_FAILURE);
            }

            if(strcmp(df,"u") == 0)
               snprintf(tbuf,sizeof(tbuf),"%s%s%s",ano,mes,dia);
            if(strcmp(df,"e") == 0)
               snprintf(tbuf,sizeof(tbuf),"%s%s%s",dia,mes,ano);
            builddia(dia,mes,ano,df,wdata);
            idata=atoi(wdata);
            tt=computedate(ano,mes,dia);
            t=localtime(&tt);
         } else if (ilf==ILF_Sarg) {
            getword_start(&gwarea,data);
            if(strcmp(df,"u") == 0) {
               if (getword(mes,sizeof(mes),&gwarea,'/')<0){
                  debuga(_("Maybe you have a broken date in your %s file\n"),arq);
                  exit(EXIT_FAILURE);
               }
               if (getword(dia,sizeof(dia),&gwarea,'/')<0){
                  debuga(_("Maybe you have a broken date in your %s file\n"),arq);
                  exit(EXIT_FAILURE);
               }
            } else {
               if (getword(dia,sizeof(dia),&gwarea,'/')<0){
                  debuga(_("Maybe you have a broken date in your %s file\n"),arq);
                  exit(EXIT_FAILURE);
               }
               if (getword(mes,sizeof(mes),&gwarea,'/')<0){
                  debuga(_("Maybe you have a broken date in your %s file\n"),arq);
                  exit(EXIT_FAILURE);
               }
            }
            if (getword(ano,sizeof(ano),&gwarea,0)<0){
               debuga(_("Maybe you have a broken date in your %s file\n"),arq);
               exit(EXIT_FAILURE);
            }
            snprintf(wdata,9,"%s%s%s",ano,mes,dia);
            idata=atoi(wdata);
            tt=computedate(ano,mes,dia);
            t=localtime(&tt);
         } else {
            t=NULL;
         }

         if(debugm)
            printf("DATE=%s IDATA=%d DFROM=%d DUNTIL=%d\n",date,idata,dfrom,duntil);

         if(date[0] != '\0'){
            if(idata < dfrom || idata > duntil) continue;
         }

         // Record only hours usage which is required
         if (t) {
            if( bsearch( &( t -> tm_wday ), weekdays.list, weekdays.len,
                                             sizeof( int ), compar ) == NULL )
               continue;

            if( bsearch( &( t -> tm_hour ), hours.list, hours.len,
                                             sizeof( int ), compar ) == NULL )
               continue;
         }


         if(strlen(user) > MAX_USER_LEN) {
            if (debugm) printf(_("User ID too long: %s\n"),user);
            totregsx++;
            continue;
         }

         // include_users
         if(IncludeUsers[0] != '\0') {
            sprintf(val1,":%s:",user);
            if((str=(char *) strstr(IncludeUsers,val1)) == (char *) NULL )
               continue;
         }

         if(vercode(code)) {
            if (debugm) printf(_("Excluded code: %s\n"),code);
            totregsx++;
            continue;
         }

         if(testvaliduserchar(user))
            continue;

#if 0
         if((str = strstr(user,"%20")) != NULL) {
            /*
            This is a patch introduced to solve bug #1624251 reported at sourceforge but
            the side effect is to truncate the name at the first space and merge the reports
            of people whose name is identical up to the first space.

            The old code used to truncate the user name at the first % if a %20 was
            found anywhere in the string. That means the string could be truncated
            at the wrong place if another % occured before the %20. This new code should
            avoid that problem and only truncate at the space. There is no bug
            report indicating that anybody noticed this.
            */
            *str='\0';
         }

         /*
         Code prior to 2.2.7 used to replace any %xx by a dot as long as a %5c was
         found in the user name.
         */
         while((str = strstr(user,"%5c")) != NULL) {
            *str='.';
            for (x=3 ; str[x] ; x++) str[x-2]=str[x];
         }
#endif

         urly=url;

         if(ilf!=ILF_Sarg) {
            /*
            The full URL is not saved in sarg log. There is no point in testing the URL to detect
            a downloaded file.
            */
            download_flag=is_download_suffix(url);
            if (download_flag) {
               download_url=url;
               download_count++;
            }
         } else
            download_flag=0;

         // remove any protocol:// at the beginning of the URL
         if ((str = strchr(url,'/')) != NULL && str[1] == '/') {
            int i;

            str+=2;
            for (i=0 ; str[i] ; i++)
               url[i]=str[i];
            url[i]='\0';
         }

         if(!LongUrl) {
            url_hostname(url,hostname,sizeof(hostname));
            url=hostname;
         }

         if(url[0] == '\0') continue;

         if(addr[0] != '\0'){
            if(strcmp(addr,ip)!=0) continue;
         }
         if(fhost) {
            if(!vhexclude(url)) {
               if (debugm) printf(_("Excluded site: %s\n"),url);
               totregsx++;
               continue;
            }
         }

         if(hm[0] != '\0') {
            hmr[0]='\0';
            chm++;
            getword_start(&gwarea,hora);
            while(chm) {
               if (getword_multisep(warea,sizeof(warea),&gwarea,':')<0){
                  debuga(_("Maybe you have a broken time in your %s file\n"),arq);
                  exit(EXIT_FAILURE);
               }
               strncat(hmr,warea,2);
               chm--;
            }
            strncat(hmr,gwarea.current,2);

            if(atoi(hmr) < atoi(hm) || atoi(hmr) > atoi(hmf)) continue;
         }

         if(site[0] != '\0'){
            if(strstr(url,site)==0) continue;
         }

         if(userip) {
            strcpy(user,ip);
            id_is_ip=1;
         } else {
            id_is_ip=0;
            if(strcmp(user,"-") == 0 || strcmp(user," ") == 0 || strcmp(user,"") == 0) {
               if(RecordsWithoutUser == RECORDWITHOUTUSER_IP) {
                  strcpy(user,ip);
                  id_is_ip=1;
               }
               if(RecordsWithoutUser == RECORDWITHOUTUSER_IGNORE)
                  continue;
               if(RecordsWithoutUser == RECORDWITHOUTUSER_EVERYBODY)
                  strcpy(user,"everybody");
            } else {
               strlow(user);
               if(NtlmUserFormat == NTLMUSERFORMAT_USER) {
                  if((str = strchr(user,'_')) != 0) {
                     strcpy(warea,str+1);
                     strcpy(user,warea);
                  }
                  if((str = strchr(user,'+')) != 0) {
                     strcpy(warea,str+1);
                     strcpy(user,warea);
                  }
               }
            }
         }

         if(us[0] != '\0'){
            if(strcmp(user,us)!=0) continue;
         }

         if(puser) {
            sprintf(wuser,":%s:",user);
            if(strstr(userfile, wuser) == 0)
               continue;
         }

         if(fuser) {
            if(!vuexclude(user)) {
               if (debugm) printf(_("Excluded user: %s\n"),user);
               totregsx++;
               continue;
            }
         }

         if(strcmp(user,"-") ==0 || strcmp(user," ") ==0 || strcmp(user,"") ==0 || strcmp(user,":") ==0)
            continue;

         if(max_elapsed) {
            if(atol(elap)>max_elapsed) {
               elap[0]='0';
               elap[1]='\0';
            }
         }

         if((str=(char *) strstr(linebuf, "[SmartFilter:")) != (char *) NULL ) {
            fixendofline(str);
            sprintf(smartfilter,"\"%s\"",str+1);
         } else sprintf(smartfilter,"\"\"");

         nopen=0;
         prev_ufile=NULL;
         for (ufile=first_user_file ; ufile && strcmp(user,ufile->user->id)!=0 ; ufile=ufile->next) {
            prev_ufile=ufile;
            if (ufile->file) nopen++;
         }
         if (!ufile) {
            ufile=malloc(sizeof(*ufile));
            if (!ufile) {
               debuga(_("Not enough memory to store the user %s\n"),user);
               exit(EXIT_FAILURE);
            }
            memset(ufile,0,sizeof(*ufile));
            ufile->next=first_user_file;
            first_user_file=ufile;
            uinfo=userinfo_create(user);
            ufile->user=uinfo;
            uinfo->id_is_ip=id_is_ip;
         } else {
            if (prev_ufile) {
               prev_ufile->next=ufile->next;
               ufile->next=first_user_file;
               first_user_file=ufile;
            }
         }

         if (ufile->file==NULL) {
            if (nopen>=maxopenfiles) {
               x=0;
               for (ufile1=first_user_file ; ufile1 ; ufile1=ufile1->next) {
                  if (ufile1->file!=NULL) {
                     if (x>=maxopenfiles) {
                        fclose(ufile1->file);
                        ufile1->file=NULL;
                     }
                     x++;
                  }
               }
            }
            if (snprintf (tmp3, sizeof(tmp3), "%s/sarg/%s.unsort", tmp, ufile->user->filename)>=sizeof(tmp3)) {
               debuga(_("Temporary user file name too long: %s/sarg/%s.unsort\n"), tmp, ufile->user->filename);
               exit(EXIT_FAILURE);
            }
            if ((ufile->file = MY_FOPEN (tmp3, "a")) == NULL) {
               debuga(_("(log) Cannot open temporary file: %s - %s\n"), tmp3, strerror(errno));
               exit (1);
            }
         }

         /*if ( strcmp ( user , sz_Last_User ) != 0 ) {
            if ( fp_Write_User )
               fclose( fp_Write_User ) ;
            sprintf (tmp3, "%s/sarg/%s.unsort", tmp, user);

            if ((fp_Write_User = MY_FOPEN (tmp3, "a")) == NULL) {
               fprintf (stderr, "%s: (log) %s: %s - %s\n", argv[0], _("Cannot open temporary file"), tmp3, strerror(errno));
               exit (1);
            }
            strcpy( sz_Last_User , user ) ;
         }*/
         fprintf(ufile->file, "%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\n",dia,hora,ip,url,tam,code,elap,smartfilter);

         if(fp_log && ilf!=ILF_Sarg)
            fprintf(fp_log, "%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\n",dia,hora,user,ip,url,tam,code,elap,smartfilter);

         totregsg++;

         if(!dataonly && download_flag && download_url && strstr(code,"DENIED") == 0) {
            ndownload = 1;

            if ( ! fp_Download_Unsort ) {
               if ((fp_Download_Unsort = MY_FOPEN ( sz_Download_Unsort, "a")) == NULL) {
                  debuga(_("(log) Cannot open temporary file: %s - %s\n"),tmp3, strerror(errno));
                  exit (1);
               }
            }
            fprintf(fp_Download_Unsort,"%s\t%s\t%s\t%s\t%s\n",dia,hora,user,ip,download_url);
         }

         if((ReportType & REPORT_TYPE_DENIED) != 0 || (ReportType & REPORT_TYPE_AUTH_FAILURES) != 0) {
            if(fp_denied && strstr(code,"DENIED/403") != 0) {
               fprintf(fp_denied, "%s\t%s\t%s\t%s\t%s\n",dia,hora,user,ip,urly);
               denied_count++;
            }
            if(fp_authfail && (strstr(code,"DENIED/401") != 0 || strstr(code,"DENIED/407") != 0)) {
               fprintf(fp_authfail, "%s\t%s\t%s\t%s\t%s\n",dia,hora,user,ip,urly);
               authfail_count++;
            }
         }

         if((!totper || idata<mindate) && ilf!=ILF_Sarg){
            totper++;
            mindate=idata;
            sprintf(period,"%s-",tbuf);
            strcpy(start_hour,tbuf2);
            if(date[0] != '\0')
               fixper(tbuf, period, cduntil);
            if(debugz){
               debugaz("tbuf",tbuf);
               debugaz("period",period);
            }
         }

         if(debugm){
            printf("IP=\t%s\n",ip);
            printf("USER=\t%s\n",user);
            printf("ELAP=\t%s\n",elap);
            printf("DATE=\t%s\n",dia);
            printf("TIME=\t%s\n",hora);
            printf("FUNC=\t%s\n",fun);
            printf("URL=\t%s\n",url);
            printf("CODE=\t%s\n",code);
            printf("LEN=\t%s\n",tam);
         }
      }
      if (!from_stdin) {
         fclose(fp_in);
         if( ShowReadStatistics )
            printf(_("SARG: Records in file: %lu, reading: %3.2f%%\n"),recs1, (float) 100 );
      }
   }

   longline_free(&line);
   if ( fp_Download_Unsort )
     fclose (fp_Download_Unsort);

   for (ufile=first_user_file ; ufile ; ufile=ufile1) {
      ufile1=ufile->next;
      if (ufile->file!=NULL) fclose(ufile->file);
      free(ufile);
   }

   free_download();
   free_excludecodes();
   free_exclude();

   if(debug) {
      int totalcount=0;

      for (ilf=0 ; ilf<ILF_Last ; ilf++) totalcount+=ilf_count[ilf];

      debuga(_("   Records read: %ld, written: %ld, excluded: %ld\n"),totregsl,totregsg,totregsx);

      if(ilf_count[ILF_Common]>0 && ilf_count[ILF_Squid]>0)
         debuga(_("Log with mixed records format (squid and common log)\n"));

      if(ilf_count[ILF_Common]>0 && ilf_count[ILF_Squid]==0)
         debuga(_("Common log format\n"));

      if(ilf_count[ILF_Common]==0 && ilf_count[ILF_Squid]>0)
         debuga(_("Squid log format\n"));

      if(ilf_count[ILF_Sarg]>0)
         debuga(_("Sarg log format\n"));

      if(totalcount==0) {
         if(!totregsg) {
            debuga(_("No records found\n"));
            debuga(_("End\n"));
         } else debuga(_("Log with invalid format\n"));
         if(fp_denied)
            fclose(fp_denied);
         if(fp_authfail)
            fclose(fp_authfail);
         userinfo_free();
         if(userfile)
            free(userfile);
         close_usertab();
         unlink(denied_unsort);
         unlink(authfail_unsort);
         unlink(tmp3);
         exit(0);
      }
   }

   if(!totregsg){
      debuga(_("No records found\n"));
      debuga(_("End\n"));
//      fclose(fp_ou);
      if(fp_denied)
         fclose(fp_denied);
      if(fp_authfail)
         fclose(fp_authfail);
      userinfo_free();
      if(userfile)
         free(userfile);
      close_usertab();
      exit(0);
   }

   if(date[0] == '\0' && ilf_count[ILF_Sarg]==0) {
      strcat(period,tbuf);
   }

   if(debugz){
      debugaz("data",dia);
      debugaz("tbuf",tbuf);
      debugaz("period",period);
   }

   if(debug)
      debuga(_("Period: %s\n"),period);

//   fclose(fp_ou);
   if(fp_denied)
      fclose(fp_denied);
   if(fp_authfail)
      fclose(fp_authfail);

   if(fp_log != NULL) {
      fclose(fp_log);
      strcpy(end_hour,tbuf2);
      getword_start(&gwarea,period);
      if (getword(val2,sizeof(val2),&gwarea,'-')<0){
         debuga(_("Maybe you have a broken date range definition.\n"));
         exit(EXIT_FAILURE);
      }
      if (getword(val1,sizeof(val1),&gwarea,'\0')<0){
         debuga(_("Maybe you have a broken date range definition.\n"));
         exit(EXIT_FAILURE);
      }
      sprintf(val4,"%s/sarg-%s_%s-%s_%s.log",ParsedOutputLog,val2,start_hour,val1,end_hour);
      if (rename(arq_log,val4)) {
         debuga(_("failed to rename %s to %s - %s\n"),arq_log,val4,strerror(errno));
      } else {
         strcpy(arq_log,val4);

         if(strcmp(ParsedOutputLogCompress,"nocompress") != 0 && ParsedOutputLogCompress[0] != '\0') {
            /*
            No double quotes around ParsedOutputLogCompress because it may contain command line options. If double quotes are
            necessary around the command name, put them in the configuration file.
            */
            sprintf(val1,"%s \"%s\"",ParsedOutputLogCompress,arq_log);
            cstatus=system(val1);
            if (!WIFEXITED(cstatus) || WEXITSTATUS(cstatus)) {
               debuga(_("command return status %d\n"),WEXITSTATUS(cstatus));
               debuga(_("command: %s\n"),val1);
               exit(EXIT_FAILURE);
            }
         }
      }
      if(debug)
         debuga(_("Sarg parsed log saved as %s\n"),arq_log);
   }

   if(DataFile[0] == '\0' && (ReportType & REPORT_TYPE_DENIED) != 0) {
      sprintf(csort,"sort -T \"%s\" -k 3,3 -k 5,5 -o \"%s\" \"%s\"",tmp,denied_sort,denied_unsort);
      cstatus=system(csort);
      if (!WIFEXITED(cstatus) || WEXITSTATUS(cstatus)) {
         debuga(_("sort command return status %d\n"),WEXITSTATUS(cstatus));
         debuga(_("sort command: %s\n"),csort);
         exit(EXIT_FAILURE);
      }
      unlink(denied_unsort);
   }

   sort_users_log(tmp, debug);

   if(DataFile[0] != '\0')
      data_file(tmp);
   else
      gerarel();

   unlink(tmp2);
   if((ReportType & REPORT_TYPE_DENIED) != 0)
      unlink(denied_sort);

   if(zip[0] != '\0' && strcmp(zip,"zcat") !=0) {
      recomp(arq, zip);
   }
//   else  unlink(arq);

   if(strcmp(tmp,"/tmp") != 0) {
      unlinkdir(tmp,0);
   }

   userinfo_free();
   if(userfile)
      free(userfile);
   close_usertab();

   if(debug)
      debuga(_("End\n"));

   exit(0);

}


static void getusers(const char *pwdfile, int debug)
{

   FILE *fp_usr;
   char buf[255];
   char *str;
   long int nreg=0;

   if(debug)
      debuga(_("Loading password file from %s\n"),pwdfile);

   if ((fp_usr = fopen(pwdfile, "r")) == NULL) {
      debuga(_("(getusers) Cannot open file %s - %s\n"),pwdfile,strerror(errno));
      exit(EXIT_FAILURE);
   }

   fseek(fp_usr, 0, SEEK_END);
   nreg = ftell(fp_usr);
   if (nreg<0) {
      debuga(_("Cannot get the size of file %s\n"),pwdfile);
      exit(EXIT_FAILURE);
   }
   nreg = nreg+5000;
   fseek(fp_usr, 0, SEEK_SET);

   if((userfile=(char *) malloc(nreg))==NULL){
      debuga(_("malloc error (%ld)\n"),nreg);
      exit(EXIT_FAILURE);
   }

   bzero(userfile,nreg);
   strcpy(userfile,":");

   while(fgets(buf,sizeof(buf),fp_usr)!=NULL) {
      str=strchr(buf,':');
      if (!str) {
         debuga(_("You have an invalid user in your %s file\n"),pwdfile);
         exit(EXIT_FAILURE);
      }
      str[1]=0;
      strcat(userfile,buf);
   }

   fclose(fp_usr);

   return;
}