Fix the creation of the datafile

diff --git a/datafile.c b/datafile.c
index 317c183570977d3c4dae56ae697b29702d9b9d93..9b2992d0a1b7264855d702aa669fe75bf7331e82 100644 (file)
--- a/datafile.c
+++ b/datafile.c
@@ -27,16 +27,17 @@
 #include "include/conf.h"
 #include "include/defs.h"
 
-void saverecs(char *dirname, const struct userinfostruct *uinfo, long long int nacc, char *url, long long int nbytes, char *ip, char *hora, char *dia, long long int nelap, long long int incache, long long int oucache);
+void saverecs(FILE *fp_ou, const struct userinfostruct *uinfo, long long int nacc, char *url, long long int nbytes, char *ip, char *hora, char *dia, long long int nelap, long long int incache, long long int oucache);
 
 void data_file(char *tmp)
 {
 
-   FILE *fp_in;
+   FILE *fp_in, *fp_ou=NULL;
 
-   char accdia[11], acchora[9], accip[MAXLEN], accurl[MAXLEN];
+   char *buf;
+   char accdia[11], acchora[9], accip[MAXLEN], *accurl;
    char oldaccdia[11], oldacchora[9], oldaccip[MAXLEN];
-   char dirname[MAXLEN], wdirname[MAXLEN], oldurl[MAXLEN];
+   char *oldurl;
    char olduser[MAXLEN], oldmsg[50], acccode[50], oldacccode[50];
    char ipantes[MAXLEN], nameantes[MAXLEN];
    char accsmart[MAXLEN];
@@ -50,15 +51,19 @@ void data_file(char *tmp)
    long long int nbytes=0;
    long long int nelap=0;
    long long int nacc=0;
-   long long int rtotal=0;
+   int rtotal=0;
    long long int incache=0;
    long long int oucache=0;
    long long int accbytes;
    long long int accelap;
    int dlen;
    int new_user;
+   int same_url;
+   int url_len;
+   int ourl_size;
    struct getwordstruct gwarea;
    struct userinfostruct *uinfo;
+   struct longlinestruct line;
    const char logext[]=".log";
 
    ipantes[0]='\0';
@@ -66,6 +71,8 @@ void data_file(char *tmp)
 
    olduser[0]='\0';
    strcat(tmp,"/sarg");
+   oldurl=NULL;
+   ourl_size=0;
 
    dirp = opendir(tmp);
    while ( (direntp = readdir( dirp )) != NULL ) {
@@ -84,33 +91,45 @@ void data_file(char *tmp)
 
       uinfo=userinfo_find_from_file(user);
       if (!uinfo) {
-         debuga(_("Ignoring unknown user file %s"),user);
+         debuga(_("Ignoring unknown user file %s\n"),user);
          continue;
       }
       strcpy(u2,uinfo->id);
-      if(Ip2Name && uinfo->id_is_ip) ip2name(u2,sizeof(u2));
+      if(Ip2Name && uinfo->id_is_ip) {
+         strcpy(ipantes,u2);
+         ip2name(u2,sizeof(u2));
+         strcpy(nameantes,u2);
+      }
       user_find(uinfo->label,MAX_USER_LEN, u2);
 
       if (snprintf(tmp3,sizeof(tmp3),"%s/%s",tmp,direntp->d_name)>=sizeof(tmp3)) {
-         debuga(_("(datafile) directory path too long: %s/%s"),tmp,direntp->d_name);
+         debuga(_("(datafile) directory path too long: %s/%s\n"),tmp,direntp->d_name);
          exit(1);
       }
 
       if((fp_in=MY_FOPEN(tmp3,"r"))==NULL){
-         fprintf(stderr, "SARG: (datafile) %s: %s\n",_("Cannot open file"),tmp);
+         debuga(_("(datafile) Cannot open file: %s\n"),tmp3);
+         exit(1);
+      }
+
+      if (longline_prepare(&line)<0) {
+         debuga(_("Not enough memory to read the downloaded files."));
          exit(1);
       }
 
       ttopen=0;
       new_user=1;
-      while(fgets(buf,sizeof(buf),fp_in)!=NULL) {
+      while((buf=longline_read(fp_in,&line))!=NULL) {
          getword_start(&gwarea,buf);
-         if (getword(accdia,sizeof(accdia),&gwarea,' ')<0 || getword(acchora,sizeof(acchora),&gwarea,' ')<0 ||
-             getword(accip,sizeof(accip),&gwarea,' ')<0 ||
-             getword(accurl,sizeof(accurl),&gwarea,' ')<0 || getword_atoll(&accbytes,&gwarea,' ')<0 ||
-             getword(acccode,sizeof(acccode),&gwarea,' ')<0 || getword_atoll(&accelap,&gwarea,' ')<0 ||
-             getword_skip(20000,&gwarea,' ')<0 || getword(accsmart,sizeof(accsmart),&gwarea,'"')<0) {
-            debuga(_("There is a broken record or garbage in file %s"),tmp3);
+         if (getword(accdia,sizeof(accdia),&gwarea,'\t')<0 || getword(acchora,sizeof(acchora),&gwarea,'\t')<0 ||
+             getword(accip,sizeof(accip),&gwarea,'\t')<0 ||
+             getword_ptr(buf,&accurl,&gwarea,'\t')<0 || getword_atoll(&accbytes,&gwarea,'\t')<0 ||
+             getword(acccode,sizeof(acccode),&gwarea,'\t')<0 || getword_atoll(&accelap,&gwarea,'\t')<0) {
+            debuga(_("There is a broken record or garbage in file %s\n"),tmp3);
+            exit(1);
+         }
+         if (getword_skip(20000,&gwarea,'"')<0 || getword(accsmart,sizeof(accsmart),&gwarea,'"')<0) {
+            debuga(_("There is an invalid smart info in file %s\n"),tmp3);
             exit(1);
          }
 
@@ -124,25 +143,41 @@ void data_file(char *tmp)
          }
 
          if(!rtotal){
+            url_len=strlen(accurl);
+            if (!oldurl || url_len>=ourl_size) {
+               ourl_size=url_len+1;
+               oldurl=realloc(oldurl,ourl_size);
+               if (!oldurl) {
+                  debuga(_("Not enough memory to store the url\n"));
+                  exit(1);
+               }
+            }
             strcpy(oldurl,accurl);
             strcpy(oldacccode,acccode);
             strcpy(oldaccip,accip);
             strcpy(oldaccdia,accdia);
             strcpy(oldacchora,acchora);
+            new_user=0;
             rtotal++;
          }
+         same_url=(strcmp(oldurl,accurl) == 0);
 
-         if(strcmp(oldurl,accurl) != 0 || new_user){
-            strcpy(oldmsg,"OK");
-            if(strstr(oldacccode,"DENIED") != 0) strcpy(oldmsg,_("DENIED"));
-            gravatmp(uinfo,oldurl,nacc,nbytes,oldmsg,nelap,indexonly,incache,oucache);
-            strcpy(wdirname,dirname);
-            saverecs(wdirname,uinfo,nacc,oldurl,nbytes,oldaccip,oldacchora,oldaccdia,nelap,incache,oucache);
+         if(!same_url || new_user){
+            if(strstr(oldacccode,"DENIED") != 0)
+               strcpy(oldmsg,_("DENIED"));
+            else
+               strcpy(oldmsg,"OK");
+            if(!fp_ou && (fp_ou=MY_FOPEN(DataFile,"w"))==NULL){
+               debuga(_("(datafile) Cannot open file: %s\n"),DataFile);
+               exit(1);
+            }
+            saverecs(fp_ou,uinfo,nacc,oldurl,nbytes,oldaccip,oldacchora,oldaccdia,nelap,incache,oucache);
             nacc=0;
             nbytes=0;
             nelap=0;
             incache=0;
             oucache=0;
+            new_user=0;
          }
 
          nacc++;
@@ -156,7 +191,18 @@ void data_file(char *tmp)
          if(strstr(crc2,"MISS") != 0) oucache+=accbytes;
          else incache+=accbytes;
 
-         strcpy(oldurl,accurl);
+         if (!same_url) {
+            url_len=strlen(accurl);
+            if (url_len>=ourl_size) {
+               ourl_size=url_len+1;
+               oldurl=realloc(oldurl,ourl_size);
+               if (!oldurl) {
+                  debuga(_("Not enough memory to store the url\n"));
+                  exit(1);
+               }
+            }
+            strcpy(oldurl,accurl);
+         }
          new_user=0;
          strcpy(oldacccode,acccode);
          strcpy(oldaccip,accip);
@@ -165,22 +211,20 @@ void data_file(char *tmp)
       }
 
       fclose(fp_in);
+      longline_free(&line);
    }
-
    (void)closedir( dirp );
+   if (oldurl) free(oldurl);
+   if (fp_ou) fclose(fp_ou);
+
    if(debug)
-      debuga("Datafile %s successfully",DataFile);
+      debuga("Datafile %s written successfully\n",DataFile);
 }
 
-void saverecs(char *dirname, const struct userinfostruct *uinfo, long long int nacc, char *url, long long int nbytes, char *ip, char *hora, char *dia, long long int nelap, long long int incache, long long int oucache)
+void saverecs(FILE *fp_ou, const struct userinfostruct *uinfo, long long int nacc, char *url, long long int nbytes, char *ip, char *hora, char *dia, long long int nelap, long long int incache, long long int oucache)
 {
-   FILE *fp_ou;
    char val[20];
-
-   if((fp_ou=MY_FOPEN(DataFile,"a"))==NULL){
-      fprintf(stderr, "SARG: (datafile) %s: %s\n",_("Cannot open file"),DataFile);
-      exit(1);
-   }
+   char name[512];
 
    if((DataFileFields & DATA_FIELD_USER) != 0) {
       fputs(uinfo->label,fp_ou);
@@ -195,8 +239,8 @@ void saverecs(char *dirname, const struct userinfostruct *uinfo, long long int n
       fputc(DataFileDelimiter[0],fp_ou);
    }
    if((DataFileFields & DATA_FIELD_URL) != 0) {
-      strcpy(name,url);
-      if (strcmp(DataFileUrl,"ip") == 0) name2ip(name);
+      url_hostname(url,name,sizeof(name));
+      if (DataFileUrl == DATAFILEURL_IP) name2ip(name);
       fputs(name,fp_ou);
       fputc(DataFileDelimiter[0],fp_ou);
    }
@@ -227,6 +271,4 @@ void saverecs(char *dirname, const struct userinfostruct *uinfo, long long int n
    }
 
    fputc('\n',fp_ou);
-
-   fclose(fp_ou);
 }

diff --git a/documentation/util.txt b/documentation/util.txt
index 56f7f18951913f4db403c32d2848775a88969923..80e9b1d040a7eb97482012bb5982654f344fb75e 100644 (file)
--- a/documentation/util.txt
+++ b/documentation/util.txt
@@ -743,6 +743,22 @@ Any temporary file created by sarg is deleted.
 
 
 
+/*! \fn void url_hostname(const char *url,char *hostname,int hostsize)
+Extract the host name from the URL.
+
+\param url The url whose host name must be extracted.
+\param hostname The buffer to store the host name.
+\param hostsize The size of the host name buffer.
+
+\note The function is stupid at this time. It just searches for the first slash
+in the URL and truncates the URL there. It doesn't take the protocol into account
+nor the port number nor any user or password information.
+*/
+
+
+
+
+
 /*! \fn void url_module(const char *url, char *w2)
 Copy at most 254 bytes from the end of the URL or stops at the first /.
 

diff --git a/getconf.c b/getconf.c
index 411f53c0dceaa80e23d1eaa78814363600dc4379..fb9ae8da59773f4b73459f4bc8f86cd81ab4dca2 100644 (file)
--- a/getconf.c
+++ b/getconf.c
@@ -124,6 +124,12 @@ struct param_list recnouser_values[]=
    {"everybody",RECORDWITHOUTUSER_EVERYBODY,~RECORDWITHOUTUSER_EVERYBODY},
 };
 
+struct param_list datafileurl_values[]=
+{
+   {"ip",DATAFILEURL_IP,~DATAFILEURL_IP},
+   {"name",DATAFILEURL_NAME,~DATAFILEURL_NAME},
+};
+
 static int is_param(const char *param,const char *buf)
 {
    int plen;
@@ -493,7 +499,7 @@ static void parmtest(char *buf)
 
    if (getparam_list("datafile_fields",SET_LIST(data_field_values),buf,&DataFileFields)>0) return;
 
-   if (getparam_string("datafile_url",buf,DataFileUrl,sizeof(DataFileUrl))>0) return;
+   if (getparam_list("datafile_url",SET_LIST(datafileurl_values),buf,&DataFileUrl)>0) return;
 
    if (getparam_string("parsed_output_log",buf,ParsedOutputLog,sizeof(ParsedOutputLog))>0) return;
 

diff --git a/include/conf.h b/include/conf.h
index 26091c845427054936953763b70712b9d5307195..01d2c211563e279a250a6aeda51c23009ffde191 100755 (executable)
--- a/include/conf.h
+++ b/include/conf.h
@@ -215,6 +215,9 @@ int mkstemps(char *template, int suffixlen);
 #define RECORDWITHOUTUSER_IGNORE    0x0002UL
 #define RECORDWITHOUTUSER_EVERYBODY 0x0004UL
 
+#define DATAFILEURL_IP   0x0001UL
+#define DATAFILEURL_NAME 0x0002UL
+
 char outdir[MAXLEN];
 char dirname[MAXLEN];
 char buf[MAXLEN];
@@ -299,7 +302,7 @@ unsigned long int UserReportFields;
 char DataFile[MAXLEN];
 char DataFileDelimiter[3];
 unsigned long int DataFileFields;
-char DataFileUrl[20];
+unsigned long int DataFileUrl;
 int ShowReadStatistics;
 char IndexSortOrder[5];
 char DansGuardianConf[MAXLEN];

diff --git a/include/defs.h b/include/defs.h
index a87a425059aff9c2b1ccea1325940a9bcf4d13b2..fbc9286c133c8bc3f69dd097e8fa6068b778b460 100755 (executable)
--- a/include/defs.h
+++ b/include/defs.h
@@ -226,6 +226,7 @@ void debuga(const char *msg,...);
 void debugaz(const char *head, const char *msg);
 void my_lltoa(unsigned long long int n, char *s, int ssize, int len);
 char *get_size(const char *path, const char *file);
+void url_hostname(const char *url,char *hostname,int hostsize);
 void url_module(const char *url, char *w2);
 void url_to_file(const char *url,char *file,int filesize);
 void strip_latin(char *line);

diff --git a/log.c b/log.c
index 7357e09429223985b8ec5cf1394d892356dda149..f12b38fa5ce65ff26e0b83129dbbef290d1172e0 100644 (file)
--- a/log.c
+++ b/log.c
@@ -105,9 +105,9 @@ int main(int argc,char *argv[])
    char *str;
    char tmp2[MAXLEN];
    char tmp3[MAXLEN];
-   char tmp4[MAXLEN];
-   char tmp5[MAXLEN];
-   char tmp6[MAXLEN];
+   char denied_unsort[MAXLEN];
+   char denied_sort[MAXLEN];
+   char authfail_unsort[MAXLEN];
    char start_hour[128];
    char end_hour[128];
    char *linebuf;
@@ -258,7 +258,7 @@ int main(int argc,char *argv[])
    strcpy(RealtimeUnauthRec,"show");
    SquidguardIgnoreDate=0;
    DansguardianIgnoreDate=0;
-   strcpy(DataFileUrl,"ip");
+   DataFileUrl=DATAFILEURL_IP;
    strcpy(MaxElapsed,"28800000");
    BytesInSitesUsersReport=0;
    UserAuthentication=0;
@@ -584,12 +584,12 @@ int main(int argc,char *argv[])
       unlinkdir(tmp3,1);
    }
    my_mkdir(tmp3);
-   strcpy(tmp4,tmp3);
-   strcpy(tmp5,tmp3);
-   strcpy(tmp6,tmp3);
-   strcat(tmp4,"/denied.log.unsort");
-   strcat(tmp5,"/denied.log");
-   strcat(tmp6,"/authfail.log.unsort");
+   strcpy(denied_unsort,tmp3);
+   strcpy(denied_sort,tmp3);
+   strcpy(authfail_unsort,tmp3);
+   strcat(denied_unsort,"/denied.log.unsort");
+   strcat(denied_sort,"/denied.log");
+   strcat(authfail_unsort,"/authfail.log.unsort");
 
    if(debug) {
       fprintf(stderr, _("SARG: Parameters:\nSARG:\n"));
@@ -696,17 +696,17 @@ int main(int argc,char *argv[])
 
    sprintf ( sz_Download_Unsort , "%s/sarg/download.unsort", tmp);
 
-   if((ReportType & REPORT_TYPE_DENIED) != 0) {
-      if((fp_denied=MY_FOPEN(tmp4,"w"))==NULL) {
-         debuga(_("SARG: (log) Cannot open file: %s - %s"),tmp4,strerror(errno));
-         exit(1);
+   if(DataFile[0]=='\0') {
+      if((ReportType & REPORT_TYPE_DENIED) != 0) {
+         if((fp_denied=MY_FOPEN(denied_unsort,"w"))==NULL) {
+            debuga(_("SARG: (log) Cannot open file: %s - %s"),denied_unsort,strerror(errno));
+            exit(1);
+         }
       }
-   }
 
-   if(DataFile[0]=='\0') {
       if((ReportType & REPORT_TYPE_DENIED) != 0 || (ReportType & REPORT_TYPE_AUTH_FAILURES) != 0) {
-         if((fp_authfail=MY_FOPEN(tmp6,"w"))==NULL) {
-            debuga(_("SARG: (log) Cannot open file: %s - %s"),tmp6,strerror(errno));
+         if((fp_authfail=MY_FOPEN(authfail_unsort,"w"))==NULL) {
+            debuga(_("SARG: (log) Cannot open file: %s - %s"),authfail_unsort,strerror(errno));
             exit(1);
          }
       }
@@ -1246,7 +1246,7 @@ int main(int argc,char *argv[])
                download_count++;
             }
          } else
-          download_flag=0;
+            download_flag=0;
 
          // remove any protocol:// at the beginning of the URL
          if ((str = strchr(url,'/')) != NULL && str[1] == '/') {
@@ -1259,11 +1259,7 @@ int main(int argc,char *argv[])
          }
 
          if(!LongUrl) {
-            int i;
-
-            for (i=0 ; i<sizeof(hostname)-1 && url[i] && url[i]!='/' ; i++)
-               hostname[i]=url[i];
-            hostname[i]='\0';
+            url_hostname(url,hostname,sizeof(hostname));
             url=hostname;
          }
 
@@ -1430,7 +1426,7 @@ int main(int argc,char *argv[])
 
          totregsg++;
 
-         if(download_flag && download_url && strstr(code,"DENIED") == 0) {
+         if(!dataonly && download_flag && download_url && strstr(code,"DENIED") == 0) {
             ndownload = 1;
 
             if ( ! fp_Download_Unsort ) {
@@ -1443,13 +1439,12 @@ int main(int argc,char *argv[])
          }
 
          if((ReportType & REPORT_TYPE_DENIED) != 0 || (ReportType & REPORT_TYPE_AUTH_FAILURES) != 0) {
-            if(strstr(code,"DENIED/403") != 0) {
+            if(fp_denied && strstr(code,"DENIED/403") != 0) {
                fprintf(fp_denied, "%s\t%s\t%s\t%s\t%s\n",dia,hora,user,ip,urly);
                denied_count++;
             }
-            if(strstr(code,"DENIED/401") != 0 || strstr(code,"DENIED/407") != 0) {
-               if(fp_authfail)
-                  fprintf(fp_authfail, "%s\t%s\t%s\t%s\t%s\n",dia,hora,user,ip,urly);
+            if(fp_authfail && (strstr(code,"DENIED/401") != 0 || strstr(code,"DENIED/407") != 0)) {
+               fprintf(fp_authfail, "%s\t%s\t%s\t%s\t%s\n",dia,hora,user,ip,urly);
                authfail_count++;
             }
          }
@@ -1532,8 +1527,8 @@ int main(int argc,char *argv[])
          if(userfile)
             free(userfile);
          close_usertab();
-         unlink(tmp4);
-         unlink(tmp6);
+         unlink(denied_unsort);
+         unlink(authfail_unsort);
          unlink(tmp3);
          exit(0);
       }
@@ -1609,15 +1604,15 @@ int main(int argc,char *argv[])
          debuga("%s %s",_("Sarg parsed log saved as"),arq_log);
    }
 
-   if((ReportType & REPORT_TYPE_DENIED) != 0) {
-      sprintf(csort,"sort -T \"%s\" -k 3,3 -k 5,5 -o \"%s\" \"%s\"",tmp,tmp5,tmp4);
+   if(DataFile[0] == '\0' && (ReportType & REPORT_TYPE_DENIED) != 0) {
+      sprintf(csort,"sort -T \"%s\" -k 3,3 -k 5,5 -o \"%s\" \"%s\"",tmp,denied_sort,denied_unsort);
       cstatus=system(csort);
       if (!WIFEXITED(cstatus) || WEXITSTATUS(cstatus)) {
          fprintf(stderr, "SARG: sort command return status %d\n",WEXITSTATUS(cstatus));
          fprintf(stderr, "SARG: sort command: %s\n",csort);
          exit(1);
       }
-      unlink(tmp4);
+      unlink(denied_unsort);
    }
 
    sort_users_log(tmp, debug);
@@ -1629,7 +1624,7 @@ int main(int argc,char *argv[])
 
    unlink(tmp2);
    if((ReportType & REPORT_TYPE_DENIED) != 0)
-      unlink(tmp5);
+      unlink(denied_sort);
 
    if(zip[0] != '\0' && strcmp(zip,"zcat") !=0) {
       recomp(arq, zip);

diff --git a/sarg.conf b/sarg.conf
index 3436f43b80294f36be73362d4ec9af4e9ba67072..eac801443c63b3d78f1694ac01f52bbf01a05a33 100644 (file)
--- a/sarg.conf
+++ b/sarg.conf
@@ -470,7 +470,7 @@
 # TAG: datafile_url ip|name
 #      Saves the URL as ip or name in datafile
 #
-#datafile ip
+#datafile_url ip
 
 # TAG: weekdays
 #      The weekdays to take account ( Sunday->0, Saturday->6 )

diff --git a/util.c b/util.c
index c1a2d4e30df52dd2a8d5b3fb61b8a6cb67da2d94..abe9841a15841d4dd6787682ffc5b3143a521031 100644 (file)
--- a/util.c
+++ b/util.c
@@ -1527,6 +1527,15 @@ void baddata(void)
    exit(1);
 }
 
+void url_hostname(const char *url,char *hostname,int hostsize)
+{
+   int i;
+
+   hostsize--;
+   for (i=0 ; i<hostsize && url[i] && url[i]!='/' ; i++)
+      hostname[i]=url[i];
+   hostname[i]='\0';
+}
 
 void url_module(const char *url, char *w2)
 {