From: Frédéric Marchal Date: Mon, 8 Mar 2010 14:09:54 +0000 (+0000) Subject: Fix the creation of the datafile X-Git-Tag: v2.3-pre2~59 X-Git-Url: http://git.ipfire.org/?a=commitdiff_plain;h=f84a35a3be0b6a02ea15facee2276b8318e30808;p=thirdparty%2Fsarg.git Fix the creation of the datafile --- diff --git a/datafile.c b/datafile.c index 317c183..9b2992d 100644 --- a/datafile.c +++ b/datafile.c @@ -27,16 +27,17 @@ #include "include/conf.h" #include "include/defs.h" -void saverecs(char *dirname, const struct userinfostruct *uinfo, long long int nacc, char *url, long long int nbytes, char *ip, char *hora, char *dia, long long int nelap, long long int incache, long long int oucache); +void saverecs(FILE *fp_ou, const struct userinfostruct *uinfo, long long int nacc, char *url, long long int nbytes, char *ip, char *hora, char *dia, long long int nelap, long long int incache, long long int oucache); void data_file(char *tmp) { - FILE *fp_in; + FILE *fp_in, *fp_ou=NULL; - char accdia[11], acchora[9], accip[MAXLEN], accurl[MAXLEN]; + char *buf; + char accdia[11], acchora[9], accip[MAXLEN], *accurl; char oldaccdia[11], oldacchora[9], oldaccip[MAXLEN]; - char dirname[MAXLEN], wdirname[MAXLEN], oldurl[MAXLEN]; + char *oldurl; char olduser[MAXLEN], oldmsg[50], acccode[50], oldacccode[50]; char ipantes[MAXLEN], nameantes[MAXLEN]; char accsmart[MAXLEN]; @@ -50,15 +51,19 @@ void data_file(char *tmp) long long int nbytes=0; long long int nelap=0; long long int nacc=0; - long long int rtotal=0; + int rtotal=0; long long int incache=0; long long int oucache=0; long long int accbytes; long long int accelap; int dlen; int new_user; + int same_url; + int url_len; + int ourl_size; struct getwordstruct gwarea; struct userinfostruct *uinfo; + struct longlinestruct line; const char logext[]=".log"; ipantes[0]='\0'; @@ -66,6 +71,8 @@ void data_file(char *tmp) olduser[0]='\0'; strcat(tmp,"/sarg"); + oldurl=NULL; + ourl_size=0; dirp = opendir(tmp); while ( (direntp = readdir( dirp )) != NULL ) { @@ -84,33 +91,45 @@ void data_file(char *tmp) uinfo=userinfo_find_from_file(user); if (!uinfo) { - debuga(_("Ignoring unknown user file %s"),user); + debuga(_("Ignoring unknown user file %s\n"),user); continue; } strcpy(u2,uinfo->id); - if(Ip2Name && uinfo->id_is_ip) ip2name(u2,sizeof(u2)); + if(Ip2Name && uinfo->id_is_ip) { + strcpy(ipantes,u2); + ip2name(u2,sizeof(u2)); + strcpy(nameantes,u2); + } user_find(uinfo->label,MAX_USER_LEN, u2); if (snprintf(tmp3,sizeof(tmp3),"%s/%s",tmp,direntp->d_name)>=sizeof(tmp3)) { - debuga(_("(datafile) directory path too long: %s/%s"),tmp,direntp->d_name); + debuga(_("(datafile) directory path too long: %s/%s\n"),tmp,direntp->d_name); exit(1); } if((fp_in=MY_FOPEN(tmp3,"r"))==NULL){ - fprintf(stderr, "SARG: (datafile) %s: %s\n",_("Cannot open file"),tmp); + debuga(_("(datafile) Cannot open file: %s\n"),tmp3); + exit(1); + } + + if (longline_prepare(&line)<0) { + debuga(_("Not enough memory to read the downloaded files.")); exit(1); } ttopen=0; new_user=1; - while(fgets(buf,sizeof(buf),fp_in)!=NULL) { + while((buf=longline_read(fp_in,&line))!=NULL) { getword_start(&gwarea,buf); - if (getword(accdia,sizeof(accdia),&gwarea,' ')<0 || getword(acchora,sizeof(acchora),&gwarea,' ')<0 || - getword(accip,sizeof(accip),&gwarea,' ')<0 || - getword(accurl,sizeof(accurl),&gwarea,' ')<0 || getword_atoll(&accbytes,&gwarea,' ')<0 || - getword(acccode,sizeof(acccode),&gwarea,' ')<0 || getword_atoll(&accelap,&gwarea,' ')<0 || - getword_skip(20000,&gwarea,' ')<0 || getword(accsmart,sizeof(accsmart),&gwarea,'"')<0) { - debuga(_("There is a broken record or garbage in file %s"),tmp3); + if (getword(accdia,sizeof(accdia),&gwarea,'\t')<0 || getword(acchora,sizeof(acchora),&gwarea,'\t')<0 || + getword(accip,sizeof(accip),&gwarea,'\t')<0 || + getword_ptr(buf,&accurl,&gwarea,'\t')<0 || getword_atoll(&accbytes,&gwarea,'\t')<0 || + getword(acccode,sizeof(acccode),&gwarea,'\t')<0 || getword_atoll(&accelap,&gwarea,'\t')<0) { + debuga(_("There is a broken record or garbage in file %s\n"),tmp3); + exit(1); + } + if (getword_skip(20000,&gwarea,'"')<0 || getword(accsmart,sizeof(accsmart),&gwarea,'"')<0) { + debuga(_("There is an invalid smart info in file %s\n"),tmp3); exit(1); } @@ -124,25 +143,41 @@ void data_file(char *tmp) } if(!rtotal){ + url_len=strlen(accurl); + if (!oldurl || url_len>=ourl_size) { + ourl_size=url_len+1; + oldurl=realloc(oldurl,ourl_size); + if (!oldurl) { + debuga(_("Not enough memory to store the url\n")); + exit(1); + } + } strcpy(oldurl,accurl); strcpy(oldacccode,acccode); strcpy(oldaccip,accip); strcpy(oldaccdia,accdia); strcpy(oldacchora,acchora); + new_user=0; rtotal++; } + same_url=(strcmp(oldurl,accurl) == 0); - if(strcmp(oldurl,accurl) != 0 || new_user){ - strcpy(oldmsg,"OK"); - if(strstr(oldacccode,"DENIED") != 0) strcpy(oldmsg,_("DENIED")); - gravatmp(uinfo,oldurl,nacc,nbytes,oldmsg,nelap,indexonly,incache,oucache); - strcpy(wdirname,dirname); - saverecs(wdirname,uinfo,nacc,oldurl,nbytes,oldaccip,oldacchora,oldaccdia,nelap,incache,oucache); + if(!same_url || new_user){ + if(strstr(oldacccode,"DENIED") != 0) + strcpy(oldmsg,_("DENIED")); + else + strcpy(oldmsg,"OK"); + if(!fp_ou && (fp_ou=MY_FOPEN(DataFile,"w"))==NULL){ + debuga(_("(datafile) Cannot open file: %s\n"),DataFile); + exit(1); + } + saverecs(fp_ou,uinfo,nacc,oldurl,nbytes,oldaccip,oldacchora,oldaccdia,nelap,incache,oucache); nacc=0; nbytes=0; nelap=0; incache=0; oucache=0; + new_user=0; } nacc++; @@ -156,7 +191,18 @@ void data_file(char *tmp) if(strstr(crc2,"MISS") != 0) oucache+=accbytes; else incache+=accbytes; - strcpy(oldurl,accurl); + if (!same_url) { + url_len=strlen(accurl); + if (url_len>=ourl_size) { + ourl_size=url_len+1; + oldurl=realloc(oldurl,ourl_size); + if (!oldurl) { + debuga(_("Not enough memory to store the url\n")); + exit(1); + } + } + strcpy(oldurl,accurl); + } new_user=0; strcpy(oldacccode,acccode); strcpy(oldaccip,accip); @@ -165,22 +211,20 @@ void data_file(char *tmp) } fclose(fp_in); + longline_free(&line); } - (void)closedir( dirp ); + if (oldurl) free(oldurl); + if (fp_ou) fclose(fp_ou); + if(debug) - debuga("Datafile %s successfully",DataFile); + debuga("Datafile %s written successfully\n",DataFile); } -void saverecs(char *dirname, const struct userinfostruct *uinfo, long long int nacc, char *url, long long int nbytes, char *ip, char *hora, char *dia, long long int nelap, long long int incache, long long int oucache) +void saverecs(FILE *fp_ou, const struct userinfostruct *uinfo, long long int nacc, char *url, long long int nbytes, char *ip, char *hora, char *dia, long long int nelap, long long int incache, long long int oucache) { - FILE *fp_ou; char val[20]; - - if((fp_ou=MY_FOPEN(DataFile,"a"))==NULL){ - fprintf(stderr, "SARG: (datafile) %s: %s\n",_("Cannot open file"),DataFile); - exit(1); - } + char name[512]; if((DataFileFields & DATA_FIELD_USER) != 0) { fputs(uinfo->label,fp_ou); @@ -195,8 +239,8 @@ void saverecs(char *dirname, const struct userinfostruct *uinfo, long long int n fputc(DataFileDelimiter[0],fp_ou); } if((DataFileFields & DATA_FIELD_URL) != 0) { - strcpy(name,url); - if (strcmp(DataFileUrl,"ip") == 0) name2ip(name); + url_hostname(url,name,sizeof(name)); + if (DataFileUrl == DATAFILEURL_IP) name2ip(name); fputs(name,fp_ou); fputc(DataFileDelimiter[0],fp_ou); } @@ -227,6 +271,4 @@ void saverecs(char *dirname, const struct userinfostruct *uinfo, long long int n } fputc('\n',fp_ou); - - fclose(fp_ou); } diff --git a/documentation/util.txt b/documentation/util.txt index 56f7f18..80e9b1d 100644 --- a/documentation/util.txt +++ b/documentation/util.txt @@ -743,6 +743,22 @@ Any temporary file created by sarg is deleted. +/*! \fn void url_hostname(const char *url,char *hostname,int hostsize) +Extract the host name from the URL. + +\param url The url whose host name must be extracted. +\param hostname The buffer to store the host name. +\param hostsize The size of the host name buffer. + +\note The function is stupid at this time. It just searches for the first slash +in the URL and truncates the URL there. It doesn't take the protocol into account +nor the port number nor any user or password information. +*/ + + + + + /*! \fn void url_module(const char *url, char *w2) Copy at most 254 bytes from the end of the URL or stops at the first /. diff --git a/getconf.c b/getconf.c index 411f53c..fb9ae8d 100644 --- a/getconf.c +++ b/getconf.c @@ -124,6 +124,12 @@ struct param_list recnouser_values[]= {"everybody",RECORDWITHOUTUSER_EVERYBODY,~RECORDWITHOUTUSER_EVERYBODY}, }; +struct param_list datafileurl_values[]= +{ + {"ip",DATAFILEURL_IP,~DATAFILEURL_IP}, + {"name",DATAFILEURL_NAME,~DATAFILEURL_NAME}, +}; + static int is_param(const char *param,const char *buf) { int plen; @@ -493,7 +499,7 @@ static void parmtest(char *buf) if (getparam_list("datafile_fields",SET_LIST(data_field_values),buf,&DataFileFields)>0) return; - if (getparam_string("datafile_url",buf,DataFileUrl,sizeof(DataFileUrl))>0) return; + if (getparam_list("datafile_url",SET_LIST(datafileurl_values),buf,&DataFileUrl)>0) return; if (getparam_string("parsed_output_log",buf,ParsedOutputLog,sizeof(ParsedOutputLog))>0) return; diff --git a/include/conf.h b/include/conf.h index 26091c8..01d2c21 100755 --- a/include/conf.h +++ b/include/conf.h @@ -215,6 +215,9 @@ int mkstemps(char *template, int suffixlen); #define RECORDWITHOUTUSER_IGNORE 0x0002UL #define RECORDWITHOUTUSER_EVERYBODY 0x0004UL +#define DATAFILEURL_IP 0x0001UL +#define DATAFILEURL_NAME 0x0002UL + char outdir[MAXLEN]; char dirname[MAXLEN]; char buf[MAXLEN]; @@ -299,7 +302,7 @@ unsigned long int UserReportFields; char DataFile[MAXLEN]; char DataFileDelimiter[3]; unsigned long int DataFileFields; -char DataFileUrl[20]; +unsigned long int DataFileUrl; int ShowReadStatistics; char IndexSortOrder[5]; char DansGuardianConf[MAXLEN]; diff --git a/include/defs.h b/include/defs.h index a87a425..fbc9286 100755 --- a/include/defs.h +++ b/include/defs.h @@ -226,6 +226,7 @@ void debuga(const char *msg,...); void debugaz(const char *head, const char *msg); void my_lltoa(unsigned long long int n, char *s, int ssize, int len); char *get_size(const char *path, const char *file); +void url_hostname(const char *url,char *hostname,int hostsize); void url_module(const char *url, char *w2); void url_to_file(const char *url,char *file,int filesize); void strip_latin(char *line); diff --git a/log.c b/log.c index 7357e09..f12b38f 100644 --- a/log.c +++ b/log.c @@ -105,9 +105,9 @@ int main(int argc,char *argv[]) char *str; char tmp2[MAXLEN]; char tmp3[MAXLEN]; - char tmp4[MAXLEN]; - char tmp5[MAXLEN]; - char tmp6[MAXLEN]; + char denied_unsort[MAXLEN]; + char denied_sort[MAXLEN]; + char authfail_unsort[MAXLEN]; char start_hour[128]; char end_hour[128]; char *linebuf; @@ -258,7 +258,7 @@ int main(int argc,char *argv[]) strcpy(RealtimeUnauthRec,"show"); SquidguardIgnoreDate=0; DansguardianIgnoreDate=0; - strcpy(DataFileUrl,"ip"); + DataFileUrl=DATAFILEURL_IP; strcpy(MaxElapsed,"28800000"); BytesInSitesUsersReport=0; UserAuthentication=0; @@ -584,12 +584,12 @@ int main(int argc,char *argv[]) unlinkdir(tmp3,1); } my_mkdir(tmp3); - strcpy(tmp4,tmp3); - strcpy(tmp5,tmp3); - strcpy(tmp6,tmp3); - strcat(tmp4,"/denied.log.unsort"); - strcat(tmp5,"/denied.log"); - strcat(tmp6,"/authfail.log.unsort"); + strcpy(denied_unsort,tmp3); + strcpy(denied_sort,tmp3); + strcpy(authfail_unsort,tmp3); + strcat(denied_unsort,"/denied.log.unsort"); + strcat(denied_sort,"/denied.log"); + strcat(authfail_unsort,"/authfail.log.unsort"); if(debug) { fprintf(stderr, _("SARG: Parameters:\nSARG:\n")); @@ -696,17 +696,17 @@ int main(int argc,char *argv[]) sprintf ( sz_Download_Unsort , "%s/sarg/download.unsort", tmp); - if((ReportType & REPORT_TYPE_DENIED) != 0) { - if((fp_denied=MY_FOPEN(tmp4,"w"))==NULL) { - debuga(_("SARG: (log) Cannot open file: %s - %s"),tmp4,strerror(errno)); - exit(1); + if(DataFile[0]=='\0') { + if((ReportType & REPORT_TYPE_DENIED) != 0) { + if((fp_denied=MY_FOPEN(denied_unsort,"w"))==NULL) { + debuga(_("SARG: (log) Cannot open file: %s - %s"),denied_unsort,strerror(errno)); + exit(1); + } } - } - if(DataFile[0]=='\0') { if((ReportType & REPORT_TYPE_DENIED) != 0 || (ReportType & REPORT_TYPE_AUTH_FAILURES) != 0) { - if((fp_authfail=MY_FOPEN(tmp6,"w"))==NULL) { - debuga(_("SARG: (log) Cannot open file: %s - %s"),tmp6,strerror(errno)); + if((fp_authfail=MY_FOPEN(authfail_unsort,"w"))==NULL) { + debuga(_("SARG: (log) Cannot open file: %s - %s"),authfail_unsort,strerror(errno)); exit(1); } } @@ -1246,7 +1246,7 @@ int main(int argc,char *argv[]) download_count++; } } else - download_flag=0; + download_flag=0; // remove any protocol:// at the beginning of the URL if ((str = strchr(url,'/')) != NULL && str[1] == '/') { @@ -1259,11 +1259,7 @@ int main(int argc,char *argv[]) } if(!LongUrl) { - int i; - - for (i=0 ; i0, Saturday->6 ) diff --git a/util.c b/util.c index c1a2d4e..abe9841 100644 --- a/util.c +++ b/util.c @@ -1527,6 +1527,15 @@ void baddata(void) exit(1); } +void url_hostname(const char *url,char *hostname,int hostsize) +{ + int i; + + hostsize--; + for (i=0 ; i