From: Frédéric Marchal <fmarchal@users.sourceforge.net>
Date: Fri, 3 Feb 2012 20:21:46 +0000 (+0100)
Subject: Don't use an intermediate file for the hourly statistics
X-Git-Tag: v2.3.3-pre1~34
X-Git-Url: http://git.ipfire.org/cgi-bin/gitweb.cgi?a=commitdiff_plain;h=026ddd8b73130df9e54da021252d17349a70005d;p=thirdparty%2Fsarg.git

Don't use an intermediate file for the hourly statistics

The data are grouped in memory and later written to the file that will
be read to produce the hourly and graph reports.
---

diff --git a/include/defs.h b/include/defs.h
index 694dca1..2eac061 100755
--- a/include/defs.h
+++ b/include/defs.h
@@ -84,6 +84,9 @@ struct globalstatstruct
 	int totuser;
 };
 
+//! The object to store the daily statistics.
+typedef struct DayStruct *DayObject;
+
 // auth.c
 void htaccess(const struct userinfostruct *uinfo);
 
@@ -198,7 +201,11 @@ void topsites(void);
 void topuser(void);
 
 // totday.c
-void day_totalize(const char *tmp, const struct userinfostruct *uinfo);
+DayObject day_prepare(void);
+void day_cleanup(DayObject ddata);
+void day_newuser(DayObject ddata);
+void day_addpoint(DayObject ddata,const char *date, const char *time, long long int elap, long long int bytes);
+void day_totalize(DayObject ddata,const char *tmp, const struct userinfostruct *uinfo);
 
 // url.c
 void read_hostalias(const char *Filename);
diff --git a/report.c b/report.c
index d6d4323..0ed0c9f 100644
--- a/report.c
+++ b/report.c
@@ -33,8 +33,6 @@ struct globalstatstruct globstat;
 static FILE *fp_tt=NULL;
 
 static FILE *maketmp(const char *user, const char *dirname, int debug);
-static FILE *maketmp_hour(const char *user, const char *dirname);
-static void gravatmp_hora(FILE *fp_ou, const char *data, const char *hora, long long int elap, long long int accbytes);
 static void gravatmp(FILE *fp_ou, const char *oldurl, long long int nacc, long long int nbytes, const char *oldmsg, long long int nelap, long long int incache, long long int oucache);
 static void gravaporuser(const struct userinfostruct *uinfo, const char *dirname, const char *url, const char *ip, const char *data, const char *hora, long long int tam, long long int elap);
 static void gravager(FILE *fp_gen,const char *filename, const struct userinfostruct *uinfo, long long int nacc, const char *url, long long int nbytes, const char *ip, const char *hora, const char *dia, long long int nelap, long long int incache, long long int oucache);
@@ -45,7 +43,6 @@ void gerarel(void)
 	FILE *fp_in;
 	FILE *fp_gen;
 	FILE *fp_tmp=NULL;
-	FILE *fp_hour=NULL;
 
 	char *buf;
 	char accdia[11], acchora[9], accip[MAXLEN], *accurl;
@@ -81,6 +78,7 @@ void gerarel(void)
 	struct getwordstruct gwarea;
 	longline line;
 	struct userinfostruct *uinfo;
+	DayObject daystat;
 
 	ipantes[0]='\0';
 	smartfilter=0;
@@ -106,6 +104,11 @@ void gerarel(void)
 	fp_tt=NULL;
 	sort_labels(&sort_field,&sort_order);
 
+	if (indexonly || datetimeby==0)
+		daystat=NULL;
+	else
+		daystat=day_prepare();
+	
 	uscan=userinfo_startscan();
 	if (uscan == NULL) {
 		debuga(_("Cannot enumerate the user list\n"));
@@ -121,6 +124,7 @@ void gerarel(void)
 			exit(EXIT_FAILURE);
 		}
 		user=uinfo->filename;
+		day_newuser(daystat);
 
 		strcpy(u2,uinfo->id);
 		if(Ip2Name && uinfo->id_is_ip) {
@@ -132,7 +136,6 @@ void gerarel(void)
 
 		if (!indexonly) {
 			fp_tmp=maketmp(user,tmp,debug);
-			fp_hour=maketmp_hour(user,tmp);
 		}
 
 		ttopen=0;
@@ -186,7 +189,7 @@ void gerarel(void)
 			}
 
 			if (!indexonly) {
-				if (fp_hour) gravatmp_hora(fp_hour,accdia,acchora,accelap,accbytes);
+				day_addpoint(daystat,accdia,acchora,accelap,accbytes);
 				if (iprel) gravaporuser(uinfo,outdirname,accurl,accip,accdia,acchora,accbytes,accelap);
 			}
 
@@ -319,13 +322,8 @@ void gerarel(void)
 			free(oldurl);
 			oldurl=NULL;
 		}
-		if (fp_hour) {
-			if (fclose(fp_hour)==EOF)
-				debuga(_("Failed to close temporary time file for user %s - %s\n"),uinfo->filename,strerror(errno));
-			fp_hour=NULL;
-		}
 		if(!new_user) {
-			if (!indexonly) day_totalize(tmp,uinfo);
+			day_totalize(daystat,tmp,uinfo);
 		}
 		if (fp_tmp) {
 			if (fclose(fp_tmp)==EOF)
@@ -338,6 +336,7 @@ void gerarel(void)
 		}
 	}
 	userinfo_stopscan(uscan);
+	day_cleanup(daystat);
 
 	totalger(fp_gen,wdirname);
 	fclose(fp_gen);
@@ -432,27 +431,6 @@ static FILE *maketmp(const char *user, const char *dirname, int debug)
 }
 
 
-static FILE *maketmp_hour(const char *user, const char *dirname)
-{
-	FILE *fp_ou;
-	char wdirname[MAXLEN];
-
-	if(datetimeby==0) return(NULL);
-
-	if (snprintf(wdirname,sizeof(wdirname),"%s/%s.htmp",tmp,user)>=sizeof(wdirname)) {
-		debuga(_("Temporary file name too long: %s/%s.htmp\n"),tmp,user);
-		exit(EXIT_FAILURE);
-	}
-
-	if((fp_ou=fopen(wdirname,"w"))==NULL){
-		debuga(_("(report-1) Cannot open file %s - %s\n"),wdirname,strerror(errno));
-		exit(EXIT_FAILURE);
-	}
-
-	return(fp_ou);
-}
-
-
 static void gravatmp(FILE *fp_ou, const char *oldurl, long long int nacc, long long int nbytes, const char *oldmsg, long long int nelap, long long int incache, long long int oucache)
 {
 	/*
@@ -473,20 +451,6 @@ static void gravatmp(FILE *fp_ou, const char *oldurl, long long int nacc, long l
 	return;
 }
 
-static void gravatmp_hora(FILE *fp_ou, const char *data, const char *hora, long long int elap, long long int bytes)
-{
-	int hour;
-
-	hour=atoi(hora);
-	fprintf(fp_ou,"%s\t%d",data,hour);
-	if((datetimeby & DATETIME_BYTE)!=0) fprintf(fp_ou,"\t%"PRIu64,(uint64_t)bytes);
-	if((datetimeby & DATETIME_ELAP)!=0) fprintf(fp_ou,"\t%"PRIu64,(uint64_t)elap);
-	fputs("\n",fp_ou);
-
-	return;
-}
-
-
 static void gravaporuser(const struct userinfostruct *uinfo, const char *dirname, const char *url, const char *ip, const char *data, const char *hora, long long int tam, long long int elap)
 {
 	FILE *fp_ou;
diff --git a/totday.c b/totday.c
index 81f1bf6..62f27b3 100644
--- a/totday.c
+++ b/totday.c
@@ -27,115 +27,153 @@
 #include "include/conf.h"
 #include "include/defs.h"
 
-void day_totalize(const char *tmp, const struct userinfostruct *uinfo)
+//! The daily statistics for one user.
+struct DayStruct
 {
-	FILE *fp_in, *fp_ou;
+	int ndaylist;
+	int maxindex;
+	int daylist[MAX_DATETIME_DAYS];
+	long long int bytes[MAX_DATETIME_DAYS*24];
+	long long int elap[MAX_DATETIME_DAYS*24];
+};
+
+/*!
+Prepare the object to store the daily statistics of one user.
+
+\return The object to pass to other functions in this module.
+The object must be freed with a call to day_cleanup().
+*/
+DayObject day_prepare(void)
+{
+	DayObject ddata;
+	
+	ddata=(DayObject)malloc(sizeof(*ddata));
+	if (!ddata)
+	{
+		debuga(_("Not enough memory to store the daily statistics\n"));
+		exit(EXIT_FAILURE);
+	}
+	
+	return(ddata);
+}
 
-	char buf[200];
-	char date[20];
+/*!
+Free the memory allocated by day_prepare().
+
+\param ddata The object returned by day_prepare().
+*/
+void day_cleanup(DayObject ddata)
+{
+	if (ddata) free(ddata);
+}
+
+/*!
+Prepare the object for a new user.
+
+\param ddata The object created by day_prepare().
+*/
+void day_newuser(DayObject ddata)
+{
+	if (ddata)
+	{
+		ddata->ndaylist=0;
+		ddata->maxindex=0;
+		memset(ddata->bytes,0,sizeof(ddata->bytes));
+		memset(ddata->elap,0,sizeof(ddata->elap));
+	}
+}
+
+/*!
+Store one data point in the statistics.
+
+\param ddata The object to store the statistics.
+\param date The date of the data point formated as day/month/year.
+\param time The time of the data point.
+\param elap The time spent processing the user's request on the proxy.
+\param bytes The number of bytes transfered by the user.
+*/
+void day_addpoint(DayObject ddata,const char *date, const char *time, long long int elap, long long int bytes)
+{
 	int hour;
-	long long int bytes;
-	long long int elap;
-	long long int tbytes[MAX_DATETIME_DAYS*24];
-	long long int telap[MAX_DATETIME_DAYS*24];
 	int day,month,year;
-	int daylist[MAX_DATETIME_DAYS];
-	int ndaylist;
 	int daynum;
 	int dayidx;
 	int i;
-	char wdirname[2048];
-	char arqout[2048];
-	char colsep;
-	struct getwordstruct gwarea;
 
-	if (datetimeby==0) return;
-
-	if (snprintf(wdirname,sizeof(wdirname),"%s/%s.htmp",tmp,uinfo->filename)>=sizeof(wdirname)) {
-		debuga(_("File name too long: %s/%s%s\n"),tmp,uinfo->filename,".htmp");
+	if (sscanf(date,"%d/%d/%d",&day,&month,&year)!=3) {
+		debuga(_("Invalid date \"%s\" for the hourly statistics\n"),date);
 		exit(EXIT_FAILURE);
 	}
-	if (snprintf(arqout,sizeof(arqout),"%s/%s.day",tmp,uinfo->filename)>=sizeof(arqout)) {
-		debuga(_("File name too long: %s/%s%s\n"),tmp,uinfo->filename,".day");
+	if (day<1 || day>31 || month<1 || month>12 || year>9999) {
+		debuga(_("Invalid date component in \"%s\" for the hourly statistics\n"),date);
 		exit(EXIT_FAILURE);
 	}
-
-	if((fp_in=fopen(wdirname,"r"))==NULL) {
-		debuga(_("(totday) Cannot open log file %s\n"),wdirname);
+	hour=atoi(time);
+	if (hour<0 || hour>=24) {
+		debuga(_("Invalid hour %d for the hourly statistics\n"),hour);
 		exit(EXIT_FAILURE);
 	}
-
-	memset(tbytes,0,sizeof(tbytes));
-	memset(telap,0,sizeof(tbytes));
-	ndaylist=0;
-
-	while(fgets(buf,sizeof(buf),fp_in)!=NULL) {
-		fixendofline(buf);
-		getword_start(&gwarea,buf);
-		if (getword(date,sizeof(date),&gwarea,'\t')<0 || getword_atoi(&hour,&gwarea,'\t')<0) {
-			debuga(_("There is a broken record or garbage in file %s\n"),wdirname);
+	daynum=(year*10000)+(month*100)+day;
+	for (dayidx=ddata->ndaylist-1 ; dayidx>=0 && daynum!=ddata->daylist[dayidx] ; dayidx--);
+	if (dayidx<0) {
+		dayidx=ddata->ndaylist++;
+		if (dayidx>=sizeof(ddata->daylist)/sizeof(*ddata->daylist)) {
+			debuga(_("Too many different dates for the hourly statistics\n"));
 			exit(EXIT_FAILURE);
 		}
-		if (sscanf(date,"%d/%d/%d",&day,&month,&year)!=3) {
-			debuga(_("Invalid date \"%s\" in %s\n"),date,wdirname);
-			exit(EXIT_FAILURE);
-		}
-		if (day<1 || day>31 || month<1 || month>12 || year>9999) {
-			debuga(_("Invalid date component in \"%s\" in %s\n"),date,wdirname);
-			exit(EXIT_FAILURE);
-		}
-		if (hour<0 || hour>=24) {
-			debuga(_("Invalid hour %d in %s\n"),hour,wdirname);
-			exit(EXIT_FAILURE);
-		}
-		daynum=(year*10000)+(month*100)+day;
-		for (dayidx=0 ; dayidx<ndaylist && daynum!=daylist[dayidx] ; dayidx++);
-		if (dayidx>=ndaylist) {
-			if (dayidx>=sizeof(daylist)/sizeof(*daylist)) {
-				debuga(_("Too many different dates in %s\n"),wdirname);
-				exit(EXIT_FAILURE);
-			}
-			daylist[ndaylist++]=daynum;
-		}
-		i=dayidx*24+hour;
-		if ((datetimeby & DATETIME_BYTE)!=0) {
-			colsep=((datetimeby & DATETIME_ELAP)!=0) ? '\t' : '\0';
-			if (getword_atoll(&bytes,&gwarea,colsep)<0) {
-				debuga(_("Invalid number of bytes in file %s\n"),wdirname);
-				exit(EXIT_FAILURE);
-			}
-			tbytes[i]+=bytes;
-		}
-		if ((datetimeby & DATETIME_ELAP)!=0) {
-			if (getword_atoll(&elap,&gwarea,'\0')<0) {
-				debuga(_("Invalid elapsed time in file %s\n"),wdirname);
-				exit(EXIT_FAILURE);
-			}
-			telap[i]+=elap;
-		}
+		ddata->daylist[dayidx]=daynum;
+	}
+	i=dayidx*24+hour;
+	if (i>=ddata->maxindex) ddata->maxindex=i+1;
+	ddata->bytes[i]+=bytes;
+	ddata->elap[i]+=elap;
+}
+
+/*!
+Store the dayly statistics in the file.
+
+\param ddata The object containing the statistics.
+\param tmp The temporary directory to store the file into.
+\param uinfo The user's data.
+*/
+void day_totalize(DayObject ddata,const char *tmp, const struct userinfostruct *uinfo)
+{
+	FILE *fp_ou;
+	int hour;
+	int day,month,year;
+	int i;
+	int daynum;
+	int dayidx;
+	char arqout[2048];
+
+	if (datetimeby==0) return;
+	if (!ddata) return;
+
+	if (snprintf(arqout,sizeof(arqout),"%s/%s.day",tmp,uinfo->filename)>=sizeof(arqout)) {
+		debuga(_("File name too long: %s/%s%s\n"),tmp,uinfo->filename,".day");
+		exit(EXIT_FAILURE);
 	}
-	fclose(fp_in);
 
 	if((fp_ou=fopen(arqout,"w"))==NULL) {
 		debuga(_("(totday) Cannot open log file %s\n"),arqout);
 		exit(EXIT_FAILURE);
 	}
 
-	for (i=0 ; i<sizeof(tbytes)/sizeof(*tbytes) ; i++) {
-		if (tbytes[i]==0 && telap[i]==0) continue;
+	for (i=0 ; i<ddata->maxindex ; i++) {
+		if (ddata->bytes[i]==0 && ddata->elap[i]==0) continue;
 		dayidx=i/24;
-		if (dayidx>=sizeof(daylist)/sizeof(*daylist)) {
-			debuga(_("Invalid day index found in %s\n"),wdirname);
+		if (dayidx>=sizeof(ddata->daylist)/sizeof(*ddata->daylist)) {
+			debuga(_("Invalid day index found in the hourly statistics\n"));
 			exit(EXIT_FAILURE);
 		}
 		hour=i%24;
-		daynum=daylist[dayidx];
+		daynum=ddata->daylist[dayidx];
 		day=daynum%100;
 		month=(daynum/100)%100;
 		year=daynum/10000;
 		fprintf(fp_ou,"%d/%d/%d\t%d",day,month,year,hour);
-		if ((datetimeby & DATETIME_BYTE)!=0) fprintf(fp_ou,"\t%"PRIu64"",(uint64_t)tbytes[i]);
-		if ((datetimeby & DATETIME_ELAP)!=0) fprintf(fp_ou,"\t%"PRIu64"",(uint64_t)telap[i]);
+		if ((datetimeby & DATETIME_BYTE)!=0) fprintf(fp_ou,"\t%"PRIu64"",(uint64_t)ddata->bytes[i]);
+		if ((datetimeby & DATETIME_ELAP)!=0) fprintf(fp_ou,"\t%"PRIu64"",(uint64_t)ddata->elap[i]);
 		fputs("\n",fp_ou);
 	}
 
@@ -143,10 +181,5 @@ void day_totalize(const char *tmp, const struct userinfostruct *uinfo)
 		debuga(_("Failed to close file %s - %s\n"),arqout,strerror(errno));
 		exit(EXIT_FAILURE);
 	}
-
-	if (unlink(wdirname)==-1) {
-		debuga(_("Cannot delete temporary file %s - %s\n"),wdirname,strerror(errno));
-		exit(EXIT_FAILURE);
-	}
 	return;
 }