Speed up the second stage of the report generation

A lot of temporary files are produced after the log is split into several
files each containing the data of the users but those temporary files were
constantly being opened and closed for each line to be written into. It
was a small waste of time.

The new processing opens the temporary user's files once per user, process
the data of the corresponding user and generate the temporary file with
those data. The gain is roughly 10% on my data set.

diff --git a/report.c b/report.c
index 625c528254aeb50f8e3ab17c78205d44c7d16e85..f6b6af919945ac768898cc1201b71ce2fafcdaea 100644 (file)
--- a/report.c
+++ b/report.c
@@ -32,10 +32,10 @@ struct globalstatstruct globstat;
 
 static FILE *fp_tt=NULL;
 
-static void maketmp(const char *user, const char *dirname, int debug);
-static void maketmp_hour(const char *user, const char *dirname);
-static void gravatmp_hora(const char *dirname, const struct userinfostruct *uinfo, const char *data, const char *hora, long long int elap, long long int accbytes);
-static void gravatmp(const struct userinfostruct *uinfo, const char *oldurl, long long int nacc, long long int nbytes, const char *oldmsg, long long int nelap, long long int incache, long long int oucache);
+static FILE *maketmp(const char *user, const char *dirname, int debug);
+static FILE *maketmp_hour(const char *user, const char *dirname);
+static void gravatmp_hora(FILE *fp_ou, const char *data, const char *hora, long long int elap, long long int accbytes);
+static void gravatmp(FILE *fp_ou, const char *oldurl, long long int nacc, long long int nbytes, const char *oldmsg, long long int nelap, long long int incache, long long int oucache);
 static void gravaporuser(const struct userinfostruct *uinfo, const char *dirname, const char *url, const char *ip, const char *data, const char *hora, long long int tam, long long int elap);
 static void gravager(FILE *fp_gen,const char *filename, const struct userinfostruct *uinfo, long long int nacc, const char *url, long long int nbytes, const char *ip, const char *hora, const char *dia, long long int nelap, long long int incache, long long int oucache);
 static void grava_SmartFilter(const char *dirname, const char *user, const char *ip, const char *data, const char *hora, const char *url, const char *smart);
@@ -44,6 +44,8 @@ void gerarel(void)
 {
        FILE *fp_in;
        FILE *fp_gen;
+       FILE *fp_tmp=NULL;
+       FILE *fp_hour=NULL;
 
        char *buf;
        char accdia[11], acchora[9], accip[MAXLEN], *accurl;
@@ -63,7 +65,6 @@ void gerarel(void)
        long long int nbytes=0;
        long long int nelap=0;
        long long int nacc=0;
-       long long int rtotal=0;
        long long int incache=0;
        long long int oucache=0;
        long long int accbytes, accelap;
@@ -76,10 +77,10 @@ void gerarel(void)
        int ourl_size=0;
        int ourltt_size=0;
        int same_url;
-       int new_user;
+       bool new_user;
        struct getwordstruct gwarea;
        longline line;
-       struct userinfostruct *uinfo,*puinfo;
+       struct userinfostruct *uinfo;
 
        ipantes[0]='\0';
        smartfilter=0;
@@ -102,7 +103,6 @@ void gerarel(void)
                exit(EXIT_FAILURE);
        }
 
-       puinfo=NULL;
        fp_tt=NULL;
        sort_labels(&sort_field,&sort_order);
 
@@ -131,16 +131,22 @@ void gerarel(void)
                user_find(uinfo->label,MAX_USER_LEN, u2);
 
                if (!indexonly) {
-                       maketmp(user,tmp,debug);
-                       maketmp_hour(user,tmp);
+                       fp_tmp=maketmp(user,tmp,debug);
+                       fp_hour=maketmp_hour(user,tmp);
                }
 
                ttopen=0;
+               oldurl=NULL;
                oldurltt=NULL;
                ourltt_size=0;
                memset(oldaccdiatt,0,sizeof(oldaccdiatt));
                memset(oldacchoratt,0,sizeof(oldacchoratt));
-               new_user=1;
+               new_user=true;
+               nacc=0;
+               nbytes=0;
+               nelap=0;
+               incache=0;
+               oucache=0;
 
                if ((line=longline_create())==NULL) {
                        debuga(_("Not enough memory to read the downloaded files\n"));
@@ -180,12 +186,11 @@ void gerarel(void)
                        }
 
                        if (!indexonly) {
-                               gravatmp_hora(outdirname,uinfo,accdia,acchora,accelap,accbytes);
-
-                               if(iprel) gravaporuser(uinfo,outdirname,accurl,accip,accdia,acchora,accbytes,accelap);
+                               if (fp_hour) gravatmp_hora(fp_hour,accdia,acchora,accelap,accbytes);
+                               if (iprel) gravaporuser(uinfo,outdirname,accurl,accip,accdia,acchora,accbytes,accelap);
                        }
 
-                       if(!rtotal){
+                       if(new_user){
                                url_len=strlen(accurl);
                                if (!oldurl || url_len>=ourl_size) {
                                        ourl_size=url_len+1;
@@ -197,37 +202,21 @@ void gerarel(void)
                                }
                                strcpy(oldurl,accurl);
                                strcpy(oldacccode,acccode);
-                               puinfo=uinfo;
                                strcpy(oldaccip,accip);
                                strcpy(oldaccdia,accdia);
                                strcpy(oldacchora,acchora);
-                               new_user=0;
-                               rtotal++;
+                               new_user=false;
                        }
                        same_url=(strcmp(oldurl,accurl) == 0);
 
-                       if(site[0] != '\0') {
-                               if(new_user){
+                       if(site[0] == '\0') {
+                               if(!same_url){
                                        if(strstr(oldacccode,"DENIED") != 0)
                                                strcpy(oldmsg,"DENIED");
                                        else
                                                strcpy(oldmsg,"OK");
-                                       if (!indexonly) gravatmp(puinfo,oldurl,nacc,nbytes,oldmsg,nelap,incache,oucache);
-                                       gravager(fp_gen,wdirname,puinfo,nacc,oldurl,nbytes,oldaccip,oldacchora,oldaccdia,nelap,incache,oucache);
-                                       nacc=0;
-                                       nbytes=0;
-                                       nelap=0;
-                                       incache=0;
-                                       oucache=0;
-                               }
-                       } else {
-                               if(!same_url || new_user){
-                                       if(strstr(oldacccode,"DENIED") != 0)
-                                               strcpy(oldmsg,"DENIED");
-                                       else
-                                               strcpy(oldmsg,"OK");
-                                       if (!indexonly) gravatmp(puinfo,oldurl,nacc,nbytes,oldmsg,nelap,incache,oucache);
-                                       gravager(fp_gen,wdirname,puinfo,nacc,oldurl,nbytes,oldaccip,oldacchora,oldaccdia,nelap,incache,oucache);
+                                       if (fp_tmp) gravatmp(fp_tmp,oldurl,nacc,nbytes,oldmsg,nelap,incache,oucache);
+                                       gravager(fp_gen,wdirname,uinfo,nacc,oldurl,nbytes,oldaccip,oldacchora,oldaccdia,nelap,incache,oucache);
                                        nacc=0;
                                        nbytes=0;
                                        nelap=0;
@@ -297,14 +286,9 @@ void gerarel(void)
                        if (str) *str='\0';
                        if(strstr(crc2,"MISS") != 0)
                                oucache+=accbytes;
-                       else incache+=accbytes;
-
-                       if(new_user) {
-                               new_user=0;
-                               if (!indexonly) day_totalize(tmp,puinfo);
-                       }
+                       else
+                               incache+=accbytes;
 
-                       puinfo=uinfo;
                        strcpy(oldacccode,acccode);
                        strcpy(oldaccip,accip);
                        if (!same_url) {
@@ -321,11 +305,33 @@ void gerarel(void)
                        }
                        strcpy(oldaccdia,accdia);
                        strcpy(oldacchora,acchora);
-
                }
                fclose(fp_in);
                longline_destroy(&line);
                if (oldurltt) free(oldurltt);
+               if (oldurl) {
+                       if(strstr(oldacccode,"DENIED") != 0)
+                               strcpy(oldmsg,"DENIED");
+                       else
+                               strcpy(oldmsg,"OK");
+                       if (fp_tmp) gravatmp(fp_tmp,oldurl,nacc,nbytes,oldmsg,nelap,incache,oucache);
+                       gravager(fp_gen,wdirname,uinfo,nacc,oldurl,nbytes,oldaccip,oldacchora,oldaccdia,nelap,incache,oucache);
+                       free(oldurl);
+                       oldurl=NULL;
+               }
+               if (fp_hour) {
+                       if (fclose(fp_hour)==EOF)
+                               debuga(_("Failed to close temporary time file for user %s - %s\n"),uinfo->filename,strerror(errno));
+                       fp_hour=NULL;
+               }
+               if(!new_user) {
+                       if (!indexonly) day_totalize(tmp,uinfo);
+               }
+               if (fp_tmp) {
+                       if (fclose(fp_tmp)==EOF)
+                               debuga(_("Failed to close temporary file for user %s - %s\n"),uinfo->filename,strerror(errno));
+                       fp_tmp=NULL;
+               }
                if (unlink(tmp3)) {
                        debuga(_("Cannot delete %s - %s\n"),tmp3,strerror(errno));
                        exit(EXIT_FAILURE);
@@ -333,18 +339,8 @@ void gerarel(void)
        }
        userinfo_stopscan(uscan);
 
-       if (oldurl) {
-               if(strstr(oldacccode,"DENIED") != 0)
-                       strcpy(oldmsg,"DENIED");
-               else
-                       strcpy(oldmsg,"OK");
-               if (!indexonly) gravatmp(puinfo,oldurl,nacc,nbytes,oldmsg,nelap,incache,oucache);
-               gravager(fp_gen,wdirname,puinfo,nacc,oldurl,nbytes,oldaccip,oldacchora,oldaccdia,nelap,incache,oucache);
-               free(oldurl);
-       }
        totalger(fp_gen,wdirname);
        fclose(fp_gen);
-       if (puinfo && !indexonly) day_totalize(tmp,puinfo);
 
        if(email[0] == '\0') {
                if (!indexonly) {
@@ -413,12 +409,13 @@ void gerarel(void)
        return;
 }
 
-static void maketmp(const char *user, const char *dirname, int debug)
+static FILE *maketmp(const char *user, const char *dirname, int debug)
 {
        FILE *fp_ou;
        char wdirname[MAXLEN];
 
-       if((ReportType & REPORT_TYPE_USERS_SITES) == 0) return;
+       if((ReportType & REPORT_TYPE_USERS_SITES) == 0) return(NULL);
+       if((ReportType & REPORT_TYPE_TOPUSERS) == 0) return(NULL);
 
        if(debug) debuga(_("Making file: %s/%s\n"),tmp,user);
        if (snprintf(wdirname,sizeof(wdirname),"%s/%s.utmp",tmp,user)>=sizeof(wdirname)) {
@@ -431,18 +428,17 @@ static void maketmp(const char *user, const char *dirname, int debug)
                exit(EXIT_FAILURE);
        }
 
-       fclose(fp_ou);
-       return;
+       return(fp_ou);
 }
 
 
-static void maketmp_hour(const char *user, const char *dirname)
+static FILE *maketmp_hour(const char *user, const char *dirname)
 {
        FILE *fp_ou;
        char wdirname[MAXLEN];
 
-       if((ReportType & REPORT_TYPE_USERS_SITES) == 0) return;
-       if(datetimeby==0) return;
+       if((ReportType & REPORT_TYPE_USERS_SITES) == 0) return(NULL);
+       if(datetimeby==0) return(NULL);
 
        if (snprintf(wdirname,sizeof(wdirname),"%s/%s.htmp",tmp,user)>=sizeof(wdirname)) {
                debuga(_("Temporary file name too long: %s/%s.htmp\n"),tmp,user);
@@ -454,38 +450,18 @@ static void maketmp_hour(const char *user, const char *dirname)
                exit(EXIT_FAILURE);
        }
 
-       fclose(fp_ou);
-       return;
+       return(fp_ou);
 }
 
 
-static void gravatmp(const struct userinfostruct *uinfo, const char *oldurl, long long int nacc, long long int nbytes, const char *oldmsg, long long int nelap, long long int incache, long long int oucache)
+static void gravatmp(FILE *fp_ou, const char *oldurl, long long int nacc, long long int nbytes, const char *oldmsg, long long int nelap, long long int incache, long long int oucache)
 {
-       FILE *fp_ou;
-       char wdirname[MAXLEN];
-
-       if((ReportType & REPORT_TYPE_USERS_SITES) == 0) return;
-
-       if (snprintf(wdirname,sizeof(wdirname),"%s/%s.utmp",tmp,uinfo->filename)>=sizeof(wdirname)) {
-               debuga(_("Temporary file name too long: %s/%s.utmp\n"),tmp,uinfo->filename);
-               exit(EXIT_FAILURE);
-       }
-
-       if((fp_ou=MY_FOPEN(wdirname,"a"))==NULL){
-               debuga(_("(report) Cannot open file %s\n"),wdirname);
-               exit(EXIT_FAILURE);
-       }
-
        /*
        This complicated printf is due to Microsoft's inability to comply with any standard. Msvcrt is unable
        to print a long long int unless it is exactly 64-bits long.
        */
        fprintf(fp_ou,"%"PRIu64"\t%"PRIu64"\t%s\t%s\t%"PRIu64"\t%"PRIu64"\t%"PRIu64"\n",(uint64_t)nacc,(uint64_t)nbytes,oldurl,oldmsg,(uint64_t)nelap,(uint64_t)incache,(uint64_t)oucache);
 
-       if (fclose(fp_ou)==EOF) {
-               debuga(_("Failed to close file %s - %s\n"),wdirname,strerror(errno));
-               exit(EXIT_FAILURE);
-       }
        ttopen=0;
 
        if(fp_tt) {
@@ -498,35 +474,16 @@ static void gravatmp(const struct userinfostruct *uinfo, const char *oldurl, lon
        return;
 }
 
-static void gravatmp_hora(const char *dirname, const struct userinfostruct *uinfo, const char *data, const char *hora, long long int elap, long long int bytes)
+static void gravatmp_hora(FILE *fp_ou, const char *data, const char *hora, long long int elap, long long int bytes)
 {
-       FILE *fp_ou;
-       char wdirname[MAXLEN];
        int hour;
 
-       if(((ReportType & REPORT_TYPE_USERS_SITES) == 0) || datetimeby==0) return;
-
-       if (snprintf(wdirname,sizeof(wdirname),"%s/%s.htmp",tmp,uinfo->filename)>=sizeof(wdirname)) {
-               debuga(_("Path too long %s/%s.htmp\n"),tmp,uinfo->filename);
-               exit(EXIT_FAILURE);
-       }
-
-       if((fp_ou=MY_FOPEN(wdirname,"a"))==NULL){
-               debuga(_("(report-2) Cannot open file %s - %s\n"),wdirname,strerror(errno));
-               exit(EXIT_FAILURE);
-       }
-
        hour=atoi(hora);
        fprintf(fp_ou,"%s\t%d",data,hour);
        if((datetimeby & DATETIME_BYTE)!=0) fprintf(fp_ou,"\t%"PRIu64,(uint64_t)bytes);
        if((datetimeby & DATETIME_ELAP)!=0) fprintf(fp_ou,"\t%"PRIu64,(uint64_t)elap);
        fputs("\n",fp_ou);
 
-       if (fclose(fp_ou)==EOF) {
-               debuga(_("Failed to close file %s - %s\n"),wdirname,strerror(errno));
-               exit(EXIT_FAILURE);
-       }
-
        return;
 }