From: Frédéric Marchal Date: Sun, 26 Dec 2010 18:28:45 +0000 (+0000) Subject: Use internal user list instead of scanning the directory for users X-Git-Tag: v2.3.2~108 X-Git-Url: http://git.ipfire.org/cgi-bin/gitweb.cgi?a=commitdiff_plain;h=935514879d80434ca731592eb2092ece100440e1;p=thirdparty%2Fsarg.git Use internal user list instead of scanning the directory for users Sarg used to scan the temporary directory for files to process and extract the user name from the files. But, now, it is a waste of time to read the directory as a list of the known users is alread in memory. This change also keep away stray files left by the crash of a previous run. --- diff --git a/CMakeLists.txt b/CMakeLists.txt index 13fa315..dd5a1b3 100755 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -3,7 +3,7 @@ PROJECT(sarg C) SET(sarg_VERSION 2) SET(sarg_REVISION "3.2-pre1") SET(sarg_BUILD "") -SET(sarg_BUILDDATE "Dec-21-2010") +SET(sarg_BUILDDATE "Dec-26-2010") INCLUDE(AddFileDependencies) INCLUDE(CheckIncludeFile) diff --git a/datafile.c b/datafile.c index 36af6ae..1b14fea 100644 --- a/datafile.c +++ b/datafile.c @@ -43,10 +43,8 @@ void data_file(char *tmp) char crc2[50]; char *str; char tmp3[MAXLEN]; - char user[MAX_USER_LEN]; char u2[MAX_USER_LEN]; - DIR *dirp; - struct dirent *direntp; + userscan uscan; long long int nbytes=0; long long int nelap=0; long long int nacc=0; @@ -55,7 +53,6 @@ void data_file(char *tmp) long long int oucache=0; long long int accbytes; long long int accelap; - int dlen; int new_user; int same_url; int url_len; @@ -63,7 +60,6 @@ void data_file(char *tmp) struct getwordstruct gwarea; struct userinfostruct *uinfo; longline line; - const char logext[]=".log"; ipantes[0]='\0'; nameantes[0]='\0'; @@ -72,29 +68,12 @@ void data_file(char *tmp) oldurl=NULL; ourl_size=0; - if ((dirp = opendir(tmp)) == NULL) { - debuga(_("Failed to open directory %s - %s\n"),tmp,strerror(errno)); + uscan=userinfo_startscan(); + if (uscan == NULL) { + debuga(_("Cannot enumerate the user list\n")); exit(EXIT_FAILURE); } - while ( (direntp = readdir( dirp )) != NULL ) { - dlen=strlen(direntp->d_name)-(sizeof(logext)-1); - if (dlen<=0) continue; - if(strcmp(direntp->d_name+dlen,logext) != 0) - continue; - - if (dlen>0) { - if (dlen>=sizeof(user)) continue; - strncpy(user,direntp->d_name,dlen); - user[dlen]=0; - } else { - user[0]='\0'; - } - - uinfo=userinfo_find_from_file(user); - if (!uinfo) { - debuga(_("Ignoring unknown user file %s\n"),user); - continue; - } + while ( (uinfo = userinfo_advancescan(uscan)) != NULL ) { strcpy(u2,uinfo->id); if(Ip2Name && uinfo->id_is_ip) { strcpy(ipantes,u2); @@ -102,9 +81,10 @@ void data_file(char *tmp) strcpy(nameantes,u2); } user_find(uinfo->label,MAX_USER_LEN, u2); + if(debug) debuga(_("Reading user file: %s/%s\n"),tmp,uinfo->filename); - if (snprintf(tmp3,sizeof(tmp3),"%s/%s",tmp,direntp->d_name)>=sizeof(tmp3)) { - debuga(_("(datafile) directory path too long: %s/%s\n"),tmp,direntp->d_name); + if (snprintf(tmp3,sizeof(tmp3),"%s/%s.log",tmp,uinfo->filename)>=sizeof(tmp3)) { + debuga(_("(datafile) directory path too long: %s/%s.log\n"),tmp,uinfo->filename); exit(EXIT_FAILURE); } @@ -210,7 +190,7 @@ void data_file(char *tmp) fclose(fp_in); longline_destroy(&line); } - (void)closedir( dirp ); + userinfo_stopscan(uscan); if (oldurl) free(oldurl); if (fp_ou) fclose(fp_ou); diff --git a/html.c b/html.c index 5e8839e..907f165 100644 --- a/html.c +++ b/html.c @@ -29,10 +29,8 @@ void htmlrel(void) { - DIR *dirp; FILE *fp_in, *fp_ou, *fp_ip, *fp_ip2, *fp_usr; - struct dirent *direntp; long long int nnbytes=0, unbytes=0, tnbytes=0, totbytes=0, totbytes2=0; long long int totelap=0, totelap2=0, nnelap=0, unelap=0, tnelap=0; long long int incache=0, oucache=0, tnincache=0, tnoucache=0, twork=0; @@ -42,10 +40,8 @@ void htmlrel(void) char *buf; char arqin[MAXLEN], arqou[MAXLEN], arqper[MAXLEN], arqip[MAXLEN]; char *url, tmsg[50], csort[MAXLEN]; - char user[MAXLEN], duser[MAXLEN]; + char duser[MAXLEN]; char user_ip[MAXLEN], olduserip[MAXLEN], tmp2[MAXLEN], tmp3[MAXLEN]; - char denied_report[255]; - char *str; char warea[MAXLEN]; char totuser[8]; char tmp6[MAXLEN]; @@ -54,15 +50,16 @@ void htmlrel(void) double perc=0, perc2=0, ouperc=0, inperc=0; int count; int cstatus; - const char txtext[]=".txt"; + int have_denied_report; const char *sort_field; const char *sort_order; - int dlen; + const char *user; char siteind[MAX_TRUNCATED_URL]; struct getwordstruct gwarea; longline line,line1; struct generalitemstruct item; const struct userinfostruct *uinfo; + userscan uscan; if((ReportType & REPORT_TYPE_USERS_SITES) == 0) return; @@ -112,35 +109,13 @@ void htmlrel(void) sort_labels(&sort_field,&sort_order); - if ((dirp = opendir(tmp)) == NULL) { - debuga(_("Failed to open directory %s - %s\n"),tmp,strerror(errno)); + uscan=userinfo_startscan(); + if (uscan == NULL) { + debuga(_("Cannot enumerate the user list\n")); exit(EXIT_FAILURE); } - while ( (direntp = readdir( dirp )) != NULL ) { - dlen=strlen(direntp->d_name)-(sizeof(txtext)-1); - if (dlen<0) continue; - if(strcmp(direntp->d_name+dlen,txtext) != 0) - continue; - - count=1; - if (dlen>0) { - if (dlen>=sizeof(user)) continue; - strncpy(user,direntp->d_name,dlen); - user[dlen]=0; - } else { - user[0]=0; - } - - uinfo=userinfo_find_from_file(user); - if (!uinfo) { - debuga(_("Unknown user ID %s in directory %s\n"),user,tmp); - exit(EXIT_FAILURE); - } - str=strrchr(user,'.'); - if (str) - strcpy(denied_report,str+1); - else - strcpy(denied_report,user); + while ( (uinfo = userinfo_advancescan(uscan)) != NULL ) { + user=uinfo->filename; if (snprintf(warea,sizeof(warea),"%s/%s",outdirname,user)>=sizeof(warea)) { debuga(_("Destination directory too long: %s/%s\n"),outdirname,user); @@ -151,25 +126,28 @@ void htmlrel(void) report_day(uinfo); greport_day(uinfo); - if (snprintf(arqin,sizeof(arqin),"%s/%s",tmp,direntp->d_name)>=sizeof(arqin)) { - debuga(_("Input file name too long: %s/%s\n"),tmp,direntp->d_name); + if (snprintf(arqin,sizeof(arqin),"%s/%s.txt",tmp,uinfo->filename)>=sizeof(arqin)) { + debuga(_("Input file name too long: %s/%s.txt\n"),tmp,uinfo->filename); + exit(EXIT_FAILURE); + } + if ((fp_in = fopen(arqin, "r")) == 0){ + if (uinfo->no_report) continue; + debuga(_("(html3) Cannot open file %s\n"),arqin); exit(EXIT_FAILURE); } + if (snprintf(arqou,sizeof(arqou),"%s/%s/%s.html",outdirname,uinfo->filename,uinfo->filename)>=sizeof(arqou)) { debuga(_("Output file name too long: %s/%s/%s.html\n"),outdirname,uinfo->filename,uinfo->filename); exit(EXIT_FAILURE); } - if (snprintf(duser,sizeof(duser),"%s/%s/denied_%s.html",tmp,direntp->d_name,denied_report)>=sizeof(duser)) { - debuga(_("File name too long: %s/%s/denied_%s.html\n"),tmp,direntp->d_name,denied_report); + if (snprintf(duser,sizeof(duser),"%s/%s/denied_%s.html",tmp,uinfo->filename,uinfo->filename)>=sizeof(duser)) { + debuga(_("File name too long: %s/%s/denied_%s.html\n"),tmp,uinfo->filename,uinfo->filename); exit(EXIT_FAILURE); } if(access(duser, R_OK) != 0) - denied_report[0]='\0'; - - if ((fp_in = fopen(arqin, "r")) == 0){ - debuga(_("(html3) Cannot open file %s\n"),arqin); - exit(EXIT_FAILURE); - } + have_denied_report=0; + else + have_denied_report=1; if ((line=longline_create())==NULL) { debuga(_("Not enough memory to read file %s\n"),arqin); @@ -241,8 +219,8 @@ void htmlrel(void) /*! \bug This line lays before the header of the table and is therefore invalid HTML. It will also break the sorttable script. Beside, I suspect it serves no purpose. */ - if(denied_report[0]!='\0') { - fprintf(fp_ou,"%s %s\n",denied_report,_("SmartFilter"),_("Report")); + if(have_denied_report) { + fprintf(fp_ou,"%s %s\n",uinfo->filename,_("SmartFilter"),_("Report")); } fputs("d_name)-(sizeof(logext)-1); - if (dlen<0) continue; - if((strcmp(direntp->d_name+dlen,logext) != 0) || - (strncmp(direntp->d_name,"download.log",12) == 0) || - (strncmp(direntp->d_name,"denied.log",10) == 0) || - (strncmp(direntp->d_name,"authfail.log.unsort",19) == 0)) - continue; - if (snprintf(tmp3,sizeof(tmp3),"%s/%s",tmp,direntp->d_name)>=sizeof(tmp3)) { - debuga(_("(report) directory entry too long: %s/%s\n"),tmp,direntp->d_name); + while ((uinfo = userinfo_advancescan(uscan)) != NULL ) { + if (snprintf(tmp3,sizeof(tmp3),"%s/%s.log",tmp,uinfo->filename)>=sizeof(tmp3)) { + debuga(_("(report) directory entry too long: %s/%s.log\n"),tmp,uinfo->filename); exit(EXIT_FAILURE); } if((fp_in=MY_FOPEN(tmp3,"r"))==NULL){ debuga(_("(report) Cannot open file %s\n"),tmp); exit(EXIT_FAILURE); } + user=uinfo->filename; - if (dlen>0) { - if (dlen>=sizeof(user)) continue; - strncpy(user,direntp->d_name,dlen); - user[dlen]=0; - } else { - user[0]='\0'; - } - - uinfo=userinfo_find_from_file(user); - if (!uinfo) { - debuga(_("Ignoring unknown user file %s\n"),user); - continue; - } strcpy(u2,uinfo->id); if(Ip2Name && uinfo->id_is_ip) { strcpy(ipantes,u2); @@ -345,7 +325,7 @@ void gerarel(void) if (oldurltt) free(oldurltt); unlink(tmp3); } - closedir(dirp); + userinfo_stopscan(uscan); if (oldurl) { if(strstr(oldacccode,"DENIED") != 0) diff --git a/sort.c b/sort.c index 3fad666..0b229a5 100644 --- a/sort.c +++ b/sort.c @@ -29,13 +29,11 @@ void tmpsort(void) { - DIR *dirp; - struct dirent *direntp; + userscan uscan; + struct userinfostruct *uinfo; int cstatus; - const char tmpext[]=".utmp"; - int dlen; char csort[MAXLEN]; - char arqou[MAXLEN], arqin[MAXLEN], wnome[MAXLEN]; + char arqou[MAXLEN], arqin[MAXLEN]; const char *field1="2,2"; const char *field2="1,1"; const char *field3="3,3"; @@ -63,35 +61,25 @@ void tmpsort(void) else order="-r"; - if ((dirp = opendir(tmp)) == NULL) { - debuga(_("Failed to open directory %s - %s\n"),tmp,strerror(errno)); + uscan=userinfo_startscan(); + if (uscan == NULL) { + debuga(_("Cannot enumerate the user list\n")); exit(EXIT_FAILURE); } - while ((direntp = readdir( dirp )) != NULL ){ - dlen=strlen(direntp->d_name)-(sizeof(tmpext)-1); - if (dlen<0) continue; - if(strcmp(direntp->d_name+dlen,tmpext) != 0) - continue; - - if (dlen>0) { - if (dlen>=sizeof(wnome)) continue; - strncpy(wnome,direntp->d_name,dlen); - wnome[dlen]='\0'; - } else { - wnome[0]='\0'; + while ((uinfo = userinfo_advancescan(uscan)) != NULL ){ + if (snprintf(arqin,sizeof(arqin),"%s/%s.utmp",tmp,uinfo->filename)>=sizeof(arqin)) { + debuga(_("file name too long: %s/%s.utmp\n"),tmp,uinfo->filename); + exit(EXIT_FAILURE); + } + if (snprintf(arqou,sizeof(arqou),"%s/%s.txt",tmp,uinfo->filename)>=sizeof(arqou)) { + debuga(_("file name too long: %s/%s.utmp\n"),tmp,uinfo->filename); + exit(EXIT_FAILURE); } - - strcpy(arqou,tmp); - strcat(arqou,"/"); - strcpy(arqin,arqou); - strcat(arqou,wnome); - strcat(arqin,direntp->d_name); if(debug) { debuga(_("Sorting file: %s\n"),arqou); } - strcat(arqou,".txt"); sprintf(csort,"sort -n -T \"%s\" %s -k %s -k %s -k %s -o \"%s\" \"%s\"",tmp,order,field1,field2,field3,arqou,arqin); cstatus=system(csort); if (!WIFEXITED(cstatus) || WEXITSTATUS(cstatus)) { @@ -100,10 +88,9 @@ void tmpsort(void) exit(EXIT_FAILURE); } unlink(arqin); - } - (void)closedir( dirp ); + userinfo_stopscan(uscan); return; } diff --git a/squidguard_log.c b/squidguard_log.c index 8d88512..4c0d7a5 100644 --- a/squidguard_log.c +++ b/squidguard_log.c @@ -251,6 +251,7 @@ static void read_log(const char *wentp, FILE *fp_ou,int dfrom,int duntil) if (!uinfo) { uinfo=userinfo_create(user); uinfo->id_is_ip=id_is_ip; + uinfo->no_report=true; if(Ip2Name && id_is_ip) ip2name(user,sizeof(user)); user_find(uinfo->label,MAX_USER_LEN, user); } diff --git a/userinfo.c b/userinfo.c index 4b315fa..6d82aa1 100644 --- a/userinfo.c +++ b/userinfo.c @@ -44,6 +44,16 @@ struct usergroupstruct int nusers; }; +/*! \brief Hold pointer to scan through the user list. +*/ +struct userscanstruct +{ + //! The group containing the user. + struct usergroupstruct *group; + //! The index of the user in the group. + int index; +}; + //! The first group of users. static struct usergroupstruct *first_user_group=NULL; @@ -148,3 +158,57 @@ struct userinfostruct *userinfo_find_from_id(const char *id) } return(NULL); } + +/*! +Start the scanning of the user list. + +\return The object to pass to subsequent scanning functions or NULL +if it failed. The object must be freed with a call to userinfo_stop(). +*/ +userscan userinfo_startscan(void) +{ + userscan uscan; + + uscan=malloc(sizeof(*uscan)); + if (!uscan) return(NULL); + uscan->group=first_user_group; + uscan->index=0; + return(uscan); +} + +/*! +Free the memory allocated by userinfo_start(). + +\param uscan The object created by userinfo_start(). +*/ +void userinfo_stopscan(userscan uscan) +{ + free(uscan); +} + +/*! +Get the user pointed to by the object and advance the object +to the next user. + +\param uscan The object created by userinfo_start(). + +\return The user in the list or NULL if the end of the list +is reached. +*/ +struct userinfostruct *userinfo_advancescan(userscan uscan) +{ + struct userinfostruct *uinfo; + + if (!uscan) return(NULL); + if (!uscan->group) return(NULL); + if (uscan->index<0 || uscan->index>=uscan->group->nusers) return(NULL); + + uinfo=uscan->group->list+uscan->index; + + ++uscan->index; + if (uscan->index>=uscan->group->nusers) { + uscan->group=uscan->group->next; + uscan->index=0; + } + return(uinfo); +}