[thirdparty/sarg.git] / datafile.c

/*
 * SARG Squid Analysis Report Generator      http://sarg.sourceforge.net
 *                                                            1998, 2010
 *
 * SARG donations:
 *      please look at http://sarg.sourceforge.net/donations.php
 * Support:
 *     http://sourceforge.net/projects/sarg/forums/forum/363374
 * ---------------------------------------------------------------------
 *
 *  This program is free software; you can redistribute it and/or modify
 *  it under the terms of the GNU General Public License as published by
 *  the Free Software Foundation; either version 2 of the License, or
 *  (at your option) any later version.
 *
 *  This program is distributed in the hope that it will be useful,
 *  but WITHOUT ANY WARRANTY; without even the implied warranty of
 *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 *  GNU General Public License for more details.
 *
 *  You should have received a copy of the GNU General Public License
 *  along with this program; if not, write to the Free Software
 *  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111, USA.
 *
 */

#include "include/conf.h"
#include "include/defs.h"

void saverecs(FILE *fp_ou, const struct userinfostruct *uinfo, long long int nacc, char *url, long long int nbytes, char *ip, char *hora, char *dia, long long int nelap, long long int incache, long long int oucache);

void data_file(char *tmp)
{
	FILE *fp_in, *fp_ou=NULL;

	char *buf;
	char accdia[11], acchora[9], accip[MAXLEN], *accurl;
	char oldaccdia[11], oldacchora[9], oldaccip[MAXLEN];
	char *oldurl;
	char olduser[MAXLEN], acccode[50], oldacccode[50];
	char ipantes[MAXLEN], nameantes[MAXLEN];
	char accsmart[MAXLEN];
	char crc2[50];
	char *str;
	char tmp3[MAXLEN];
	char user[MAX_USER_LEN];
	char u2[MAX_USER_LEN];
	DIR *dirp;
	struct dirent *direntp;
	long long int nbytes=0;
	long long int nelap=0;
	long long int nacc=0;
	int rtotal=0;
	long long int incache=0;
	long long int oucache=0;
	long long int accbytes;
	long long int accelap;
	int dlen;
	int new_user;
	int same_url;
	int url_len;
	int ourl_size;
	struct getwordstruct gwarea;
	struct userinfostruct *uinfo;
	longline line;
	const char logext[]=".log";

	ipantes[0]='\0';
	nameantes[0]='\0';

	olduser[0]='\0';
	oldurl=NULL;
	ourl_size=0;

	if ((dirp = opendir(tmp)) == NULL) {
		debuga(_("Failed to open directory %s - %s\n"),tmp,strerror(errno));
		exit(EXIT_FAILURE);
	}
	while ( (direntp = readdir( dirp )) != NULL ) {
		dlen=strlen(direntp->d_name)-(sizeof(logext)-1);
		if (dlen<=0) continue;
		if(strcmp(direntp->d_name+dlen,logext) != 0)
			continue;

		if (dlen>0) {
			if (dlen>=sizeof(user)) continue;
			strncpy(user,direntp->d_name,dlen);
			user[dlen]=0;
		} else {
			user[0]='\0';
		}

		uinfo=userinfo_find_from_file(user);
		if (!uinfo) {
			debuga(_("Ignoring unknown user file %s\n"),user);
			continue;
		}
		strcpy(u2,uinfo->id);
		if(Ip2Name && uinfo->id_is_ip) {
			strcpy(ipantes,u2);
			ip2name(u2,sizeof(u2));
			strcpy(nameantes,u2);
		}
		user_find(uinfo->label,MAX_USER_LEN, u2);

		if (snprintf(tmp3,sizeof(tmp3),"%s/%s",tmp,direntp->d_name)>=sizeof(tmp3)) {
			debuga(_("(datafile) directory path too long: %s/%s\n"),tmp,direntp->d_name);
			exit(EXIT_FAILURE);
		}

		if((fp_in=MY_FOPEN(tmp3,"r"))==NULL){
			debuga(_("(datafile) Cannot open file %s\n"),tmp3);
			exit(EXIT_FAILURE);
		}

		if ((line=longline_create())==NULL) {
			debuga(_("Not enough memory to read the downloaded files.\n"));
			exit(EXIT_FAILURE);
		}

		ttopen=0;
		new_user=1;
		while((buf=longline_read(fp_in,line))!=NULL) {
			getword_start(&gwarea,buf);
			if (getword(accdia,sizeof(accdia),&gwarea,'\t')<0 || getword(acchora,sizeof(acchora),&gwarea,'\t')<0 ||
				getword(accip,sizeof(accip),&gwarea,'\t')<0 ||
				getword_ptr(buf,&accurl,&gwarea,'\t')<0 || getword_atoll(&accbytes,&gwarea,'\t')<0 ||
				getword(acccode,sizeof(acccode),&gwarea,'\t')<0 || getword_atoll(&accelap,&gwarea,'\t')<0) {
				debuga(_("There is a broken record or garbage in file %s\n"),tmp3);
				exit(EXIT_FAILURE);
			}
			if (getword_skip(20000,&gwarea,'"')<0 || getword(accsmart,sizeof(accsmart),&gwarea,'"')<0) {
				debuga(_("There is an invalid smart info in file %s\n"),tmp3);
				exit(EXIT_FAILURE);
			}

			if(Ip2Name) {
				if(strcmp(accip,ipantes) != 0) {
					strcpy(ipantes,accip);
					ip2name(accip,sizeof(accip));
					strcpy(nameantes,accip);
				}
				else strcpy(accip,nameantes);
			}

			if(!rtotal){
				url_len=strlen(accurl);
				if (!oldurl || url_len>=ourl_size) {
					ourl_size=url_len+1;
					oldurl=realloc(oldurl,ourl_size);
					if (!oldurl) {
						debuga(_("Not enough memory to store the url\n"));
						exit(EXIT_FAILURE);
					}
				}
				strcpy(oldurl,accurl);
				strcpy(oldacccode,acccode);
				strcpy(oldaccip,accip);
				strcpy(oldaccdia,accdia);
				strcpy(oldacchora,acchora);
				new_user=0;
				rtotal++;
			}
			same_url=(strcmp(oldurl,accurl) == 0);

			if(!same_url || new_user){
				if(!fp_ou && (fp_ou=MY_FOPEN(DataFile,"w"))==NULL){
					debuga(_("(datafile) Cannot open file %s\n"),DataFile);
					exit(EXIT_FAILURE);
				}
				saverecs(fp_ou,uinfo,nacc,oldurl,nbytes,oldaccip,oldacchora,oldaccdia,nelap,incache,oucache);
				nacc=0;
				nbytes=0;
				nelap=0;
				incache=0;
				oucache=0;
				new_user=0;
			}

			nacc++;
			nbytes+=accbytes;
			nelap+=accelap;

			strcpy(crc2,acccode);
			str=strchr(crc2,'/');
			if (str) *str='\0';

			if(strstr(crc2,"MISS") != 0) oucache+=accbytes;
			else incache+=accbytes;

			if (!same_url) {
				url_len=strlen(accurl);
				if (url_len>=ourl_size) {
					ourl_size=url_len+1;
					oldurl=realloc(oldurl,ourl_size);
					if (!oldurl) {
						debuga(_("Not enough memory to store the url\n"));
						exit(EXIT_FAILURE);
					}
				}
				strcpy(oldurl,accurl);
			}
			new_user=0;
			strcpy(oldacccode,acccode);
			strcpy(oldaccip,accip);
			strcpy(oldaccdia,accdia);
			strcpy(oldacchora,acchora);
		}

		fclose(fp_in);
		longline_destroy(&line);
	}
	(void)closedir( dirp );
	if (oldurl) free(oldurl);
	if (fp_ou) fclose(fp_ou);

	if(debug)
		debuga(_("Datafile %s written successfully\n"),DataFile);
}

void saverecs(FILE *fp_ou, const struct userinfostruct *uinfo, long long int nacc, char *url, long long int nbytes, char *ip, char *hora, char *dia, long long int nelap, long long int incache, long long int oucache)
{
	char val[20];
	char name[512];

	if((DataFileFields & DATA_FIELD_USER) != 0) {
		fputs(uinfo->label,fp_ou);
		fputc(DataFileDelimiter[0],fp_ou);
	}
	if((DataFileFields & DATA_FIELD_DATE) != 0) {
		fputs(dia,fp_ou);
		fputc(DataFileDelimiter[0],fp_ou);
	}
	if((DataFileFields & DATA_FIELD_TIME) != 0) {
		fputs(hora,fp_ou);
		fputc(DataFileDelimiter[0],fp_ou);
	}
	if((DataFileFields & DATA_FIELD_URL) != 0) {
		url_hostname(url,name,sizeof(name));
		if (DataFileUrl == DATAFILEURL_IP) name2ip(name);
		fputs(name,fp_ou);
		fputc(DataFileDelimiter[0],fp_ou);
	}
	if((DataFileFields & DATA_FIELD_CONNECT) != 0) {
		my_lltoa(nacc,val,sizeof(val),0);
		fputs(val,fp_ou);
		fputc(DataFileDelimiter[0],fp_ou);
	}
	if((DataFileFields & DATA_FIELD_BYTES) != 0) {
		my_lltoa(nbytes,val,sizeof(val),0);
		fputs(val,fp_ou);
		fputc(DataFileDelimiter[0],fp_ou);
	}
	if((DataFileFields & DATA_FIELD_IN_CACHE) != 0) {
		my_lltoa(incache,val,sizeof(val),0);
		fputs(val,fp_ou);
		fputc(DataFileDelimiter[0],fp_ou);
	}
	if((DataFileFields & DATA_FIELD_OUT_CACHE) != 0) {
		my_lltoa(oucache,val,sizeof(val),0);
		fputs(val,fp_ou);
		fputc(DataFileDelimiter[0],fp_ou);
	}
	if((DataFileFields & DATA_FIELD_ELAPSED) != 0) {
		my_lltoa(nelap,val,sizeof(val),0);
		fputs(val,fp_ou);
		fputc(DataFileDelimiter[0],fp_ou);
	}

	fputc('\n',fp_ou);
}
Commit	Line	Data
25697a35	1	/*
94ff9470	2	* SARG Squid Analysis Report Generator http://sarg.sourceforge.net
1164c474	3	* 1998, 2010
25697a35 GS	4	*
	5	* SARG donations:
	6	* please look at http://sarg.sourceforge.net/donations.php
1164c474 FM	7	* Support:
1164c474 FM	8	* http://sourceforge.net/projects/sarg/forums/forum/363374
25697a35 GS	9	* ---------------------------------------------------------------------
	10	*
	11	* This program is free software; you can redistribute it and/or modify
	12	* it under the terms of the GNU General Public License as published by
	13	* the Free Software Foundation; either version 2 of the License, or
	14	* (at your option) any later version.
	15	*
	16	* This program is distributed in the hope that it will be useful,
	17	* but WITHOUT ANY WARRANTY; without even the implied warranty of
	18	* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
	19	* GNU General Public License for more details.
	20	*
	21	* You should have received a copy of the GNU General Public License
	22	* along with this program; if not, write to the Free Software
	23	* Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111, USA.
	24	*
	25	*/
	26
	27	#include "include/conf.h"
5f3cfd1d	28	#include "include/defs.h"
25697a35	29
f84a35a3	30	void saverecs(FILE fp_ou, const struct userinfostruct uinfo, long long int nacc, char url, long long int nbytes, char ip, char hora, char dia, long long int nelap, long long int incache, long long int oucache);
25697a35 GS	31
	32	void data_file(char *tmp)
	33	{
9bd92830	34	FILE fp_in, fp_ou=NULL;
25697a35	35
9bd92830 FM	36	char *buf;
	37	char accdia[11], acchora[9], accip[MAXLEN], *accurl;
	38	char oldaccdia[11], oldacchora[9], oldaccip[MAXLEN];
	39	char *oldurl;
	40	char olduser[MAXLEN], acccode[50], oldacccode[50];
	41	char ipantes[MAXLEN], nameantes[MAXLEN];
	42	char accsmart[MAXLEN];
	43	char crc2[50];
	44	char *str;
	45	char tmp3[MAXLEN];
	46	char user[MAX_USER_LEN];
	47	char u2[MAX_USER_LEN];
	48	DIR *dirp;
	49	struct dirent *direntp;
	50	long long int nbytes=0;
	51	long long int nelap=0;
	52	long long int nacc=0;
	53	int rtotal=0;
	54	long long int incache=0;
	55	long long int oucache=0;
	56	long long int accbytes;
	57	long long int accelap;
	58	int dlen;
	59	int new_user;
	60	int same_url;
	61	int url_len;
	62	int ourl_size;
	63	struct getwordstruct gwarea;
	64	struct userinfostruct *uinfo;
	65	longline line;
	66	const char logext[]=".log";
25697a35	67
9bd92830 FM	68	ipantes[0]='\0';
9bd92830 FM	69	nameantes[0]='\0';
25697a35	70
9bd92830 FM	71	olduser[0]='\0';
	72	oldurl=NULL;
	73	ourl_size=0;
25697a35	74
9bd92830 FM	75	if ((dirp = opendir(tmp)) == NULL) {
	76	debuga(_("Failed to open directory %s - %s\n"),tmp,strerror(errno));
	77	exit(EXIT_FAILURE);
	78	}
	79	while ( (direntp = readdir( dirp )) != NULL ) {
	80	dlen=strlen(direntp->d_name)-(sizeof(logext)-1);
	81	if (dlen<=0) continue;
	82	if(strcmp(direntp->d_name+dlen,logext) != 0)
	83	continue;
25697a35	84
9bd92830 FM	85	if (dlen>0) {
	86	if (dlen>=sizeof(user)) continue;
	87	strncpy(user,direntp->d_name,dlen);
	88	user[dlen]=0;
	89	} else {
	90	user[0]='\0';
	91	}
c0ec9cc7	92
9bd92830 FM	93	uinfo=userinfo_find_from_file(user);
	94	if (!uinfo) {
	95	debuga(_("Ignoring unknown user file %s\n"),user);
	96	continue;
	97	}
	98	strcpy(u2,uinfo->id);
	99	if(Ip2Name && uinfo->id_is_ip) {
	100	strcpy(ipantes,u2);
	101	ip2name(u2,sizeof(u2));
	102	strcpy(nameantes,u2);
	103	}
	104	user_find(uinfo->label,MAX_USER_LEN, u2);
f2ec8c75	105
9bd92830 FM	106	if (snprintf(tmp3,sizeof(tmp3),"%s/%s",tmp,direntp->d_name)>=sizeof(tmp3)) {
	107	debuga(_("(datafile) directory path too long: %s/%s\n"),tmp,direntp->d_name);
	108	exit(EXIT_FAILURE);
	109	}
f2ec8c75	110
9bd92830 FM	111	if((fp_in=MY_FOPEN(tmp3,"r"))==NULL){
	112	debuga(_("(datafile) Cannot open file %s\n"),tmp3);
	113	exit(EXIT_FAILURE);
	114	}
25697a35	115
9bd92830 FM	116	if ((line=longline_create())==NULL) {
	117	debuga(_("Not enough memory to read the downloaded files.\n"));
	118	exit(EXIT_FAILURE);
	119	}
f84a35a3	120
9bd92830 FM	121	ttopen=0;
	122	new_user=1;
	123	while((buf=longline_read(fp_in,line))!=NULL) {
	124	getword_start(&gwarea,buf);
	125	if (getword(accdia,sizeof(accdia),&gwarea,'\t')<0 \|\| getword(acchora,sizeof(acchora),&gwarea,'\t')<0 \|\|
	126	getword(accip,sizeof(accip),&gwarea,'\t')<0 \|\|
	127	getword_ptr(buf,&accurl,&gwarea,'\t')<0 \|\| getword_atoll(&accbytes,&gwarea,'\t')<0 \|\|
	128	getword(acccode,sizeof(acccode),&gwarea,'\t')<0 \|\| getword_atoll(&accelap,&gwarea,'\t')<0) {
	129	debuga(_("There is a broken record or garbage in file %s\n"),tmp3);
	130	exit(EXIT_FAILURE);
	131	}
	132	if (getword_skip(20000,&gwarea,'"')<0 \|\| getword(accsmart,sizeof(accsmart),&gwarea,'"')<0) {
	133	debuga(_("There is an invalid smart info in file %s\n"),tmp3);
	134	exit(EXIT_FAILURE);
	135	}
25697a35	136
9bd92830 FM	137	if(Ip2Name) {
	138	if(strcmp(accip,ipantes) != 0) {
	139	strcpy(ipantes,accip);
	140	ip2name(accip,sizeof(accip));
	141	strcpy(nameantes,accip);
	142	}
	143	else strcpy(accip,nameantes);
	144	}
5542cb59	145
9bd92830 FM	146	if(!rtotal){
	147	url_len=strlen(accurl);
	148	if (!oldurl \|\| url_len>=ourl_size) {
	149	ourl_size=url_len+1;
	150	oldurl=realloc(oldurl,ourl_size);
	151	if (!oldurl) {
	152	debuga(_("Not enough memory to store the url\n"));
	153	exit(EXIT_FAILURE);
	154	}
	155	}
	156	strcpy(oldurl,accurl);
	157	strcpy(oldacccode,acccode);
	158	strcpy(oldaccip,accip);
	159	strcpy(oldaccdia,accdia);
	160	strcpy(oldacchora,acchora);
	161	new_user=0;
	162	rtotal++;
	163	}
	164	same_url=(strcmp(oldurl,accurl) == 0);
5542cb59	165
9bd92830 FM	166	if(!same_url \|\| new_user){
	167	if(!fp_ou && (fp_ou=MY_FOPEN(DataFile,"w"))==NULL){
	168	debuga(_("(datafile) Cannot open file %s\n"),DataFile);
	169	exit(EXIT_FAILURE);
	170	}
	171	saverecs(fp_ou,uinfo,nacc,oldurl,nbytes,oldaccip,oldacchora,oldaccdia,nelap,incache,oucache);
	172	nacc=0;
	173	nbytes=0;
	174	nelap=0;
	175	incache=0;
	176	oucache=0;
	177	new_user=0;
	178	}
5542cb59	179
9bd92830 FM	180	nacc++;
	181	nbytes+=accbytes;
	182	nelap+=accelap;
25697a35	183
9bd92830 FM	184	strcpy(crc2,acccode);
	185	str=strchr(crc2,'/');
	186	if (str) *str='\0';
5542cb59	187
9bd92830 FM	188	if(strstr(crc2,"MISS") != 0) oucache+=accbytes;
9bd92830 FM	189	else incache+=accbytes;
25697a35	190
9bd92830 FM	191	if (!same_url) {
	192	url_len=strlen(accurl);
	193	if (url_len>=ourl_size) {
	194	ourl_size=url_len+1;
	195	oldurl=realloc(oldurl,ourl_size);
	196	if (!oldurl) {
	197	debuga(_("Not enough memory to store the url\n"));
	198	exit(EXIT_FAILURE);
	199	}
	200	}
	201	strcpy(oldurl,accurl);
	202	}
	203	new_user=0;
	204	strcpy(oldacccode,acccode);
	205	strcpy(oldaccip,accip);
	206	strcpy(oldaccdia,accdia);
	207	strcpy(oldacchora,acchora);
	208	}
5542cb59	209
9bd92830 FM	210	fclose(fp_in);
	211	longline_destroy(&line);
	212	}
	213	(void)closedir( dirp );
	214	if (oldurl) free(oldurl);
	215	if (fp_ou) fclose(fp_ou);
5542cb59	216
9bd92830 FM	217	if(debug)
9bd92830 FM	218	debuga(_("Datafile %s written successfully\n"),DataFile);
25697a35 GS	219	}
25697a35 GS	220
f84a35a3	221	void saverecs(FILE fp_ou, const struct userinfostruct uinfo, long long int nacc, char url, long long int nbytes, char ip, char hora, char dia, long long int nelap, long long int incache, long long int oucache)
25697a35	222	{
9bd92830 FM	223	char val[20];
9bd92830 FM	224	char name[512];
25697a35	225
9bd92830 FM	226	if((DataFileFields & DATA_FIELD_USER) != 0) {
	227	fputs(uinfo->label,fp_ou);
	228	fputc(DataFileDelimiter[0],fp_ou);
	229	}
	230	if((DataFileFields & DATA_FIELD_DATE) != 0) {
	231	fputs(dia,fp_ou);
	232	fputc(DataFileDelimiter[0],fp_ou);
	233	}
	234	if((DataFileFields & DATA_FIELD_TIME) != 0) {
	235	fputs(hora,fp_ou);
	236	fputc(DataFileDelimiter[0],fp_ou);
	237	}
	238	if((DataFileFields & DATA_FIELD_URL) != 0) {
	239	url_hostname(url,name,sizeof(name));
	240	if (DataFileUrl == DATAFILEURL_IP) name2ip(name);
	241	fputs(name,fp_ou);
	242	fputc(DataFileDelimiter[0],fp_ou);
	243	}
	244	if((DataFileFields & DATA_FIELD_CONNECT) != 0) {
	245	my_lltoa(nacc,val,sizeof(val),0);
	246	fputs(val,fp_ou);
	247	fputc(DataFileDelimiter[0],fp_ou);
	248	}
	249	if((DataFileFields & DATA_FIELD_BYTES) != 0) {
	250	my_lltoa(nbytes,val,sizeof(val),0);
	251	fputs(val,fp_ou);
	252	fputc(DataFileDelimiter[0],fp_ou);
	253	}
	254	if((DataFileFields & DATA_FIELD_IN_CACHE) != 0) {
	255	my_lltoa(incache,val,sizeof(val),0);
	256	fputs(val,fp_ou);
	257	fputc(DataFileDelimiter[0],fp_ou);
	258	}
	259	if((DataFileFields & DATA_FIELD_OUT_CACHE) != 0) {
	260	my_lltoa(oucache,val,sizeof(val),0);
	261	fputs(val,fp_ou);
	262	fputc(DataFileDelimiter[0],fp_ou);
	263	}
	264	if((DataFileFields & DATA_FIELD_ELAPSED) != 0) {
	265	my_lltoa(nelap,val,sizeof(val),0);
	266	fputs(val,fp_ou);
	267	fputc(DataFileDelimiter[0],fp_ou);
	268	}
25697a35	269
9bd92830	270	fputc('\n',fp_ou);
25697a35	271	}