2 * SARG Squid Analysis Report Generator http://sarg.sourceforge.net
6 * please look at http://sarg.sourceforge.net/donations.php
8 * http://sourceforge.net/projects/sarg/forums/forum/363374
9 * ---------------------------------------------------------------------
11 * This program is free software; you can redistribute it and/or modify
12 * it under the terms of the GNU General Public License as published by
13 * the Free Software Foundation; either version 2 of the License, or
14 * (at your option) any later version.
16 * This program is distributed in the hope that it will be useful,
17 * but WITHOUT ANY WARRANTY; without even the implied warranty of
18 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
19 * GNU General Public License for more details.
21 * You should have received a copy of the GNU General Public License
22 * along with this program; if not, write to the Free Software
23 * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111, USA.
27 #include "include/conf.h"
28 #include "include/defs.h"
30 //! The global statistics of the whole log read.
31 struct globalstatstruct globstat
;
33 static FILE *fp_tt
=NULL
;
35 static FILE *maketmp(const char *user
, const char *dirname
, int debug
);
36 static void gravatmp(FILE *fp_ou
, const char *oldurl
, long long int nacc
, long long int nbytes
, const char *oldmsg
, long long int nelap
, long long int incache
, long long int oucache
);
37 static void gravaporuser(const struct userinfostruct
*uinfo
, const char *dirname
, const char *url
, const char *ip
, const char *data
, const char *hora
, long long int tam
, long long int elap
);
38 static void gravager(FILE *fp_gen
,const char *filename
, const struct userinfostruct
*uinfo
, long long int nacc
, const char *url
, long long int nbytes
, const char *ip
, const char *hora
, const char *dia
, long long int nelap
, long long int incache
, long long int oucache
);
39 static void grava_SmartFilter(const char *dirname
, const char *user
, const char *ip
, const char *data
, const char *hora
, const char *url
, const char *smart
);
48 char accdia
[11], acchora
[9], accip
[MAXLEN
], *accurl
;
49 char oldaccdia
[11], oldacchora
[9], oldaccip
[MAXLEN
];
50 char wdirname
[MAXLEN
];
52 char oldmsg
[50], acccode
[MAXLEN
/2 - 1], oldacccode
[MAXLEN
/2 - 1];
53 char ipantes
[MAXLEN
], nameantes
[MAXLEN
];
54 char accsmart
[MAXLEN
];
55 char crc2
[MAXLEN
/2 -1];
56 char siteind
[MAX_TRUNCATED_URL
];
59 char oldaccdiatt
[11],oldacchoratt
[9];
61 char u2
[MAX_USER_LEN
];
62 long long int nbytes
=0;
63 long long int nelap
=0;
65 long long int incache
=0;
66 long long int oucache
=0;
67 long long int accbytes
, accelap
;
70 const char *sort_field
;
71 const char *sort_order
;
78 struct getwordstruct gwarea
;
80 struct userinfostruct
*uinfo
;
85 memset(&globstat
,0,sizeof(globstat
));
87 if (vrfydir(&period
, addr
, site
, us
, email
)<0) {
88 debuga(_("Cannot create the output directory name containing the period as part of the name\n"));
93 debugaz(_("outdirname=%s\n"),outdirname
);
96 if(UserAgentLog
[0] != '\0' && email
[0] == '\0') useragent();
98 snprintf(wdirname
,sizeof(wdirname
),"%s/sarg-general",outdirname
);
99 if((fp_gen
=MY_FOPEN(wdirname
,"w"))==NULL
){
100 debuga(_("(report) Cannot open file %s\n"),wdirname
);
105 sort_labels(&sort_field
,&sort_order
);
107 if (indexonly
|| datetimeby
==0)
110 daystat
=day_prepare();
112 uscan
=userinfo_startscan();
114 debuga(_("Cannot enumerate the user list\n"));
117 while ((uinfo
= userinfo_advancescan(uscan
)) != NULL
) {
118 sort_users_log(tmp
,debug
,uinfo
);
119 if (snprintf(tmp3
,sizeof(tmp3
),"%s/%s.user_log",tmp
,uinfo
->filename
)>=sizeof(tmp3
)) {
120 debuga(_("(report) directory entry too long: %s/%s.user_log\n"),tmp
,uinfo
->filename
);
123 if((fp_in
=MY_FOPEN(tmp3
,"r"))==NULL
){
124 debuga(_("(report) Cannot open file %s\n"),tmp
);
127 user
=uinfo
->filename
;
128 day_newuser(daystat
);
130 strcpy(u2
,uinfo
->id
);
131 if(Ip2Name
&& uinfo
->id_is_ip
) {
133 ip2name(u2
,sizeof(u2
));
134 strcpy(nameantes
,u2
);
136 user_find(uinfo
->label
,MAX_USER_LEN
, u2
);
139 fp_tmp
=maketmp(user
,tmp
,debug
);
146 memset(oldaccdiatt
,0,sizeof(oldaccdiatt
));
147 memset(oldacchoratt
,0,sizeof(oldacchoratt
));
155 if ((line
=longline_create())==NULL
) {
156 debuga(_("Not enough memory to read the downloaded files\n"));
160 while((buf
=longline_read(fp_in
,line
))!=NULL
) {
161 getword_start(&gwarea
,buf
);
162 if (getword(accdia
,sizeof(accdia
),&gwarea
,'\t')<0 || getword(acchora
,sizeof(acchora
),&gwarea
,'\t')<0 ||
163 getword(accip
,sizeof(accip
),&gwarea
,'\t')<0 ||
164 getword_ptr(buf
,&accurl
,&gwarea
,'\t')<0 || getword_atoll(&accbytes
,&gwarea
,'\t')<0 ||
165 getword(acccode
,sizeof(acccode
),&gwarea
,'\t')<0) {
166 debuga(_("There is a broken record or garbage in file %s\n"),tmp3
);
169 if(strncmp(acccode
,"TCP_DENIED/407",14) == 0) continue;
170 if (getword_atoll(&accelap
,&gwarea
,'\t')<0) {
171 debuga(_("There is a broken elapsed time in file %s\n"),tmp3
);
174 if (getword_skip(20000,&gwarea
,'"')<0 || getword(accsmart
,sizeof(accsmart
),&gwarea
,'"')<0) {
175 debuga(_("There is a broken smart info in file %s\n"),tmp3
);
179 if(accsmart
[0] != '\0') {
181 grava_SmartFilter(tmp
,uinfo
->id
,accip
,accdia
,acchora
,accurl
,accsmart
);
185 if(strcmp(accip
,ipantes
) != 0) {
186 strcpy(ipantes
,accip
);
187 ip2name(accip
,sizeof(accip
));
188 strcpy(nameantes
,accip
);
189 } else strcpy(accip
,nameantes
);
193 day_addpoint(daystat
,accdia
,acchora
,accelap
,accbytes
);
194 if (iprel
) gravaporuser(uinfo
,outdirname
,accurl
,accip
,accdia
,acchora
,accbytes
,accelap
);
198 url_len
=strlen(accurl
);
199 if (!oldurl
|| url_len
>=ourl_size
) {
201 oldurl
=realloc(oldurl
,ourl_size
);
203 debuga(_("Not enough memory to store the url\n"));
207 strcpy(oldurl
,accurl
);
208 strcpy(oldacccode
,acccode
);
209 strcpy(oldaccip
,accip
);
210 strcpy(oldaccdia
,accdia
);
211 strcpy(oldacchora
,acchora
);
214 same_url
=(strcmp(oldurl
,accurl
) == 0);
216 if(site
[0] == '\0') {
218 if(strstr(oldacccode
,"DENIED") != 0)
219 strcpy(oldmsg
,"DENIED");
222 if (fp_tmp
) gravatmp(fp_tmp
,oldurl
,nacc
,nbytes
,oldmsg
,nelap
,incache
,oucache
);
223 gravager(fp_gen
,wdirname
,uinfo
,nacc
,oldurl
,nbytes
,oldaccip
,oldacchora
,oldaccdia
,nelap
,incache
,oucache
);
235 if ((ReportType
& REPORT_TYPE_SITE_USER_TIME_DATE
) != 0 && !indexonly
&&
236 (!oldurltt
|| strcmp(oldurltt
,accurl
) || strcmp(oldaccdiatt
,accdia
) || strcmp(oldacchoratt
,acchora
))) {
239 snprintf(arqtt
,sizeof(arqtt
),"%s/%s",outdirname
,uinfo
->filename
);
240 if(access(arqtt
, R_OK
) != 0)
242 url_to_file(accurl
,siteind
,sizeof(siteind
));
243 snprintf(arqtt
,sizeof(arqtt
),"%s/%s/tt%s-%s.html",outdirname
,uinfo
->filename
,uinfo
->filename
,siteind
);
244 if ((fp_tt
= fopen(arqtt
, "w")) == 0) {
245 debuga(_("(report) Cannot open file %s\n"),arqtt
);
252 sprintf(httplink,"<font size=%s color=%s><href=http://%s>%s",FontSize,PrivacyStringColor,PrivacyString,PrivacyString);
254 sprintf(httplink,"<font size=%s><a href=\"http://%s\">%s</a>",FontSize,accurl,accurl);
257 write_html_header(fp_tt
,(IndexTree
== INDEX_TREE_DATE
) ? 4 : 2,_("Site access report"),HTML_JS_NONE
);
258 fprintf(fp_tt
,"<tr><td class=\"header_c\">%s: %s</td></tr>\n",_("Period"),period
.html
);
259 fprintf(fp_tt
,"<tr><td class=\"header_c\">%s: %s</td></tr>\n",_("User"),uinfo
->label
);
260 fputs("<tr><td class=\"header_c\">",fp_tt
);
261 fprintf(fp_tt
,_("Sort: %s, %s"),sort_field
,sort_order
);
262 fputs("</td></tr>\n",fp_tt
);
263 fprintf(fp_tt
,"<tr><th class=\"header_c\">%s</th></tr>\n",_("User"));
264 close_html_header(fp_tt
);
266 fputs("<div class=\"report\"><table cellpadding=\"0\" cellspacing=\"2\">\n",fp_tt
);
267 fprintf(fp_tt
,"<tr><th class=\"header_l\">%s</th><th class=\"header_l\">%s</th><th class=\"header_l\">%s</th></tr>\n",_("ACCESSED SITE"),_("DATE"),_("TIME"));
270 fputs("<tr><td class=\"data\">",fp_tt
);
271 output_html_string(fp_tt
,accurl
,100);
272 fprintf(fp_tt
,"</td><td class=\"data\">%s</td><td class=\"data\">%s</td></tr>\n",accdia
,acchora
);
274 url_len
=strlen(accurl
);
275 if (!oldurltt
|| url_len
>=ourltt_size
) {
276 ourltt_size
=url_len
+1;
277 oldurltt
=realloc(oldurltt
,ourltt_size
);
279 debuga(_("Not enough memory to store the url\n"));
283 strcpy(oldurltt
,accurl
);
284 strcpy(oldaccdiatt
,accdia
);
285 strcpy(oldacchoratt
,acchora
);
288 strcpy(crc2
,acccode
);
289 str
=strchr(crc2
,'/');
291 if(strstr(crc2
,"MISS") != 0)
296 strcpy(oldacccode
,acccode
);
297 strcpy(oldaccip
,accip
);
299 url_len
=strlen(accurl
);
300 if (url_len
>=ourl_size
) {
302 oldurl
=realloc(oldurl
,ourl_size
);
304 debuga(_("Not enough memory to store the url\n"));
308 strcpy(oldurl
,accurl
);
310 strcpy(oldaccdia
,accdia
);
311 strcpy(oldacchora
,acchora
);
314 longline_destroy(&line
);
315 if (oldurltt
) free(oldurltt
);
317 if(strstr(oldacccode
,"DENIED") != 0)
318 strcpy(oldmsg
,"DENIED");
321 if (fp_tmp
) gravatmp(fp_tmp
,oldurl
,nacc
,nbytes
,oldmsg
,nelap
,incache
,oucache
);
322 gravager(fp_gen
,wdirname
,uinfo
,nacc
,oldurl
,nbytes
,oldaccip
,oldacchora
,oldaccdia
,nelap
,incache
,oucache
);
327 day_totalize(daystat
,tmp
,uinfo
);
330 if (fclose(fp_tmp
)==EOF
)
331 debuga(_("Failed to close temporary file for user %s - %s\n"),uinfo
->filename
,strerror(errno
));
334 if (!KeepTempLog
&& unlink(tmp3
)) {
335 debuga(_("Cannot delete \"%s\": %s\n"),tmp3
,strerror(errno
));
339 userinfo_stopscan(uscan
);
340 day_cleanup(daystat
);
342 totalger(fp_gen
,wdirname
);
345 if(email
[0] == '\0') {
347 if(DansGuardianConf
[0] != '\0')
350 debugaz(_("Dansguardian report not produced because no dansguardian configuration file was provided\n"));
358 if((ReportType
& REPORT_TYPE_DOWNLOADS
) != 0)
361 debugaz(_("Downloaded files report not requested in report_type\n"));
363 if((ReportType
& REPORT_TYPE_TOPSITES
) != 0)
366 debugaz(_("Top sites report not requested in report_type\n"));
368 if((ReportType
& REPORT_TYPE_SITES_USERS
) != 0)
371 debugaz(_("Sites & users report not requested in report_type\n"));
373 if ((ReportType
& REPORT_TYPE_DENIED
) != 0)
376 debugaz(_("Denied accesses report not requested in report_type\n"));
378 if ((ReportType
& REPORT_TYPE_AUTH_FAILURES
) != 0)
381 debugaz(_("Authentication failures report not requested in report_type\n"));
383 if(smartfilter
) smartfilter_report();
385 if(DansGuardianConf
[0] != '\0')
386 dansguardian_report();
390 if((ReportType
& REPORT_TYPE_USERS_SITES
) != 0)
393 debugaz(_("User's detailed report not requested in report_type\n"));
398 if(SuccessfulMsg
) debuga(_("Successful report generated on %s\n"),outdirname
);
400 geramail(outdirname
, debug
, outdir
, email
, tmp
);
402 if((strcmp(email
,"stdout") != 0) && SuccessfulMsg
)
403 debuga(_("Successful report generated and sent to %s\n"),email
);
406 if(indexonly
) index_only(outdirname
, debug
);
408 removetmp(outdirname
);
412 static FILE *maketmp(const char *user
, const char *dirname
, int debug
)
415 char wdirname
[MAXLEN
];
417 if((ReportType
& REPORT_TYPE_USERS_SITES
) == 0) return(NULL
);
418 if((ReportType
& REPORT_TYPE_TOPUSERS
) == 0) return(NULL
);
420 if(debug
) debuga(_("Making file: %s/%s\n"),tmp
,user
);
421 if (snprintf(wdirname
,sizeof(wdirname
),"%s/%s.utmp",tmp
,user
)>=sizeof(wdirname
)) {
422 debuga(_("Temporary file name too long: %s/%s.utmp\n"),tmp
,user
);
426 if((fp_ou
=fopen(wdirname
,"w"))==NULL
){
427 debuga(_("(report) Cannot open file %s\n"),wdirname
);
435 static void gravatmp(FILE *fp_ou
, const char *oldurl
, long long int nacc
, long long int nbytes
, const char *oldmsg
, long long int nelap
, long long int incache
, long long int oucache
)
438 This complicated printf is due to Microsoft's inability to comply with any standard. Msvcrt is unable
439 to print a long long int unless it is exactly 64-bits long.
441 fprintf(fp_ou
,"%"PRIu64
"\t%"PRIu64
"\t%s\t%s\t%"PRIu64
"\t%"PRIu64
"\t%"PRIu64
"\n",(uint64_t)nacc
,(uint64_t)nbytes
,oldurl
,oldmsg
,(uint64_t)nelap
,(uint64_t)incache
,(uint64_t)oucache
);
446 fputs("</table>\n</div>\n",fp_tt
);
447 fputs("</body>\n</html>\n",fp_tt
);
455 static void gravaporuser(const struct userinfostruct
*uinfo
, const char *dirname
, const char *url
, const char *ip
, const char *data
, const char *hora
, long long int tam
, long long int elap
)
458 char wdirname
[MAXLEN
];
460 if((ReportType
& REPORT_TYPE_USERS_SITES
) == 0) return;
462 if (snprintf(wdirname
,sizeof(wdirname
),"%s/%s.ip",tmp
,uinfo
->filename
)>=sizeof(wdirname
)) {
463 debuga(_("Path too long %s/%s.ip\n"),tmp
,uinfo
->filename
);
467 if((fp_ou
=MY_FOPEN(wdirname
,"a"))==NULL
){
468 debuga(_("(report) Cannot open file %s\n"),wdirname
);
473 This complicated printf is due to Microsoft's inability to comply with any standard. Msvcrt is unable
474 to print a long long int unless it is exactly 64-bits long.
476 fprintf(fp_ou
,"%s\t%s\t%s\t%s\t%"PRIu64
"\t%"PRIu64
"\n",ip
,url
,data
,hora
,(uint64_t)tam
,(uint64_t)elap
);
484 static void gravager(FILE *fp_gen
,const char *filename
, const struct userinfostruct
*uinfo
, long long int nacc
, const char *url
, long long int nbytes
, const char *ip
, const char *hora
, const char *dia
, long long int nelap
, long long int incache
, long long int oucache
)
487 This complicated printf is due to Microsoft's inability to comply with any standard. Msvcrt is unable
488 to print a long long int unless it is exactly 64-bits long.
490 if (fprintf(fp_gen
,"%s\t%"PRIu64
"\t%"PRIu64
"\t%s\t%s\t%s\t%s\t%"PRIu64
"\t%"PRIu64
"\t%"PRIu64
"\n",uinfo
->id
,(uint64_t)nacc
,(uint64_t)nbytes
,url
,ip
,hora
,dia
,(uint64_t)nelap
,(uint64_t)incache
,(uint64_t)oucache
)<0) {
491 debuga(_("Failed to write a line in %s\n"),filename
);
496 globstat
.nbytes
+=nbytes
;
497 globstat
.elap
+=nelap
;
498 globstat
.incache
+=incache
;
499 globstat
.oucache
+=oucache
;
504 Write the total line at the end of the general file.
506 void totalger(FILE *fp_gen
,const char *filename
)
509 This complicated printf is due to Microsoft's inability to comply with any standard. Msvcrt is unable
510 to print a long long int unless it is exactly 64-bits long.
512 if (fprintf(fp_gen
,"TOTAL\t%"PRIu64
"\t%"PRIu64
"\t%"PRIu64
"\t%"PRIu64
"\t%"PRIu64
"\n",(uint64_t)globstat
.nacc
,(uint64_t)globstat
.nbytes
,(uint64_t)globstat
.elap
,(uint64_t)globstat
.incache
,(uint64_t)globstat
.oucache
)<0) {
513 debuga(_("Failed to write the total line in %s\n"),filename
);
518 int ger_read(char *buffer
,struct generalitemstruct
*item
,const char *filename
)
522 long long int number
;
524 if (strncmp(buffer
,"TOTAL\t",6)==0) {
529 if (*buffer
== '-') {
532 } else if (*buffer
== '+') {
536 while (isdigit(*buffer
))
537 number
=(number
* 10) + (*buffer
++)-'0';
539 debuga(_("Invalid total number of accesses in %s\n"),filename
);
543 item
->nacc
=number
*sign
;
546 if (*buffer
== '-') {
549 } else if (*buffer
== '+') {
553 while (isdigit(*buffer
))
554 number
=(number
* 10) + (*buffer
++)-'0';
556 debuga(_("Invalid total size in %s\n"),filename
);
560 item
->nbytes
=number
*sign
;
563 if (*buffer
== '-') {
566 } else if (*buffer
== '+') {
570 while (isdigit(*buffer
))
571 number
=(number
* 10) + (*buffer
++)-'0';
573 debuga(_("Invalid total elapsed time in %s\n"),filename
);
577 item
->nelap
=number
*sign
;
580 if (*buffer
== '-') {
583 } else if (*buffer
== '+') {
587 while (isdigit(*buffer
))
588 number
=(number
* 10) + (*buffer
++)-'0';
590 debuga(_("Invalid total cache hit in %s\n"),filename
);
594 item
->incache
=number
*sign
;
597 if (*buffer
== '-') {
600 } else if (*buffer
== '+') {
604 while (isdigit(*buffer
))
605 number
=(number
* 10) + (*buffer
++)-'0';
607 debuga(_("Invalid total cache miss in %s\n"),filename
);
610 item
->oucache
=number
*sign
;
615 for (i
=0 ; i
<MAX_USER_LEN
-1 && (unsigned char)*buffer
>=' ' ; i
++) buffer
++;
617 debuga(_("User name too long or invalid in %s\n"),filename
);
623 if (*buffer
== '-') {
626 } else if (*buffer
== '+') {
630 while (isdigit(*buffer
))
631 number
=(number
* 10) + (*buffer
++)-'0';
633 debuga(_("Invalid number of accesses in %s\n"),filename
);
637 item
->nacc
=number
*sign
;
640 if (*buffer
== '-') {
643 } else if (*buffer
== '+') {
647 while (isdigit(*buffer
))
648 number
=(number
* 10) + (*buffer
++)-'0';
650 debuga(_("Invalid number of bytes in %s\n"),filename
);
654 item
->nbytes
=number
*sign
;
657 while ((unsigned char)*buffer
>=' ') buffer
++;
659 debuga(_("URL too long or invalid in %s\n"),filename
);
665 for (i
=0 ; i
<MAX_IP_LEN
-1 && (unsigned char)*buffer
>=' ' ; i
++) buffer
++;
667 debuga(_("IP address too long or invalid in %s\n"),filename
);
673 for (i
=0 ; i
<MAX_DATETIME_LEN
-1 && (unsigned char)*buffer
>=' ' ; i
++) buffer
++;
675 debuga(_("Time too long or invalid in %s\n"),filename
);
681 for (i
=0 ; i
<MAX_DATETIME_LEN
-1 && (unsigned char)*buffer
>=' ' ; i
++) buffer
++;
683 debuga(_("Date too long or invalid in %s\n"),filename
);
689 if (*buffer
== '-') {
692 } else if (*buffer
== '+') {
696 while (isdigit(*buffer
))
697 number
=(number
* 10) + (*buffer
++)-'0';
699 debuga(_("Invalid elapsed time in %s\n"),filename
);
703 item
->nelap
=number
*sign
;
706 if (*buffer
== '-') {
709 } else if (*buffer
== '+') {
713 while (isdigit(*buffer
))
714 number
=(number
* 10) + (*buffer
++)-'0';
716 debuga(_("Invalid cache hit size in %s\n"),filename
);
720 item
->incache
=number
*sign
;
723 if (*buffer
== '-') {
726 } else if (*buffer
== '+') {
730 while (isdigit(*buffer
))
731 number
=(number
* 10) + (*buffer
++)-'0';
733 debuga(_("Invalid cache miss size in %s\n"),filename
);
736 item
->oucache
=number
*sign
;
741 static void grava_SmartFilter(const char *dirname
, const char *user
, const char *ip
, const char *data
, const char *hora
, const char *url
, const char *smart
)
744 char wdirname
[MAXLEN
];
746 if (snprintf(wdirname
,sizeof(wdirname
),"%s/smartfilter.int_unsort",dirname
)>=sizeof(wdirname
)) {
747 debuga(_("File name too long for %s/smartfilter.int_unsort\n"),dirname
);
751 if((fp_ou
=MY_FOPEN(wdirname
,"a"))==NULL
){
752 debuga(_("(report) Cannot open file %s\n"),wdirname
);
756 fprintf(fp_ou
,"%s\t%s\t%s\t%s\t%s\t%s\n",user
,data
,hora
,ip
,url
,smart
);
757 fputs("</body>\n</html>\n",fp_tt
);