2 * SARG Squid Analysis Report Generator http://sarg.sourceforge.net
6 * please look at http://sarg.sourceforge.net/donations.php
8 * http://sourceforge.net/projects/sarg/forums/forum/363374
9 * ---------------------------------------------------------------------
11 * This program is free software; you can redistribute it and/or modify
12 * it under the terms of the GNU General Public License as published by
13 * the Free Software Foundation; either version 2 of the License, or
14 * (at your option) any later version.
16 * This program is distributed in the hope that it will be useful,
17 * but WITHOUT ANY WARRANTY; without even the implied warranty of
18 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
19 * GNU General Public License for more details.
21 * You should have received a copy of the GNU General Public License
22 * along with this program; if not, write to the Free Software
23 * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111, USA.
27 #include "include/conf.h"
28 #include "include/defs.h"
30 //! The global statistics of the whole log read.
31 struct globalstatstruct globstat
;
32 //! \c True to enable the smart filter.
33 bool smartfilter
=false;
35 static FILE *fp_tt
=NULL
;
37 static FILE *maketmp(const char *user
, const char *dirname
, int debug
);
38 static void gravatmp(FILE *fp_ou
, const char *oldurl
, long long int nacc
, long long int nbytes
, const char *oldmsg
, long long int nelap
, long long int incache
, long long int oucache
);
39 static void gravaporuser(const struct userinfostruct
*uinfo
, const char *dirname
, const char *url
, const char *ip
, const char *data
, const char *hora
, long long int tam
, long long int elap
);
40 static void gravager(FILE *fp_gen
,const char *filename
, const struct userinfostruct
*uinfo
, long long int nacc
, const char *url
, long long int nbytes
, const char *ip
, const char *hora
, const char *dia
, long long int nelap
, long long int incache
, long long int oucache
);
41 static void grava_SmartFilter(const char *dirname
, const char *user
, const char *ip
, const char *data
, const char *hora
, const char *url
, const char *smart
);
50 char accdia
[11], acchora
[9], accip
[MAXLEN
], *accurl
;
51 char oldaccdia
[11], oldacchora
[9], oldaccip
[MAXLEN
];
52 char wdirname
[MAXLEN
];
54 char oldmsg
[50], acccode
[MAXLEN
/2 - 1], oldacccode
[MAXLEN
/2 - 1];
55 char ipantes
[MAXLEN
], nameantes
[MAXLEN
];
56 char accsmart
[MAXLEN
];
57 char crc2
[MAXLEN
/2 -1];
58 char siteind
[MAX_TRUNCATED_URL
];
61 char oldaccdiatt
[11],oldacchoratt
[9];
63 char u2
[MAX_USER_LEN
];
64 long long int nbytes
=0;
65 long long int nelap
=0;
67 long long int incache
=0;
68 long long int oucache
=0;
69 long long int accbytes
, accelap
;
72 const char *sort_field
;
73 const char *sort_order
;
80 struct getwordstruct gwarea
;
82 struct userinfostruct
*uinfo
;
87 memset(&globstat
,0,sizeof(globstat
));
89 if (vrfydir(&period
, addr
, site
, us
, email
)<0) {
90 debuga(_("Cannot create the output directory name containing the period as part of the name\n"));
95 debugaz(_("outdirname=%s\n"),outdirname
);
98 if(UserAgentLog
[0] != '\0' && email
[0] == '\0') useragent();
100 snprintf(wdirname
,sizeof(wdirname
),"%s/sarg-general",outdirname
);
101 if((fp_gen
=MY_FOPEN(wdirname
,"w"))==NULL
){
102 debuga(_("(report) Cannot open file %s\n"),wdirname
);
107 sort_labels(&sort_field
,&sort_order
);
109 if (indexonly
|| datetimeby
==0)
112 daystat
=day_prepare();
114 uscan
=userinfo_startscan();
116 debuga(_("Cannot enumerate the user list\n"));
119 while ((uinfo
= userinfo_advancescan(uscan
)) != NULL
) {
120 sort_users_log(tmp
,debug
,uinfo
);
121 if (snprintf(tmp3
,sizeof(tmp3
),"%s/%s.user_log",tmp
,uinfo
->filename
)>=sizeof(tmp3
)) {
122 debuga(_("(report) directory entry too long: %s/%s.user_log\n"),tmp
,uinfo
->filename
);
125 if((fp_in
=MY_FOPEN(tmp3
,"r"))==NULL
){
126 debuga(_("(report) Cannot open file %s\n"),tmp
);
129 user
=uinfo
->filename
;
130 day_newuser(daystat
);
132 strcpy(u2
,uinfo
->id
);
133 if(Ip2Name
&& uinfo
->id_is_ip
) {
135 ip2name(u2
,sizeof(u2
));
136 strcpy(nameantes
,u2
);
138 user_find(uinfo
->label
,MAX_USER_LEN
, u2
);
141 fp_tmp
=maketmp(user
,tmp
,debug
);
148 memset(oldaccdiatt
,0,sizeof(oldaccdiatt
));
149 memset(oldacchoratt
,0,sizeof(oldacchoratt
));
157 if ((line
=longline_create())==NULL
) {
158 debuga(_("Not enough memory to read the downloaded files\n"));
162 while((buf
=longline_read(fp_in
,line
))!=NULL
) {
163 getword_start(&gwarea
,buf
);
164 if (getword(accdia
,sizeof(accdia
),&gwarea
,'\t')<0 || getword(acchora
,sizeof(acchora
),&gwarea
,'\t')<0 ||
165 getword(accip
,sizeof(accip
),&gwarea
,'\t')<0 ||
166 getword_ptr(buf
,&accurl
,&gwarea
,'\t')<0 || getword_atoll(&accbytes
,&gwarea
,'\t')<0 ||
167 getword(acccode
,sizeof(acccode
),&gwarea
,'\t')<0) {
168 debuga(_("There is a broken record or garbage in file %s\n"),tmp3
);
171 if(strncmp(acccode
,"TCP_DENIED/407",14) == 0) continue;
172 if (getword_atoll(&accelap
,&gwarea
,'\t')<0) {
173 debuga(_("There is a broken elapsed time in file %s\n"),tmp3
);
176 if (getword_skip(20000,&gwarea
,'"')<0 || getword(accsmart
,sizeof(accsmart
),&gwarea
,'"')<0) {
177 debuga(_("There is a broken smart info in file %s\n"),tmp3
);
181 if(accsmart
[0] != '\0') {
183 grava_SmartFilter(tmp
,uinfo
->id
,accip
,accdia
,acchora
,accurl
,accsmart
);
187 if(strcmp(accip
,ipantes
) != 0) {
188 strcpy(ipantes
,accip
);
189 ip2name(accip
,sizeof(accip
));
190 strcpy(nameantes
,accip
);
191 } else strcpy(accip
,nameantes
);
195 day_addpoint(daystat
,accdia
,acchora
,accelap
,accbytes
);
196 if (iprel
) gravaporuser(uinfo
,outdirname
,accurl
,accip
,accdia
,acchora
,accbytes
,accelap
);
200 url_len
=strlen(accurl
);
201 if (!oldurl
|| url_len
>=ourl_size
) {
203 oldurl
=realloc(oldurl
,ourl_size
);
205 debuga(_("Not enough memory to store the url\n"));
209 strcpy(oldurl
,accurl
);
210 strcpy(oldacccode
,acccode
);
211 strcpy(oldaccip
,accip
);
212 strcpy(oldaccdia
,accdia
);
213 strcpy(oldacchora
,acchora
);
216 same_url
=(strcmp(oldurl
,accurl
) == 0);
218 if(site
[0] == '\0') {
220 if(strstr(oldacccode
,"DENIED") != 0)
221 strcpy(oldmsg
,"DENIED");
224 if (fp_tmp
) gravatmp(fp_tmp
,oldurl
,nacc
,nbytes
,oldmsg
,nelap
,incache
,oucache
);
225 gravager(fp_gen
,wdirname
,uinfo
,nacc
,oldurl
,nbytes
,oldaccip
,oldacchora
,oldaccdia
,nelap
,incache
,oucache
);
237 if ((ReportType
& REPORT_TYPE_SITE_USER_TIME_DATE
) != 0 && !indexonly
&&
238 (!oldurltt
|| strcmp(oldurltt
,accurl
) || strcmp(oldaccdiatt
,accdia
) || strcmp(oldacchoratt
,acchora
))) {
241 snprintf(arqtt
,sizeof(arqtt
),"%s/%s",outdirname
,uinfo
->filename
);
242 if(access(arqtt
, R_OK
) != 0)
244 url_to_file(accurl
,siteind
,sizeof(siteind
));
245 snprintf(arqtt
,sizeof(arqtt
),"%s/%s/tt%s-%s.html",outdirname
,uinfo
->filename
,uinfo
->filename
,siteind
);
246 if ((fp_tt
= fopen(arqtt
, "w")) == 0) {
247 debuga(_("(report) Cannot open file %s\n"),arqtt
);
254 sprintf(httplink,"<font size=%s color=%s><href=http://%s>%s",FontSize,PrivacyStringColor,PrivacyString,PrivacyString);
256 sprintf(httplink,"<font size=%s><a href=\"http://%s\">%s</a>",FontSize,accurl,accurl);
259 write_html_header(fp_tt
,(IndexTree
== INDEX_TREE_DATE
) ? 4 : 2,_("Site access report"),HTML_JS_NONE
);
260 fprintf(fp_tt
,"<tr><td class=\"header_c\">%s: %s</td></tr>\n",_("Period"),period
.html
);
261 fprintf(fp_tt
,"<tr><td class=\"header_c\">%s: %s</td></tr>\n",_("User"),uinfo
->label
);
262 fputs("<tr><td class=\"header_c\">",fp_tt
);
263 fprintf(fp_tt
,_("Sort: %s, %s"),sort_field
,sort_order
);
264 fputs("</td></tr>\n",fp_tt
);
265 fprintf(fp_tt
,"<tr><th class=\"header_c\">%s</th></tr>\n",_("User"));
266 close_html_header(fp_tt
);
268 fputs("<div class=\"report\"><table cellpadding=\"0\" cellspacing=\"2\">\n",fp_tt
);
269 fprintf(fp_tt
,"<tr><th class=\"header_l\">%s</th><th class=\"header_l\">%s</th><th class=\"header_l\">%s</th></tr>\n",_("ACCESSED SITE"),_("DATE"),_("TIME"));
272 fputs("<tr><td class=\"data\">",fp_tt
);
273 output_html_string(fp_tt
,accurl
,100);
274 fprintf(fp_tt
,"</td><td class=\"data\">%s</td><td class=\"data\">%s</td></tr>\n",accdia
,acchora
);
276 url_len
=strlen(accurl
);
277 if (!oldurltt
|| url_len
>=ourltt_size
) {
278 ourltt_size
=url_len
+1;
279 oldurltt
=realloc(oldurltt
,ourltt_size
);
281 debuga(_("Not enough memory to store the url\n"));
285 strcpy(oldurltt
,accurl
);
286 strcpy(oldaccdiatt
,accdia
);
287 strcpy(oldacchoratt
,acchora
);
290 strcpy(crc2
,acccode
);
291 str
=strchr(crc2
,'/');
293 if(strstr(crc2
,"MISS") != 0)
298 strcpy(oldacccode
,acccode
);
299 strcpy(oldaccip
,accip
);
301 url_len
=strlen(accurl
);
302 if (url_len
>=ourl_size
) {
304 oldurl
=realloc(oldurl
,ourl_size
);
306 debuga(_("Not enough memory to store the url\n"));
310 strcpy(oldurl
,accurl
);
312 strcpy(oldaccdia
,accdia
);
313 strcpy(oldacchora
,acchora
);
316 longline_destroy(&line
);
317 if (oldurltt
) free(oldurltt
);
319 if(strstr(oldacccode
,"DENIED") != 0)
320 strcpy(oldmsg
,"DENIED");
323 if (fp_tmp
) gravatmp(fp_tmp
,oldurl
,nacc
,nbytes
,oldmsg
,nelap
,incache
,oucache
);
324 gravager(fp_gen
,wdirname
,uinfo
,nacc
,oldurl
,nbytes
,oldaccip
,oldacchora
,oldaccdia
,nelap
,incache
,oucache
);
329 day_totalize(daystat
,tmp
,uinfo
);
332 if (fclose(fp_tmp
)==EOF
)
333 debuga(_("Failed to close temporary file for user %s - %s\n"),uinfo
->filename
,strerror(errno
));
336 if (!KeepTempLog
&& unlink(tmp3
)) {
337 debuga(_("Cannot delete \"%s\": %s\n"),tmp3
,strerror(errno
));
341 userinfo_stopscan(uscan
);
342 day_cleanup(daystat
);
344 totalger(fp_gen
,wdirname
);
347 if(email
[0] == '\0') {
349 if(DansGuardianConf
[0] != '\0')
352 debugaz(_("Dansguardian report not produced because no dansguardian configuration file was provided\n"));
360 if((ReportType
& REPORT_TYPE_DOWNLOADS
) != 0)
363 debugaz(_("Downloaded files report not requested in report_type\n"));
365 if((ReportType
& REPORT_TYPE_TOPSITES
) != 0)
368 debugaz(_("Top sites report not requested in report_type\n"));
370 if((ReportType
& REPORT_TYPE_SITES_USERS
) != 0)
373 debugaz(_("Sites & users report not requested in report_type\n"));
375 if ((ReportType
& REPORT_TYPE_DENIED
) != 0)
378 debugaz(_("Denied accesses report not requested in report_type\n"));
380 if ((ReportType
& REPORT_TYPE_AUTH_FAILURES
) != 0)
383 debugaz(_("Authentication failures report not requested in report_type\n"));
385 if(smartfilter
) smartfilter_report();
387 if(DansGuardianConf
[0] != '\0')
388 dansguardian_report();
392 if((ReportType
& REPORT_TYPE_USERS_SITES
) != 0)
395 debugaz(_("User's detailed report not requested in report_type\n"));
400 if(SuccessfulMsg
) debuga(_("Successful report generated on %s\n"),outdirname
);
402 geramail(outdirname
, debug
, outdir
, email
, tmp
);
404 if((strcmp(email
,"stdout") != 0) && SuccessfulMsg
)
405 debuga(_("Successful report generated and sent to %s\n"),email
);
408 if(indexonly
) index_only(outdirname
, debug
);
410 removetmp(outdirname
);
414 static FILE *maketmp(const char *user
, const char *dirname
, int debug
)
417 char wdirname
[MAXLEN
];
419 if((ReportType
& REPORT_TYPE_USERS_SITES
) == 0) return(NULL
);
420 if((ReportType
& REPORT_TYPE_TOPUSERS
) == 0) return(NULL
);
422 if(debug
) debuga(_("Making file: %s/%s\n"),tmp
,user
);
423 if (snprintf(wdirname
,sizeof(wdirname
),"%s/%s.utmp",tmp
,user
)>=sizeof(wdirname
)) {
424 debuga(_("Temporary file name too long: %s/%s.utmp\n"),tmp
,user
);
428 if((fp_ou
=fopen(wdirname
,"w"))==NULL
){
429 debuga(_("(report) Cannot open file %s\n"),wdirname
);
437 static void gravatmp(FILE *fp_ou
, const char *oldurl
, long long int nacc
, long long int nbytes
, const char *oldmsg
, long long int nelap
, long long int incache
, long long int oucache
)
440 This complicated printf is due to Microsoft's inability to comply with any standard. Msvcrt is unable
441 to print a long long int unless it is exactly 64-bits long.
443 fprintf(fp_ou
,"%"PRIu64
"\t%"PRIu64
"\t%s\t%s\t%"PRIu64
"\t%"PRIu64
"\t%"PRIu64
"\n",(uint64_t)nacc
,(uint64_t)nbytes
,oldurl
,oldmsg
,(uint64_t)nelap
,(uint64_t)incache
,(uint64_t)oucache
);
448 fputs("</table>\n</div>\n",fp_tt
);
449 fputs("</body>\n</html>\n",fp_tt
);
457 static void gravaporuser(const struct userinfostruct
*uinfo
, const char *dirname
, const char *url
, const char *ip
, const char *data
, const char *hora
, long long int tam
, long long int elap
)
460 char wdirname
[MAXLEN
];
462 if((ReportType
& REPORT_TYPE_USERS_SITES
) == 0) return;
464 if (snprintf(wdirname
,sizeof(wdirname
),"%s/%s.ip",tmp
,uinfo
->filename
)>=sizeof(wdirname
)) {
465 debuga(_("Path too long %s/%s.ip\n"),tmp
,uinfo
->filename
);
469 if((fp_ou
=MY_FOPEN(wdirname
,"a"))==NULL
){
470 debuga(_("(report) Cannot open file %s\n"),wdirname
);
475 This complicated printf is due to Microsoft's inability to comply with any standard. Msvcrt is unable
476 to print a long long int unless it is exactly 64-bits long.
478 fprintf(fp_ou
,"%s\t%s\t%s\t%s\t%"PRIu64
"\t%"PRIu64
"\n",ip
,url
,data
,hora
,(uint64_t)tam
,(uint64_t)elap
);
486 static void gravager(FILE *fp_gen
,const char *filename
, const struct userinfostruct
*uinfo
, long long int nacc
, const char *url
, long long int nbytes
, const char *ip
, const char *hora
, const char *dia
, long long int nelap
, long long int incache
, long long int oucache
)
489 This complicated printf is due to Microsoft's inability to comply with any standard. Msvcrt is unable
490 to print a long long int unless it is exactly 64-bits long.
492 if (fprintf(fp_gen
,"%s\t%"PRIu64
"\t%"PRIu64
"\t%s\t%s\t%s\t%s\t%"PRIu64
"\t%"PRIu64
"\t%"PRIu64
"\n",uinfo
->id
,(uint64_t)nacc
,(uint64_t)nbytes
,url
,ip
,hora
,dia
,(uint64_t)nelap
,(uint64_t)incache
,(uint64_t)oucache
)<0) {
493 debuga(_("Failed to write a line in %s\n"),filename
);
498 globstat
.nbytes
+=nbytes
;
499 globstat
.elap
+=nelap
;
500 globstat
.incache
+=incache
;
501 globstat
.oucache
+=oucache
;
506 Write the total line at the end of the general file.
508 void totalger(FILE *fp_gen
,const char *filename
)
511 This complicated printf is due to Microsoft's inability to comply with any standard. Msvcrt is unable
512 to print a long long int unless it is exactly 64-bits long.
514 if (fprintf(fp_gen
,"TOTAL\t%"PRIu64
"\t%"PRIu64
"\t%"PRIu64
"\t%"PRIu64
"\t%"PRIu64
"\n",(uint64_t)globstat
.nacc
,(uint64_t)globstat
.nbytes
,(uint64_t)globstat
.elap
,(uint64_t)globstat
.incache
,(uint64_t)globstat
.oucache
)<0) {
515 debuga(_("Failed to write the total line in %s\n"),filename
);
520 int ger_read(char *buffer
,struct generalitemstruct
*item
,const char *filename
)
524 long long int number
;
526 if (strncmp(buffer
,"TOTAL\t",6)==0) {
531 if (*buffer
== '-') {
534 } else if (*buffer
== '+') {
538 while (isdigit(*buffer
))
539 number
=(number
* 10) + (*buffer
++)-'0';
541 debuga(_("Invalid total number of accesses in %s\n"),filename
);
545 item
->nacc
=number
*sign
;
548 if (*buffer
== '-') {
551 } else if (*buffer
== '+') {
555 while (isdigit(*buffer
))
556 number
=(number
* 10) + (*buffer
++)-'0';
558 debuga(_("Invalid total size in %s\n"),filename
);
562 item
->nbytes
=number
*sign
;
565 if (*buffer
== '-') {
568 } else if (*buffer
== '+') {
572 while (isdigit(*buffer
))
573 number
=(number
* 10) + (*buffer
++)-'0';
575 debuga(_("Invalid total elapsed time in %s\n"),filename
);
579 item
->nelap
=number
*sign
;
582 if (*buffer
== '-') {
585 } else if (*buffer
== '+') {
589 while (isdigit(*buffer
))
590 number
=(number
* 10) + (*buffer
++)-'0';
592 debuga(_("Invalid total cache hit in %s\n"),filename
);
596 item
->incache
=number
*sign
;
599 if (*buffer
== '-') {
602 } else if (*buffer
== '+') {
606 while (isdigit(*buffer
))
607 number
=(number
* 10) + (*buffer
++)-'0';
609 debuga(_("Invalid total cache miss in %s\n"),filename
);
612 item
->oucache
=number
*sign
;
617 for (i
=0 ; i
<MAX_USER_LEN
-1 && (unsigned char)*buffer
>=' ' ; i
++) buffer
++;
619 debuga(_("User name too long or invalid in %s\n"),filename
);
625 if (*buffer
== '-') {
628 } else if (*buffer
== '+') {
632 while (isdigit(*buffer
))
633 number
=(number
* 10) + (*buffer
++)-'0';
635 debuga(_("Invalid number of accesses in %s\n"),filename
);
639 item
->nacc
=number
*sign
;
642 if (*buffer
== '-') {
645 } else if (*buffer
== '+') {
649 while (isdigit(*buffer
))
650 number
=(number
* 10) + (*buffer
++)-'0';
652 debuga(_("Invalid number of bytes in %s\n"),filename
);
656 item
->nbytes
=number
*sign
;
659 while ((unsigned char)*buffer
>=' ') buffer
++;
661 debuga(_("URL too long or invalid in %s\n"),filename
);
667 for (i
=0 ; i
<MAX_IP_LEN
-1 && (unsigned char)*buffer
>=' ' ; i
++) buffer
++;
669 debuga(_("IP address too long or invalid in %s\n"),filename
);
675 for (i
=0 ; i
<MAX_DATETIME_LEN
-1 && (unsigned char)*buffer
>=' ' ; i
++) buffer
++;
677 debuga(_("Time too long or invalid in %s\n"),filename
);
683 for (i
=0 ; i
<MAX_DATETIME_LEN
-1 && (unsigned char)*buffer
>=' ' ; i
++) buffer
++;
685 debuga(_("Date too long or invalid in %s\n"),filename
);
691 if (*buffer
== '-') {
694 } else if (*buffer
== '+') {
698 while (isdigit(*buffer
))
699 number
=(number
* 10) + (*buffer
++)-'0';
701 debuga(_("Invalid elapsed time in %s\n"),filename
);
705 item
->nelap
=number
*sign
;
708 if (*buffer
== '-') {
711 } else if (*buffer
== '+') {
715 while (isdigit(*buffer
))
716 number
=(number
* 10) + (*buffer
++)-'0';
718 debuga(_("Invalid cache hit size in %s\n"),filename
);
722 item
->incache
=number
*sign
;
725 if (*buffer
== '-') {
728 } else if (*buffer
== '+') {
732 while (isdigit(*buffer
))
733 number
=(number
* 10) + (*buffer
++)-'0';
735 debuga(_("Invalid cache miss size in %s\n"),filename
);
738 item
->oucache
=number
*sign
;
743 static void grava_SmartFilter(const char *dirname
, const char *user
, const char *ip
, const char *data
, const char *hora
, const char *url
, const char *smart
)
746 char wdirname
[MAXLEN
];
748 if (snprintf(wdirname
,sizeof(wdirname
),"%s/smartfilter.int_unsort",dirname
)>=sizeof(wdirname
)) {
749 debuga(_("File name too long for %s/smartfilter.int_unsort\n"),dirname
);
753 if((fp_ou
=MY_FOPEN(wdirname
,"a"))==NULL
){
754 debuga(_("(report) Cannot open file %s\n"),wdirname
);
758 fprintf(fp_ou
,"%s\t%s\t%s\t%s\t%s\t%s\n",user
,data
,hora
,ip
,url
,smart
);
759 fputs("</body>\n</html>\n",fp_tt
);