2 * SARG Squid Analysis Report Generator http://sarg.sourceforge.net
6 * please look at http://sarg.sourceforge.net/donations.php
8 * http://sourceforge.net/projects/sarg/forums/forum/363374
9 * ---------------------------------------------------------------------
11 * This program is free software; you can redistribute it and/or modify
12 * it under the terms of the GNU General Public License as published by
13 * the Free Software Foundation; either version 2 of the License, or
14 * (at your option) any later version.
16 * This program is distributed in the hope that it will be useful,
17 * but WITHOUT ANY WARRANTY; without even the implied warranty of
18 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
19 * GNU General Public License for more details.
21 * You should have received a copy of the GNU General Public License
22 * along with this program; if not, write to the Free Software
23 * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111, USA.
27 #include "include/conf.h"
28 #include "include/defs.h"
30 //! The global statistics of the whole log read.
31 struct globalstatstruct globstat
;
33 static FILE *fp_tt
=NULL
;
35 static FILE *maketmp(const char *user
, const char *dirname
, int debug
);
36 static void gravatmp(FILE *fp_ou
, const char *oldurl
, long long int nacc
, long long int nbytes
, const char *oldmsg
, long long int nelap
, long long int incache
, long long int oucache
);
37 static void gravaporuser(const struct userinfostruct
*uinfo
, const char *dirname
, const char *url
, const char *ip
, const char *data
, const char *hora
, long long int tam
, long long int elap
);
38 static void gravager(FILE *fp_gen
,const char *filename
, const struct userinfostruct
*uinfo
, long long int nacc
, const char *url
, long long int nbytes
, const char *ip
, const char *hora
, const char *dia
, long long int nelap
, long long int incache
, long long int oucache
);
39 static void grava_SmartFilter(const char *dirname
, const char *user
, const char *ip
, const char *data
, const char *hora
, const char *url
, const char *smart
);
48 char accdia
[11], acchora
[9], accip
[MAXLEN
], *accurl
;
49 char oldaccdia
[11], oldacchora
[9], oldaccip
[MAXLEN
];
50 char wdirname
[MAXLEN
];
52 char oldmsg
[50], acccode
[MAXLEN
/2 - 1], oldacccode
[MAXLEN
/2 - 1];
53 char ipantes
[MAXLEN
], nameantes
[MAXLEN
];
54 char accsmart
[MAXLEN
];
55 char crc2
[MAXLEN
/2 -1];
56 char siteind
[MAX_TRUNCATED_URL
];
59 char oldaccdiatt
[11],oldacchoratt
[9];
61 char u2
[MAX_USER_LEN
];
62 long long int nbytes
=0;
63 long long int nelap
=0;
65 long long int incache
=0;
66 long long int oucache
=0;
67 long long int accbytes
, accelap
;
70 const char *sort_field
;
71 const char *sort_order
;
78 struct getwordstruct gwarea
;
80 struct userinfostruct
*uinfo
;
85 memset(&globstat
,0,sizeof(globstat
));
87 if (vrfydir(&period
, addr
, site
, us
, email
)<0) {
88 debuga(_("Cannot create the output directory name containing the period as part of the name\n"));
93 debugaz(_("outdirname=%s\n"),outdirname
);
96 if(UserAgentLog
[0] != '\0' && email
[0] == '\0') useragent();
98 snprintf(wdirname
,sizeof(wdirname
),"%s/sarg-general",outdirname
);
99 if((fp_gen
=MY_FOPEN(wdirname
,"w"))==NULL
){
100 debuga(_("(report) Cannot open file %s\n"),wdirname
);
105 sort_labels(&sort_field
,&sort_order
);
107 if (indexonly
|| datetimeby
==0)
110 daystat
=day_prepare();
112 uscan
=userinfo_startscan();
114 debuga(_("Cannot enumerate the user list\n"));
117 while ((uinfo
= userinfo_advancescan(uscan
)) != NULL
) {
118 if (snprintf(tmp3
,sizeof(tmp3
),"%s/%s.log",tmp
,uinfo
->filename
)>=sizeof(tmp3
)) {
119 debuga(_("(report) directory entry too long: %s/%s.log\n"),tmp
,uinfo
->filename
);
122 if((fp_in
=MY_FOPEN(tmp3
,"r"))==NULL
){
123 debuga(_("(report) Cannot open file %s\n"),tmp
);
126 user
=uinfo
->filename
;
127 day_newuser(daystat
);
129 strcpy(u2
,uinfo
->id
);
130 if(Ip2Name
&& uinfo
->id_is_ip
) {
132 ip2name(u2
,sizeof(u2
));
133 strcpy(nameantes
,u2
);
135 user_find(uinfo
->label
,MAX_USER_LEN
, u2
);
138 fp_tmp
=maketmp(user
,tmp
,debug
);
145 memset(oldaccdiatt
,0,sizeof(oldaccdiatt
));
146 memset(oldacchoratt
,0,sizeof(oldacchoratt
));
154 if ((line
=longline_create())==NULL
) {
155 debuga(_("Not enough memory to read the downloaded files\n"));
159 while((buf
=longline_read(fp_in
,line
))!=NULL
) {
160 getword_start(&gwarea
,buf
);
161 if (getword(accdia
,sizeof(accdia
),&gwarea
,'\t')<0 || getword(acchora
,sizeof(acchora
),&gwarea
,'\t')<0 ||
162 getword(accip
,sizeof(accip
),&gwarea
,'\t')<0 ||
163 getword_ptr(buf
,&accurl
,&gwarea
,'\t')<0 || getword_atoll(&accbytes
,&gwarea
,'\t')<0 ||
164 getword(acccode
,sizeof(acccode
),&gwarea
,'\t')<0) {
165 debuga(_("There is a broken record or garbage in file %s\n"),tmp3
);
168 if(strncmp(acccode
,"TCP_DENIED/407",14) == 0) continue;
169 if (getword_atoll(&accelap
,&gwarea
,'\t')<0) {
170 debuga(_("There is a broken elapsed time in file %s\n"),tmp3
);
173 if (getword_skip(20000,&gwarea
,'"')<0 || getword(accsmart
,sizeof(accsmart
),&gwarea
,'"')<0) {
174 debuga(_("There is a broken smart info in file %s\n"),tmp3
);
178 if(accsmart
[0] != '\0') {
180 grava_SmartFilter(outdirname
,uinfo
->id
,accip
,accdia
,acchora
,accurl
,accsmart
);
184 if(strcmp(accip
,ipantes
) != 0) {
185 strcpy(ipantes
,accip
);
186 ip2name(accip
,sizeof(accip
));
187 strcpy(nameantes
,accip
);
188 } else strcpy(accip
,nameantes
);
192 day_addpoint(daystat
,accdia
,acchora
,accelap
,accbytes
);
193 if (iprel
) gravaporuser(uinfo
,outdirname
,accurl
,accip
,accdia
,acchora
,accbytes
,accelap
);
197 url_len
=strlen(accurl
);
198 if (!oldurl
|| url_len
>=ourl_size
) {
200 oldurl
=realloc(oldurl
,ourl_size
);
202 debuga(_("Not enough memory to store the url\n"));
206 strcpy(oldurl
,accurl
);
207 strcpy(oldacccode
,acccode
);
208 strcpy(oldaccip
,accip
);
209 strcpy(oldaccdia
,accdia
);
210 strcpy(oldacchora
,acchora
);
213 same_url
=(strcmp(oldurl
,accurl
) == 0);
215 if(site
[0] == '\0') {
217 if(strstr(oldacccode
,"DENIED") != 0)
218 strcpy(oldmsg
,"DENIED");
221 if (fp_tmp
) gravatmp(fp_tmp
,oldurl
,nacc
,nbytes
,oldmsg
,nelap
,incache
,oucache
);
222 gravager(fp_gen
,wdirname
,uinfo
,nacc
,oldurl
,nbytes
,oldaccip
,oldacchora
,oldaccdia
,nelap
,incache
,oucache
);
234 if ((ReportType
& REPORT_TYPE_SITE_USER_TIME_DATE
) != 0 && !indexonly
&&
235 (!oldurltt
|| strcmp(oldurltt
,accurl
) || strcmp(oldaccdiatt
,accdia
) || strcmp(oldacchoratt
,acchora
))) {
238 snprintf(arqtt
,sizeof(arqtt
),"%s/%s",outdirname
,uinfo
->filename
);
239 if(access(arqtt
, R_OK
) != 0)
241 url_to_file(accurl
,siteind
,sizeof(siteind
));
242 snprintf(arqtt
,sizeof(arqtt
),"%s/%s/tt%s-%s.html",outdirname
,uinfo
->filename
,uinfo
->filename
,siteind
);
243 if ((fp_tt
= fopen(arqtt
, "w")) == 0) {
244 debuga(_("(report) Cannot open file %s\n"),arqtt
);
251 sprintf(httplink,"<font size=%s color=%s><href=http://%s>%s",FontSize,PrivacyStringColor,PrivacyString,PrivacyString);
253 sprintf(httplink,"<font size=%s><a href=\"http://%s\">%s</a>",FontSize,accurl,accurl);
256 write_html_header(fp_tt
,(IndexTree
== INDEX_TREE_DATE
) ? 4 : 2,_("Site access report"),HTML_JS_NONE
);
257 fprintf(fp_tt
,"<tr><td class=\"header_c\">%s: %s</td></tr>\n",_("Period"),period
.html
);
258 fprintf(fp_tt
,"<tr><td class=\"header_c\">%s: %s</td></tr>\n",_("User"),uinfo
->label
);
259 fputs("<tr><td class=\"header_c\">",fp_tt
);
260 fprintf(fp_tt
,_("Sort: %s, %s"),sort_field
,sort_order
);
261 fputs("</td></tr>\n",fp_tt
);
262 fprintf(fp_tt
,"<tr><th class=\"header_c\">%s</th></tr>\n",_("User"));
263 close_html_header(fp_tt
);
265 fputs("<div class=\"report\"><table cellpadding=\"0\" cellspacing=\"2\">\n",fp_tt
);
266 fprintf(fp_tt
,"<tr><th class=\"header_l\">%s</th><th class=\"header_l\">%s</th><th class=\"header_l\">%s</th></tr>\n",_("ACCESSED SITE"),_("DATE"),_("TIME"));
269 fputs("<tr><td class=\"data\">",fp_tt
);
270 output_html_string(fp_tt
,accurl
,100);
271 fprintf(fp_tt
,"</td><td class=\"data\">%s</td><td class=\"data\">%s</td></tr>\n",accdia
,acchora
);
273 url_len
=strlen(accurl
);
274 if (!oldurltt
|| url_len
>=ourltt_size
) {
275 ourltt_size
=url_len
+1;
276 oldurltt
=realloc(oldurltt
,ourltt_size
);
278 debuga(_("Not enough memory to store the url\n"));
282 strcpy(oldurltt
,accurl
);
283 strcpy(oldaccdiatt
,accdia
);
284 strcpy(oldacchoratt
,acchora
);
287 strcpy(crc2
,acccode
);
288 str
=strchr(crc2
,'/');
290 if(strstr(crc2
,"MISS") != 0)
295 strcpy(oldacccode
,acccode
);
296 strcpy(oldaccip
,accip
);
298 url_len
=strlen(accurl
);
299 if (url_len
>=ourl_size
) {
301 oldurl
=realloc(oldurl
,ourl_size
);
303 debuga(_("Not enough memory to store the url\n"));
307 strcpy(oldurl
,accurl
);
309 strcpy(oldaccdia
,accdia
);
310 strcpy(oldacchora
,acchora
);
313 longline_destroy(&line
);
314 if (oldurltt
) free(oldurltt
);
316 if(strstr(oldacccode
,"DENIED") != 0)
317 strcpy(oldmsg
,"DENIED");
320 if (fp_tmp
) gravatmp(fp_tmp
,oldurl
,nacc
,nbytes
,oldmsg
,nelap
,incache
,oucache
);
321 gravager(fp_gen
,wdirname
,uinfo
,nacc
,oldurl
,nbytes
,oldaccip
,oldacchora
,oldaccdia
,nelap
,incache
,oucache
);
326 day_totalize(daystat
,tmp
,uinfo
);
329 if (fclose(fp_tmp
)==EOF
)
330 debuga(_("Failed to close temporary file for user %s - %s\n"),uinfo
->filename
,strerror(errno
));
334 debuga(_("Cannot delete %s - %s\n"),tmp3
,strerror(errno
));
338 userinfo_stopscan(uscan
);
339 day_cleanup(daystat
);
341 totalger(fp_gen
,wdirname
);
344 if(email
[0] == '\0') {
346 if(DansGuardianConf
[0] != '\0')
349 debugaz(_("Dansguardian report not produced because no dansguardian configuration file was provided\n"));
357 if((ReportType
& REPORT_TYPE_DOWNLOADS
) != 0)
360 debugaz(_("Downloaded files report not requested in report_type\n"));
362 if((ReportType
& REPORT_TYPE_TOPSITES
) != 0)
365 debugaz(_("Top sites report not requested in report_type\n"));
367 if((ReportType
& REPORT_TYPE_SITES_USERS
) != 0)
370 debugaz(_("Sites & users report not requested in report_type\n"));
372 if ((ReportType
& REPORT_TYPE_DENIED
) != 0)
375 debugaz(_("Denied accesses report not requested in report_type\n"));
377 if ((ReportType
& REPORT_TYPE_AUTH_FAILURES
) != 0)
380 debugaz(_("Authentication failures report not requested in report_type\n"));
382 if(smartfilter
) smartfilter_report();
384 if(DansGuardianConf
[0] != '\0')
385 dansguardian_report();
389 if((ReportType
& REPORT_TYPE_USERS_SITES
) != 0)
392 debugaz(_("User's detailed report not requested in report_type\n"));
397 if(SuccessfulMsg
) debuga(_("Successful report generated on %s\n"),outdirname
);
399 geramail(outdirname
, debug
, outdir
, email
, tmp
);
401 if((strcmp(email
,"stdout") != 0) && SuccessfulMsg
)
402 debuga(_("Successful report generated and sent to %s\n"),email
);
405 if(indexonly
) index_only(outdirname
, debug
);
407 removetmp(outdirname
);
411 static FILE *maketmp(const char *user
, const char *dirname
, int debug
)
414 char wdirname
[MAXLEN
];
416 if((ReportType
& REPORT_TYPE_USERS_SITES
) == 0) return(NULL
);
417 if((ReportType
& REPORT_TYPE_TOPUSERS
) == 0) return(NULL
);
419 if(debug
) debuga(_("Making file: %s/%s\n"),tmp
,user
);
420 if (snprintf(wdirname
,sizeof(wdirname
),"%s/%s.utmp",tmp
,user
)>=sizeof(wdirname
)) {
421 debuga(_("Temporary file name too long: %s/%s.utmp\n"),tmp
,user
);
425 if((fp_ou
=fopen(wdirname
,"w"))==NULL
){
426 debuga(_("(report) Cannot open file %s\n"),wdirname
);
434 static void gravatmp(FILE *fp_ou
, const char *oldurl
, long long int nacc
, long long int nbytes
, const char *oldmsg
, long long int nelap
, long long int incache
, long long int oucache
)
437 This complicated printf is due to Microsoft's inability to comply with any standard. Msvcrt is unable
438 to print a long long int unless it is exactly 64-bits long.
440 fprintf(fp_ou
,"%"PRIu64
"\t%"PRIu64
"\t%s\t%s\t%"PRIu64
"\t%"PRIu64
"\t%"PRIu64
"\n",(uint64_t)nacc
,(uint64_t)nbytes
,oldurl
,oldmsg
,(uint64_t)nelap
,(uint64_t)incache
,(uint64_t)oucache
);
445 fputs("</table>\n</div>\n",fp_tt
);
446 fputs("</body>\n</html>\n",fp_tt
);
454 static void gravaporuser(const struct userinfostruct
*uinfo
, const char *dirname
, const char *url
, const char *ip
, const char *data
, const char *hora
, long long int tam
, long long int elap
)
457 char wdirname
[MAXLEN
];
459 if((ReportType
& REPORT_TYPE_USERS_SITES
) == 0) return;
461 if (snprintf(wdirname
,sizeof(wdirname
),"%s/%s.ip",tmp
,uinfo
->filename
)>=sizeof(wdirname
)) {
462 debuga(_("Path too long %s/%s.ip\n"),tmp
,uinfo
->filename
);
466 if((fp_ou
=MY_FOPEN(wdirname
,"a"))==NULL
){
467 debuga(_("(report) Cannot open file %s\n"),wdirname
);
472 This complicated printf is due to Microsoft's inability to comply with any standard. Msvcrt is unable
473 to print a long long int unless it is exactly 64-bits long.
475 fprintf(fp_ou
,"%s\t%s\t%s\t%s\t%"PRIu64
"\t%"PRIu64
"\n",ip
,url
,data
,hora
,(uint64_t)tam
,(uint64_t)elap
);
483 static void gravager(FILE *fp_gen
,const char *filename
, const struct userinfostruct
*uinfo
, long long int nacc
, const char *url
, long long int nbytes
, const char *ip
, const char *hora
, const char *dia
, long long int nelap
, long long int incache
, long long int oucache
)
486 This complicated printf is due to Microsoft's inability to comply with any standard. Msvcrt is unable
487 to print a long long int unless it is exactly 64-bits long.
489 if (fprintf(fp_gen
,"%s\t%"PRIu64
"\t%"PRIu64
"\t%s\t%s\t%s\t%s\t%"PRIu64
"\t%"PRIu64
"\t%"PRIu64
"\n",uinfo
->id
,(uint64_t)nacc
,(uint64_t)nbytes
,url
,ip
,hora
,dia
,(uint64_t)nelap
,(uint64_t)incache
,(uint64_t)oucache
)<0) {
490 debuga(_("Failed to write a line in %s\n"),filename
);
495 globstat
.nbytes
+=nbytes
;
496 globstat
.elap
+=nelap
;
497 globstat
.incache
+=incache
;
498 globstat
.oucache
+=oucache
;
503 Write the total line at the end of the general file.
505 void totalger(FILE *fp_gen
,const char *filename
)
508 This complicated printf is due to Microsoft's inability to comply with any standard. Msvcrt is unable
509 to print a long long int unless it is exactly 64-bits long.
511 if (fprintf(fp_gen
,"TOTAL\t%"PRIu64
"\t%"PRIu64
"\t%"PRIu64
"\t%"PRIu64
"\t%"PRIu64
"\n",(uint64_t)globstat
.nacc
,(uint64_t)globstat
.nbytes
,(uint64_t)globstat
.elap
,(uint64_t)globstat
.incache
,(uint64_t)globstat
.oucache
)<0) {
512 debuga(_("Failed to write the total line in %s\n"),filename
);
517 int ger_read(char *buffer
,struct generalitemstruct
*item
,const char *filename
)
521 long long int number
;
523 if (strncmp(buffer
,"TOTAL\t",6)==0) {
528 if (*buffer
== '-') {
531 } else if (*buffer
== '+') {
535 while (isdigit(*buffer
))
536 number
=(number
* 10) + (*buffer
++)-'0';
538 debuga(_("Invalid total number of accesses in %s\n"),filename
);
542 item
->nacc
=number
*sign
;
545 if (*buffer
== '-') {
548 } else if (*buffer
== '+') {
552 while (isdigit(*buffer
))
553 number
=(number
* 10) + (*buffer
++)-'0';
555 debuga(_("Invalid total size in %s\n"),filename
);
559 item
->nbytes
=number
*sign
;
562 if (*buffer
== '-') {
565 } else if (*buffer
== '+') {
569 while (isdigit(*buffer
))
570 number
=(number
* 10) + (*buffer
++)-'0';
572 debuga(_("Invalid total elapsed time in %s\n"),filename
);
576 item
->nelap
=number
*sign
;
579 if (*buffer
== '-') {
582 } else if (*buffer
== '+') {
586 while (isdigit(*buffer
))
587 number
=(number
* 10) + (*buffer
++)-'0';
589 debuga(_("Invalid total cache hit in %s\n"),filename
);
593 item
->incache
=number
*sign
;
596 if (*buffer
== '-') {
599 } else if (*buffer
== '+') {
603 while (isdigit(*buffer
))
604 number
=(number
* 10) + (*buffer
++)-'0';
606 debuga(_("Invalid total cache miss in %s\n"),filename
);
609 item
->oucache
=number
*sign
;
614 for (i
=0 ; i
<MAX_USER_LEN
-1 && (unsigned char)*buffer
>=' ' ; i
++) buffer
++;
616 debuga(_("User name too long or invalid in %s\n"),filename
);
622 if (*buffer
== '-') {
625 } else if (*buffer
== '+') {
629 while (isdigit(*buffer
))
630 number
=(number
* 10) + (*buffer
++)-'0';
632 debuga(_("Invalid number of accesses in %s\n"),filename
);
636 item
->nacc
=number
*sign
;
639 if (*buffer
== '-') {
642 } else if (*buffer
== '+') {
646 while (isdigit(*buffer
))
647 number
=(number
* 10) + (*buffer
++)-'0';
649 debuga(_("Invalid number of bytes in %s\n"),filename
);
653 item
->nbytes
=number
*sign
;
656 while ((unsigned char)*buffer
>=' ') buffer
++;
658 debuga(_("URL too long or invalid in %s\n"),filename
);
664 for (i
=0 ; i
<MAX_IP_LEN
-1 && (unsigned char)*buffer
>=' ' ; i
++) buffer
++;
666 debuga(_("IP address too long or invalid in %s\n"),filename
);
672 for (i
=0 ; i
<MAX_DATETIME_LEN
-1 && (unsigned char)*buffer
>=' ' ; i
++) buffer
++;
674 debuga(_("Time too long or invalid in %s\n"),filename
);
680 for (i
=0 ; i
<MAX_DATETIME_LEN
-1 && (unsigned char)*buffer
>=' ' ; i
++) buffer
++;
682 debuga(_("Date too long or invalid in %s\n"),filename
);
688 if (*buffer
== '-') {
691 } else if (*buffer
== '+') {
695 while (isdigit(*buffer
))
696 number
=(number
* 10) + (*buffer
++)-'0';
698 debuga(_("Invalid elapsed time in %s\n"),filename
);
702 item
->nelap
=number
*sign
;
705 if (*buffer
== '-') {
708 } else if (*buffer
== '+') {
712 while (isdigit(*buffer
))
713 number
=(number
* 10) + (*buffer
++)-'0';
715 debuga(_("Invalid cache hit size in %s\n"),filename
);
719 item
->incache
=number
*sign
;
722 if (*buffer
== '-') {
725 } else if (*buffer
== '+') {
729 while (isdigit(*buffer
))
730 number
=(number
* 10) + (*buffer
++)-'0';
732 debuga(_("Invalid cache miss size in %s\n"),filename
);
735 item
->oucache
=number
*sign
;
740 static void grava_SmartFilter(const char *dirname
, const char *user
, const char *ip
, const char *data
, const char *hora
, const char *url
, const char *smart
)
743 char wdirname
[MAXLEN
];
745 sprintf(wdirname
,"%s/smartfilter.unsort",dirname
);
747 if((fp_ou
=MY_FOPEN(wdirname
,"a"))==NULL
){
748 debuga(_("(report) Cannot open file %s\n"),wdirname
);
752 fprintf(fp_ou
,"%s\t%s\t%s\t%s\t%s\t%s\n",user
,data
,hora
,ip
,url
,smart
);
753 fputs("</body>\n</html>\n",fp_tt
);