]> git.ipfire.org Git - thirdparty/sarg.git/blob - log.c
projects / thirdparty / sarg.git / blob
? search:
summary | shortlog | log | commit | commitdiff | tree
blame | history | raw | HEAD
Add option to hide index column
[thirdparty/sarg.git] / log.c
1 /*
2 * SARG Squid Analysis Report Generator http://sarg.sourceforge.net
3 * 1998, 2010
4 *
5 * SARG donations:
6 * please look at http://sarg.sourceforge.net/donations.php
7 * Support:
8 * http://sourceforge.net/projects/sarg/forums/forum/363374
9 * ---------------------------------------------------------------------
10 *
11 * This program is free software; you can redistribute it and/or modify
12 * it under the terms of the GNU General Public License as published by
13 * the Free Software Foundation; either version 2 of the License, or
14 * (at your option) any later version.
15 *
16 * This program is distributed in the hope that it will be useful,
17 * but WITHOUT ANY WARRANTY; without even the implied warranty of
18 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
19 * GNU General Public License for more details.
20 *
21 * You should have received a copy of the GNU General Public License
22 * along with this program; if not, write to the Free Software
23 * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111, USA.
24 *
25 */
26
27 #include "include/conf.h"
28 #include "include/defs.h"
29
30 #ifdef HAVE_GETOPT_H
31 #include <getopt.h>
32 #endif
33
34 #define REPORT_EVERY_X_LINES 5000
35 #define MAX_OPEN_USER_FILES 10
36
37 struct userfilestruct
38 {
39 struct userfilestruct *next;
40 struct userinfostruct *user;
41 FILE *file;
42 };
43
44 /*@null@*/static char *userfile=NULL;
45
46 numlist weekdays = { { 0, 1, 2, 3, 4, 5, 6 }, 7 };
47 numlist hours = { { 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12,
48 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23 }, 24 };
49
50 static void getusers(const char *pwdfile, int debug);
51
52 int main(int argc,char *argv[])
53 {
54 enum isa_col_id {
55 ISACOL_Ip,
56 ISACOL_UserName,
57 ISACOL_Date,
58 ISACOL_Time,
59 ISACOL_TimeTaken,
60 ISACOL_Bytes,
61 ISACOL_Uri,
62 ISACOL_Status,
63 ISACOL_Last //last entry of the list !
64 };
65 enum InputLogFormat {
66 ILF_Unknown,
67 ILF_Squid,
68 ILF_Common,
69 ILF_Sarg,
70 ILF_Isa,
71 ILF_Last //last entry of the list !
72 };
73
74 FILE *fp_in = NULL, *fp_denied=NULL, *fp_authfail=NULL, *fp_log=NULL;
75
76 char sz_Download_Unsort[ 20000 ] ;
77 FILE * fp_Download_Unsort = NULL ;
78
79 extern int optind;
80 extern int optopt;
81 extern char *optarg;
82
83 char data[255];
84 char elap[255];
85 char ip[MAXLEN];
86 char tam[255];
87 char fun[MAXLEN];
88 char wuser[MAXLEN];
89 char smartfilter[MAXLEN];
90 char dia[128];
91 char mes[30];
92 char ano[30];
93 char hora[30];
94 char wtemp[MAXLEN];
95 char date[255];
96 char arq[255];
97 char arq_log[255];
98 int hm, hmf, hmr;
99 char hm_str[15];
100 char uagent[MAXLEN];
101 char hexclude[MAXLEN];
102 char csort[MAXLEN];
103 int cstatus;
104 char tbuf2[128];
105 char zip[20];
106 char *str;
107 char tmp3[MAXLEN];
108 char denied_unsort[MAXLEN];
109 char denied_sort[MAXLEN];
110 char authfail_unsort[MAXLEN];
111 char start_hour[128];
112 char end_hour[128];
113 char *linebuf;
114 char hostname[512];
115 char *url;
116 char *urly;
117 char user[MAX_USER_LEN];
118 enum InputLogFormat ilf;
119 int ilf_count[ILF_Last];
120 int ch;
121 int x;
122 int errflg=0;
123 int puser=0;
124 bool fhost=false;
125 bool dns=false;
126 bool fuser=false;
127 int idata=0;
128 int mindate=0;
129 int maxdate=0;
130 int iarq=0;
131 int isa_ncols=0,isa_cols[ISACOL_Last];
132 int lastlog=-1;
133 bool from_stdin;
134 bool from_pipe;
135 int blen;
136 int maxopenfiles;
137 int nopen;
138 bool id_is_ip;
139 long totregsl=0;
140 long totregsg=0;
141 long totregsx=0;
142 bool totper=false;
143 long int max_elapsed=0;
144 long long int iyear, imonth, iday;
145 bool realt;
146 bool userip;
147 struct tm tt;
148 struct tm *t;
149 unsigned long recs1=0UL;
150 unsigned long recs2=0UL;
151 int OutputNonZero = REPORT_EVERY_X_LINES ;
152 bool download_flag=false;
153 char *download_url=NULL;
154 struct getwordstruct gwarea;
155 longline line;
156 time_t tnum;
157 struct stat logstat;
158 struct userinfostruct *uinfo;
159 struct userfilestruct *first_user_file, *ufile, *ufile1, *prev_ufile;
160 static int split=0;
161 static int convert=0;
162 static int output_css=0;
163 int option_index;
164 static struct option long_options[]=
165 {
166 {"convert",no_argument,&convert,1},
167 {"css",no_argument,&output_css,1},
168 {"lastlog",required_argument,NULL,2},
169 {"keeplogs",no_argument,NULL,3},
170 {"split",no_argument,&split,1},
171 {0,0,0,0}
172 };
173
174 #ifdef HAVE_LOCALE_H
175 setlocale(LC_TIME,"");
176 #endif
177
178 #if defined(ENABLE_NLS) && defined(HAVE_LOCALE_H)
179 if (!setlocale (LC_ALL, "")) {
180 fprintf(stderr,"SARG: Cannot set the locale LC_ALL to the environment variable\n");
181 exit(EXIT_FAILURE);
182 }
183 if (!bindtextdomain (PACKAGE_NAME, LOCALEDIR)) {
184 fprintf(stderr,"SARG: Cannot bind to text domain %s in directory %s (%s)\n",PACKAGE_NAME,LOCALEDIR,strerror(errno));
185 exit(EXIT_FAILURE);
186 }
187 if (!textdomain (PACKAGE_NAME)) {
188 fprintf(stderr,"SARG: Cannot set gettext domain for %s PACKAGE_NAME (%s)\n",PACKAGE_NAME,strerror(errno));
189 exit(EXIT_FAILURE);
190 }
191 #endif //ENABLE_NLS
192
193 BgImage[0]='\0';
194 LogoImage[0]='\0';
195 LogoText[0]='\0';
196 PasswdFile[0]='\0';
197 OutputEmail[0]='\0';
198 UserAgentLog[0]='\0';
199 ExcludeHosts[0]='\0';
200 ExcludeUsers[0]='\0';
201 ConfigFile[0]='\0';
202 code[0]='\0';
203 LastLog=0;
204 ReportType=0UL;
205 UserTabFile[0]='\0';
206 BlockIt[0]='\0';
207 ExternalCSSFile[0]='\0';
208 RedirectorLogFormat[0]='\0';
209 NRedirectorLogs=0;
210 for (ilf=0 ; ilf<ILF_Last ; ilf++) ilf_count[ilf]=0;
211
212 sprintf(ExcludeCodes,"%s/exclude_codes",SYSCONFDIR);
213 strcpy(GraphDaysBytesBarColor,"orange");
214 strcpy(BgColor,"#ffffff");
215 strcpy(TxColor,"#000000");
216 strcpy(TxBgColor,"lavender");
217 strcpy(TiColor,"darkblue");
218 strcpy(Width,"80");
219 strcpy(Height,"45");
220 strcpy(LogoTextColor,"#000000");
221 strcpy(HeaderColor,"darkblue");
222 strcpy(HeaderBgColor,"#dddddd");
223 strcpy(LogoTextColor,"#006699");
224 strcpy(FontSize,"9px");
225 strcpy(TempDir,"/tmp");
226 strcpy(OutputDir,"/var/www/html/squid-reports");
227 Ip2Name=false;
228 strcpy(DateFormat,"u");
229 OverwriteReport=false;
230 RemoveTempFiles=true;
231 strcpy(ReplaceIndex,"index.html");
232 Index=INDEX_YES;
233 RecordsWithoutUser=RECORDWITHOUTUSER_IP;
234 UseComma=0;
235 strcpy(MailUtility,"mailx");
236 TopSitesNum=100;
237 TopUsersNum=0;
238 UserIp=0;
239 TopuserSort=TOPUSER_SORT_BYTES | TOPUSER_SORT_REVERSE;
240 UserSort=USER_SORT_BYTES | USER_SORT_REVERSE;
241 TopsitesSort=TOPSITE_SORT_CONNECT | TOPSITE_SORT_REVERSE;
242 LongUrl=0;
243 strcpy(FontFace,"Verdana,Tahoma,Arial");
244 datetimeby=DATETIME_BYTE;
245 strcpy(CharSet,"ISO-8859-1");
246 Privacy=0;
247 strcpy(PrivacyString,"***.***.***.***");
248 strcpy(PrivacyStringColor,"blue");
249 SuccessfulMsg=true;
250 TopUserFields=TOPUSERFIELDS_NUM | TOPUSERFIELDS_DATE_TIME | TOPUSERFIELDS_USERID | TOPUSERFIELDS_CONNECT |
251 TOPUSERFIELDS_BYTES | TOPUSERFIELDS_SETYB | TOPUSERFIELDS_IN_CACHE_OUT |
252 TOPUSERFIELDS_USED_TIME | TOPUSERFIELDS_MILISEC | TOPUSERFIELDS_PTIME |
253 TOPUSERFIELDS_TOTAL | TOPUSERFIELDS_AVERAGE;
254 UserReportFields=USERREPORTFIELDS_CONNECT | USERREPORTFIELDS_BYTES | USERREPORTFIELDS_SETYB |
255 USERREPORTFIELDS_IN_CACHE_OUT | USERREPORTFIELDS_USED_TIME | USERREPORTFIELDS_MILISEC |
256 USERREPORTFIELDS_PTIME | USERREPORTFIELDS_TOTAL | USERREPORTFIELDS_AVERAGE;
257 strcpy(DataFileDelimiter,";");
258 DataFileFields=DATA_FIELD_USER | DATA_FIELD_DATE | DATA_FIELD_TIME | DATA_FIELD_URL | DATA_FIELD_CONNECT |
259 DATA_FIELD_BYTES | DATA_FIELD_IN_CACHE | DATA_FIELD_OUT_CACHE | DATA_FIELD_ELAPSED;
260 ShowReadStatistics=true;
261 strcpy(IndexSortOrder,"D");
262 ShowSargInfo=true;
263 ShowSargLogo=true;
264 strcpy(ParsedOutputLog,"no");
265 strcpy(ParsedOutputLogCompress,"/bin/gzip -f");
266 DisplayedValues=DISPLAY_ABBREV;
267 strcpy(HeaderFontSize,"9px");
268 strcpy(TitleFontSize,"11px");
269 strcpy(AuthUserTemplateFile,"sarg_htaccess");
270 set_download_suffix("7z,ace,arj,avi,bat,bin,bz2,bzip,cab,com,cpio,dll,doc,dot,exe,gz,iso,lha,lzh,mdb,mov,mp3,mpeg,mpg,mso,nrg,ogg,ppt,rar,rtf,shs,src,sys,tar,tgz,vcd,vob,wma,wmv,zip");
271 Graphs=true;
272 #if defined(FONTDIR)
273 strcpy(GraphFont,FONTDIR"/DejaVuSans.ttf");
274 #else
275 GraphFont[0]='\0';
276 #endif
277 strcpy(Ulimit,"20000");
278 NtlmUserFormat=NTLMUSERFORMAT_DOMAINUSER;
279 IndexTree=INDEX_TREE_FILE;
280 IndexFields=INDEXFIELDS_DIRSIZE;
281 strcpy(RealtimeTypes,"GET,PUT,CONNECT");
282 RealtimeUnauthRec=REALTIME_UNAUTH_REC_SHOW;
283 RedirectorFilterOutDate=true;
284 DansguardianFilterOutDate=true;
285 DataFileUrl=DATAFILEURL_IP;
286 strcpy(MaxElapsed,"28800000");
287 BytesInSitesUsersReport=0;
288 UserAuthentication=0;
289 strcpy(LDAPHost,"127.0.0.1");
290 LDAPPort=389;
291 LDAPProtocolVersion=3;
292 LDAPBindDN[0]='\0';
293 LDAPBindPW[0]='\0';
294 LDAPBaseSearch[0]='\0';
295 strcpy(LDAPFilterSearch, "(uid=%s)");
296 strcpy(LDAPTargetAttr, "cn");
297 SortTableJs[0]='\0';
298
299 dia[0]='\0';
300 mes[0]='\0';
301 ano[0]='\0';
302 hora[0]='\0';
303 tmp[0]='\0';
304 tmp3[0]='\0';
305 wtemp[0]='\0';
306 us[0]='\0';
307 date[0]='\0';
308 df[0]='\0';
309 uagent[0]='\0';
310 hexclude[0]='\0';
311 addr[0]='\0';
312 hm=-1;
313 hmf=-1;
314 site[0]='\0';
315 outdir[0]='\0';
316 elap[0]='\0';
317 email[0]='\0';
318 zip[0]='\0';
319 UserInvalidChar[0]='\0';
320 DataFile[0]='\0';
321 SquidGuardConf[0]='\0';
322 DansGuardianConf[0]='\0';
323 start_hour[0]='\0';
324 end_hour[0]='\0';
325 hm_str[0]='\0';
326
327 denied_count=0;
328 download_count=0;
329 authfail_count=0;
330 dansguardian_count=0;
331 squidguard_count=0;
332 useragent_count=0;
333 DeniedReportLimit=10;
334 AuthfailReportLimit=10;
335 DansGuardianReportLimit=10;
336 SquidGuardReportLimit=10;
337 DownloadReportLimit=50;
338 UserReportLimit=0;
339 debug=0;
340 debugz=0;
341 debugm=0;
342 iprel=false;
343 userip=false;
344 realt=false;
345 realtime_refresh=3;
346 realtime_access_log_lines=1000;
347 cost=0.01;
348 nocost=50000000;
349 ndownload=0;
350 squid24=false;
351 dfrom=0;
352 duntil=0;
353
354 bzero(IncludeUsers, sizeof(IncludeUsers));
355 bzero(ExcludeString, sizeof(ExcludeString));
356 first_user_file=NULL;
357 memset(&period,0,sizeof(period));
358
359 NAccessLog=0;
360 for(x=0; x<MAXLOGS; x++)
361 AccessLog[x][0]='\0';
362 AccessLogFromCmdLine=0;
363 RedirectorLogFromCmdLine=0;
364
365 strcpy(Title,_("Squid User Access Report"));
366
367 while((ch = getopt_long_only(argc, argv, "a:b:c:d:e:f:g:u:l:L:o:s:t:w:hijmnprvxyz",long_options,&option_index)) != -1){
368 switch(ch)
369 {
370 case 0:
371 break;
372 case 2:
373 lastlog=atoi(optarg);
374 break;
375 case 3:
376 lastlog=0;
377 break;
378 case 'a':
379 strcpy(addr,optarg);
380 break;
381 case 'b':
382 strcpy(uagent,optarg);
383 break;
384 case 'c':
385 strcpy(hexclude,optarg);
386 break;
387 case 'd':
388 strncpy(date,optarg,sizeof(date)-1);
389 date[sizeof(date)-1]='\0';
390 date_from(date, &dfrom, &duntil);
391 break;
392 case 'e':
393 strcpy(email,optarg);
394 break;
395 case 'f':
396 strcpy(ConfigFile,optarg);
397 break;
398 case 'g':
399 strcpy(df,optarg);
400 break;
401 case 'h':
402 usage(argv[0]);
403 exit(EXIT_SUCCESS);
404 case 'i':
405 iprel=true;
406 break;
407 case 'l':
408 if (NAccessLog>=MAXLOGS) {
409 debuga(_("Too many log files passed on command line with option -l.\n"));
410 exit(EXIT_FAILURE);
411 }
412 if (strlen(optarg)>=MAX_LOG_FILELEN) {
413 debuga(_("Log file name too long passed on command line with option -l: %s\n"),optarg);
414 exit(EXIT_FAILURE);
415 }
416 strcpy(AccessLog[NAccessLog],optarg);
417 NAccessLog++;
418 AccessLogFromCmdLine++;
419 break;
420 case 'L':
421 if (NRedirectorLogs>MAX_REDIRECTOR_LOGS) {
422 debuga(_("Too many redirector logs passed on command line with option -L.\n"));
423 exit(EXIT_FAILURE);
424 }
425 if (strlen(optarg)>=MAX_REDIRECTOR_FILELEN) {
426 debuga(_("Redirector log file name too long passed on command line with opton -L: %s\n"),optarg);
427 exit(EXIT_FAILURE);
428 }
429 strcpy(RedirectorLogs[NRedirectorLogs],optarg);
430 NRedirectorLogs++;
431 RedirectorLogFromCmdLine++;
432 break;
433 case 'm':
434 debugm++;
435 break;
436 case 'n':
437 dns=true;
438 break;
439 case 'o':
440 strcpy(outdir,optarg);
441 break;
442 case 'p':
443 userip=true;
444 break;
445 case 'r':
446 realt=true;
447 break;
448 case 's':
449 strcpy(site,optarg);
450 break;
451 case 't':
452 {
453 int h1,m1,h2,m2;
454
455 if(strstr(optarg,"-") == 0) {
456 if(sscanf(optarg,"%d:%d",&h1,&m1)!=2) {
457 debuga(_("Time period passed on the command line with option -t must be MM:SS\n"));
458 exit(EXIT_FAILURE);
459 }
460 hm=h1*100+m1;
461 hmf=hm;
462 snprintf(hm_str,sizeof(hm_str),"%02d:%02d",h1,m1);
463 } else {
464 if(sscanf(optarg,"%d:%d-%d:%d",&h1,&m1,&h2,&m2)!=4) {
465 debuga(_("Time range passed on the command line with option -t must be MM:SS-MM:SS\n"));
466 exit(EXIT_FAILURE);
467 }
468 hm=h1*100+m1;
469 hmf=h2*100+m2;
470 snprintf(hm_str,sizeof(hm_str),"%02d:%02d-%02d:%02d",h1,m1,h2,m2);
471 }
472 break;
473 }
474 case 'u':
475 strcpy(us,optarg);
476 break;
477 case 'v':
478 version();
479 break;
480 case 'w':
481 strcpy(tmp,optarg);
482 break;
483 case 'x':
484 debug++;
485 break;
486 case 'y':
487 langcode++;
488 break;
489 case 'z':
490 debugz++;
491 break;
492 /*case ':':
493 debuga(_("Option -%c require an argument\n"),optopt);
494 errflg++;
495 break;*/
496 case '?':
497 usage(argv[0]);
498 exit(EXIT_FAILURE);
499 default:
500 abort();
501 }
502 }
503
504 if (errflg>0) {
505 usage(argv[0]);
506 exit(2);
507 }
508
509 if (optind<argc) {
510 for (iarq=optind ; iarq<argc ; iarq++) {
511 if (NAccessLog>=MAXLOGS) {
512 debuga(_("Too many log files passed on command line.\n"));
513 exit(EXIT_FAILURE);
514 }
515 if (strlen(argv[iarq])>=MAX_LOG_FILELEN) {
516 debuga(_("Log file name too long passed on command line: %s\n"),argv[iarq]);
517 exit(EXIT_FAILURE);
518 }
519 strcpy(AccessLog[NAccessLog],argv[iarq]);
520 NAccessLog++;
521 AccessLogFromCmdLine++;
522 }
523 }
524
525 if(debug) debuga(_("Init\n"));
526
527 if(ConfigFile[0] == '\0') snprintf(ConfigFile,sizeof(ConfigFile),"%s/sarg.conf",SYSCONFDIR);
528 if(access(ConfigFile, R_OK) != 0) {
529 debuga(_("Cannot open config file: %s - %s\n"),ConfigFile,strerror(errno));
530 exit(EXIT_FAILURE);
531 }
532
533 if(access(ConfigFile, R_OK) == 0)
534 getconf();
535
536 if(userip) UserIp=true;
537
538 if(dns) Ip2Name=true;
539
540 if (lastlog>=0) LastLog=lastlog;
541
542 if(realt) {
543 realtime();
544 exit(EXIT_SUCCESS);
545 }
546
547 if(IndexTree == INDEX_TREE_FILE)
548 strcpy(ImageFile,"../images");
549 else
550 strcpy(ImageFile,"../../../images");
551
552 dataonly=0;
553 if(DataFile[0] != '\0')
554 dataonly++;
555
556 if(NAccessLog == 0) {
557 strcpy(AccessLog[0],"/var/log/squid/access.log");
558 NAccessLog++;
559 }
560
561 if(output_css) {
562 css_content(stdout);
563 exit(EXIT_SUCCESS);
564 }
565 if(split) {
566 for (iarq=0 ; iarq<NAccessLog ; iarq++)
567 splitlog(AccessLog[iarq], df, dfrom, duntil, convert);
568 exit(EXIT_SUCCESS);
569 }
570 if(convert) {
571 for (iarq=0 ; iarq<NAccessLog ; iarq++)
572 convlog(AccessLog[iarq], df, dfrom, duntil);
573 exit(EXIT_SUCCESS);
574 }
575
576 load_excludecodes(ExcludeCodes);
577
578 if(access(PasswdFile, R_OK) == 0) {
579 getusers(PasswdFile,debug);
580 puser++;
581 }
582
583 if(hexclude[0] == '\0')
584 strcpy(hexclude,ExcludeHosts);
585 if(hexclude[0] != '\0') {
586 gethexclude(hexclude,debug);
587 fhost=true;
588 }
589
590 if(ReportType == 0) {
591 ReportType=REPORT_TYPE_TOPUSERS | REPORT_TYPE_TOPSITES | REPORT_TYPE_USERS_SITES |
592 REPORT_TYPE_SITES_USERS | REPORT_TYPE_DATE_TIME | REPORT_TYPE_DENIED |
593 REPORT_TYPE_AUTH_FAILURES | REPORT_TYPE_SITE_USER_TIME_DATE | REPORT_TYPE_DOWNLOADS;
594 }
595
596 if(access(ExcludeUsers, R_OK) == 0) {
597 getuexclude(ExcludeUsers,debug);
598 fuser=true;
599 }
600
601 indexonly=0;
602 if(fuser) {
603 if(is_indexonly())
604 indexonly++;
605 }
606 if(strcmp(ExcludeUsers,"indexonly") == 0) indexonly++;
607 if(Index == INDEX_ONLY) indexonly++;
608
609 if(MaxElapsed[0] != '\0') max_elapsed=atol(MaxElapsed);
610
611 if(outdir[0] == '\0') strcpy(outdir,OutputDir);
612 strcat(outdir,"/");
613
614 if(uagent[0] == '\0') strcpy(uagent,UserAgentLog);
615
616 if(tmp[0] == '\0') strcpy(tmp,TempDir);
617 else strcpy(TempDir,tmp);
618 /*
619 For historical reasons, the temporary directory is the subdirectory "sarg" of the path
620 provided by the user.
621 */
622 strcat(tmp,"/sarg");
623
624 if (tmp[0]!='\0' && strncmp(outdir,tmp,strlen(tmp))==0) {
625 debuga(_("The output directory \"%s\" must be outside of the temporary directory \"%s\"\n"),outdir,tmp);
626 exit(EXIT_FAILURE);
627 }
628
629 if(df[0] == '\0') strcpy(df,DateFormat);
630 else strcpy(DateFormat,df);
631
632 if(df[0] == '\0') {
633 strcpy(df,"u");
634 strcpy(DateFormat,"u");
635 }
636 if (df[0]=='w')
637 IndexTree=INDEX_TREE_FILE;
638
639 if(email[0] == '\0' && OutputEmail[0] != '\0') strcpy(email,OutputEmail);
640
641 if(email[0] != '\0') {
642 my_mkdir(tmp);
643 strcpy(outdir,tmp);
644 strcat(outdir,"/");
645 }
646
647 if(access(tmp, R_OK) == 0) {
648 unlinkdir(tmp,1);
649 }
650 my_mkdir(tmp);
651 snprintf(denied_unsort,sizeof(denied_unsort),"%s/denied.log.unsort",tmp);
652 snprintf(denied_sort,sizeof(denied_sort),"%s/denied.log",tmp);
653 snprintf(authfail_unsort,sizeof(authfail_unsort),"%s/authfail.log.unsort",tmp);
654
655 if(debug) {
656 debuga(_("Parameters:\n"));
657 debuga(_(" Hostname or IP address (-a) = %s\n"),addr);
658 debuga(_(" Useragent log (-b) = %s\n"),uagent);
659 debuga(_(" Exclude file (-c) = %s\n"),hexclude);
660 debuga(_(" Date from-until (-d) = %s\n"),date);
661 debuga(_(" Email address to send reports (-e) = %s\n"),email);
662 debuga(_(" Config file (-f) = %s\n"),ConfigFile);
663 if(strcmp(df,"e") == 0)
664 debuga(_(" Date format (-g) = Europe (dd/mm/yyyy)\n"));
665 if(strcmp(df,"u") == 0)
666 debuga(_(" Date format (-g) = USA (mm/dd/yyyy)\n"));
667 if(strcmp(df,"w") == 0)
668 debuga(_(" Date format (-g) = Sites & Users (yyyy/ww)\n"));
669 debuga(_(" IP report (-i) = %s\n"),(iprel) ? _("Yes") : _("No"));
670 for (iarq=0 ; iarq<NAccessLog ; iarq++)
671 debuga(_(" Input log (-l) = %s\n"),AccessLog[iarq]);
672 for (iarq=0 ; iarq<NRedirectorLogs ; iarq++)
673 debuga(_(" Redirector log (-L) = %s\n"),RedirectorLogs[iarq]);
674 debuga(_(" Resolve IP Address (-n) = %s\n"),(Ip2Name) ? _("Yes") : _("No"));
675 debuga(_(" Output dir (-o) = %s\n"),outdir);
676 debuga(_("Use Ip Address instead of userid (-p) = %s\n"),(UserIp) ? _("Yes") : _("No"));
677 debuga(_(" Accessed site (-s) = %s\n"),site);
678 debuga(_(" Time (-t) = %s\n"),hm_str);
679 debuga(_(" User (-u) = %s\n"),us);
680 debuga(_(" Temporary dir (-w) = %s\n"),tmp);
681 debuga(_(" Debug messages (-x) = %s\n"),(debug) ? _("Yes") : _("No"));
682 debuga(_(" Process messages (-z) = %s\n"),(debugz) ? _("Yes") : _("No"));
683 debuga(_(" Previous reports to keep (--lastlog) = %d\n"),LastLog);
684 debuga("\n");
685 }
686
687 if(debugm) {
688 printf(_("Parameters:\n"));
689 printf(_(" Hostname or IP address (-a) = %s\n"),addr);
690 printf(_(" Useragent log (-b) = %s\n"),uagent);
691 printf(_(" Exclude file (-c) = %s\n"),hexclude);
692 printf(_(" Date from-until (-d) = %s\n"),date);
693 printf(_(" Email address to send reports (-e) = %s\n"),email);
694 printf(_(" Config file (-f) = %s\n"),ConfigFile);
695 if(strcmp(df,"e") == 0)
696 printf(_(" Date format (-g) = Europe (dd/mm/yyyy)\n"));
697 if(strcmp(df,"u") == 0)
698 printf(_(" Date format (-g) = USA (mm/dd/yyyy)\n"));
699 if(strcmp(df,"w") == 0)
700 printf(_(" Date format (-g) = Sites & Users (yyyy/ww)\n"));
701 printf(_(" IP report (-i) = %s\n"),(iprel) ? _("Yes") : _("No"));
702 for (iarq=0 ; iarq<NAccessLog ; iarq++)
703 printf(_(" Input log (-l) = %s\n"),AccessLog[iarq]);
704 for (iarq=0 ; iarq<NRedirectorLogs ; iarq++)
705 printf(_(" Redirector log (-L) = %s\n"),RedirectorLogs[iarq]);
706 printf(_(" Resolve IP Address (-n) = %s\n"),(Ip2Name) ? _("Yes") : _("No"));
707 printf(_(" Output dir (-o) = %s\n"),outdir);
708 printf(_("Use Ip Address instead of userid (-p) = %s\n"),(UserIp) ? _("Yes") : _("No"));
709 printf(_(" Accessed site (-s) = %s\n"),site);
710 printf(_(" Time (-t) = %s\n"),hm_str);
711 printf(_(" User (-u) = %s\n"),us);
712 printf(_(" Temporary dir (-w) = %s\n"),tmp);
713 printf(_(" Debug messages (-x) = %s\n"),(debug) ? _("Yes") : _("No"));
714 printf(_(" Process messages (-z) = %s\n"),(debugz) ? _("Yes") : _("No"));
715 printf(_(" Previous reports to keep (--lastlog) = %d\n"),LastLog);
716 printf(_("sarg version: %s\n"),VERSION);
717 }
718
719 if(debug)
720 debuga(_("sarg version: %s\n"),VERSION);
721
722 #ifdef ENABLE_DOUBLE_CHECK_DATA
723 debuga(_("Sarg compiled to report warnings if the output is inconsistent\n"));
724 #endif
725
726 maxopenfiles=MAX_OPEN_USER_FILES;
727 #ifdef HAVE_RLIM_T
728 if (Ulimit[0] != '\0') {
729 struct rlimit rl;
730 long l1, l2;
731 int rc=0;
732
733 #if defined(RLIMIT_NOFILE)
734 getrlimit (RLIMIT_NOFILE, &rl);
735 #elif defined(RLIMIT_OFILE)
736 getrlimit (RLIMIT_OFILE, &rl);
737 #else
738 #warning "No rlimit resource for the number of open files"
739 #endif
740 l1 = rl.rlim_cur;
741 l2 = rl.rlim_max;
742
743 rl.rlim_cur = atol(Ulimit);
744 rl.rlim_max = atol(Ulimit);
745 #if defined(RLIMIT_NOFILE)
746 rc=setrlimit (RLIMIT_NOFILE, &rl);
747 #elif defined(RLIMIT_OFILE)
748 rc=setrlimit (RLIMIT_OFILE, &rl);
749 #else
750 #warning "No rlimit resource for the number of open files"
751 #endif
752 if(rc == -1) {
753 debuga(_("setrlimit error - %s\n"),strerror(errno));
754 }
755
756 if(debug)
757 debuga("Maximum file descriptor: cur=%ld max=%ld, changed to cur="RLIM_STRING" max="RLIM_STRING,l1,l2,rl.rlim_cur,rl.rlim_max);
758 }
759 #endif
760
761 init_usertab(UserTabFile);
762
763 if ((line=longline_create())==NULL) {
764 debuga(_("Not enough memory to read a log file\n"));
765 exit(EXIT_FAILURE);
766 }
767
768 snprintf(sz_Download_Unsort,sizeof(sz_Download_Unsort),"%s/download.unsort", tmp);
769
770 if(DataFile[0]=='\0') {
771 if((ReportType & REPORT_TYPE_DENIED) != 0) {
772 if((fp_denied=MY_FOPEN(denied_unsort,"w"))==NULL) {
773 debuga(_("(log) Cannot open file: %s - %s\n"),denied_unsort,strerror(errno));
774 exit(EXIT_FAILURE);
775 }
776 }
777
778 if((ReportType & REPORT_TYPE_DENIED) != 0 || (ReportType & REPORT_TYPE_AUTH_FAILURES) != 0) {
779 if((fp_authfail=MY_FOPEN(authfail_unsort,"w"))==NULL) {
780 debuga(_("(log) Cannot open file: %s - %s\n"),authfail_unsort,strerror(errno));
781 exit(EXIT_FAILURE);
782 }
783 }
784 }
785
786 for (iarq=0 ; iarq<NAccessLog ; iarq++) {
787 strcpy(arq,AccessLog[iarq]);
788
789 strcpy(arqtt,arq);
790
791 if(strcmp(arq,"-")==0) {
792 if(debug)
793 debuga(_("Reading access log file: from stdin\n"));
794 fp_in=stdin;
795 from_stdin=true;
796 } else {
797 if (date[0]!='\0') {
798 if (stat(arq,&logstat)!=0) {
799 debuga(_("Cannot get the modification time of input log file %s (%s). Processing it anyway\n"),arq,strerror(errno));
800 } else {
801 struct tm *logtime=localtime(&logstat.st_mtime);
802 if ((logtime->tm_year+1900)*10000+(logtime->tm_mon+1)*100+logtime->tm_mday<dfrom) {
803 debuga(_("Ignoring old log file %s\n"),arq);
804 continue;
805 }
806 }
807 }
808 fp_in=decomp(arq,&from_pipe);
809 if(fp_in==NULL) {
810 debuga(_("(log) Cannot open log file: %s - %s\n"),arq,strerror(errno));
811 exit(EXIT_FAILURE);
812 }
813 if(debug) debuga(_("Reading access log file: %s\n"),arq);
814 from_stdin=false;
815 }
816 ilf=ILF_Unknown;
817 download_flag=false;
818 // pre-read the file only if we have to show stats
819 if(ShowReadStatistics && !from_stdin && !from_pipe) {
820 size_t nread,i;
821 bool skipcr=false;
822 char tmp4[MAXLEN];
823
824 recs1=0UL;
825 recs2=0UL;
826
827 while ((nread=fread(tmp4,1,sizeof(tmp4),fp_in))>0) {
828 for (i=0 ; i<nread ; i++)
829 if (skipcr) {
830 if (tmp4[i]!='\n' && tmp4[i]!='\r') {
831 skipcr=false;
832 }
833 } else {
834 if (tmp4[i]=='\n' || tmp4[i]=='\r') {
835 skipcr=true;
836 recs1++;
837 }
838 }
839 }
840 rewind(fp_in);
841 printf(_("SARG: Records in file: %lu, reading: %3.2f%%"),recs1,(float) 0);
842 putchar('\r');
843 fflush( stdout ) ;
844 }
845
846 longline_reset(line);
847
848 while ((linebuf=longline_read(fp_in,line))!=NULL) {
849 blen=strlen(linebuf);
850
851 if (ilf==ILF_Unknown) {
852 if(strncmp(linebuf,"#Software: Mic",14) == 0) {
853 fixendofline(linebuf);
854 if (debug)
855 debuga(_("Log is from Microsoft ISA: %s\n"),linebuf);
856 ilf=ILF_Isa;
857 ilf_count[ilf]++;
858 continue;
859 }
860
861 if(strncmp(linebuf,"*** SARG Log ***",16) == 0) {
862 if (getperiod_fromsarglog(arqtt,&period)<0) {
863 debuga(_("The name of the file is invalid: %s\n"),arq);
864 exit(EXIT_FAILURE);
865 }
866 ilf=ILF_Sarg;
867 ilf_count[ilf]++;
868 continue;
869 }
870 }
871
872 if(!fp_log && strcmp(ParsedOutputLog, "no") != 0 && ilf!=ILF_Sarg) {
873 if(access(ParsedOutputLog,R_OK) != 0) {
874 my_mkdir(ParsedOutputLog);
875 }
876 sprintf(arq_log,"%s/sarg_temp.log",ParsedOutputLog);
877 if((fp_log=MY_FOPEN(arq_log,"w"))==NULL) {
878 debuga(_("(log) Cannot open log file: %s - %s\n"),arq_log,strerror(errno));
879 exit(EXIT_FAILURE);
880 }
881 fputs("*** SARG Log ***\n",fp_log);
882 }
883
884 recs2++;
885 if( ShowReadStatistics && !from_stdin && !from_pipe && --OutputNonZero<=0) {
886 double perc = recs2 * 100. / recs1 ;
887 printf(_("SARG: Records in file: %lu, reading: %3.2lf%%"),recs2,perc);
888 putchar('\r');
889 fflush (stdout);
890 OutputNonZero = REPORT_EVERY_X_LINES ;
891 }
892 if(blen < 58) continue;
893 if(strstr(linebuf,"HTTP/0.0") != 0) continue;
894 if(strstr(linebuf,"logfile turned over") != 0) continue;
895 if(linebuf[0] == ' ') continue;
896
897 // exclude_string
898 if(ExcludeString[0] != '\0') {
899 bool exstring=false;
900 getword_start(&gwarea,ExcludeString);
901 while(strchr(gwarea.current,':') != 0) {
902 if (getword_multisep(val1,sizeof(val1),&gwarea,':')<0) {
903 debuga(_("Maybe you have a broken record or garbage in your exclusion string\n"));
904 exit(EXIT_FAILURE);
905 }
906 if((str=(char *) strstr(linebuf,val1)) != (char *) NULL ) {
907 exstring=true;
908 break;
909 }
910 }
911 if(!exstring && (str=(char *) strstr(linebuf,gwarea.current)) != (char *) NULL )
912 exstring=true;
913 if(exstring) continue;
914 }
915
916 totregsl++;
917 if(debugm)
918 printf("BUF=%s\n",linebuf);
919
920 t=NULL;
921 if (ilf==ILF_Squid || ilf==ILF_Common || ilf==ILF_Unknown) {
922 getword_start(&gwarea,linebuf);
923 if (getword(data,sizeof(data),&gwarea,' ')<0) {
924 debuga(_("Maybe you have a broken time in your access.log file\n"));
925 exit(EXIT_FAILURE);
926 }
927 if((str=(char *) strchr(data, '.')) != (char *) NULL && (str=(char *) strchr(str+1, '.')) != (char *) NULL ) {
928 strcpy(ip,data);
929 strcpy(elap,"0");
930 if(squid24) {
931 if (getword(user,sizeof(user),&gwarea,' ')<0 || getword_skip(255,&gwarea,' ')<0) {
932 debuga(_("Maybe you have a broken record or garbage in your %s file\n"),arq);
933 exit(EXIT_FAILURE);
934 }
935 } else {
936 if (getword_skip(255,&gwarea,' ')<0 || getword(user,sizeof(user),&gwarea,' ')<0) {
937 debuga(_("Maybe you have a broken record or garbage in your %s file\n"),arq);
938 exit(EXIT_FAILURE);
939 }
940 }
941 if (getword(data,sizeof(data),&gwarea,']')<0 || getword_skip(MAXLEN,&gwarea,'"')<0 ||
942 getword(fun,sizeof(fun),&gwarea,' ')<0) {
943 debuga(_("Maybe you have a broken record or garbage in your %s file\n"),arq);
944 exit(EXIT_FAILURE);
945 }
946 if (getword_ptr(linebuf,&url,&gwarea,' ')<0) {
947 debuga(_("Maybe you have a broken url in your %s file\n"),arq);
948 exit(EXIT_FAILURE);
949 }
950 if (getword_skip(MAXLEN,&gwarea,' ')<0) {
951 debuga(_("Maybe you have a broken record or garbage in your %s file\n"),arq);
952 exit(EXIT_FAILURE);
953 }
954 if (getword(code2,sizeof(code2),&gwarea,' ')<0) {
955 debuga(_("Maybe you have a broken record or garbage in your %s file\n"),arq);
956 exit(EXIT_FAILURE);
957 }
958 if (getword(tam,sizeof(tam),&gwarea,' ')<0) {
959 debuga(_("Maybe you have a broken record or garbage in your %s file\n"),arq);
960 exit(EXIT_FAILURE);
961 }
962 if((str=(char *) strchr(gwarea.current, ' ')) != (char *) NULL ) {
963 if (getword(code,sizeof(code),&gwarea,' ')<0) {
964 debuga(_("Maybe you have a broken record or garbage in your %s file\n"),arq);
965 exit(EXIT_FAILURE);
966 }
967 } else {
968 if (getword(code,sizeof(code),&gwarea,'\0')<0) {
969 debuga(_("Maybe you have a broken record or garbage in your %s file\n"),arq);
970 exit(EXIT_FAILURE);
971 }
972 }
973
974 if ((str = strchr(code, ':')) != NULL)
975 *str = '/';
976
977 if(strcmp(tam,"\0") == 0)
978 strcpy(tam,"0");
979
980 ilf=ILF_Common;
981 ilf_count[ilf]++;
982
983 getword_start(&gwarea,data+1);
984 if (getword_multisep(data,sizeof(data),&gwarea,':')<0){
985 debuga(_("Maybe you have a broken date in your %s file\n"),arq);
986 exit(EXIT_FAILURE);
987 }
988 if (getword_multisep(hora,sizeof(hora),&gwarea,' ')<0){
989 debuga(_("Maybe you have a broken date in your %s file\n"),arq);
990 exit(EXIT_FAILURE);
991 }
992 getword_start(&gwarea,data);
993 if (getword_atoll(&iday,&gwarea,'/')<0){
994 debuga(_("Maybe you have a broken date in your %s file\n"),arq);
995 exit(EXIT_FAILURE);
996 }
997 if (getword(mes,sizeof(mes),&gwarea,'/')<0){
998 debuga(_("Maybe you have a broken date in your %s file\n"),arq);
999 exit(EXIT_FAILURE);
1000 }
1001 if (getword_atoll(&iyear,&gwarea,'/')<0){
1002 debuga(_("Maybe you have a broken date in your %s file\n"),arq);
1003 exit(EXIT_FAILURE);
1004 }
1005
1006 imonth=month2num(mes)+1;
1007 idata=builddia(iday,imonth,iyear);
1008 computedate(iyear,imonth,iday,&tt);
1009 t=&tt;
1010 }
1011
1012 if(ilf==ILF_Unknown || ilf==ILF_Squid) {
1013 if (getword(elap,sizeof(elap),&gwarea,' ')<0) {
1014 debuga(_("Maybe you have a broken elapsed time in your %s file\n"),arq);
1015 exit(EXIT_FAILURE);
1016 }
1017 while(strcmp(elap,"") == 0 && gwarea.current[0] != '\0')
1018 if (getword(elap,sizeof(elap),&gwarea,' ')<0) {
1019 debuga(_("Maybe you have a broken elapsed time in your %s file\n"),arq);
1020 exit(EXIT_FAILURE);
1021 }
1022 if(strlen(elap) < 1) continue;
1023 if (getword(ip,sizeof(ip),&gwarea,' ')<0){
1024 debuga(_("Maybe you have a broken client IP address in your %s file\n"),arq);
1025 exit(EXIT_FAILURE);
1026 }
1027 if (getword(code,sizeof(code),&gwarea,' ')<0){
1028 debuga(_("Maybe you have a broken result code in your %s file\n"),arq);
1029 exit(EXIT_FAILURE);
1030 }
1031 if (getword(tam,sizeof(tam),&gwarea,' ')<0){
1032 debuga(_("Maybe you have a broken amount of data in your %s file\n"),arq);
1033 exit(EXIT_FAILURE);
1034 }
1035 if (getword(fun,sizeof(fun),&gwarea,' ')<0){
1036 debuga(_("Maybe you have a broken request method in your %s file\n"),arq);
1037 exit(EXIT_FAILURE);
1038 }
1039 if (getword_ptr(linebuf,&url,&gwarea,' ')<0){
1040 debuga(_("Maybe you have a broken url in your %s file\n"),arq);
1041 exit(EXIT_FAILURE);
1042 }
1043 if (getword(user,sizeof(user),&gwarea,' ')<0){
1044 debuga(_("Maybe you have a broken user ID in your %s file\n"),arq);
1045 exit(EXIT_FAILURE);
1046 }
1047 ilf=ILF_Squid;
1048 ilf_count[ilf]++;
1049
1050 tnum=atoi(data);
1051 t=localtime(&tnum);
1052 if (t == NULL) {
1053 debuga(_("Cannot convert the timestamp from the squid log file\n"));
1054 exit(EXIT_FAILURE);
1055 }
1056
1057 strftime(tbuf2, sizeof(tbuf2), "%H%M", t);
1058
1059 idata=(t->tm_year+1900)*10000+(t->tm_mon+1)*100+t->tm_mday;
1060 }
1061 }
1062 if (ilf==ILF_Sarg) {
1063 getword_start(&gwarea,linebuf);
1064 if (getword(data,sizeof(data),&gwarea,'\t')<0){
1065 debuga(_("Maybe you have a broken record or garbage in your %s file\n"),arq);
1066 exit(EXIT_FAILURE);
1067 }
1068 if (getword(hora,sizeof(hora),&gwarea,'\t')<0) {
1069 debuga(_("Maybe you have a broken record or garbage in your %s file\n"),arq);
1070 exit(EXIT_FAILURE);
1071 }
1072 if (getword(user,sizeof(user),&gwarea,'\t')<0) {
1073 debuga(_("Maybe you have a broken record or garbage in your %s file\n"),arq);
1074 exit(EXIT_FAILURE);
1075 }
1076 if (getword(ip,sizeof(ip),&gwarea,'\t')<0) {
1077 debuga(_("Maybe you have a broken record or garbage in your %s file\n"),arq);
1078 exit(EXIT_FAILURE);
1079 }
1080 if (getword_ptr(linebuf,&url,&gwarea,'\t')<0){
1081 debuga(_("Maybe you have a broken record or garbage in your %s file\n"),arq);
1082 exit(EXIT_FAILURE);
1083 }
1084 if (getword(tam,sizeof(tam),&gwarea,'\t')<0){
1085 debuga(_("Maybe you have a broken record or garbage in your %s file\n"),arq);
1086 exit(EXIT_FAILURE);
1087 }
1088 if (getword(code,sizeof(code),&gwarea,'\t')<0){
1089 debuga(_("Maybe you have a broken record or garbage in your %s file\n"),arq);
1090 exit(EXIT_FAILURE);
1091 }
1092 if (getword(elap,sizeof(elap),&gwarea,'\t')<0){
1093 debuga(_("Maybe you have a broken record or garbage in your %s file\n"),arq);
1094 exit(EXIT_FAILURE);
1095 }
1096 if (getword(smartfilter,sizeof(smartfilter),&gwarea,'\0')<0){
1097 debuga(_("Maybe you have a broken record or garbage in your %s file\n"),arq);
1098 exit(EXIT_FAILURE);
1099 }
1100 getword_start(&gwarea,data);
1101 if (getword_atoll(&iday,&gwarea,'/')<0 || iday<1 || iday>31){
1102 debuga(_("Maybe you have a broken date in your %s file\n"),arq);
1103 exit(EXIT_FAILURE);
1104 }
1105 if (getword_atoll(&imonth,&gwarea,'/')<0 || imonth<1 || imonth>12){
1106 debuga(_("Maybe you have a broken date in your %s file\n"),arq);
1107 exit(EXIT_FAILURE);
1108 }
1109 if (getword_atoll(&iyear,&gwarea,'\0')<0){
1110 debuga(_("Maybe you have a broken date in your %s file\n"),arq);
1111 exit(EXIT_FAILURE);
1112 }
1113 idata=builddia(iday,imonth,iyear);
1114 computedate(iyear,imonth,iday,&tt);
1115 t=&tt;
1116 }
1117 if (ilf==ILF_Isa) {
1118 if (linebuf[0] == '#') {
1119 int ncols,cols[ISACOL_Last];
1120
1121 fixendofline(linebuf);
1122 getword_start(&gwarea,linebuf);
1123 // remove the #Fields: column at the beginning of the line
1124 if (getword_skip(1000,&gwarea,' ')<0){
1125 debuga(_("Maybe you have a broken record or garbage in your %s file\n"),arq);
1126 exit(EXIT_FAILURE);
1127 }
1128 for (ncols=0 ; ncols<ISACOL_Last ; ncols++) cols[ncols]=-1;
1129 ncols=0;
1130 while(gwarea.current[0] != '\0') {
1131 if (getword(val1,sizeof(val1),&gwarea,'\t')<0){
1132 debuga(_("Maybe you have a broken record or garbage in your %s file\n"),arq);
1133 exit(EXIT_FAILURE);
1134 }
1135 if(strcmp(val1,"c-ip") == 0) cols[ISACOL_Ip]=ncols;
1136 if(strcmp(val1,"cs-username") == 0) cols[ISACOL_UserName]=ncols;
1137 if(strcmp(val1,"date") == 0) cols[ISACOL_Date]=ncols;
1138 if(strcmp(val1,"time") == 0) cols[ISACOL_Time]=ncols;
1139 if(strcmp(val1,"time-taken") == 0) cols[ISACOL_TimeTaken]=ncols;
1140 if(strcmp(val1,"sc-bytes") == 0) cols[ISACOL_Bytes]=ncols;
1141 if(strcmp(val1,"cs-uri") == 0) cols[ISACOL_Uri]=ncols;
1142 if(strcmp(val1,"sc-status") == 0) cols[ISACOL_Status]=ncols;
1143 ncols++;
1144 }
1145 if (cols[ISACOL_Ip]>=0) {
1146 isa_ncols=ncols;
1147 for (ncols=0 ; ncols<ISACOL_Last ; ncols++)
1148 isa_cols[ncols]=cols[ncols];
1149 }
1150 continue;
1151 }
1152 if (!isa_ncols) continue;
1153 getword_start(&gwarea,linebuf);
1154 for (x=0 ; x<isa_ncols ; x++) {
1155 if (getword_ptr(linebuf,&str,&gwarea,'\t')<0) {
1156 debuga(_("Maybe you have a broken record or garbage in your %s file\n"),arq);
1157 exit(EXIT_FAILURE);
1158 }
1159 if (x==isa_cols[ISACOL_Ip]) {
1160 if (strlen(str)>=sizeof(ip)) {
1161 debuga(_("Maybe you have a broken IP in your %s file\n"),arq);
1162 exit(EXIT_FAILURE);
1163 }
1164 strcpy(ip,str);
1165 } else if (x==isa_cols[ISACOL_UserName]) {
1166 if (strlen(str)>=sizeof(user)) {
1167 debuga(_("Maybe you have a broken user ID in your %s file\n"),arq);
1168 exit(EXIT_FAILURE);
1169 }
1170 strcpy(user,str);
1171 } else if (x==isa_cols[ISACOL_Date]) {
1172 if (strlen(str)>=sizeof(data)) {
1173 debuga(_("Maybe you have a broken date in your %s file\n"),arq);
1174 exit(EXIT_FAILURE);
1175 }
1176 strcpy(data,str);
1177 } else if (x==isa_cols[ISACOL_Time]) {
1178 if (strlen(str)>=sizeof(hora)) {
1179 debuga(_("Maybe you have a broken time in your %s file\n"),arq);
1180 exit(EXIT_FAILURE);
1181 }
1182 strcpy(hora,str);
1183 } else if (x==isa_cols[ISACOL_TimeTaken]) {
1184 if (strlen(str)>=sizeof(elap)) {
1185 debuga(_("Maybe you have a broken download duration in your %s file\n"),arq);
1186 exit(EXIT_FAILURE);
1187 }
1188 strcpy(elap,str);
1189 } else if (x==isa_cols[ISACOL_Bytes]) {
1190 if (strlen(str)>=sizeof(tam)) {
1191 debuga(_("Maybe you have a broken download size in your %s file\n"),arq);
1192 exit(EXIT_FAILURE);
1193 }
1194 strcpy(tam,str);
1195 } else if (x==isa_cols[ISACOL_Uri]) {
1196 url=str;
1197 } else if (x==isa_cols[ISACOL_Status]) {
1198 if (strlen(str)>=sizeof(code)) {
1199 debuga(_("Maybe you have a broken access code in your %s file\n"),arq);
1200 exit(EXIT_FAILURE);
1201 }
1202 strcpy(code,str);
1203 }
1204 }
1205
1206 if(strcmp(code,"401") == 0 || strcmp(code,"403") == 0 || strcmp(code,"407") == 0) {
1207 sprintf(val1,"DENIED/%s",code);
1208 strcpy(code,val1);
1209 }
1210 getword_start(&gwarea,data);
1211 if (getword_atoll(&iyear,&gwarea,'-')<0){
1212 debuga(_("Maybe you have a broken year in your %s file\n"),arq);
1213 exit(EXIT_FAILURE);
1214 }
1215 if (getword_atoll(&imonth,&gwarea,'-')<0){
1216 debuga(_("Maybe you have a broken month in your %s file\n"),arq);
1217 exit(EXIT_FAILURE);
1218 }
1219 if (getword_atoll(&iday,&gwarea,'\0')<0){
1220 debuga(_("Maybe you have a broken day in your %s file\n"),arq);
1221 exit(EXIT_FAILURE);
1222 }
1223
1224 idata=builddia(iday,imonth,iyear);
1225 computedate(iyear,imonth,iday,&tt);
1226 t=&tt;
1227 }
1228 if (t==NULL) {
1229 debuga(_("Unknown input log file format\n"));
1230 break;
1231 }
1232
1233 strftime(dia, sizeof(dia), "%d/%m/%Y", t);
1234 snprintf(hora,sizeof(hora),"%02d:%02d:%02d",t->tm_hour,t->tm_min,t->tm_sec);
1235
1236 if(debugm)
1237 printf("DATE=%s IDATA=%d DFROM=%d DUNTIL=%d\n",date,idata,dfrom,duntil);
1238
1239 if(date[0] != '\0'){
1240 if(idata < dfrom || idata > duntil) continue;
1241 }
1242
1243 // Record only hours usage which is required
1244 if (t) {
1245 if( bsearch( &( t -> tm_wday ), weekdays.list, weekdays.len,
1246 sizeof( int ), compar ) == NULL )
1247 continue;
1248
1249 if( bsearch( &( t -> tm_hour ), hours.list, hours.len,
1250 sizeof( int ), compar ) == NULL )
1251 continue;
1252 }
1253
1254
1255 if(strlen(user) > MAX_USER_LEN) {
1256 if (debugm) printf(_("User ID too long: %s\n"),user);
1257 totregsx++;
1258 continue;
1259 }
1260
1261 // include_users
1262 if(IncludeUsers[0] != '\0') {
1263 sprintf(val1,":%s:",user);
1264 if((str=(char *) strstr(IncludeUsers,val1)) == (char *) NULL )
1265 continue;
1266 }
1267
1268 if(vercode(code)) {
1269 if (debugm) printf(_("Excluded code: %s\n"),code);
1270 totregsx++;
1271 continue;
1272 }
1273
1274 if(testvaliduserchar(user))
1275 continue;
1276
1277 #if 0
1278 if((str = strstr(user,"%20")) != NULL) {
1279 /*
1280 This is a patch introduced to solve bug #1624251 reported at sourceforge but
1281 the side effect is to truncate the name at the first space and merge the reports
1282 of people whose name is identical up to the first space.
1283
1284 The old code used to truncate the user name at the first % if a %20 was
1285 found anywhere in the string. That means the string could be truncated
1286 at the wrong place if another % occured before the %20. This new code should
1287 avoid that problem and only truncate at the space. There is no bug
1288 report indicating that anybody noticed this.
1289 */
1290 *str='\0';
1291 }
1292
1293 /*
1294 Code prior to 2.2.7 used to replace any %xx by a dot as long as a %5c was
1295 found in the user name.
1296 */
1297 while((str = strstr(user,"%5c")) != NULL) {
1298 *str='.';
1299 for (x=3 ; str[x] ; x++) str[x-2]=str[x];
1300 }
1301 #endif
1302
1303 urly=url;
1304
1305 if(ilf!=ILF_Sarg) {
1306 /*
1307 The full URL is not saved in sarg log. There is no point in testing the URL to detect
1308 a downloaded file.
1309 */
1310 download_flag=is_download_suffix(url);
1311 if (download_flag) {
1312 download_url=url;
1313 download_count++;
1314 }
1315 } else
1316 download_flag=false;
1317
1318 // remove any protocol:// at the beginning of the URL
1319 if ((str = strchr(url,'/')) != NULL && str[1] == '/') {
1320 int i;
1321
1322 str+=2;
1323 for (i=0 ; str[i] ; i++)
1324 url[i]=str[i];
1325 url[i]='\0';
1326 }
1327
1328 if(!LongUrl) {
1329 url_hostname(url,hostname,sizeof(hostname));
1330 url=hostname;
1331 }
1332
1333 if(url[0] == '\0') continue;
1334
1335 if(addr[0] != '\0'){
1336 if(strcmp(addr,ip)!=0) continue;
1337 }
1338 if(fhost) {
1339 if(!vhexclude(url)) {
1340 if (debugm) printf(_("Excluded site: %s\n"),url);
1341 totregsx++;
1342 continue;
1343 }
1344 }
1345
1346 if(hm >= 0 && hmf >= 0) {
1347 hmr=t->tm_hour*100+t->tm_min;
1348 if(hmr < hm || hmr > hmf) continue;
1349 }
1350
1351 if(site[0] != '\0'){
1352 if(strstr(url,site)==0) continue;
1353 }
1354
1355 if(UserIp) {
1356 strcpy(user,ip);
1357 id_is_ip=true;
1358 } else {
1359 id_is_ip=false;
1360 if(strcmp(user,"-") == 0 || strcmp(user," ") == 0 || strcmp(user,"") == 0) {
1361 if(RecordsWithoutUser == RECORDWITHOUTUSER_IP) {
1362 strcpy(user,ip);
1363 id_is_ip=true;
1364 }
1365 if(RecordsWithoutUser == RECORDWITHOUTUSER_IGNORE)
1366 continue;
1367 if(RecordsWithoutUser == RECORDWITHOUTUSER_EVERYBODY)
1368 strcpy(user,"everybody");
1369 } else {
1370 strlow(user);
1371 if(NtlmUserFormat == NTLMUSERFORMAT_USER) {
1372 if((str = strchr(user,'_')) != 0) {
1373 strcpy(warea,str+1);
1374 strcpy(user,warea);
1375 }
1376 if((str = strchr(user,'+')) != 0) {
1377 strcpy(warea,str+1);
1378 strcpy(user,warea);
1379 }
1380 }
1381 }
1382 }
1383
1384 if(us[0] != '\0'){
1385 if(strcmp(user,us)!=0) continue;
1386 }
1387
1388 if(puser) {
1389 sprintf(wuser,":%s:",user);
1390 if(strstr(userfile, wuser) == 0)
1391 continue;
1392 }
1393
1394 if(fuser) {
1395 if(!vuexclude(user)) {
1396 if (debugm) printf(_("Excluded user: %s\n"),user);
1397 totregsx++;
1398 continue;
1399 }
1400 }
1401
1402 if(strcmp(user,"-") ==0 || strcmp(user," ") ==0 || strcmp(user,"") ==0 || strcmp(user,":") ==0)
1403 continue;
1404
1405 if(max_elapsed) {
1406 if(atol(elap)>max_elapsed) {
1407 elap[0]='0';
1408 elap[1]='\0';
1409 }
1410 }
1411
1412 if((str=(char *) strstr(linebuf, "[SmartFilter:")) != (char *) NULL ) {
1413 fixendofline(str);
1414 sprintf(smartfilter,"\"%s\"",str+1);
1415 } else sprintf(smartfilter,"\"\"");
1416
1417 nopen=0;
1418 prev_ufile=NULL;
1419 for (ufile=first_user_file ; ufile && strcmp(user,ufile->user->id)!=0 ; ufile=ufile->next) {
1420 prev_ufile=ufile;
1421 if (ufile->file) nopen++;
1422 }
1423 if (!ufile) {
1424 ufile=malloc(sizeof(*ufile));
1425 if (!ufile) {
1426 debuga(_("Not enough memory to store the user %s\n"),user);
1427 exit(EXIT_FAILURE);
1428 }
1429 memset(ufile,0,sizeof(*ufile));
1430 ufile->next=first_user_file;
1431 first_user_file=ufile;
1432 uinfo=userinfo_create(user);
1433 ufile->user=uinfo;
1434 uinfo->id_is_ip=id_is_ip;
1435 } else {
1436 if (prev_ufile) {
1437 prev_ufile->next=ufile->next;
1438 ufile->next=first_user_file;
1439 first_user_file=ufile;
1440 }
1441 }
1442
1443 if (ufile->file==NULL) {
1444 if (nopen>=maxopenfiles) {
1445 x=0;
1446 for (ufile1=first_user_file ; ufile1 ; ufile1=ufile1->next) {
1447 if (ufile1->file!=NULL) {
1448 if (x>=maxopenfiles) {
1449 if (fclose(ufile1->file)==EOF) {
1450 debuga(_("Failed to close the log file of user %s - %s\n"),ufile1->user->id,strerror(errno));
1451 exit(EXIT_FAILURE);
1452 }
1453 ufile1->file=NULL;
1454 }
1455 x++;
1456 }
1457 }
1458 }
1459 if (snprintf (tmp3, sizeof(tmp3), "%s/%s.unsort", tmp, ufile->user->filename)>=sizeof(tmp3)) {
1460 debuga(_("Temporary user file name too long: %s/%s.unsort\n"), tmp, ufile->user->filename);
1461 exit(EXIT_FAILURE);
1462 }
1463 if ((ufile->file = MY_FOPEN (tmp3, "a")) == NULL) {
1464 debuga(_("(log) Cannot open temporary file: %s - %s\n"), tmp3, strerror(errno));
1465 exit (1);
1466 }
1467 }
1468
1469 /*if ( strcmp ( user , sz_Last_User ) != 0 ) {
1470 if ( fp_Write_User )
1471 fclose( fp_Write_User ) ;
1472 sprintf (tmp3, "%s/%s.unsort", tmp, user);
1473
1474 if ((fp_Write_User = MY_FOPEN (tmp3, "a")) == NULL) {
1475 fprintf (stderr, "%s: (log) %s: %s - %s\n", argv[0], _("Cannot open temporary file"), tmp3, strerror(errno));
1476 exit (1);
1477 }
1478 strcpy( sz_Last_User , user ) ;
1479 }*/
1480 if (fprintf(ufile->file, "%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\n",dia,hora,ip,url,tam,code,elap,smartfilter)<=0) {
1481 debuga(_("Write error in the log file of user %s\n"),user);
1482 exit(EXIT_FAILURE);
1483 }
1484
1485 if(fp_log && ilf!=ILF_Sarg)
1486 fprintf(fp_log, "%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\n",dia,hora,user,ip,url,tam,code,elap,smartfilter);
1487
1488 totregsg++;
1489
1490 if(!dataonly && download_flag && download_url && strstr(code,"DENIED") == 0) {
1491 ndownload = 1;
1492
1493 if ( ! fp_Download_Unsort ) {
1494 if ((fp_Download_Unsort = MY_FOPEN ( sz_Download_Unsort, "a")) == NULL) {
1495 debuga(_("(log) Cannot open temporary file: %s - %s\n"),sz_Download_Unsort, strerror(errno));
1496 exit (1);
1497 }
1498 }
1499 fprintf(fp_Download_Unsort,"%s\t%s\t%s\t%s\t%s\n",dia,hora,user,ip,download_url);
1500 }
1501
1502 if((ReportType & REPORT_TYPE_DENIED) != 0 || (ReportType & REPORT_TYPE_AUTH_FAILURES) != 0) {
1503 if(fp_denied && strstr(code,"DENIED/403") != 0) {
1504 fprintf(fp_denied, "%s\t%s\t%s\t%s\t%s\n",dia,hora,user,ip,urly);
1505 denied_count++;
1506 }
1507 if(fp_authfail && (strstr(code,"DENIED/401") != 0 || strstr(code,"DENIED/407") != 0)) {
1508 fprintf(fp_authfail, "%s\t%s\t%s\t%s\t%s\n",dia,hora,user,ip,urly);
1509 authfail_count++;
1510 }
1511 }
1512
1513 if (ilf!=ILF_Sarg) {
1514 if(!totper || idata<mindate){
1515 mindate=idata;
1516 memcpy(&period.start,t,sizeof(*t));
1517 strcpy(start_hour,tbuf2);
1518 }
1519 if (!totper || idata>maxdate) {
1520 maxdate=idata;
1521 memcpy(&period.end,t,sizeof(*t));
1522 }
1523 totper=true;
1524 }
1525
1526 if(debugm){
1527 printf("IP=\t%s\n",ip);
1528 printf("USER=\t%s\n",user);
1529 printf("ELAP=\t%s\n",elap);
1530 printf("DATE=\t%s\n",dia);
1531 printf("TIME=\t%s\n",hora);
1532 printf("FUNC=\t%s\n",fun);
1533 printf("URL=\t%s\n",url);
1534 printf("CODE=\t%s\n",code);
1535 printf("LEN=\t%s\n",tam);
1536 }
1537 }
1538 if (!from_stdin) {
1539 if (from_pipe)
1540 pclose(fp_in);
1541 else {
1542 fclose(fp_in);
1543 if( ShowReadStatistics )
1544 printf(_("SARG: Records in file: %lu, reading: %3.2f%%\n"),recs1, (float) 100 );
1545 }
1546 }
1547 }
1548
1549 if (debug)
1550 debuga(_(" Records read: %ld, written: %ld, excluded: %ld\n"),totregsl,totregsg,totregsx);
1551
1552 longline_destroy(&line);
1553 if ( fp_Download_Unsort )
1554 fclose (fp_Download_Unsort);
1555
1556 for (ufile=first_user_file ; ufile ; ufile=ufile1) {
1557 ufile1=ufile->next;
1558 if (ufile->file!=NULL) fclose(ufile->file);
1559 free(ufile);
1560 }
1561
1562 free_download();
1563 free_excludecodes();
1564 free_exclude();
1565
1566 if(debug) {
1567 int totalcount=0;
1568
1569 for (ilf=0 ; ilf<ILF_Last ; ilf++) totalcount+=ilf_count[ilf];
1570
1571 if(ilf_count[ILF_Common]>0 && ilf_count[ILF_Squid]>0)
1572 debuga(_("Log with mixed records format (squid and common log)\n"));
1573
1574 if(ilf_count[ILF_Common]>0 && ilf_count[ILF_Squid]==0)
1575 debuga(_("Common log format\n"));
1576
1577 if(ilf_count[ILF_Common]==0 && ilf_count[ILF_Squid]>0)
1578 debuga(_("Squid log format\n"));
1579
1580 if(ilf_count[ILF_Sarg]>0)
1581 debuga(_("Sarg log format\n"));
1582
1583 if(totalcount==0 && totregsg)
1584 debuga(_("Log with invalid format\n"));
1585 }
1586
1587 if(!totregsg){
1588 debuga(_("No records found\n"));
1589 debuga(_("End\n"));
1590 if(fp_denied) fclose(fp_denied);
1591 if(fp_authfail) fclose(fp_authfail);
1592 userinfo_free();
1593 if(userfile) free(userfile);
1594 close_usertab();
1595 exit(EXIT_SUCCESS);
1596 }
1597
1598 if (date[0]!='\0') {
1599 char date0[30], date1[30];
1600
1601 strftime(date0,sizeof(date0),"%d/%m/%Y",&period.start);
1602 strftime(date1,sizeof(date1),"%d/%m/%Y",&period.end);
1603 debuga(_("Period covered by log files: %s-%s\n"),date0,date1);
1604 getperiod_fromrange(&period,dfrom,duntil);
1605 }
1606 if (getperiod_buildtext(&period)<0) {
1607 debuga(_("Failed to build the string representation of the date range\n"));
1608 exit(EXIT_FAILURE);
1609 }
1610
1611 if(debugz){
1612 debugaz("data",dia);
1613 debugaz("period",period.text);
1614 }
1615
1616 if(debug)
1617 debuga(_("Period: %s\n"),period.text);
1618
1619 // fclose(fp_ou);
1620 if(fp_denied)
1621 fclose(fp_denied);
1622 if(fp_authfail)
1623 fclose(fp_authfail);
1624
1625 if(fp_log != NULL) {
1626 fclose(fp_log);
1627 strcpy(end_hour,tbuf2);
1628 strftime(val2,sizeof(val2),"%d%m%Y",&period.start);
1629 strftime(val1,sizeof(val1),"%d%m%Y",&period.end);
1630 sprintf(val4,"%s/sarg-%s_%s-%s_%s.log",ParsedOutputLog,val2,start_hour,val1,end_hour);
1631 if (rename(arq_log,val4)) {
1632 debuga(_("failed to rename %s to %s - %s\n"),arq_log,val4,strerror(errno));
1633 } else {
1634 strcpy(arq_log,val4);
1635
1636 if(strcmp(ParsedOutputLogCompress,"nocompress") != 0 && ParsedOutputLogCompress[0] != '\0') {
1637 /*
1638 No double quotes around ParsedOutputLogCompress because it may contain command line options. If double quotes are
1639 necessary around the command name, put them in the configuration file.
1640 */
1641 sprintf(val1,"%s \"%s\"",ParsedOutputLogCompress,arq_log);
1642 cstatus=system(val1);
1643 if (!WIFEXITED(cstatus) || WEXITSTATUS(cstatus)) {
1644 debuga(_("command return status %d\n"),WEXITSTATUS(cstatus));
1645 debuga(_("command: %s\n"),val1);
1646 exit(EXIT_FAILURE);
1647 }
1648 }
1649 }
1650 if(debug)
1651 debuga(_("Sarg parsed log saved as %s\n"),arq_log);
1652 }
1653
1654 if(DataFile[0] == '\0' && (ReportType & REPORT_TYPE_DENIED) != 0) {
1655 sprintf(csort,"sort -T \"%s\" -k 3,3 -k 5,5 -o \"%s\" \"%s\"",tmp,denied_sort,denied_unsort);
1656 cstatus=system(csort);
1657 if (!WIFEXITED(cstatus) || WEXITSTATUS(cstatus)) {
1658 debuga(_("sort command return status %d\n"),WEXITSTATUS(cstatus));
1659 debuga(_("sort command: %s\n"),csort);
1660 exit(EXIT_FAILURE);
1661 }
1662 unlink(denied_unsort);
1663 }
1664
1665 sort_users_log(tmp, debug);
1666
1667 if(DataFile[0] != '\0')
1668 data_file(tmp);
1669 else
1670 gerarel();
1671
1672 if((ReportType & REPORT_TYPE_DENIED) != 0)
1673 unlink(denied_sort);
1674
1675 if(strcmp(tmp,"/tmp") != 0) {
1676 unlinkdir(tmp,0);
1677 }
1678
1679 userinfo_free();
1680 if(userfile)
1681 free(userfile);
1682 close_usertab();
1683
1684 if(debug)
1685 debuga(_("End\n"));
1686
1687 exit(EXIT_SUCCESS);
1688 }
1689
1690
1691 static void getusers(const char *pwdfile, int debug)
1692 {
1693 FILE *fp_usr;
1694 char buf[255];
1695 char *str;
1696 long int nreg=0;
1697
1698 if(debug)
1699 debuga(_("Loading password file from %s\n"),pwdfile);
1700
1701 if ((fp_usr = fopen(pwdfile, "r")) == NULL) {
1702 debuga(_("(getusers) Cannot open file %s - %s\n"),pwdfile,strerror(errno));
1703 exit(EXIT_FAILURE);
1704 }
1705
1706 if (fseek(fp_usr, 0, SEEK_END)==-1) {
1707 debuga(_("Failed to move till the end of the users file %s: %s\n"),pwdfile,strerror(errno));
1708 exit(EXIT_FAILURE);
1709 }
1710 nreg = ftell(fp_usr);
1711 if (nreg<0) {
1712 debuga(_("Cannot get the size of file %s\n"),pwdfile);
1713 exit(EXIT_FAILURE);
1714 }
1715 nreg = nreg+5000;
1716 if (fseek(fp_usr, 0, SEEK_SET)==-1) {
1717 debuga(_("Failed to rewind the users file %s: %s\n"),pwdfile,strerror(errno));
1718 exit(EXIT_FAILURE);
1719 }
1720
1721 if((userfile=(char *) malloc(nreg))==NULL){
1722 debuga(_("malloc error (%ld)\n"),nreg);
1723 exit(EXIT_FAILURE);
1724 }
1725
1726 bzero(userfile,nreg);
1727 strcpy(userfile,":");
1728
1729 while(fgets(buf,sizeof(buf),fp_usr)!=NULL) {
1730 str=strchr(buf,':');
1731 if (!str) {
1732 debuga(_("You have an invalid user in your %s file\n"),pwdfile);
1733 exit(EXIT_FAILURE);
1734 }
1735 str[1]='\0';
1736 strcat(userfile,buf);
1737 }
1738
1739 fclose(fp_usr);
1740
1741 return;
1742 }
Unnamed repository; edit this file 'description' to name the repository.