]> git.ipfire.org Git - thirdparty/sarg.git/blob - log.c
projects / thirdparty / sarg.git / blob
? search:
summary | shortlog | log | commit | commitdiff | tree
blame | history | raw | HEAD
Don't try to produce a parsed log if parsed_output_log is none
[thirdparty/sarg.git] / log.c
1 /*
2 * SARG Squid Analysis Report Generator http://sarg.sourceforge.net
3 * 1998, 2011
4 *
5 * SARG donations:
6 * please look at http://sarg.sourceforge.net/donations.php
7 * Support:
8 * http://sourceforge.net/projects/sarg/forums/forum/363374
9 * ---------------------------------------------------------------------
10 *
11 * This program is free software; you can redistribute it and/or modify
12 * it under the terms of the GNU General Public License as published by
13 * the Free Software Foundation; either version 2 of the License, or
14 * (at your option) any later version.
15 *
16 * This program is distributed in the hope that it will be useful,
17 * but WITHOUT ANY WARRANTY; without even the implied warranty of
18 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
19 * GNU General Public License for more details.
20 *
21 * You should have received a copy of the GNU General Public License
22 * along with this program; if not, write to the Free Software
23 * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111, USA.
24 *
25 */
26
27 #include "include/conf.h"
28 #include "include/defs.h"
29
30 #ifdef HAVE_GETOPT_H
31 #include <getopt.h>
32 #endif
33
34 #define REPORT_EVERY_X_LINES 5000
35 #define MAX_OPEN_USER_FILES 10
36
37 struct userfilestruct
38 {
39 struct userfilestruct *next;
40 struct userinfostruct *user;
41 FILE *file;
42 };
43
44 /*@null@*/static char *userfile=NULL;
45
46 numlist weekdays = { { 0, 1, 2, 3, 4, 5, 6 }, 7 };
47 numlist hours = { { 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23 }, 24 };
48
49 static void getusers(const char *pwdfile, int debug);
50
51 int main(int argc,char *argv[])
52 {
53 enum isa_col_id {
54 ISACOL_Ip,
55 ISACOL_UserName,
56 ISACOL_Date,
57 ISACOL_Time,
58 ISACOL_TimeTaken,
59 ISACOL_Bytes,
60 ISACOL_Uri,
61 ISACOL_Status,
62 ISACOL_Last //last entry of the list !
63 };
64 enum InputLogFormat {
65 ILF_Unknown,
66 ILF_Squid,
67 ILF_Common,
68 ILF_Sarg,
69 ILF_Isa,
70 ILF_Last //last entry of the list !
71 };
72
73 FILE *fp_in = NULL, *fp_denied=NULL, *fp_authfail=NULL, *fp_log=NULL;
74
75 char sz_Download_Unsort[ 20000 ] ;
76 FILE * fp_Download_Unsort = NULL ;
77
78 extern int optind;
79 extern int optopt;
80 extern char *optarg;
81
82 char data[255];
83 char elap[255];
84 char ip[MAXLEN];
85 char tam[255];
86 char fun[MAXLEN];
87 char wuser[MAXLEN];
88 char smartfilter[MAXLEN];
89 char dia[128];
90 char mes[30];
91 char ano[30];
92 char hora[30];
93 char wtemp[MAXLEN];
94 char date[255];
95 char arq[255];
96 char arq_log[255];
97 int hm, hmf, hmr;
98 char hm_str[15];
99 char uagent[MAXLEN];
100 char hexclude[MAXLEN];
101 char csort[MAXLEN];
102 int cstatus;
103 char tbuf2[128];
104 char zip[20];
105 char *str;
106 char tmp3[MAXLEN];
107 char denied_unsort[MAXLEN];
108 char denied_sort[MAXLEN];
109 char authfail_unsort[MAXLEN];
110 char start_hour[128];
111 char end_hour[128];
112 char *linebuf;
113 char hostname[512];
114 char *url;
115 char *urly;
116 char user[MAX_USER_LEN];
117 enum InputLogFormat ilf;
118 int ilf_count[ILF_Last];
119 int ch;
120 int x;
121 int errflg=0;
122 int puser=0;
123 bool fhost=false;
124 bool dns=false;
125 bool fuser=false;
126 int idata=0;
127 int mindate=0;
128 int maxdate=0;
129 int iarq=0;
130 int isa_ncols=0,isa_cols[ISACOL_Last];
131 int lastlog=-1;
132 bool from_stdin;
133 bool from_pipe;
134 int blen;
135 int maxopenfiles;
136 int nopen;
137 bool id_is_ip;
138 long totregsl=0;
139 long totregsg=0;
140 long totregsx=0;
141 bool totper=false;
142 long int max_elapsed=0;
143 long long int iyear, imonth, iday;
144 bool realt;
145 bool userip;
146 struct tm tt;
147 struct tm *t;
148 unsigned long recs1=0UL;
149 unsigned long recs2=0UL;
150 int OutputNonZero = REPORT_EVERY_X_LINES ;
151 bool download_flag=false;
152 char *download_url=NULL;
153 struct getwordstruct gwarea;
154 longline line;
155 time_t tnum;
156 struct stat logstat;
157 struct userinfostruct *uinfo;
158 struct userfilestruct *first_user_file, *ufile, *ufile1, *prev_ufile;
159 static int split=0;
160 static int convert=0;
161 static int output_css=0;
162 int option_index;
163 static struct option long_options[]=
164 {
165 {"convert",no_argument,&convert,1},
166 {"css",no_argument,&output_css,1},
167 {"lastlog",required_argument,NULL,2},
168 {"keeplogs",no_argument,NULL,3},
169 {"split",no_argument,&split,1},
170 {0,0,0,0}
171 };
172
173 #ifdef HAVE_LOCALE_H
174 setlocale(LC_TIME,"");
175 #endif
176
177 #if defined(ENABLE_NLS) && defined(HAVE_LOCALE_H)
178 if (!setlocale (LC_ALL, "")) {
179 fprintf(stderr,"SARG: Cannot set the locale LC_ALL to the environment variable\n");
180 exit(EXIT_FAILURE);
181 }
182 if (!bindtextdomain (PACKAGE_NAME, LOCALEDIR)) {
183 fprintf(stderr,"SARG: Cannot bind to text domain %s in directory %s (%s)\n",PACKAGE_NAME,LOCALEDIR,strerror(errno));
184 exit(EXIT_FAILURE);
185 }
186 if (!textdomain (PACKAGE_NAME)) {
187 fprintf(stderr,"SARG: Cannot set gettext domain for %s PACKAGE_NAME (%s)\n",PACKAGE_NAME,strerror(errno));
188 exit(EXIT_FAILURE);
189 }
190 #endif //ENABLE_NLS
191
192 BgImage[0]='\0';
193 LogoImage[0]='\0';
194 LogoText[0]='\0';
195 PasswdFile[0]='\0';
196 OutputEmail[0]='\0';
197 UserAgentLog[0]='\0';
198 ExcludeHosts[0]='\0';
199 ExcludeUsers[0]='\0';
200 ConfigFile[0]='\0';
201 code[0]='\0';
202 LastLog=0;
203 ReportType=0UL;
204 UserTabFile[0]='\0';
205 BlockIt[0]='\0';
206 ExternalCSSFile[0]='\0';
207 RedirectorLogFormat[0]='\0';
208 NRedirectorLogs=0;
209 for (ilf=0 ; ilf<ILF_Last ; ilf++) ilf_count[ilf]=0;
210
211 sprintf(ExcludeCodes,"%s/exclude_codes",SYSCONFDIR);
212 strcpy(GraphDaysBytesBarColor,"orange");
213 strcpy(BgColor,"#ffffff");
214 strcpy(TxColor,"#000000");
215 strcpy(TxBgColor,"lavender");
216 strcpy(TiColor,"darkblue");
217 strcpy(Width,"80");
218 strcpy(Height,"45");
219 strcpy(LogoTextColor,"#000000");
220 strcpy(HeaderColor,"darkblue");
221 strcpy(HeaderBgColor,"#dddddd");
222 strcpy(LogoTextColor,"#006699");
223 strcpy(FontSize,"9px");
224 strcpy(TempDir,"/tmp");
225 strcpy(OutputDir,"/var/www/html/squid-reports");
226 Ip2Name=false;
227 strcpy(DateFormat,"u");
228 OverwriteReport=false;
229 RemoveTempFiles=true;
230 strcpy(ReplaceIndex,"index.html");
231 Index=INDEX_YES;
232 RecordsWithoutUser=RECORDWITHOUTUSER_IP;
233 UseComma=0;
234 strcpy(MailUtility,"mailx");
235 TopSitesNum=100;
236 TopUsersNum=0;
237 UserIp=0;
238 TopuserSort=TOPUSER_SORT_BYTES | TOPUSER_SORT_REVERSE;
239 UserSort=USER_SORT_BYTES | USER_SORT_REVERSE;
240 TopsitesSort=TOPSITE_SORT_CONNECT | TOPSITE_SORT_REVERSE;
241 LongUrl=0;
242 strcpy(FontFace,"Verdana,Tahoma,Arial");
243 datetimeby=DATETIME_BYTE;
244 strcpy(CharSet,"ISO-8859-1");
245 Privacy=0;
246 strcpy(PrivacyString,"***.***.***.***");
247 strcpy(PrivacyStringColor,"blue");
248 SuccessfulMsg=true;
249 TopUserFields=TOPUSERFIELDS_NUM | TOPUSERFIELDS_DATE_TIME | TOPUSERFIELDS_USERID | TOPUSERFIELDS_CONNECT |
250 TOPUSERFIELDS_BYTES | TOPUSERFIELDS_SETYB | TOPUSERFIELDS_IN_CACHE_OUT |
251 TOPUSERFIELDS_USED_TIME | TOPUSERFIELDS_MILISEC | TOPUSERFIELDS_PTIME |
252 TOPUSERFIELDS_TOTAL | TOPUSERFIELDS_AVERAGE;
253 UserReportFields=USERREPORTFIELDS_CONNECT | USERREPORTFIELDS_BYTES | USERREPORTFIELDS_SETYB |
254 USERREPORTFIELDS_IN_CACHE_OUT | USERREPORTFIELDS_USED_TIME | USERREPORTFIELDS_MILISEC |
255 USERREPORTFIELDS_PTIME | USERREPORTFIELDS_TOTAL | USERREPORTFIELDS_AVERAGE;
256 strcpy(DataFileDelimiter,";");
257 DataFileFields=DATA_FIELD_USER | DATA_FIELD_DATE | DATA_FIELD_TIME | DATA_FIELD_URL | DATA_FIELD_CONNECT |
258 DATA_FIELD_BYTES | DATA_FIELD_IN_CACHE | DATA_FIELD_OUT_CACHE | DATA_FIELD_ELAPSED;
259 ShowReadStatistics=true;
260 strcpy(IndexSortOrder,"D");
261 ShowSargInfo=true;
262 ShowSargLogo=true;
263 ParsedOutputLog[0]='\0';
264 strcpy(ParsedOutputLogCompress,"/bin/gzip -f");
265 DisplayedValues=DISPLAY_ABBREV;
266 strcpy(HeaderFontSize,"9px");
267 strcpy(TitleFontSize,"11px");
268 strcpy(AuthUserTemplateFile,"sarg_htaccess");
269 set_download_suffix("7z,ace,arj,avi,bat,bin,bz2,bzip,cab,com,cpio,dll,doc,dot,exe,gz,iso,lha,lzh,mdb,mov,mp3,mpeg,mpg,mso,nrg,ogg,ppt,rar,rtf,shs,src,sys,tar,tgz,vcd,vob,wma,wmv,zip");
270 Graphs=true;
271 #if defined(FONTDIR)
272 strcpy(GraphFont,FONTDIR"/DejaVuSans.ttf");
273 #else
274 GraphFont[0]='\0';
275 #endif
276 strcpy(Ulimit,"20000");
277 NtlmUserFormat=NTLMUSERFORMAT_DOMAINUSER;
278 IndexTree=INDEX_TREE_FILE;
279 IndexFields=INDEXFIELDS_DIRSIZE;
280 strcpy(RealtimeTypes,"GET,PUT,CONNECT");
281 RealtimeUnauthRec=REALTIME_UNAUTH_REC_SHOW;
282 RedirectorFilterOutDate=true;
283 DansguardianFilterOutDate=true;
284 DataFileUrl=DATAFILEURL_IP;
285 strcpy(MaxElapsed,"28800000");
286 BytesInSitesUsersReport=0;
287 UserAuthentication=0;
288 strcpy(LDAPHost,"127.0.0.1");
289 LDAPPort=389;
290 LDAPProtocolVersion=3;
291 LDAPBindDN[0]='\0';
292 LDAPBindPW[0]='\0';
293 LDAPBaseSearch[0]='\0';
294 strcpy(LDAPFilterSearch, "(uid=%s)");
295 strcpy(LDAPTargetAttr, "cn");
296 SortTableJs[0]='\0';
297
298 dia[0]='\0';
299 mes[0]='\0';
300 ano[0]='\0';
301 hora[0]='\0';
302 tmp[0]='\0';
303 tmp3[0]='\0';
304 wtemp[0]='\0';
305 us[0]='\0';
306 date[0]='\0';
307 df[0]='\0';
308 uagent[0]='\0';
309 hexclude[0]='\0';
310 addr[0]='\0';
311 hm=-1;
312 hmf=-1;
313 site[0]='\0';
314 outdir[0]='\0';
315 elap[0]='\0';
316 email[0]='\0';
317 zip[0]='\0';
318 UserInvalidChar[0]='\0';
319 DataFile[0]='\0';
320 SquidGuardConf[0]='\0';
321 DansGuardianConf[0]='\0';
322 start_hour[0]='\0';
323 end_hour[0]='\0';
324 hm_str[0]='\0';
325
326 denied_count=0;
327 download_count=0;
328 authfail_count=0;
329 dansguardian_count=0;
330 squidguard_count=0;
331 useragent_count=0;
332 DeniedReportLimit=10;
333 AuthfailReportLimit=10;
334 DansGuardianReportLimit=10;
335 SquidGuardReportLimit=10;
336 DownloadReportLimit=50;
337 UserReportLimit=0;
338 debug=0;
339 debugz=0;
340 debugm=0;
341 iprel=false;
342 userip=false;
343 realt=false;
344 realtime_refresh=3;
345 realtime_access_log_lines=1000;
346 cost=0.01;
347 nocost=50000000;
348 ndownload=0;
349 squid24=false;
350 dfrom=0;
351 duntil=0;
352
353 bzero(IncludeUsers, sizeof(IncludeUsers));
354 bzero(ExcludeString, sizeof(ExcludeString));
355 first_user_file=NULL;
356 memset(&period,0,sizeof(period));
357
358 NAccessLog=0;
359 for(x=0; x<MAXLOGS; x++)
360 AccessLog[x][0]='\0';
361 AccessLogFromCmdLine=0;
362 RedirectorLogFromCmdLine=0;
363
364 strcpy(Title,_("Squid User Access Report"));
365
366 while((ch = getopt_long_only(argc, argv, "a:b:c:d:e:f:g:u:l:L:o:s:t:w:hijmnprvxyz",long_options,&option_index)) != -1){
367 switch(ch)
368 {
369 case 0:
370 break;
371 case 2:
372 lastlog=atoi(optarg);
373 break;
374 case 3:
375 lastlog=0;
376 break;
377 case 'a':
378 strcpy(addr,optarg);
379 break;
380 case 'b':
381 strcpy(uagent,optarg);
382 break;
383 case 'c':
384 strcpy(hexclude,optarg);
385 break;
386 case 'd':
387 strncpy(date,optarg,sizeof(date)-1);
388 date[sizeof(date)-1]='\0';
389 date_from(date, &dfrom, &duntil);
390 break;
391 case 'e':
392 strcpy(email,optarg);
393 break;
394 case 'f':
395 strcpy(ConfigFile,optarg);
396 break;
397 case 'g':
398 strcpy(df,optarg);
399 break;
400 case 'h':
401 usage(argv[0]);
402 exit(EXIT_SUCCESS);
403 case 'i':
404 iprel=true;
405 break;
406 case 'l':
407 if (NAccessLog>=MAXLOGS) {
408 debuga(_("Too many log files passed on command line with option -l.\n"));
409 exit(EXIT_FAILURE);
410 }
411 if (strlen(optarg)>=MAX_LOG_FILELEN) {
412 debuga(_("Log file name too long passed on command line with option -l: %s\n"),optarg);
413 exit(EXIT_FAILURE);
414 }
415 strcpy(AccessLog[NAccessLog],optarg);
416 NAccessLog++;
417 AccessLogFromCmdLine++;
418 break;
419 case 'L':
420 if (NRedirectorLogs>MAX_REDIRECTOR_LOGS) {
421 debuga(_("Too many redirector logs passed on command line with option -L.\n"));
422 exit(EXIT_FAILURE);
423 }
424 if (strlen(optarg)>=MAX_REDIRECTOR_FILELEN) {
425 debuga(_("Redirector log file name too long passed on command line with opton -L: %s\n"),optarg);
426 exit(EXIT_FAILURE);
427 }
428 strcpy(RedirectorLogs[NRedirectorLogs],optarg);
429 NRedirectorLogs++;
430 RedirectorLogFromCmdLine++;
431 break;
432 case 'm':
433 debugm++;
434 break;
435 case 'n':
436 dns=true;
437 break;
438 case 'o':
439 strcpy(outdir,optarg);
440 break;
441 case 'p':
442 userip=true;
443 break;
444 case 'r':
445 realt=true;
446 break;
447 case 's':
448 strcpy(site,optarg);
449 break;
450 case 't':
451 {
452 int h1,m1,h2,m2;
453
454 if(strstr(optarg,"-") == 0) {
455 if(sscanf(optarg,"%d:%d",&h1,&m1)!=2) {
456 debuga(_("Time period passed on the command line with option -t must be MM:SS\n"));
457 exit(EXIT_FAILURE);
458 }
459 hm=h1*100+m1;
460 hmf=hm;
461 snprintf(hm_str,sizeof(hm_str),"%02d:%02d",h1,m1);
462 } else {
463 if(sscanf(optarg,"%d:%d-%d:%d",&h1,&m1,&h2,&m2)!=4) {
464 debuga(_("Time range passed on the command line with option -t must be MM:SS-MM:SS\n"));
465 exit(EXIT_FAILURE);
466 }
467 hm=h1*100+m1;
468 hmf=h2*100+m2;
469 snprintf(hm_str,sizeof(hm_str),"%02d:%02d-%02d:%02d",h1,m1,h2,m2);
470 }
471 break;
472 }
473 case 'u':
474 strcpy(us,optarg);
475 break;
476 case 'v':
477 version();
478 break;
479 case 'w':
480 strcpy(tmp,optarg);
481 break;
482 case 'x':
483 debug++;
484 break;
485 case 'y':
486 langcode++;
487 break;
488 case 'z':
489 debugz++;
490 break;
491 case ':':
492 debuga(_("Option -%c requires an argument\n"),optopt);
493 exit(EXIT_FAILURE);
494 case '?':
495 usage(argv[0]);
496 exit(EXIT_FAILURE);
497 default:
498 abort();
499 }
500 }
501
502 if (errflg>0) {
503 usage(argv[0]);
504 exit(2);
505 }
506
507 if (optind<argc) {
508 for (iarq=optind ; iarq<argc ; iarq++) {
509 if (NAccessLog>=MAXLOGS) {
510 debuga(_("Too many log files passed on command line.\n"));
511 exit(EXIT_FAILURE);
512 }
513 if (strlen(argv[iarq])>=MAX_LOG_FILELEN) {
514 debuga(_("Log file name too long passed on command line: %s\n"),argv[iarq]);
515 exit(EXIT_FAILURE);
516 }
517 strcpy(AccessLog[NAccessLog],argv[iarq]);
518 NAccessLog++;
519 AccessLogFromCmdLine++;
520 }
521 }
522
523 if(debug) debuga(_("Init\n"));
524
525 if(ConfigFile[0] == '\0') snprintf(ConfigFile,sizeof(ConfigFile),"%s/sarg.conf",SYSCONFDIR);
526 if(access(ConfigFile, R_OK) != 0) {
527 debuga(_("Cannot open config file: %s - %s\n"),ConfigFile,strerror(errno));
528 exit(EXIT_FAILURE);
529 }
530
531 if(access(ConfigFile, R_OK) == 0)
532 getconf();
533
534 if(userip) UserIp=true;
535
536 if(dns) Ip2Name=true;
537
538 if (lastlog>=0) LastLog=lastlog;
539
540 if(realt) {
541 realtime();
542 exit(EXIT_SUCCESS);
543 }
544
545 if(IndexTree == INDEX_TREE_FILE)
546 strcpy(ImageFile,"../images");
547 else
548 strcpy(ImageFile,"../../../images");
549
550 dataonly=0;
551 if(DataFile[0] != '\0')
552 dataonly++;
553
554 if(NAccessLog == 0) {
555 strcpy(AccessLog[0],"/var/log/squid/access.log");
556 NAccessLog++;
557 }
558
559 if(output_css) {
560 css_content(stdout);
561 exit(EXIT_SUCCESS);
562 }
563 if(split) {
564 for (iarq=0 ; iarq<NAccessLog ; iarq++)
565 splitlog(AccessLog[iarq], df, dfrom, duntil, convert);
566 exit(EXIT_SUCCESS);
567 }
568 if(convert) {
569 for (iarq=0 ; iarq<NAccessLog ; iarq++)
570 convlog(AccessLog[iarq], df, dfrom, duntil);
571 exit(EXIT_SUCCESS);
572 }
573
574 load_excludecodes(ExcludeCodes);
575
576 if(access(PasswdFile, R_OK) == 0) {
577 getusers(PasswdFile,debug);
578 puser++;
579 }
580
581 if(hexclude[0] == '\0')
582 strcpy(hexclude,ExcludeHosts);
583 if(hexclude[0] != '\0') {
584 gethexclude(hexclude,debug);
585 fhost=true;
586 }
587
588 if(ReportType == 0) {
589 ReportType=REPORT_TYPE_TOPUSERS | REPORT_TYPE_TOPSITES | REPORT_TYPE_USERS_SITES |
590 REPORT_TYPE_SITES_USERS | REPORT_TYPE_DATE_TIME | REPORT_TYPE_DENIED |
591 REPORT_TYPE_AUTH_FAILURES | REPORT_TYPE_SITE_USER_TIME_DATE | REPORT_TYPE_DOWNLOADS;
592 }
593
594 if(access(ExcludeUsers, R_OK) == 0) {
595 getuexclude(ExcludeUsers,debug);
596 fuser=true;
597 }
598
599 indexonly=0;
600 if(fuser) {
601 if(is_indexonly())
602 indexonly++;
603 }
604 if(strcmp(ExcludeUsers,"indexonly") == 0) indexonly++;
605 if(Index == INDEX_ONLY) indexonly++;
606
607 if(MaxElapsed[0] != '\0') max_elapsed=atol(MaxElapsed);
608
609 if(outdir[0] == '\0') strcpy(outdir,OutputDir);
610 if(outdir[0] != '\0') strcat(outdir,"/");
611
612 if(uagent[0] == '\0') strcpy(uagent,UserAgentLog);
613
614 if(tmp[0] == '\0') strcpy(tmp,TempDir);
615 else strcpy(TempDir,tmp);
616 /*
617 For historical reasons, the temporary directory is the subdirectory "sarg" of the path
618 provided by the user.
619 */
620 strcat(tmp,"/sarg");
621
622 if (tmp[0]!='\0' && strncmp(outdir,tmp,strlen(tmp))==0) {
623 debuga(_("The output directory \"%s\" must be outside of the temporary directory \"%s\"\n"),outdir,tmp);
624 exit(EXIT_FAILURE);
625 }
626
627 if(df[0] == '\0') strcpy(df,DateFormat);
628 else strcpy(DateFormat,df);
629
630 if(df[0] == '\0') {
631 strcpy(df,"u");
632 strcpy(DateFormat,"u");
633 }
634 if (df[0]=='w')
635 IndexTree=INDEX_TREE_FILE;
636
637 if(email[0] == '\0' && OutputEmail[0] != '\0') strcpy(email,OutputEmail);
638
639 if(email[0] != '\0') {
640 my_mkdir(tmp);
641 strcpy(outdir,tmp);
642 strcat(outdir,"/");
643 }
644
645 if(access(tmp, R_OK) == 0) {
646 unlinkdir(tmp,1);
647 }
648 my_mkdir(tmp);
649 snprintf(denied_unsort,sizeof(denied_unsort),"%s/denied.log.unsort",tmp);
650 snprintf(denied_sort,sizeof(denied_sort),"%s/denied.log",tmp);
651 snprintf(authfail_unsort,sizeof(authfail_unsort),"%s/authfail.log.unsort",tmp);
652
653 if(debug) {
654 debuga(_("Parameters:\n"));
655 debuga(_(" Hostname or IP address (-a) = %s\n"),addr);
656 debuga(_(" Useragent log (-b) = %s\n"),uagent);
657 debuga(_(" Exclude file (-c) = %s\n"),hexclude);
658 debuga(_(" Date from-until (-d) = %s\n"),date);
659 debuga(_(" Email address to send reports (-e) = %s\n"),email);
660 debuga(_(" Config file (-f) = %s\n"),ConfigFile);
661 if(strcmp(df,"e") == 0)
662 debuga(_(" Date format (-g) = Europe (dd/mm/yyyy)\n"));
663 if(strcmp(df,"u") == 0)
664 debuga(_(" Date format (-g) = USA (mm/dd/yyyy)\n"));
665 if(strcmp(df,"w") == 0)
666 debuga(_(" Date format (-g) = Sites & Users (yyyy/ww)\n"));
667 debuga(_(" IP report (-i) = %s\n"),(iprel) ? _("Yes") : _("No"));
668 for (iarq=0 ; iarq<NAccessLog ; iarq++)
669 debuga(_(" Input log (-l) = %s\n"),AccessLog[iarq]);
670 for (iarq=0 ; iarq<NRedirectorLogs ; iarq++)
671 debuga(_(" Redirector log (-L) = %s\n"),RedirectorLogs[iarq]);
672 debuga(_(" Resolve IP Address (-n) = %s\n"),(Ip2Name) ? _("Yes") : _("No"));
673 debuga(_(" Output dir (-o) = %s\n"),outdir);
674 debuga(_("Use Ip Address instead of userid (-p) = %s\n"),(UserIp) ? _("Yes") : _("No"));
675 debuga(_(" Accessed site (-s) = %s\n"),site);
676 debuga(_(" Time (-t) = %s\n"),hm_str);
677 debuga(_(" User (-u) = %s\n"),us);
678 debuga(_(" Temporary dir (-w) = %s\n"),tmp);
679 debuga(_(" Debug messages (-x) = %s\n"),(debug) ? _("Yes") : _("No"));
680 debuga(_(" Process messages (-z) = %s\n"),(debugz) ? _("Yes") : _("No"));
681 debuga(_(" Previous reports to keep (--lastlog) = %d\n"),LastLog);
682 debuga("\n");
683 }
684
685 if(debugm) {
686 printf(_("Parameters:\n"));
687 printf(_(" Hostname or IP address (-a) = %s\n"),addr);
688 printf(_(" Useragent log (-b) = %s\n"),uagent);
689 printf(_(" Exclude file (-c) = %s\n"),hexclude);
690 printf(_(" Date from-until (-d) = %s\n"),date);
691 printf(_(" Email address to send reports (-e) = %s\n"),email);
692 printf(_(" Config file (-f) = %s\n"),ConfigFile);
693 if(strcmp(df,"e") == 0)
694 printf(_(" Date format (-g) = Europe (dd/mm/yyyy)\n"));
695 if(strcmp(df,"u") == 0)
696 printf(_(" Date format (-g) = USA (mm/dd/yyyy)\n"));
697 if(strcmp(df,"w") == 0)
698 printf(_(" Date format (-g) = Sites & Users (yyyy/ww)\n"));
699 printf(_(" IP report (-i) = %s\n"),(iprel) ? _("Yes") : _("No"));
700 for (iarq=0 ; iarq<NAccessLog ; iarq++)
701 printf(_(" Input log (-l) = %s\n"),AccessLog[iarq]);
702 for (iarq=0 ; iarq<NRedirectorLogs ; iarq++)
703 printf(_(" Redirector log (-L) = %s\n"),RedirectorLogs[iarq]);
704 printf(_(" Resolve IP Address (-n) = %s\n"),(Ip2Name) ? _("Yes") : _("No"));
705 printf(_(" Output dir (-o) = %s\n"),outdir);
706 printf(_("Use Ip Address instead of userid (-p) = %s\n"),(UserIp) ? _("Yes") : _("No"));
707 printf(_(" Accessed site (-s) = %s\n"),site);
708 printf(_(" Time (-t) = %s\n"),hm_str);
709 printf(_(" User (-u) = %s\n"),us);
710 printf(_(" Temporary dir (-w) = %s\n"),tmp);
711 printf(_(" Debug messages (-x) = %s\n"),(debug) ? _("Yes") : _("No"));
712 printf(_(" Process messages (-z) = %s\n"),(debugz) ? _("Yes") : _("No"));
713 printf(_(" Previous reports to keep (--lastlog) = %d\n"),LastLog);
714 printf(_("sarg version: %s\n"),VERSION);
715 }
716
717 if(debug)
718 debuga(_("sarg version: %s\n"),VERSION);
719
720 #ifdef ENABLE_DOUBLE_CHECK_DATA
721 debuga(_("Sarg compiled to report warnings if the output is inconsistent\n"));
722 #endif
723
724 maxopenfiles=MAX_OPEN_USER_FILES;
725 #ifdef HAVE_RLIM_T
726 if (Ulimit[0] != '\0') {
727 struct rlimit rl;
728 long l1, l2;
729 int rc=0;
730
731 #if defined(RLIMIT_NOFILE)
732 getrlimit (RLIMIT_NOFILE, &rl);
733 #elif defined(RLIMIT_OFILE)
734 getrlimit (RLIMIT_OFILE, &rl);
735 #else
736 #warning "No rlimit resource for the number of open files"
737 #endif
738 l1 = rl.rlim_cur;
739 l2 = rl.rlim_max;
740
741 rl.rlim_cur = atol(Ulimit);
742 rl.rlim_max = atol(Ulimit);
743 #if defined(RLIMIT_NOFILE)
744 rc=setrlimit (RLIMIT_NOFILE, &rl);
745 #elif defined(RLIMIT_OFILE)
746 rc=setrlimit (RLIMIT_OFILE, &rl);
747 #else
748 #warning "No rlimit resource for the number of open files"
749 #endif
750 if(rc == -1) {
751 debuga(_("setrlimit error - %s\n"),strerror(errno));
752 }
753
754 if(debug)
755 debuga("Maximum file descriptor: cur=%ld max=%ld, changed to cur="RLIM_STRING" max="RLIM_STRING,l1,l2,rl.rlim_cur,rl.rlim_max);
756 }
757 #endif
758
759 init_usertab(UserTabFile);
760
761 if ((line=longline_create())==NULL) {
762 debuga(_("Not enough memory to read a log file\n"));
763 exit(EXIT_FAILURE);
764 }
765
766 snprintf(sz_Download_Unsort,sizeof(sz_Download_Unsort),"%s/download.unsort", tmp);
767
768 if(DataFile[0]=='\0') {
769 if((ReportType & REPORT_TYPE_DENIED) != 0) {
770 if((fp_denied=MY_FOPEN(denied_unsort,"w"))==NULL) {
771 debuga(_("(log) Cannot open file: %s - %s\n"),denied_unsort,strerror(errno));
772 exit(EXIT_FAILURE);
773 }
774 }
775
776 if((ReportType & REPORT_TYPE_DENIED) != 0 || (ReportType & REPORT_TYPE_AUTH_FAILURES) != 0) {
777 if((fp_authfail=MY_FOPEN(authfail_unsort,"w"))==NULL) {
778 debuga(_("(log) Cannot open file: %s - %s\n"),authfail_unsort,strerror(errno));
779 exit(EXIT_FAILURE);
780 }
781 }
782 }
783
784 for (iarq=0 ; iarq<NAccessLog ; iarq++) {
785 strcpy(arq,AccessLog[iarq]);
786
787 strcpy(arqtt,arq);
788
789 if(strcmp(arq,"-")==0) {
790 if(debug)
791 debuga(_("Reading access log file: from stdin\n"));
792 fp_in=stdin;
793 from_stdin=true;
794 } else {
795 if (date[0]!='\0') {
796 if (stat(arq,&logstat)!=0) {
797 debuga(_("Cannot get the modification time of input log file %s (%s). Processing it anyway\n"),arq,strerror(errno));
798 } else {
799 struct tm *logtime=localtime(&logstat.st_mtime);
800 if ((logtime->tm_year+1900)*10000+(logtime->tm_mon+1)*100+logtime->tm_mday<dfrom) {
801 debuga(_("Ignoring old log file %s\n"),arq);
802 continue;
803 }
804 }
805 }
806 fp_in=decomp(arq,&from_pipe);
807 if(fp_in==NULL) {
808 debuga(_("(log) Cannot open log file: %s - %s\n"),arq,strerror(errno));
809 exit(EXIT_FAILURE);
810 }
811 if(debug) debuga(_("Reading access log file: %s\n"),arq);
812 from_stdin=false;
813 }
814 ilf=ILF_Unknown;
815 download_flag=false;
816 // pre-read the file only if we have to show stats
817 if(ShowReadStatistics && !from_stdin && !from_pipe) {
818 size_t nread,i;
819 bool skipcr=false;
820 char tmp4[MAXLEN];
821
822 recs1=0UL;
823 recs2=0UL;
824
825 while ((nread=fread(tmp4,1,sizeof(tmp4),fp_in))>0) {
826 for (i=0 ; i<nread ; i++)
827 if (skipcr) {
828 if (tmp4[i]!='\n' && tmp4[i]!='\r') {
829 skipcr=false;
830 }
831 } else {
832 if (tmp4[i]=='\n' || tmp4[i]=='\r') {
833 skipcr=true;
834 recs1++;
835 }
836 }
837 }
838 rewind(fp_in);
839 printf(_("SARG: Records in file: %lu, reading: %3.2f%%"),recs1,(float) 0);
840 putchar('\r');
841 fflush( stdout ) ;
842 }
843
844 longline_reset(line);
845
846 while ((linebuf=longline_read(fp_in,line))!=NULL) {
847 blen=strlen(linebuf);
848
849 if (ilf==ILF_Unknown) {
850 if(strncmp(linebuf,"#Software: Mic",14) == 0) {
851 fixendofline(linebuf);
852 if (debug)
853 debuga(_("Log is from Microsoft ISA: %s\n"),linebuf);
854 ilf=ILF_Isa;
855 ilf_count[ilf]++;
856 continue;
857 }
858
859 if(strncmp(linebuf,"*** SARG Log ***",16) == 0) {
860 if (getperiod_fromsarglog(arqtt,&period)<0) {
861 debuga(_("The name of the file is invalid: %s\n"),arq);
862 exit(EXIT_FAILURE);
863 }
864 ilf=ILF_Sarg;
865 ilf_count[ilf]++;
866 continue;
867 }
868 }
869
870 if(!fp_log && ParsedOutputLog[0] && ilf!=ILF_Sarg) {
871 if(access(ParsedOutputLog,R_OK) != 0) {
872 my_mkdir(ParsedOutputLog);
873 }
874 sprintf(arq_log,"%s/sarg_temp.log",ParsedOutputLog);
875 if((fp_log=MY_FOPEN(arq_log,"w"))==NULL) {
876 debuga(_("(log) Cannot open log file: %s - %s\n"),arq_log,strerror(errno));
877 exit(EXIT_FAILURE);
878 }
879 fputs("*** SARG Log ***\n",fp_log);
880 }
881
882 recs2++;
883 if( ShowReadStatistics && !from_stdin && !from_pipe && --OutputNonZero<=0) {
884 double perc = recs2 * 100. / recs1 ;
885 printf(_("SARG: Records in file: %lu, reading: %3.2lf%%"),recs2,perc);
886 putchar('\r');
887 fflush (stdout);
888 OutputNonZero = REPORT_EVERY_X_LINES ;
889 }
890 if(blen < 58) continue;
891 if(strstr(linebuf,"HTTP/0.0") != 0) continue;
892 if(strstr(linebuf,"logfile turned over") != 0) continue;
893 if(linebuf[0] == ' ') continue;
894
895 // exclude_string
896 if(ExcludeString[0] != '\0') {
897 bool exstring=false;
898 getword_start(&gwarea,ExcludeString);
899 while(strchr(gwarea.current,':') != 0) {
900 if (getword_multisep(val1,sizeof(val1),&gwarea,':')<0) {
901 debuga(_("Maybe you have a broken record or garbage in your exclusion string\n"));
902 exit(EXIT_FAILURE);
903 }
904 if((str=(char *) strstr(linebuf,val1)) != (char *) NULL ) {
905 exstring=true;
906 break;
907 }
908 }
909 if(!exstring && (str=(char *) strstr(linebuf,gwarea.current)) != (char *) NULL )
910 exstring=true;
911 if(exstring) continue;
912 }
913
914 totregsl++;
915 if(debugm)
916 printf("BUF=%s\n",linebuf);
917
918 t=NULL;
919 if (ilf==ILF_Squid || ilf==ILF_Common || ilf==ILF_Unknown) {
920 getword_start(&gwarea,linebuf);
921 if (getword(data,sizeof(data),&gwarea,' ')<0) {
922 debuga(_("Maybe you have a broken time in your access.log file\n"));
923 exit(EXIT_FAILURE);
924 }
925 if((str=(char *) strchr(data, '.')) != (char *) NULL && (str=(char *) strchr(str+1, '.')) != (char *) NULL ) {
926 strcpy(ip,data);
927 strcpy(elap,"0");
928 if(squid24) {
929 if (getword(user,sizeof(user),&gwarea,' ')<0 || getword_skip(255,&gwarea,' ')<0) {
930 debuga(_("Maybe you have a broken record or garbage in your %s file\n"),arq);
931 exit(EXIT_FAILURE);
932 }
933 } else {
934 if (getword_skip(255,&gwarea,' ')<0 || getword(user,sizeof(user),&gwarea,' ')<0) {
935 debuga(_("Maybe you have a broken record or garbage in your %s file\n"),arq);
936 exit(EXIT_FAILURE);
937 }
938 }
939 if (getword(data,sizeof(data),&gwarea,']')<0 || getword_skip(MAXLEN,&gwarea,'"')<0 ||
940 getword(fun,sizeof(fun),&gwarea,' ')<0) {
941 debuga(_("Maybe you have a broken record or garbage in your %s file\n"),arq);
942 exit(EXIT_FAILURE);
943 }
944 if (getword_ptr(linebuf,&url,&gwarea,' ')<0) {
945 debuga(_("Maybe you have a broken url in your %s file\n"),arq);
946 exit(EXIT_FAILURE);
947 }
948 if (getword_skip(MAXLEN,&gwarea,' ')<0) {
949 debuga(_("Maybe you have a broken record or garbage in your %s file\n"),arq);
950 exit(EXIT_FAILURE);
951 }
952 if (getword(code2,sizeof(code2),&gwarea,' ')<0) {
953 debuga(_("Maybe you have a broken record or garbage in your %s file\n"),arq);
954 exit(EXIT_FAILURE);
955 }
956 if (getword(tam,sizeof(tam),&gwarea,' ')<0) {
957 debuga(_("Maybe you have a broken record or garbage in your %s file\n"),arq);
958 exit(EXIT_FAILURE);
959 }
960 if((str=(char *) strchr(gwarea.current, ' ')) != (char *) NULL ) {
961 if (getword(code,sizeof(code),&gwarea,' ')<0) {
962 debuga(_("Maybe you have a broken record or garbage in your %s file\n"),arq);
963 exit(EXIT_FAILURE);
964 }
965 } else {
966 if (getword(code,sizeof(code),&gwarea,'\0')<0) {
967 debuga(_("Maybe you have a broken record or garbage in your %s file\n"),arq);
968 exit(EXIT_FAILURE);
969 }
970 }
971
972 if ((str = strchr(code, ':')) != NULL)
973 *str = '/';
974
975 if(strcmp(tam,"\0") == 0)
976 strcpy(tam,"0");
977
978 ilf=ILF_Common;
979 ilf_count[ilf]++;
980
981 getword_start(&gwarea,data+1);
982 if (getword_multisep(data,sizeof(data),&gwarea,':')<0){
983 debuga(_("Maybe you have a broken date in your %s file\n"),arq);
984 exit(EXIT_FAILURE);
985 }
986 if (getword_multisep(hora,sizeof(hora),&gwarea,' ')<0){
987 debuga(_("Maybe you have a broken date in your %s file\n"),arq);
988 exit(EXIT_FAILURE);
989 }
990 getword_start(&gwarea,data);
991 if (getword_atoll(&iday,&gwarea,'/')<0){
992 debuga(_("Maybe you have a broken date in your %s file\n"),arq);
993 exit(EXIT_FAILURE);
994 }
995 if (getword(mes,sizeof(mes),&gwarea,'/')<0){
996 debuga(_("Maybe you have a broken date in your %s file\n"),arq);
997 exit(EXIT_FAILURE);
998 }
999 if (getword_atoll(&iyear,&gwarea,'/')<0){
1000 debuga(_("Maybe you have a broken date in your %s file\n"),arq);
1001 exit(EXIT_FAILURE);
1002 }
1003
1004 imonth=month2num(mes)+1;
1005 idata=builddia(iday,imonth,iyear);
1006 computedate(iyear,imonth,iday,&tt);
1007 t=&tt;
1008 }
1009
1010 if(ilf==ILF_Unknown || ilf==ILF_Squid) {
1011 if (getword(elap,sizeof(elap),&gwarea,' ')<0) {
1012 debuga(_("Maybe you have a broken elapsed time in your %s file\n"),arq);
1013 exit(EXIT_FAILURE);
1014 }
1015 while(strcmp(elap,"") == 0 && gwarea.current[0] != '\0')
1016 if (getword(elap,sizeof(elap),&gwarea,' ')<0) {
1017 debuga(_("Maybe you have a broken elapsed time in your %s file\n"),arq);
1018 exit(EXIT_FAILURE);
1019 }
1020 if(strlen(elap) < 1) continue;
1021 if (getword(ip,sizeof(ip),&gwarea,' ')<0){
1022 debuga(_("Maybe you have a broken client IP address in your %s file\n"),arq);
1023 exit(EXIT_FAILURE);
1024 }
1025 if (getword(code,sizeof(code),&gwarea,' ')<0){
1026 debuga(_("Maybe you have a broken result code in your %s file\n"),arq);
1027 exit(EXIT_FAILURE);
1028 }
1029 if (getword(tam,sizeof(tam),&gwarea,' ')<0){
1030 debuga(_("Maybe you have a broken amount of data in your %s file\n"),arq);
1031 exit(EXIT_FAILURE);
1032 }
1033 if (getword(fun,sizeof(fun),&gwarea,' ')<0){
1034 debuga(_("Maybe you have a broken request method in your %s file\n"),arq);
1035 exit(EXIT_FAILURE);
1036 }
1037 if (getword_ptr(linebuf,&url,&gwarea,' ')<0){
1038 debuga(_("Maybe you have a broken url in your %s file\n"),arq);
1039 exit(EXIT_FAILURE);
1040 }
1041 if (getword(user,sizeof(user),&gwarea,' ')<0){
1042 debuga(_("Maybe you have a broken user ID in your %s file\n"),arq);
1043 exit(EXIT_FAILURE);
1044 }
1045 ilf=ILF_Squid;
1046 ilf_count[ilf]++;
1047
1048 tnum=atoi(data);
1049 t=localtime(&tnum);
1050 if (t == NULL) {
1051 debuga(_("Cannot convert the timestamp from the squid log file\n"));
1052 exit(EXIT_FAILURE);
1053 }
1054
1055 strftime(tbuf2, sizeof(tbuf2), "%H%M", t);
1056
1057 idata=(t->tm_year+1900)*10000+(t->tm_mon+1)*100+t->tm_mday;
1058 }
1059 }
1060 if (ilf==ILF_Sarg) {
1061 getword_start(&gwarea,linebuf);
1062 if (getword(data,sizeof(data),&gwarea,'\t')<0){
1063 debuga(_("Maybe you have a broken record or garbage in your %s file\n"),arq);
1064 exit(EXIT_FAILURE);
1065 }
1066 if (getword(hora,sizeof(hora),&gwarea,'\t')<0) {
1067 debuga(_("Maybe you have a broken record or garbage in your %s file\n"),arq);
1068 exit(EXIT_FAILURE);
1069 }
1070 if (getword(user,sizeof(user),&gwarea,'\t')<0) {
1071 debuga(_("Maybe you have a broken record or garbage in your %s file\n"),arq);
1072 exit(EXIT_FAILURE);
1073 }
1074 if (getword(ip,sizeof(ip),&gwarea,'\t')<0) {
1075 debuga(_("Maybe you have a broken record or garbage in your %s file\n"),arq);
1076 exit(EXIT_FAILURE);
1077 }
1078 if (getword_ptr(linebuf,&url,&gwarea,'\t')<0){
1079 debuga(_("Maybe you have a broken record or garbage in your %s file\n"),arq);
1080 exit(EXIT_FAILURE);
1081 }
1082 if (getword(tam,sizeof(tam),&gwarea,'\t')<0){
1083 debuga(_("Maybe you have a broken record or garbage in your %s file\n"),arq);
1084 exit(EXIT_FAILURE);
1085 }
1086 if (getword(code,sizeof(code),&gwarea,'\t')<0){
1087 debuga(_("Maybe you have a broken record or garbage in your %s file\n"),arq);
1088 exit(EXIT_FAILURE);
1089 }
1090 if (getword(elap,sizeof(elap),&gwarea,'\t')<0){
1091 debuga(_("Maybe you have a broken record or garbage in your %s file\n"),arq);
1092 exit(EXIT_FAILURE);
1093 }
1094 if (getword(smartfilter,sizeof(smartfilter),&gwarea,'\0')<0){
1095 debuga(_("Maybe you have a broken record or garbage in your %s file\n"),arq);
1096 exit(EXIT_FAILURE);
1097 }
1098 getword_start(&gwarea,data);
1099 if (getword_atoll(&iday,&gwarea,'/')<0 || iday<1 || iday>31){
1100 debuga(_("Maybe you have a broken date in your %s file\n"),arq);
1101 exit(EXIT_FAILURE);
1102 }
1103 if (getword_atoll(&imonth,&gwarea,'/')<0 || imonth<1 || imonth>12){
1104 debuga(_("Maybe you have a broken date in your %s file\n"),arq);
1105 exit(EXIT_FAILURE);
1106 }
1107 if (getword_atoll(&iyear,&gwarea,'\0')<0){
1108 debuga(_("Maybe you have a broken date in your %s file\n"),arq);
1109 exit(EXIT_FAILURE);
1110 }
1111 idata=builddia(iday,imonth,iyear);
1112 computedate(iyear,imonth,iday,&tt);
1113 if (sscanf(hora,"%d:%d:%d",&tt.tm_hour,&tt.tm_min,&tt.tm_sec)!=3 || tt.tm_hour<0 || tt.tm_hour>=24 ||
1114 tt.tm_min<0 || tt.tm_min>=60 || tt.tm_sec<0 || tt.tm_sec>=60) {
1115 debuga(_("Invalid time found in %s\n"),arq);
1116 exit(EXIT_FAILURE);
1117 }
1118 t=&tt;
1119 }
1120 if (ilf==ILF_Isa) {
1121 if (linebuf[0] == '#') {
1122 int ncols,cols[ISACOL_Last];
1123
1124 fixendofline(linebuf);
1125 getword_start(&gwarea,linebuf);
1126 // remove the #Fields: column at the beginning of the line
1127 if (getword_skip(1000,&gwarea,' ')<0){
1128 debuga(_("Maybe you have a broken record or garbage in your %s file\n"),arq);
1129 exit(EXIT_FAILURE);
1130 }
1131 for (ncols=0 ; ncols<ISACOL_Last ; ncols++) cols[ncols]=-1;
1132 ncols=0;
1133 while(gwarea.current[0] != '\0') {
1134 if (getword(val1,sizeof(val1),&gwarea,'\t')<0){
1135 debuga(_("Maybe you have a broken record or garbage in your %s file\n"),arq);
1136 exit(EXIT_FAILURE);
1137 }
1138 if(strcmp(val1,"c-ip") == 0) cols[ISACOL_Ip]=ncols;
1139 if(strcmp(val1,"cs-username") == 0) cols[ISACOL_UserName]=ncols;
1140 if(strcmp(val1,"date") == 0) cols[ISACOL_Date]=ncols;
1141 if(strcmp(val1,"time") == 0) cols[ISACOL_Time]=ncols;
1142 if(strcmp(val1,"time-taken") == 0) cols[ISACOL_TimeTaken]=ncols;
1143 if(strcmp(val1,"sc-bytes") == 0) cols[ISACOL_Bytes]=ncols;
1144 if(strcmp(val1,"cs-uri") == 0) cols[ISACOL_Uri]=ncols;
1145 if(strcmp(val1,"sc-status") == 0) cols[ISACOL_Status]=ncols;
1146 ncols++;
1147 }
1148 if (cols[ISACOL_Ip]>=0) {
1149 isa_ncols=ncols;
1150 for (ncols=0 ; ncols<ISACOL_Last ; ncols++)
1151 isa_cols[ncols]=cols[ncols];
1152 }
1153 continue;
1154 }
1155 if (!isa_ncols) continue;
1156 getword_start(&gwarea,linebuf);
1157 for (x=0 ; x<isa_ncols ; x++) {
1158 if (getword_ptr(linebuf,&str,&gwarea,'\t')<0) {
1159 debuga(_("Maybe you have a broken record or garbage in your %s file\n"),arq);
1160 exit(EXIT_FAILURE);
1161 }
1162 if (x==isa_cols[ISACOL_Ip]) {
1163 if (strlen(str)>=sizeof(ip)) {
1164 debuga(_("Maybe you have a broken IP in your %s file\n"),arq);
1165 exit(EXIT_FAILURE);
1166 }
1167 strcpy(ip,str);
1168 } else if (x==isa_cols[ISACOL_UserName]) {
1169 if (strlen(str)>=sizeof(user)) {
1170 debuga(_("Maybe you have a broken user ID in your %s file\n"),arq);
1171 exit(EXIT_FAILURE);
1172 }
1173 strcpy(user,str);
1174 } else if (x==isa_cols[ISACOL_Date]) {
1175 if (strlen(str)>=sizeof(data)) {
1176 debuga(_("Maybe you have a broken date in your %s file\n"),arq);
1177 exit(EXIT_FAILURE);
1178 }
1179 strcpy(data,str);
1180 } else if (x==isa_cols[ISACOL_Time]) {
1181 if (strlen(str)>=sizeof(hora)) {
1182 debuga(_("Maybe you have a broken time in your %s file\n"),arq);
1183 exit(EXIT_FAILURE);
1184 }
1185 strcpy(hora,str);
1186 } else if (x==isa_cols[ISACOL_TimeTaken]) {
1187 if (strlen(str)>=sizeof(elap)) {
1188 debuga(_("Maybe you have a broken download duration in your %s file\n"),arq);
1189 exit(EXIT_FAILURE);
1190 }
1191 strcpy(elap,str);
1192 } else if (x==isa_cols[ISACOL_Bytes]) {
1193 if (strlen(str)>=sizeof(tam)) {
1194 debuga(_("Maybe you have a broken download size in your %s file\n"),arq);
1195 exit(EXIT_FAILURE);
1196 }
1197 strcpy(tam,str);
1198 } else if (x==isa_cols[ISACOL_Uri]) {
1199 url=str;
1200 } else if (x==isa_cols[ISACOL_Status]) {
1201 if (strlen(str)>=sizeof(code)) {
1202 debuga(_("Maybe you have a broken access code in your %s file\n"),arq);
1203 exit(EXIT_FAILURE);
1204 }
1205 strcpy(code,str);
1206 }
1207 }
1208
1209 if(strcmp(code,"401") == 0 || strcmp(code,"403") == 0 || strcmp(code,"407") == 0) {
1210 sprintf(val1,"DENIED/%s",code);
1211 strcpy(code,val1);
1212 }
1213 getword_start(&gwarea,data);
1214 if (getword_atoll(&iyear,&gwarea,'-')<0){
1215 debuga(_("Maybe you have a broken year in your %s file\n"),arq);
1216 exit(EXIT_FAILURE);
1217 }
1218 if (getword_atoll(&imonth,&gwarea,'-')<0){
1219 debuga(_("Maybe you have a broken month in your %s file\n"),arq);
1220 exit(EXIT_FAILURE);
1221 }
1222 if (getword_atoll(&iday,&gwarea,'\0')<0){
1223 debuga(_("Maybe you have a broken day in your %s file\n"),arq);
1224 exit(EXIT_FAILURE);
1225 }
1226
1227 idata=builddia(iday,imonth,iyear);
1228 computedate(iyear,imonth,iday,&tt);
1229 if (isa_cols[ISACOL_Time]>=0) {
1230 if (sscanf(hora,"%d:%d:%d",&tt.tm_hour,&tt.tm_min,&tt.tm_sec)!=3 || tt.tm_hour<0 || tt.tm_hour>=24 ||
1231 tt.tm_min<0 || tt.tm_min>=60 || tt.tm_sec<0 || tt.tm_sec>=60) {
1232 debuga(_("Invalid time found in %s\n"),arq);
1233 exit(EXIT_FAILURE);
1234 }
1235 }
1236 t=&tt;
1237 }
1238 if (t==NULL) {
1239 debuga(_("Unknown input log file format\n"));
1240 break;
1241 }
1242
1243 strftime(dia, sizeof(dia), "%d/%m/%Y", t);
1244 snprintf(hora,sizeof(hora),"%02d:%02d:%02d",t->tm_hour,t->tm_min,t->tm_sec);
1245
1246 if(debugm)
1247 printf("DATE=%s IDATA=%d DFROM=%d DUNTIL=%d\n",date,idata,dfrom,duntil);
1248
1249 if(date[0] != '\0'){
1250 if(idata < dfrom || idata > duntil) continue;
1251 }
1252
1253 // Record only hours usage which is required
1254 if (t) {
1255 if( bsearch( &( t -> tm_wday ), weekdays.list, weekdays.len, sizeof( int ), compar ) == NULL )
1256 continue;
1257
1258 if( bsearch( &( t -> tm_hour ), hours.list, hours.len, sizeof( int ), compar ) == NULL )
1259 continue;
1260 }
1261
1262
1263 if(strlen(user) > MAX_USER_LEN) {
1264 if (debugm) printf(_("User ID too long: %s\n"),user);
1265 totregsx++;
1266 continue;
1267 }
1268
1269 // include_users
1270 if(IncludeUsers[0] != '\0') {
1271 sprintf(val1,":%s:",user);
1272 if((str=(char *) strstr(IncludeUsers,val1)) == (char *) NULL )
1273 continue;
1274 }
1275
1276 if(vercode(code)) {
1277 if (debugm) printf(_("Excluded code: %s\n"),code);
1278 totregsx++;
1279 continue;
1280 }
1281
1282 if(testvaliduserchar(user))
1283 continue;
1284
1285 #if 0
1286 if((str = strstr(user,"%20")) != NULL) {
1287 /*
1288 This is a patch introduced to solve bug #1624251 reported at sourceforge but
1289 the side effect is to truncate the name at the first space and merge the reports
1290 of people whose name is identical up to the first space.
1291
1292 The old code used to truncate the user name at the first % if a %20 was
1293 found anywhere in the string. That means the string could be truncated
1294 at the wrong place if another % occured before the %20. This new code should
1295 avoid that problem and only truncate at the space. There is no bug
1296 report indicating that anybody noticed this.
1297 */
1298 *str='\0';
1299 }
1300
1301 /*
1302 Code prior to 2.2.7 used to replace any %xx by a dot as long as a %5c was
1303 found in the user name.
1304 */
1305 while((str = strstr(user,"%5c")) != NULL) {
1306 *str='.';
1307 for (x=3 ; str[x] ; x++) str[x-2]=str[x];
1308 }
1309 #endif
1310
1311 urly=url;
1312
1313 if(ilf!=ILF_Sarg) {
1314 /*
1315 The full URL is not saved in sarg log. There is no point in testing the URL to detect
1316 a downloaded file.
1317 */
1318 download_flag=is_download_suffix(url);
1319 if (download_flag) {
1320 download_url=url;
1321 download_count++;
1322 }
1323 } else
1324 download_flag=false;
1325
1326 // remove any protocol:// at the beginning of the URL
1327 if ((str = strchr(url,'/')) != NULL && str[1] == '/') {
1328 int i;
1329
1330 str+=2;
1331 for (i=0 ; str[i] ; i++)
1332 url[i]=str[i];
1333 url[i]='\0';
1334 }
1335
1336 if(!LongUrl) {
1337 url_hostname(url,hostname,sizeof(hostname));
1338 url=hostname;
1339 }
1340
1341 if(url[0] == '\0') continue;
1342
1343 if(addr[0] != '\0'){
1344 if(strcmp(addr,ip)!=0) continue;
1345 }
1346 if(fhost) {
1347 if(!vhexclude(url)) {
1348 if (debugm) printf(_("Excluded site: %s\n"),url);
1349 totregsx++;
1350 continue;
1351 }
1352 }
1353
1354 if(hm >= 0 && hmf >= 0) {
1355 hmr=t->tm_hour*100+t->tm_min;
1356 if(hmr < hm || hmr > hmf) continue;
1357 }
1358
1359 if(site[0] != '\0'){
1360 if(strstr(url,site)==0) continue;
1361 }
1362
1363 if(UserIp) {
1364 strcpy(user,ip);
1365 id_is_ip=true;
1366 } else {
1367 id_is_ip=false;
1368 if(strcmp(user,"-") == 0 || strcmp(user," ") == 0 || strcmp(user,"") == 0) {
1369 if(RecordsWithoutUser == RECORDWITHOUTUSER_IP) {
1370 strcpy(user,ip);
1371 id_is_ip=true;
1372 }
1373 if(RecordsWithoutUser == RECORDWITHOUTUSER_IGNORE)
1374 continue;
1375 if(RecordsWithoutUser == RECORDWITHOUTUSER_EVERYBODY)
1376 strcpy(user,"everybody");
1377 } else {
1378 strlow(user);
1379 if(NtlmUserFormat == NTLMUSERFORMAT_USER) {
1380 if((str = strchr(user,'_')) != 0) {
1381 strcpy(warea,str+1);
1382 strcpy(user,warea);
1383 }
1384 if((str = strchr(user,'+')) != 0) {
1385 strcpy(warea,str+1);
1386 strcpy(user,warea);
1387 }
1388 }
1389 }
1390 }
1391
1392 if(us[0] != '\0'){
1393 if(strcmp(user,us)!=0) continue;
1394 }
1395
1396 if(puser) {
1397 sprintf(wuser,":%s:",user);
1398 if(strstr(userfile, wuser) == 0)
1399 continue;
1400 }
1401
1402 if(fuser) {
1403 if(!vuexclude(user)) {
1404 if (debugm) printf(_("Excluded user: %s\n"),user);
1405 totregsx++;
1406 continue;
1407 }
1408 }
1409
1410 if(strcmp(user,"-") ==0 || strcmp(user," ") ==0 || strcmp(user,"") ==0 || strcmp(user,":") ==0)
1411 continue;
1412
1413 if(max_elapsed) {
1414 if(atol(elap)>max_elapsed) {
1415 elap[0]='0';
1416 elap[1]='\0';
1417 }
1418 }
1419
1420 if((str=(char *) strstr(linebuf, "[SmartFilter:")) != (char *) NULL ) {
1421 fixendofline(str);
1422 sprintf(smartfilter,"\"%s\"",str+1);
1423 } else sprintf(smartfilter,"\"\"");
1424
1425 nopen=0;
1426 prev_ufile=NULL;
1427 for (ufile=first_user_file ; ufile && strcmp(user,ufile->user->id)!=0 ; ufile=ufile->next) {
1428 prev_ufile=ufile;
1429 if (ufile->file) nopen++;
1430 }
1431 if (!ufile) {
1432 ufile=malloc(sizeof(*ufile));
1433 if (!ufile) {
1434 debuga(_("Not enough memory to store the user %s\n"),user);
1435 exit(EXIT_FAILURE);
1436 }
1437 memset(ufile,0,sizeof(*ufile));
1438 ufile->next=first_user_file;
1439 first_user_file=ufile;
1440 uinfo=userinfo_create(user);
1441 ufile->user=uinfo;
1442 uinfo->id_is_ip=id_is_ip;
1443 } else {
1444 if (prev_ufile) {
1445 prev_ufile->next=ufile->next;
1446 ufile->next=first_user_file;
1447 first_user_file=ufile;
1448 }
1449 }
1450
1451 if (ufile->file==NULL) {
1452 if (nopen>=maxopenfiles) {
1453 x=0;
1454 for (ufile1=first_user_file ; ufile1 ; ufile1=ufile1->next) {
1455 if (ufile1->file!=NULL) {
1456 if (x>=maxopenfiles) {
1457 if (fclose(ufile1->file)==EOF) {
1458 debuga(_("Failed to close the log file of user %s - %s\n"),ufile1->user->id,strerror(errno));
1459 exit(EXIT_FAILURE);
1460 }
1461 ufile1->file=NULL;
1462 }
1463 x++;
1464 }
1465 }
1466 }
1467 if (snprintf (tmp3, sizeof(tmp3), "%s/%s.unsort", tmp, ufile->user->filename)>=sizeof(tmp3)) {
1468 debuga(_("Temporary user file name too long: %s/%s.unsort\n"), tmp, ufile->user->filename);
1469 exit(EXIT_FAILURE);
1470 }
1471 if ((ufile->file = MY_FOPEN (tmp3, "a")) == NULL) {
1472 debuga(_("(log) Cannot open temporary file: %s - %s\n"), tmp3, strerror(errno));
1473 exit (1);
1474 }
1475 }
1476
1477 /*if ( strcmp ( user , sz_Last_User ) != 0 ) {
1478 if ( fp_Write_User )
1479 fclose( fp_Write_User ) ;
1480 sprintf (tmp3, "%s/%s.unsort", tmp, user);
1481
1482 if ((fp_Write_User = MY_FOPEN (tmp3, "a")) == NULL) {
1483 fprintf (stderr, "%s: (log) %s: %s - %s\n", argv[0], _("Cannot open temporary file"), tmp3, strerror(errno));
1484 exit (1);
1485 }
1486 strcpy( sz_Last_User , user ) ;
1487 }*/
1488 if (fprintf(ufile->file, "%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\n",dia,hora,ip,url,tam,code,elap,smartfilter)<=0) {
1489 debuga(_("Write error in the log file of user %s\n"),user);
1490 exit(EXIT_FAILURE);
1491 }
1492
1493 if(fp_log && ilf!=ILF_Sarg)
1494 fprintf(fp_log, "%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\n",dia,hora,user,ip,url,tam,code,elap,smartfilter);
1495
1496 totregsg++;
1497
1498 if(!dataonly && download_flag && download_url && strstr(code,"DENIED") == 0) {
1499 ndownload = 1;
1500
1501 if ( ! fp_Download_Unsort ) {
1502 if ((fp_Download_Unsort = MY_FOPEN ( sz_Download_Unsort, "a")) == NULL) {
1503 debuga(_("(log) Cannot open temporary file: %s - %s\n"),sz_Download_Unsort, strerror(errno));
1504 exit (1);
1505 }
1506 }
1507 fprintf(fp_Download_Unsort,"%s\t%s\t%s\t%s\t%s\n",dia,hora,user,ip,download_url);
1508 }
1509
1510 if((ReportType & REPORT_TYPE_DENIED) != 0 || (ReportType & REPORT_TYPE_AUTH_FAILURES) != 0) {
1511 if(fp_denied && strstr(code,"DENIED/403") != 0) {
1512 fprintf(fp_denied, "%s\t%s\t%s\t%s\t%s\n",dia,hora,user,ip,urly);
1513 denied_count++;
1514 }
1515 if(fp_authfail && (strstr(code,"DENIED/401") != 0 || strstr(code,"DENIED/407") != 0)) {
1516 fprintf(fp_authfail, "%s\t%s\t%s\t%s\t%s\n",dia,hora,user,ip,urly);
1517 authfail_count++;
1518 }
1519 }
1520
1521 if (ilf!=ILF_Sarg) {
1522 if(!totper || idata<mindate){
1523 mindate=idata;
1524 memcpy(&period.start,t,sizeof(*t));
1525 strcpy(start_hour,tbuf2);
1526 }
1527 if (!totper || idata>maxdate) {
1528 maxdate=idata;
1529 memcpy(&period.end,t,sizeof(*t));
1530 }
1531 totper=true;
1532 }
1533
1534 if(debugm){
1535 printf("IP=\t%s\n",ip);
1536 printf("USER=\t%s\n",user);
1537 printf("ELAP=\t%s\n",elap);
1538 printf("DATE=\t%s\n",dia);
1539 printf("TIME=\t%s\n",hora);
1540 printf("FUNC=\t%s\n",fun);
1541 printf("URL=\t%s\n",url);
1542 printf("CODE=\t%s\n",code);
1543 printf("LEN=\t%s\n",tam);
1544 }
1545 }
1546 if (!from_stdin) {
1547 if (from_pipe)
1548 pclose(fp_in);
1549 else {
1550 fclose(fp_in);
1551 if( ShowReadStatistics )
1552 printf(_("SARG: Records in file: %lu, reading: %3.2f%%\n"),recs1, (float) 100 );
1553 }
1554 }
1555 }
1556
1557 if (debug)
1558 debuga(_(" Records read: %ld, written: %ld, excluded: %ld\n"),totregsl,totregsg,totregsx);
1559
1560 longline_destroy(&line);
1561 if ( fp_Download_Unsort )
1562 fclose (fp_Download_Unsort);
1563
1564 for (ufile=first_user_file ; ufile ; ufile=ufile1) {
1565 ufile1=ufile->next;
1566 if (ufile->file!=NULL) fclose(ufile->file);
1567 free(ufile);
1568 }
1569
1570 free_download();
1571 free_excludecodes();
1572 free_exclude();
1573
1574 if(debug) {
1575 int totalcount=0;
1576
1577 for (ilf=0 ; ilf<ILF_Last ; ilf++) totalcount+=ilf_count[ilf];
1578
1579 if(ilf_count[ILF_Common]>0 && ilf_count[ILF_Squid]>0)
1580 debuga(_("Log with mixed records format (squid and common log)\n"));
1581
1582 if(ilf_count[ILF_Common]>0 && ilf_count[ILF_Squid]==0)
1583 debuga(_("Common log format\n"));
1584
1585 if(ilf_count[ILF_Common]==0 && ilf_count[ILF_Squid]>0)
1586 debuga(_("Squid log format\n"));
1587
1588 if(ilf_count[ILF_Sarg]>0)
1589 debuga(_("Sarg log format\n"));
1590
1591 if(totalcount==0 && totregsg)
1592 debuga(_("Log with invalid format\n"));
1593 }
1594
1595 if(!totregsg){
1596 debuga(_("No records found\n"));
1597 debuga(_("End\n"));
1598 if(fp_denied) fclose(fp_denied);
1599 if(fp_authfail) fclose(fp_authfail);
1600 userinfo_free();
1601 if(userfile) free(userfile);
1602 close_usertab();
1603 exit(EXIT_SUCCESS);
1604 }
1605
1606 if (date[0]!='\0') {
1607 char date0[30], date1[30];
1608
1609 strftime(date0,sizeof(date0),"%d/%m/%Y",&period.start);
1610 strftime(date1,sizeof(date1),"%d/%m/%Y",&period.end);
1611 debuga(_("Period covered by log files: %s-%s\n"),date0,date1);
1612 getperiod_fromrange(&period,dfrom,duntil);
1613 }
1614 if (getperiod_buildtext(&period)<0) {
1615 debuga(_("Failed to build the string representation of the date range\n"));
1616 exit(EXIT_FAILURE);
1617 }
1618
1619 if(debugz){
1620 debugaz("data",dia);
1621 debugaz("period",period.text);
1622 }
1623
1624 if(debug)
1625 debuga(_("Period: %s\n"),period.text);
1626
1627 if(fp_denied)
1628 fclose(fp_denied);
1629 if(fp_authfail)
1630 fclose(fp_authfail);
1631
1632 if(fp_log != NULL) {
1633 fclose(fp_log);
1634 strcpy(end_hour,tbuf2);
1635 strftime(val2,sizeof(val2),"%d%m%Y",&period.start);
1636 strftime(val1,sizeof(val1),"%d%m%Y",&period.end);
1637 sprintf(val4,"%s/sarg-%s_%s-%s_%s.log",ParsedOutputLog,val2,start_hour,val1,end_hour);
1638 if (rename(arq_log,val4)) {
1639 debuga(_("failed to rename %s to %s - %s\n"),arq_log,val4,strerror(errno));
1640 } else {
1641 strcpy(arq_log,val4);
1642
1643 if(strcmp(ParsedOutputLogCompress,"nocompress") != 0 && ParsedOutputLogCompress[0] != '\0') {
1644 /*
1645 No double quotes around ParsedOutputLogCompress because it may contain command line options. If double quotes are
1646 necessary around the command name, put them in the configuration file.
1647 */
1648 sprintf(val1,"%s \"%s\"",ParsedOutputLogCompress,arq_log);
1649 cstatus=system(val1);
1650 if (!WIFEXITED(cstatus) || WEXITSTATUS(cstatus)) {
1651 debuga(_("command return status %d\n"),WEXITSTATUS(cstatus));
1652 debuga(_("command: %s\n"),val1);
1653 exit(EXIT_FAILURE);
1654 }
1655 }
1656 }
1657 if(debug)
1658 debuga(_("Sarg parsed log saved as %s\n"),arq_log);
1659 }
1660
1661 if(DataFile[0] == '\0' && (ReportType & REPORT_TYPE_DENIED) != 0) {
1662 sprintf(csort,"sort -T \"%s\" -k 3,3 -k 5,5 -o \"%s\" \"%s\"",tmp,denied_sort,denied_unsort);
1663 cstatus=system(csort);
1664 if (!WIFEXITED(cstatus) || WEXITSTATUS(cstatus)) {
1665 debuga(_("sort command return status %d\n"),WEXITSTATUS(cstatus));
1666 debuga(_("sort command: %s\n"),csort);
1667 exit(EXIT_FAILURE);
1668 }
1669 if (unlink(denied_unsort)) {
1670 debuga(_("Cannot delete %s - %s\n"),denied_unsort,strerror(errno));
1671 exit(EXIT_FAILURE);
1672 }
1673 }
1674
1675 sort_users_log(tmp, debug);
1676
1677 if(DataFile[0] != '\0')
1678 data_file(tmp);
1679 else
1680 gerarel();
1681
1682 if((ReportType & REPORT_TYPE_DENIED) != 0) {
1683 unlink(denied_sort);
1684 }
1685
1686 if(strcmp(tmp,"/tmp") != 0) {
1687 unlinkdir(tmp,0);
1688 }
1689
1690 userinfo_free();
1691 if(userfile)
1692 free(userfile);
1693 close_usertab();
1694
1695 if(debug)
1696 debuga(_("End\n"));
1697
1698 exit(EXIT_SUCCESS);
1699 }
1700
1701
1702 static void getusers(const char *pwdfile, int debug)
1703 {
1704 FILE *fp_usr;
1705 char buf[255];
1706 char *str;
1707 long int nreg=0;
1708
1709 if(debug)
1710 debuga(_("Loading password file from %s\n"),pwdfile);
1711
1712 if ((fp_usr = fopen(pwdfile, "r")) == NULL) {
1713 debuga(_("(getusers) Cannot open file %s - %s\n"),pwdfile,strerror(errno));
1714 exit(EXIT_FAILURE);
1715 }
1716
1717 if (fseek(fp_usr, 0, SEEK_END)==-1) {
1718 debuga(_("Failed to move till the end of the users file %s: %s\n"),pwdfile,strerror(errno));
1719 exit(EXIT_FAILURE);
1720 }
1721 nreg = ftell(fp_usr);
1722 if (nreg<0) {
1723 debuga(_("Cannot get the size of file %s\n"),pwdfile);
1724 exit(EXIT_FAILURE);
1725 }
1726 nreg = nreg+5000;
1727 if (fseek(fp_usr, 0, SEEK_SET)==-1) {
1728 debuga(_("Failed to rewind the users file %s: %s\n"),pwdfile,strerror(errno));
1729 exit(EXIT_FAILURE);
1730 }
1731
1732 if((userfile=(char *) malloc(nreg))==NULL){
1733 debuga(_("malloc error (%ld)\n"),nreg);
1734 exit(EXIT_FAILURE);
1735 }
1736
1737 bzero(userfile,nreg);
1738 strcpy(userfile,":");
1739
1740 while(fgets(buf,sizeof(buf),fp_usr)!=NULL) {
1741 str=strchr(buf,':');
1742 if (!str) {
1743 debuga(_("You have an invalid user in your %s file\n"),pwdfile);
1744 exit(EXIT_FAILURE);
1745 }
1746 str[1]='\0';
1747 strcat(userfile,buf);
1748 }
1749
1750 fclose(fp_usr);
1751
1752 return;
1753 }
Unnamed repository; edit this file 'description' to name the repository.