Keep reading the log files even with a small number of errors

Sarg tolerates a few errors in the input log files. The number of errors
can be configured. Sarg can stop on some consecutive errors and on the
total number of errors.

diff --git a/getconf.c b/getconf.c
index be25115f167e38f9558585016319a1d604d609ae..df6795c9e2c2fbf909221a98f67a2fd411867f95 100644 (file)
--- a/getconf.c
+++ b/getconf.c
@@ -752,7 +752,11 @@ static void parmtest(char *buf)
        if (getparam_string("hostalias",buf,HostAliasFile,sizeof(HostAliasFile))>0) return;
 
        if (getparam_bool("keep_temp_log",buf,&KeepTempLog)>0) return;
-       
+
+       if (getparam_int("max_successive_log_errors",buf,&NumLogSuccessiveErrors)>0) return;
+
+       if (getparam_int("max_total_log_errors",buf,&NumLogTotalErrors)>0) return;
+
        if(strstr(buf,"squid24") != 0) {
                squid24=true;
                return;

diff --git a/include/conf.h b/include/conf.h
index dd80cd22bed8e054ef01567261598475bdf2e2a9..15a7a00a5802a96f577e70b61f1e6ba48691e3a0 100755 (executable)
--- a/include/conf.h
+++ b/include/conf.h
@@ -446,6 +446,14 @@ char GraphFont[MAXLEN];
 char SortTableJs[256];
 //! The name of the file containing the host names to replace by an alias in the report.
 char HostAliasFile[512];
+//! The number of consecutive errors allowed in an input log file before the process is interrupted.
+int NumLogSuccessiveErrors;
+/*!
+The total number of errors allowed in an input log file before the process is interrupted. A negative
+value means the process should never fails irrespective of the number of errors found in the input
+log files.
+*/
+int NumLogTotalErrors;
 
 int  idate;
 int  download_count;

diff --git a/log.c b/log.c
index 6dcc08d1f4780ad766a485747e2ca495c204c328..30bf27524379a16d8c758b87e1e6521abd50ede8 100644 (file)
--- a/log.c
+++ b/log.c
@@ -246,6 +246,8 @@ int main(int argc,char *argv[])
        dfrom=0;
        duntil=0;
        KeepTempLog=false;
+       NumLogSuccessiveErrors=3;
+       NumLogTotalErrors=50;
 
        bzero(IncludeUsers, sizeof(IncludeUsers));
        bzero(ExcludeString, sizeof(ExcludeString));

diff --git a/readlog.c b/readlog.c
index 8558f5d53f595bd52357cd507b40e400735b3cca..16e68596f8fb1da2144c9019f9ba1f99285e15c7 100644 (file)
--- a/readlog.c
+++ b/readlog.c
@@ -95,6 +95,8 @@ int ReadLogFile(struct ReadLogDataStruct *Filter)
        int maxdate=0;
        int cstatus;
        int current_format_idx;
+       int successive_errors=0;
+       int total_errors=0;
        int format_count[sizeof(LogFormats)/sizeof(*LogFormats)];
        unsigned long int recs1=0UL;
        unsigned long int recs2=0UL;
@@ -268,8 +270,18 @@ int ReadLogFile(struct ReadLogDataStruct *Filter)
                                        if (log_entry_status!=RLRC_Unknown) break;
                                }
                                if (x>=(int)(sizeof(LogFormats)/sizeof(*LogFormats))) {
-                                       debuga(_("Unknown line format found in input log file %s:\n%s\n"),arq,linebuf);
-                                       exit(EXIT_FAILURE);
+                                       if (++successive_errors>NumLogSuccessiveErrors) {
+                                               debuga(ngettext("%d consecutive error found in the input log file %s\n",
+                                                                                                               "%d consecutive errors found in the input log file %s\n",successive_errors),successive_errors,arq);
+                                               exit(EXIT_FAILURE);
+                                       }
+                                       if (NumLogTotalErrors>=0 && ++total_errors>NumLogTotalErrors) {
+                                               debuga(ngettext("%d error found in the input log file (last in %s)\n",
+                                                                                                               "%d errors found in the input log file (last in %s)\n",total_errors),total_errors,arq);
+                                               exit(EXIT_FAILURE);
+                                       }
+                                       debuga(_("The following line read from %s could not be parsed and is ignored\n%s\n"),arq,linebuf);
+                                       continue;
                                }
                                current_format=LogFormats[x];
                                current_format_idx=x;
@@ -277,12 +289,13 @@ int ReadLogFile(struct ReadLogDataStruct *Filter)
                                        /* TRANSLATORS: The argument is the log format name as translated by you. */
                                        debuga(_("Log format identified as \"%s\" for %s\n"),_(current_format->Name),arq);
                                }
+                               successive_errors=0;
                        }
                        if (log_entry_status==RLRC_Ignore) {
                                continue;
                        }
                        if (current_format_idx<0 || current_format==NULL) {
-                               debuga(_("Sarg couldn't determine the format of the input log file %s\n"),arq);
+                               debuga(_("Sarg failed to determine the format of the input log file %s\n"),arq);
                                exit(EXIT_FAILURE);
                        }
                        if (log_entry_status==RLRC_InternalError) {

diff --git a/sarg.conf b/sarg.conf
index 5206fc7787df0c5ae719a1e998146ad6fa58badd..d4a8f677ab7f14a333d598ad1748353b2fb483f4 100644 (file)
--- a/sarg.conf
+++ b/sarg.conf
@@ -809,3 +809,18 @@
 #      Use this option only to diagnose a problem with your reports. A better
 #      alternative is to run sarg from the command line with optino -k.
 #keep_temp_log no
+
+# TAG: max_successive_log_errors n
+#      Set the number of consecutive errors allowed in the input log file before
+#      the reading is aborted with an error.
+#max_successive_log_errors 3
+
+# TAG: max_total_log_errors n
+#      The reading of the input log file is interrupted if too many errors are found
+#      in the log file. This parameter set the number of errors before the reading
+#      is aborted. Set it to -1 to keep reading the logs irrespective of the
+#      errors found.
+#
+#      Note that the max_successive_log_errors is still taken into account and
+#      cannot be disabled.
+#max_total_log_errors 50