From: Frédéric Marchal Date: Sun, 16 Dec 2012 17:11:55 +0000 (+0100) Subject: Get rid of the limit on the number of input files X-Git-Url: http://git.ipfire.org/?a=commitdiff_plain;h=6068ae5629f8781540100bcc0459b5f7944533a1;p=thirdparty%2Fsarg.git Get rid of the limit on the number of input files Previous versions were limited to 255 access log files with a maximum length of 1024 characters. That limit has been alleviated. --- diff --git a/CMakeLists.txt b/CMakeLists.txt index 3a91760..7b50395 100755 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -52,8 +52,9 @@ SET(SRC util.c log.c report.c topuser.c email.c sort.c html.c smartfilter.c denied.c authfail.c charset.c dichotomic.c redirector.c auth.c download.c grepday.c ip2name_exec.c dansguardian_log.c dansguardian_report.c realtime.c btree_cache.c - usertab.c userinfo.c longline.c url.c readlog.c readlog_squid.c - readlog_sarg.c readlog_extlog.c readlog_common.c) + usertab.c userinfo.c longline.c url.c fnmatch.c stringbuffer.c + filelist.c readlog.c + readlog_squid.c readlog_sarg.c readlog_extlog.c readlog_common.c) FOREACH(f ${SRC}) ADD_FILE_DEPENDENCIES(${f} ${CMAKE_BINARY_DIR}/config.h ${CMAKE_SOURCE_DIR}/include/conf.h ${CMAKE_SOURCE_DIR}/include/info.h ${CMAKE_SOURCE_DIR}/include/defs.h) @@ -116,6 +117,7 @@ CHECK_INCLUDE_FILE(libgen.h HAVE_LIBGEN_H) CHECK_INCLUDE_FILE(stdbool.h HAVE_STDBOOL_H) CHECK_INCLUDE_FILE(getopt.h HAVE_GETOPT_H) CHECK_INCLUDE_FILE(fcntl.h HAVE_FCNTL_H) +CHECK_INCLUDE_FILE(fnmatch.h HAVE_FNMATCH_H) IF(!HAVE_GETOPT_H) MESSAGE(SEND_ERROR "getopt.h is required to compile sarg") @@ -149,6 +151,7 @@ CHECK_FUNCTION_EXISTS(lstat HAVE_LSTAT) CHECK_FUNCTION_EXISTS(getnameinfo HAVE_GETNAMEINFO) CHECK_FUNCTION_EXISTS(getaddrinfo HAVE_GETADDRINFO) CHECK_FUNCTION_EXISTS(inet_aton HAVE_INET_ATON) +CHECK_FUNCTION_EXISTS(fnmatch HAVE_FNMATCH) CHECK_STRUCT_HAS_MEMBER("struct sockaddr_storage" ss_len sys/socket.h HAVE_SOCKADDR_SA_LEN) diff --git a/Makefile.in b/Makefile.in index 40e86b1..e05044d 100644 --- a/Makefile.in +++ b/Makefile.in @@ -40,8 +40,9 @@ SRCS = util.c log.c report.c topuser.c email.c sort.c html.c \ smartfilter.c denied.c authfail.c charset.c dichotomic.c \ redirector.c auth.c download.c grepday.c ip2name_exec.c \ dansguardian_log.c dansguardian_report.c realtime.c btree_cache.c \ - usertab.c userinfo.c longline.c url.c readlog.c readlog_squid.c \ - readlog_sarg.c readlog_extlog.c readlog_common.c + usertab.c userinfo.c longline.c url.c fnmatch.c stringbuffer.c \ + filelist.c readlog.c \ + readlog_squid.c readlog_sarg.c readlog_extlog.c readlog_common.c OBJS = $(SRCS:.c=.o) diff --git a/configure.in b/configure.in index c02a3b2..44eb2ce 100644 --- a/configure.in +++ b/configure.in @@ -76,7 +76,7 @@ AC_HEADER_STDC AC_CHECK_HEADERS(stdio.h stdlib.h string.h strings.h sys/time.h time.h unistd.h sys/dirent.h \ dirent.h sys/types.h sys/socket.h netdb.h arpa/inet.h netinet/in.h sys/stat.h \ ctype.h errno.h sys/resource.h sys/wait.h stdarg.h inttypes.h limits.h locale.h \ - execinfo.h math.h libintl.h libgen.h stdbool.h getopt.h fcntl.h) + execinfo.h math.h libintl.h libgen.h stdbool.h getopt.h fcntl.h fnmatch.h) if test $ac_cv_header_getopt_h = "no" ; then AC_MSG_ERROR("getopt.h is required to compile sarg") @@ -191,6 +191,7 @@ AC_CHECK_FUNCS(lstat) AC_CHECK_FUNCS(getnameinfo) AC_CHECK_FUNCS(getaddrinfo) AC_CHECK_FUNCS(mkstemp) +AC_CHECK_FUNCS(fnmatch) dnl check for structure members AC_CHECK_MEMBER([struct sockaddr_storage.ss_len],[AC_DEFINE([HAVE_SOCKADDR_SA_LEN],1,[ss_len in sockaddr_storage])]) diff --git a/filelist.c b/filelist.c new file mode 100644 index 0000000..221d461 --- /dev/null +++ b/filelist.c @@ -0,0 +1,502 @@ +/* + * SARG Squid Analysis Report Generator http://sarg.sourceforge.net + * 1998, 2012 + * + * SARG donations: + * please look at http://sarg.sourceforge.net/donations.php + * Support: + * http://sourceforge.net/projects/sarg/forums/forum/363374 + * --------------------------------------------------------------------- + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111, USA. + * + */ + +#include "include/conf.h" +#include "include/defs.h" +#include "include/stringbuffer.h" +#include "include/filelist.h" + +struct DirEntryStruct +{ + //! Next entry at the same level. + struct DirEntryStruct *Sibbling; + //! First child of this entry. + struct DirEntryStruct *Child; + //! Name of this entry. + char *Name; + //! \c True if it contains any wildcard. + bool IsMask; +}; + +/*! + * \brief List of files. + * + * The list may contain wildcards. + */ +struct FileListStruct +{ + //! Root of the tree. + struct DirEntryStruct *First; + //! Buffer containing the file name strings. + StringBufferObject Buffer; + //! Deepest level of the tree. + int TreeDepth; + //! \c True if the tree depth is correct. + bool TreeDepthOk; +}; + +struct DirEntryIterator +{ + //! The current node at each level. + struct DirEntryStruct *Dir; + //! Length of the path up to that level. + int PathLength; +}; + +/*! + * \brief Iterator of the file list. + */ +struct _FileListIterator +{ + //! File list object from which we are iterating. + FileListObject Parent; + //! Current path being stored in the object. + char *CurrentPath; + //! Number of bytes allocated for the current path. + int CurrentPathSize; + //! Level known to be stored in the path. + int CurrentPathLevel; + //! Tree depth when the iteration started. + int TreeDepth; + //! Current level being iterated over. + int Level; + //! The current node at each level. + struct DirEntryIterator *DirNodes; +}; + + +/*! + * Create an object to store the files to process. + * + * \return The created object or NULL if it failed. + * The object must be destroyed with a call to FileList_Destroy(). + */ +FileListObject FileList_Create(void) +{ + FileListObject FObj; + + FObj=(FileListObject)calloc(1,sizeof(*FObj)); + if (!FObj) + return(NULL); + + return(FObj); +} + +/*! + * Destroy the entries tree. + */ +static void FileList_DestroyEntry(struct DirEntryStruct *Entry) +{ + struct DirEntryStruct *Next; + + while (Entry) + { + if (Entry->Child) + FileList_DestroyEntry(Entry->Child); + Next=Entry->Sibbling; + free(Entry); + Entry=Next; + } +} + +/*! + * Destroy the object created by FileList_Create(). + * + * \param FPtr A pointer to the object to destroy. It is + * reset to NULL before the function returns. + */ +void FileList_Destroy(FileListObject *FPtr) +{ + FileListObject FObj; + + if (!FPtr || !*FPtr) return; + FObj=*FPtr; + *FPtr=NULL; + + FileList_DestroyEntry(FObj->First); + StringBuffer_Destroy(&FObj->Buffer); + free(FObj); +} + +/*! + * Store an entry in the tree. + * + * \param FObj The file list object created by FileList_Create(). + * \param FileName Name of the file to store recursively. + * + * \return The branch created with all the entries in \c FileName. + * The returned value is NULL if \c FileName could not be added. + */ +static struct DirEntryStruct *FileList_StoreEntry(FileListObject FObj,const char *FileName) +{ + struct DirEntryStruct *Entry; + int i; + bool IsMask=false; + int LastDir=-1; + int Next=-1; + + Entry=(struct DirEntryStruct *)calloc(1,sizeof(*Entry)); + if (!Entry) return(NULL); + for (i=0 ; FileName[i] ; i++) + { + if (FileName[i]=='/') + { + if (IsMask) + { + /* The path contains a wildcard. There are no directories + * before this path or it would have been caught by the other + * break in this loop. We store it. + */ + Next=i; + break; + } + LastDir=i; + } + else if (FileName[i]=='*' || FileName[i]=='?') + { + if (LastDir>=0) + { + /* Some directories without wildcards before this directory + * with wildcard. We store the previous directories in one + * entry and disregard, for now, the current path level. + */ + Next=LastDir; + break; + } + IsMask=true; + } + } + Entry->Name=StringBuffer_StoreLength(FObj->Buffer,FileName,(Next<0) ? i : Next); + if (!Entry->Name) + { + free(Entry); + return(NULL); + } + Entry->IsMask=IsMask; + if (Next>0) + { + FObj->TreeDepthOk=false; //it will have to be recomputed + Entry->Child=FileList_StoreEntry(FObj,FileName+Next+1); + if (!Entry->Child) + { + free(Entry); + return(NULL); + } + } + return(Entry); +} + +/*! + * Store a file in the internal data structure. + * + * \param FObj The file list object created by FileList_Create(). + * \param EntryPtr Pointer to the tree node to add or create. + * \param FileName The name of the file. + * + * \return \c True on success or \c false on failure. + */ +static bool FileList_AddFileRecursive(FileListObject FObj,struct DirEntryStruct **EntryPtr,const char *FileName) +{ + int i; + struct DirEntryStruct *Entry; + struct DirEntryStruct *Last; + int LastDir; + + if (!*EntryPtr) + { + Entry=FileList_StoreEntry(FObj,FileName); + if (!Entry) return(false); + *EntryPtr=Entry; + return(true); + } + + // find where to store the file name in the existing tree + Last=NULL; + for (Entry=*EntryPtr ; Entry ; Entry=Entry->Sibbling) + { + LastDir=-1; + for (i=0 ; Entry->Name[i] && FileName[i] && Entry->Name[i]==FileName[i] ; i++) + { + if (FileName[i]=='/') + LastDir=i; + } + if (FileName[i]=='/' && Entry->Name[i]=='\0') + { + //root is matching, check sub level + return(FileList_AddFileRecursive(FObj,&Entry->Child,FileName+i+1)); + } + if (LastDir>0) + { + //paths begin with the same directory but diverges at LastDir + struct DirEntryStruct *Split; + + Split=(struct DirEntryStruct *)calloc(1,sizeof(*Split)); + if (!Split) return(false); + Split->Name=Entry->Name+LastDir+1; + Split->Child=Entry->Child; + Entry->Name[LastDir]='\0'; + Entry->Child=Split; + return(FileList_AddFileRecursive(FObj,&Entry->Child,FileName+LastDir+1)); + } + Last=Entry; + } + + // add a new entry + Entry=FileList_StoreEntry(FObj,FileName); + if (!Entry) return(false); + Last->Sibbling=Entry; + + return(true); +} + +/*! + * Add a file to the object. + * + * \param FObj The object created by FileList_Create(). + * \param FileName The file name to add to the list. + * + * \return \c True if the file was added or \c false if it + * failed. The function may fail if a parameter is invalid. + * It will also fail if the memory cannot be allocated. + */ +bool FileList_AddFile(FileListObject FObj,const char *FileName) +{ + if (!FObj || !FileName) return(false); + + if (!FObj->Buffer) + { + FObj->Buffer=StringBuffer_Create(); + if (!FObj->Buffer) + return(false); + } + + return(FileList_AddFileRecursive(FObj,&FObj->First,FileName)); +} + +/*! + * Recursively measure the tree depth. + * + * \param FObj File list object created by FileList_Create(). + * \param Entry Node whose child are to be processed. + * \param Level Current level. + */ +static void FileList_SetDepth(FileListObject FObj,struct DirEntryStruct *Entry,int Level) +{ + if (Level>FObj->TreeDepth) FObj->TreeDepth=Level; + while (Entry) + { + if (Entry->Child) + FileList_SetDepth(FObj,Entry->Child,Level+1); + Entry=Entry->Sibbling; + } +} + +/*! + * Start the iteration over the files in the list. + * + * \param FObj The object to iterate over. + * + * \return The iterator structure to pass ot FileListIter_Next() + * to get the first file name or NULL if an error occured. + */ +FileListIterator FileListIter_Open(FileListObject FObj) +{ + struct _FileListIterator *FIter; + struct DirEntryStruct *Dir; + + FIter=(FileListIterator)calloc(1,sizeof(*FIter)); + if (!FIter) return(NULL); + FIter->Parent=FObj; + + // compute the depth of the tree. + /* + * The tree depth computation is not thread safe. A lock is necessary around + * the following code to make it thread safe. + */ + if (!FObj->TreeDepthOk) + { + FObj->TreeDepth=0; + if (FObj->First) FileList_SetDepth(FObj,FObj->First,1); + FObj->TreeDepthOk=true; + } + FIter->TreeDepth=FObj->TreeDepth; + FIter->Level=-1; + FIter->CurrentPathSize=0; + FIter->CurrentPathLevel=0; + if (FIter->TreeDepth>0) + { + FIter->DirNodes=(struct DirEntryIterator *)calloc(FIter->TreeDepth,sizeof(struct DirEntryIterator)); + if (!FIter->DirNodes) + { + FileListIter_Close(FIter); + return(NULL); + } + for (Dir=FObj->First ; Dir ; Dir=Dir->Child) + { + FIter->DirNodes[++FIter->Level].Dir=Dir; + } + } + + return(FIter); +} + +/*! + * Get the next entry in the directory tree. + */ +static void FileListIter_GetNext(struct _FileListIterator *FIter) +{ + struct DirEntryStruct *Dir; + + FIter->CurrentPathLevel=0; + while (FIter->Level>=0) + { + Dir=FIter->DirNodes[FIter->Level].Dir; + if (Dir->Sibbling) + { + Dir=Dir->Sibbling; + FIter->DirNodes[FIter->Level].Dir=Dir; + FIter->CurrentPathLevel=FIter->Level; + while (Dir->Child) + { + if (FIter->Level>=FIter->TreeDepth) break; + Dir=Dir->Child; + FIter->DirNodes[FIter->Level++].Dir=Dir; + } + break; + } + FIter->Level--; + } +} + +/*! + * Get the next file in the list. + * + * \param FIter The iterator created by FileListIter_Open(). + * + * \return The iterator function containing the next file name or NULL + * if there are no more files. + */ +const char *FileListIter_Next(struct _FileListIterator *FIter) +{ + int Length; + int Level; + struct DirEntryIterator *DIter; + + if (!FIter) return(NULL); + if (!FIter->DirNodes) return(NULL); + if (FIter->Level<0 || FIter->Level>=FIter->TreeDepth) return(NULL); + + // how much space to store the path + Length=FIter->DirNodes[FIter->CurrentPathLevel].PathLength; + for (Level=FIter->CurrentPathLevel ; Level<=FIter->Level ; Level++) + { + DIter=FIter->DirNodes+Level; + DIter->PathLength=Length;; + Length+=strlen(DIter->Dir->Name)+1; + } + + // get the memory to store the path + if (Length>FIter->CurrentPathSize) + { + char *temp=realloc(FIter->CurrentPath,Length); + if (!temp) return(NULL); + FIter->CurrentPath=temp; + FIter->CurrentPathSize=Length; + } + + for (Level=FIter->CurrentPathLevel ; Level<=FIter->Level ; Level++) + { + DIter=FIter->DirNodes+Level; + if (Level>0) FIter->CurrentPath[DIter->PathLength-1]='/'; + strcpy(FIter->CurrentPath+DIter->PathLength,DIter->Dir->Name); + } + FIter->CurrentPathLevel=Level; + + FileListIter_GetNext(FIter); + return(FIter->CurrentPath); +} + +/*! + * Get the next file entry in the list without expanding the + * wildcards. + * + * \param FIter The iterator created by FileListIter_Open(). + * + * \return The iterator function containing the next file name or NULL + * if there are no more files. + */ +const char *FileListIter_NextWithMask(struct _FileListIterator *FIter) +{ + int Length; + int Level; + struct DirEntryIterator *DIter; + + if (!FIter) return(NULL); + if (!FIter->DirNodes) return(NULL); + if (FIter->Level<0 || FIter->Level>=FIter->TreeDepth) return(NULL); + + // how much space to store the path + Length=FIter->DirNodes[FIter->CurrentPathLevel].PathLength; + for (Level=FIter->CurrentPathLevel ; Level<=FIter->Level ; Level++) + { + DIter=FIter->DirNodes+Level; + DIter->PathLength=Length;; + Length+=strlen(DIter->Dir->Name)+1; + } + + // get the memory to store the path + if (Length>FIter->CurrentPathSize) + { + char *temp=realloc(FIter->CurrentPath,Length); + if (!temp) return(NULL); + FIter->CurrentPath=temp; + FIter->CurrentPathSize=Length; + } + + for (Level=FIter->CurrentPathLevel ; Level<=FIter->Level ; Level++) + { + DIter=FIter->DirNodes+Level; + if (Level>0) FIter->CurrentPath[DIter->PathLength-1]='/'; + strcpy(FIter->CurrentPath+DIter->PathLength,DIter->Dir->Name); + } + FIter->CurrentPathLevel=Level; + + FileListIter_GetNext(FIter); + return(FIter->CurrentPath); +} + +/*! + * Destroy the iterator created by FileListIter_Open(). + */ +void FileListIter_Close(struct _FileListIterator *FIter) +{ + if (FIter) + { + if (FIter->CurrentPath) free(FIter->CurrentPath); + if (FIter->DirNodes) free(FIter->DirNodes); + free(FIter); + } +} diff --git a/fnmatch.c b/fnmatch.c new file mode 100644 index 0000000..c1f71c5 --- /dev/null +++ b/fnmatch.c @@ -0,0 +1,44 @@ +/* + * SARG Squid Analysis Report Generator http://sarg.sourceforge.net + * 1998, 2012 + * + * SARG donations: + * please look at http://sarg.sourceforge.net/donations.php + * Support: + * http://sourceforge.net/projects/sarg/forums/forum/363374 + * --------------------------------------------------------------------- + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111, USA. + * + */ + +#ifndef HAVE_FNMATCH +#include "include/conf.h" +#include "include/defs.h" + +/*! +Simplified function to match a pattern against a file name. It is provided +here for MingW which doesn't have that function. + +\param pattern The pattern to match. It can contain ? and *. +\param string The file name to match. Use slash as directory separator. +\param flags +*/ +int fnmatch(const char *pattern,const char *string,int flags) +{ + return(0); +} + +#endif //HAVE_FNMATCH diff --git a/getconf.c b/getconf.c index df6795c..e7b86c5 100644 --- a/getconf.c +++ b/getconf.c @@ -26,10 +26,12 @@ #include "include/conf.h" #include "include/defs.h" +#include "include/filelist.h" #define SET_LIST(list) list,sizeof(list)/sizeof(list[0]) extern numlist hours, weekdays; +extern FileListObject AccessLog; struct param_list { @@ -215,6 +217,31 @@ static int getparam_string(const char *param,char *buf,char *value,int value_siz return(1); } +/*! + * Get a pointer to the beginning of the string value defined by the + * parameter. The returned value is NULL if the buffer doesn't contain + * the parameter. + * + * \param param The parameter to look for. + * \param buf The buffer containing the line read from the configuration + * file. + * + * \return A pointer to the beginning of the parameter value or NULL if + * the line is not for the requtested parameter. + */ +static char *getparam_stringptr(const char *param,char *buf) +{ + int plen; + + plen=strlen(param); + if (strncmp(buf,param,plen) != 0) return(NULL); + buf+=plen; + if ((unsigned char)*buf>' ') return(NULL); + while (*buf && (unsigned char)*buf<=' ') buf++; + + return(buf); +} + static int getparam_quoted(const char *param,char *buf,char *value,int value_size) { int plen; @@ -497,12 +524,13 @@ static void parmtest(char *buf) if (is_param("access_log",buf)>0) { if (AccessLogFromCmdLine==0) { - if (NAccessLog>=MAXLOGS) { - debuga(_("Too many log files in configuration file\n")); + char *FileName=getparam_stringptr("access_log",buf); + if (!AccessLog) + AccessLog=FileList_Create(); + if (!FileList_AddFile(AccessLog,FileName)) { + debuga(_("Not enough memory to store the input log file names\n")); exit(EXIT_FAILURE); } - getparam_string("access_log",buf,AccessLog[NAccessLog],MAX_LOG_FILELEN); - NAccessLog++; } return; } diff --git a/include/conf.h b/include/conf.h index baf1355..07acf54 100755 --- a/include/conf.h +++ b/include/conf.h @@ -115,6 +115,9 @@ typedef int bool; #ifdef HAVE_FCNTL_H #include #endif +#ifdef HAVE_FNMATCH_H +#include +#endif #if defined(HAVE_FOPEN64) #define _FILE_OFFSET_BITS 64 @@ -172,8 +175,6 @@ int mkstemps(char *template, int suffixlen); #define MAX_USER_FNAME_LEN 128 #define MAX_IP_LEN 64 #define MAX_DATETIME_LEN 32 -#define MAXLOGS 255 -#define MAX_LOG_FILELEN 1024 #define MAX_REDIRECTOR_LOGS 64 #define MAX_REDIRECTOR_FILELEN 1024 /*! @@ -321,8 +322,6 @@ char warea[MAXLEN]; char name[MAXLEN]; bool LongUrl; bool Ip2Name; -int NAccessLog; -char AccessLog[MAXLOGS][MAX_LOG_FILELEN]; int AccessLogFromCmdLine; char Title[MAXLEN]; char BgColor[MAXLEN]; diff --git a/include/config.h.in b/include/config.h.in index c042e9c..f4d2d95 100644 --- a/include/config.h.in +++ b/include/config.h.in @@ -50,6 +50,7 @@ #cmakedefine HAVE_WS2TCPIP_H #cmakedefine HAVE_FCNTL_H #cmakedefine HAVE_PCRE_H +#cmakedefine HAVE_FNMATCH_H #cmakedefine IBERTY_LIB @@ -62,6 +63,7 @@ #cmakedefine HAVE_GETNAMEINFO #cmakedefine HAVE_GETADDRINFO #cmakedefine HAVE_INET_ATON +#cmakedefine HAVE_FNMATCH #cmakedefine HAVE_SOCKADDR_SA_LEN diff --git a/include/defs.h b/include/defs.h index 68d235f..6703159 100755 --- a/include/defs.h +++ b/include/defs.h @@ -173,6 +173,11 @@ int vuexclude(const char *user); bool is_indexonly(void); void free_exclude(void); +#ifndef HAVE_FNMATCH +// fnmtach.c +int fnmatch(const char *pattern,const char *string,int flags); +#endif + // getconf.c void getconf(void); diff --git a/include/filelist.h b/include/filelist.h new file mode 100644 index 0000000..22392c3 --- /dev/null +++ b/include/filelist.h @@ -0,0 +1,20 @@ +#ifndef FILELIST_HEADER +#define FILELIST_HEADER + +//! Store the files to read. +typedef struct FileListStruct *FileListObject; + +//! Iterator over the file list. +typedef struct _FileListIterator *FileListIterator; + +FileListObject FileList_Create(void); +void FileList_Destroy(FileListObject *FPtr); + +bool FileList_AddFile(FileListObject FObj,const char *FileName); + +FileListIterator FileListIter_Open(FileListObject FObj); +const char *FileListIter_Next(FileListIterator FIter); +const char *FileListIter_NextWithMask(FileListIterator FIter); +void FileListIter_Close(FileListIterator FIter); + +#endif //FILELIST_HEADER diff --git a/include/stringbuffer.h b/include/stringbuffer.h new file mode 100644 index 0000000..86660a5 --- /dev/null +++ b/include/stringbuffer.h @@ -0,0 +1,13 @@ +#ifndef STRINGBUFFER_HEADER +#define STRINGBUFFER_HEADER + +//! Object created by the string buffer module. +typedef struct StringBufferStruct *StringBufferObject; + +StringBufferObject StringBuffer_Create(void); +void StringBuffer_Destroy(StringBufferObject *SPtr); + +char *StringBuffer_StoreLength(StringBufferObject SObj,const char *String,int Length); +char *StringBuffer_Store(StringBufferObject SObj,const char *String); + +#endif //STRINGBUFFER_HEADER diff --git a/log.c b/log.c index 4e6053e..7c8be7f 100644 --- a/log.c +++ b/log.c @@ -27,6 +27,7 @@ #include "include/conf.h" #include "include/defs.h" #include "include/readlog.h" +#include "include/filelist.h" #ifdef HAVE_GETOPT_H #include @@ -38,6 +39,10 @@ struct ReadLogDataStruct ReadFilter; //! The list of the system users. /*@null@*/char *userfile=NULL; +//! List of the input log files to process. +FileListObject AccessLog=NULL; + + static void getusers(const char *pwdfile, int debug); int main(int argc,char *argv[]) @@ -51,7 +56,6 @@ int main(int argc,char *argv[]) char hexclude[MAXLEN]; char splitprefix[MAXLEN]; int ch; - int x; int errflg=0; bool dns=false; int iarq=0; @@ -67,6 +71,7 @@ int main(int argc,char *argv[]) time_t process_end_time; double read_elapsed; double process_elapsed; + FileListIterator FIter; static int split=0; static int convert=0; static int output_css=0; @@ -266,9 +271,6 @@ int main(int argc,char *argv[]) bzero(ExcludeString, sizeof(ExcludeString)); memset(&period,0,sizeof(period)); - NAccessLog=0; - for(x=0; x=MAXLOGS) { - debuga(_("Too many log files passed on command line with option -l.\n")); + if (!AccessLog) + AccessLog=FileList_Create(); + if (!FileList_AddFile(AccessLog,optarg)) { + debuga(_("Not enough memory to store the input log file names\n")); exit(EXIT_FAILURE); } - if (strlen(optarg)>=MAX_LOG_FILELEN) { - debuga(_("Log file name too long passed on command line with option -l: %s\n"),optarg); - exit(EXIT_FAILURE); - } - strcpy(AccessLog[NAccessLog],optarg); - NAccessLog++; AccessLogFromCmdLine++; break; case 'L': @@ -420,18 +418,19 @@ int main(int argc,char *argv[]) exit(2); } + if(output_css) { + css_content(stdout); + exit(EXIT_SUCCESS); + } + if (optind=MAXLOGS) { - debuga(_("Too many log files passed on command line.\n")); - exit(EXIT_FAILURE); - } - if (strlen(argv[iarq])>=MAX_LOG_FILELEN) { - debuga(_("Log file name too long passed on command line: %s\n"),argv[iarq]); + if (!FileList_AddFile(AccessLog,argv[iarq])) { + debuga(_("Not enough memory to store the input log file names\n")); exit(EXIT_FAILURE); } - strcpy(AccessLog[NAccessLog],argv[iarq]); - NAccessLog++; AccessLogFromCmdLine++; } } @@ -473,23 +472,30 @@ int main(int argc,char *argv[]) if (df=='w') IndexTree=INDEX_TREE_FILE; - if(NAccessLog == 0) { - strcpy(AccessLog[0],"/var/log/squid/access.log"); - NAccessLog++; + if(AccessLog==NULL) { + AccessLog=FileList_Create(); + if (!FileList_AddFile(AccessLog,"/var/log/squid/access.log")) { + debuga(_("Not enough memory to store the input log file names\n")); + exit(EXIT_FAILURE); + } } - if(output_css) { - css_content(stdout); - exit(EXIT_SUCCESS); - } if(split) { - for (iarq=0 ; iarq=sizeof(cmd)) { + debuga(_("Input log file name too long: %s\n"),file); + exit(EXIT_FAILURE); + } fp = popen(cmd, "r"); if (!fp) { - debuga(_("Failed to get the %d trailing lines of %s: %s\n"),realtime_access_log_lines,AccessLog[0],strerror(errno)); + debuga(_("Failed to get %d trailing lines from %s: %s\n"),realtime_access_log_lines,file,strerror(errno)); debuga(_("tail command: %s\n"),cmd); exit(EXIT_FAILURE); } + FileListIter_Close(FIter); while((buf=longline_read(fp,line)) != NULL ) if (getdata(buf,tmp)<0) { debuga(_("Maybe a broken record or garbage was returned by %s\n"),cmd); diff --git a/stringbuffer.c b/stringbuffer.c new file mode 100644 index 0000000..47ba985 --- /dev/null +++ b/stringbuffer.c @@ -0,0 +1,182 @@ +/* + * SARG Squid Analysis Report Generator http://sarg.sourceforge.net + * 1998, 2012 + * + * SARG donations: + * please look at http://sarg.sourceforge.net/donations.php + * Support: + * http://sourceforge.net/projects/sarg/forums/forum/363374 + * --------------------------------------------------------------------- + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111, USA. + * + */ +/*!\file +\brief Efficient? strings storage + +Store strings in a globaly allocated memory to avoid memory waste and +memory fragmentation. +*/ + +#include "include/conf.h" +#include "include/stringbuffer.h" + +//! Default size of the string buffer (I hope it fits inside one memory page). +#define STRINGBUFFER_SIZE (4096-sizeof(struct StringBufferStruct)) + +/*! + * \brief String storage data. + * + * Strings are concatenated in fixed size buffers. The buffers are linked + * in a list. New buffers are added as the previous buffers are filled. + */ +struct StringBufferStruct +{ + //! Next buffer in the chained list. + StringBufferObject Next; + //! How many buffer bytes are left. + int BytesLeft; + //! Where the strings are stored. + char *Buffer; +}; + +/*! + * Create an object to store constant strings. + * + * \return The created object or NULL if it failed. + */ +StringBufferObject StringBuffer_Create(void) +{ + StringBufferObject SObj; + + SObj=(StringBufferObject)calloc(1,sizeof(*SObj)); + if (!SObj) return(NULL); + return(SObj); +} + +/*! + * Destroy the object created by StringBuffer_Create(). + * + * Any string pointer to the destroyed object becomes invalid. + * + * \param SPtr A pointer to the object created by StringBuffer_Create(). + * The pointer is reset to NULL before the functrion returns to prevent + * subsequent use of the freed pointer. + */ +void StringBuffer_Destroy(StringBufferObject *SPtr) +{ + StringBufferObject SObj; + StringBufferObject Next; + + if (!SPtr || !*SPtr) return; + SObj=*SPtr; + *SPtr=NULL; + + while (SObj) + { + Next=SObj->Next; + if (SObj->Buffer) free(SObj->Buffer); + free(SObj); + SObj=Next; + } +} + +/*! + * Store a string in an existing buffer. + */ +static char *StringBuffer_StoreInBuffer(StringBufferObject SObj,const char *String,int Length) +{ + int Start=0; + + if (SObj->Buffer) + { + Start=STRINGBUFFER_SIZE-SObj->BytesLeft; + } + else if (Length>=STRINGBUFFER_SIZE) + { + SObj->BytesLeft=Length+1; + SObj->Buffer=malloc(SObj->BytesLeft); + } + else + { + SObj->BytesLeft=STRINGBUFFER_SIZE; + SObj->Buffer=malloc(SObj->BytesLeft); + } + if (!SObj->Buffer) return(NULL); + strncpy(SObj->Buffer+Start,String,Length); + SObj->Buffer[Start+Length]='\0'; + SObj->BytesLeft-=Length+1; + return(SObj->Buffer+Start); +} + +/*! + * Add a string to the buffer. Duplicate strings are not merged. + * Each call to this function stores one copy of the string. + * + * \param SObj The string buffer object. + * \param String The string to store. + * \param Length The length of the string. + * + * \return The pointer to the stored string or NULL if the function + * failed. The returned string may be altered or truncated but not + * appended to. + */ +char *StringBuffer_StoreLength(StringBufferObject SObj,const char *String,int Length) +{ + StringBufferObject SLast; + char *Ptr; + + if (!SObj) return(NULL); + + // find a suitable buffer + SLast=NULL; + while (SObj) + { + if (!SObj->Buffer || LengthBytesLeft) + { + return(StringBuffer_StoreInBuffer(SObj,String,Length)); + } + SLast=SObj; + SObj=SObj->Next; + } + + // create a new buffer + SObj=(StringBufferObject)calloc(1,sizeof(*SObj)); + if (!SObj) return(NULL); + Ptr=StringBuffer_StoreInBuffer(SObj,String,Length); + if (!Ptr) + { + free(SObj); + return(NULL); + } + SLast->Next=SObj; + return(Ptr); +} + +/*! + * Add a string to the buffer. Duplicate strings are not merged. + * Each call to this function stores one copy of the string. + * + * \param SObj The string buffer object. + * \param String The string to store. + * + * \return The pointer to the stored string or NULL if the function + * failed. The returned string may be altered or truncated but not + * appended to. + */ +char *StringBuffer_Store(StringBufferObject SObj,const char *String) +{ + return(StringBuffer_StoreLength(SObj,String,strlen(String))); +}