Add --unittests-coverage option to list how many code modules have tests

diff --git a/src/app-layer-dcerpc-udp.c b/src/app-layer-dcerpc-udp.c
index 9079e8216692b5cf5ecf95c81c161285cd6d71e6..bf8777c1835cb66800a718f37db343704f17151c 100644 (file)
--- a/src/app-layer-dcerpc-udp.c
+++ b/src/app-layer-dcerpc-udp.c
@@ -729,6 +729,9 @@ void RegisterDCERPCUDPParsers(void) {
                        DCERPCUDPParse);
        AppLayerRegisterStateFuncs(ALPROTO_DCERPC_UDP, DCERPCUDPStateAlloc,
                        DCERPCUDPStateFree);
+#ifdef UNITTESTS
+    AppLayerRegisterUnittests(ALPROTO_DCERPC_UDP, DCERPCUDPParserRegisterTests);
+#endif
 }
 
 /* UNITTESTS */

diff --git a/src/app-layer-dcerpc.c b/src/app-layer-dcerpc.c
index 49a90a926991c1535e3909e56583e8e1c7e1388d..e9374e129f068aa1000fb800241db7daff006ab4 100644 (file)
--- a/src/app-layer-dcerpc.c
+++ b/src/app-layer-dcerpc.c
@@ -1878,6 +1878,9 @@ void RegisterDCERPCParsers(void) {
             DCERPCParseResponse);
     AppLayerRegisterStateFuncs(ALPROTO_DCERPC, DCERPCStateAlloc,
             DCERPCStateFree);
+#ifdef UNITTESTS
+    AppLayerRegisterUnittests(ALPROTO_DCERPC, DCERPCParserRegisterTests);
+#endif
 }
 
 /* UNITTESTS */

diff --git a/src/app-layer-ftp.c b/src/app-layer-ftp.c
index 403f3c4eaedb5535fd1aa219a0c886854c1e637d..69ec7fb76ecf075531f6e4432e30ceecf6731177 100644 (file)
--- a/src/app-layer-ftp.c
+++ b/src/app-layer-ftp.c
@@ -279,6 +279,9 @@ void RegisterFTPParsers(void) {
                            FTP_FIELD_REQUEST_LINE, FTPParseRequestCommandLine,
                            "ftp");
     AppLayerRegisterStateFuncs(ALPROTO_FTP, FTPStateAlloc, FTPStateFree);
+#ifdef UNITTESTS
+    AppLayerRegisterUnittests(ALPROTO_FTP, FTPParserRegisterTests);
+#endif
 }
 
 void FTPAtExitPrintStats(void) {

diff --git a/src/app-layer-htp.c b/src/app-layer-htp.c
index 05a7f6c1f4fc2eff38e3cbf34f6d4cd719dbb4e2..4d8462b6db3bbc92c4c5b6a79d037adf7193e25b 100644 (file)
--- a/src/app-layer-htp.c
+++ b/src/app-layer-htp.c
@@ -2425,6 +2425,9 @@ void RegisterHTPParsers(void)
                           HTPHandleRequestData);
     AppLayerRegisterProto(proto_name, ALPROTO_HTTP, STREAM_TOCLIENT,
                           HTPHandleResponseData);
+#ifdef UNITTESTS
+    AppLayerRegisterUnittests(ALPROTO_HTTP, HTPParserRegisterTests);
+#endif
 
     SC_ATOMIC_INIT(htp_config_flags);
     HTPConfigure();

diff --git a/src/app-layer-parser.c b/src/app-layer-parser.c
index 41a899d29d341c7d2bda00f21ec5979a50728664..0f1135d6943e9979f04e90695d853d55c5739c08 100644 (file)
--- a/src/app-layer-parser.c
+++ b/src/app-layer-parser.c
@@ -45,6 +45,7 @@
 #include "app-layer-protos.h"
 #include "app-layer-parser.h"
 #include "app-layer-smb.h"
+#include "app-layer-smb2.h"
 #include "app-layer-dcerpc.h"
 #include "app-layer-dcerpc-udp.h"
 #include "app-layer-htp.h"
@@ -727,6 +728,12 @@ int AppLayerRegisterProto(char *name, uint8_t proto, uint8_t flags,
     return 0;
 }
 
+#ifdef UNITTESTS
+void AppLayerRegisterUnittests(uint16_t proto, void (*RegisterUnittests)(void)) {
+    al_proto_table[proto].RegisterUnittests = RegisterUnittests;
+}
+#endif
+
 void AppLayerRegisterStateFuncs(uint16_t proto, void *(*StateAlloc)(void),
                                 void (*StateFree)(void *))
 {
@@ -1337,6 +1344,8 @@ void RegisterAppLayerParsers(void)
     RegisterHTPParsers();
     RegisterSSLParsers();
     RegisterSMBParsers();
+    /** \todo bug 719 */
+    //RegisterSMB2Parsers();
     RegisterDCERPCParsers();
     RegisterDCERPCUDPParsers();
     RegisterFTPParsers();
@@ -5869,6 +5878,25 @@ static int AppLayerProbingParserTest15(void)
 void AppLayerParserRegisterTests(void)
 {
 #ifdef UNITTESTS
+    int i;
+    for (i = 0; i < ALPROTO_MAX; i++) {
+        AppLayerProto *p = &al_proto_table[i];
+
+        if (p->name == NULL)
+            continue;
+
+        g_ut_modules++;
+
+        if (p->RegisterUnittests != NULL) {
+            p->RegisterUnittests();
+            g_ut_covered++;
+        } else {
+            if (coverage_unittests)
+                SCLogWarning(SC_WARN_NO_UNITTESTS, "app layer module %s has no "
+                        "unittests", p->name);
+        }
+    }
+
     UtRegisterTest("AppLayerParserTest01", AppLayerParserTest01, 1);
     UtRegisterTest("AppLayerParserTest02", AppLayerParserTest02, 1);
     UtRegisterTest("AppLayerProbingParserTest01", AppLayerProbingParserTest01, 1);

diff --git a/src/app-layer-parser.h b/src/app-layer-parser.h
index 7232d5bfe6efd3a93200d9ac62c17205081fdbdb..dfd22e6bb0e6d1def89e46160720c852c8a431f1 100644 (file)
--- a/src/app-layer-parser.h
+++ b/src/app-layer-parser.h
@@ -65,6 +65,10 @@ typedef struct AppLayerProto_ {
     uint64_t (*StateGetTxCnt)(void *alstate);
     void *(*StateGetTx)(void *alstate, uint64_t tx_id);
     int (*StateGetAlstateProgressCompletionStatus)(uint8_t direction);
+
+#ifdef UNITTESTS
+    void (*RegisterUnittests)(void);
+#endif
 } AppLayerProto;
 
 /** flags for the result elmts */
@@ -249,6 +253,9 @@ void AppLayerRegisterProbingParser(struct AlpProtoDetectCtx_ *, uint16_t, uint16
                                    uint16_t, uint16_t, uint8_t, uint8_t,
                                    uint8_t,
                                    uint16_t (*ProbingParser)(uint8_t *, uint32_t));
+#ifdef UNITTESTS
+void AppLayerRegisterUnittests(uint16_t proto, void (*RegisterUnittests)(void));
+#endif
 void AppLayerRegisterStateFuncs(uint16_t proto, void *(*StateAlloc)(void),
                                 void (*StateFree)(void *));
 void AppLayerRegisterLocalStorageFunc(uint16_t proto,

diff --git a/src/app-layer-smb.c b/src/app-layer-smb.c
index bc95303f8cbac01c7026bb86758d58e467ce7b9c..e957b12976083aee251a34275c40a2dd90071cd3 100644 (file)
--- a/src/app-layer-smb.c
+++ b/src/app-layer-smb.c
@@ -1419,7 +1419,9 @@ void RegisterSMBParsers(void) {
                                   STREAM_TOSERVER,
                                   APP_LAYER_PROBING_PARSER_PRIORITY_HIGH, 1,
                                   SMBProbingParser);
-
+#ifdef UNITTESTS
+    AppLayerRegisterUnittests(ALPROTO_SMB, SMBParserRegisterTests);
+#endif
     return;
 }
 

diff --git a/src/app-layer-smb2.c b/src/app-layer-smb2.c
index a2dc7c611b066d2c8dd4301b0b51a395bd2f356a..866caca5fe4651f07f6e9e177df86627a4085417 100644 (file)
--- a/src/app-layer-smb2.c
+++ b/src/app-layer-smb2.c
@@ -521,6 +521,9 @@ void RegisterSMB2Parsers(void) {
     AppLayerRegisterProto("smb", ALPROTO_SMB2, STREAM_TOSERVER, SMB2Parse);
     AppLayerRegisterProto("smb", ALPROTO_SMB2, STREAM_TOCLIENT, SMB2Parse);
     AppLayerRegisterStateFuncs(ALPROTO_SMB2, SMB2StateAlloc, SMB2StateFree);
+#ifdef UNITTESTS
+    AppLayerRegisterUnittests(ALPROTO_SMB2, SMB2ParserRegisterTests);
+#endif
 }
 
 /* UNITTESTS */
@@ -585,7 +588,6 @@ end:
 }
 
 void SMB2ParserRegisterTests(void) {
-    printf("SMB2ParserRegisterTests\n");
     UtRegisterTest("SMB2ParserTest01", SMB2ParserTest01, 1);
 }
 #endif

diff --git a/src/app-layer-smtp.c b/src/app-layer-smtp.c
index b058cebc6223f2043ab6861c7c0f6a1f23bfd73c..f58bbc4bda51d68d039d161b8061ebf6c9b4d222 100644 (file)
--- a/src/app-layer-smtp.c
+++ b/src/app-layer-smtp.c
@@ -865,6 +865,9 @@ void RegisterSMTPParsers(void)
 
     SMTPSetMpmState();
 
+#ifdef UNITTESTS
+    AppLayerRegisterUnittests(ALPROTO_SMTP, SMTPParserRegisterTests);
+#endif
     return;
 }
 

diff --git a/src/app-layer-ssh.c b/src/app-layer-ssh.c
index 2fd7db48841688a8f8302f8d18d6e06c1c233be1..9c69fd20f126cdaad2b5eb7a9b7ef5d7722ef80e 100644 (file)
--- a/src/app-layer-ssh.c
+++ b/src/app-layer-ssh.c
@@ -747,7 +747,9 @@ void RegisterSSHParsers(void)
                             SSHParseClientRecord);
 
     AppLayerRegisterStateFuncs(ALPROTO_SSH, SSHStateAlloc, SSHStateFree);
-
+#ifdef UNITTESTS
+    AppLayerRegisterUnittests(ALPROTO_SSH, SSHParserRegisterTests);
+#endif
 }
 
 /* UNITTESTS */

diff --git a/src/app-layer-ssl.c b/src/app-layer-ssl.c
index 9e1413851ea554a7a271fa3f06b54423ec346242..2eaa39f57cedda9cbae68bbf10779a7432109846 100644 (file)
--- a/src/app-layer-ssl.c
+++ b/src/app-layer-ssl.c
@@ -989,6 +989,9 @@ void RegisterSSLParsers(void)
                                   STREAM_TOSERVER,
                                   APP_LAYER_PROBING_PARSER_PRIORITY_HIGH, 1,
                                   SSLProbingParser);
+#ifdef UNITTESTS
+    AppLayerRegisterUnittests(ALPROTO_TLS, SSLParserRegisterTests);
+#endif
 
     /* Get the value of no reassembly option from the config file */
     if (ConfGetBool("tls.no-reassemble", &ssl_config.no_reassemble) != 1)

diff --git a/src/detect.c b/src/detect.c
index 8eb1a2325c24c283f6db1e5190b418973335d785..8c80eb62389bb3b263aed3323ac82983b4364a53 100644 (file)
--- a/src/detect.c
+++ b/src/detect.c
@@ -4903,22 +4903,23 @@ void SigTableSetup(void) {
     DetectLuajitRegister();
     DetectIPRepRegister();
     DetectDnsQueryRegister();
-
-    uint8_t i = 0;
-    for (i = 0; i < DETECT_TBLSIZE; i++) {
-        if (sigmatch_table[i].RegisterTests == NULL) {
-            SCLogDebug("detection plugin %s has no unittest "
-                   "registration function.", sigmatch_table[i].name);
-        }
-    }
 }
 
 void SigTableRegisterTests(void) {
     /* register the tests */
-    uint8_t i = 0;
+    int i = 0;
     for (i = 0; i < DETECT_TBLSIZE; i++) {
+        g_ut_modules++;
         if (sigmatch_table[i].RegisterTests != NULL) {
             sigmatch_table[i].RegisterTests();
+            g_ut_covered++;
+        } else {
+            SCLogDebug("detection plugin %s has no unittest "
+                   "registration function.", sigmatch_table[i].name);
+
+            if (coverage_unittests)
+                SCLogWarning(SC_WARN_NO_UNITTESTS, "detection plugin %s has no unittest "
+                        "registration function.", sigmatch_table[i].name);
         }
     }
 }

diff --git a/src/suricata-common.h b/src/suricata-common.h
index a7555ec80e2fcd291de71972530a499eb4fb6a47..7db178a07e8149222c2f0355a06976212a391527 100644 (file)
--- a/src/suricata-common.h
+++ b/src/suricata-common.h
@@ -313,5 +313,8 @@ typedef enum PacketProfileDetectId_ {
 size_t strlcat(char *, const char *src, size_t siz);
 size_t strlcpy(char *dst, const char *src, size_t siz);
 
+extern int coverage_unittests;
+extern int g_ut_modules;
+extern int g_ut_covered;
 #endif /* __SURICATA_COMMON_H__ */
 

diff --git a/src/suricata.c b/src/suricata.c
index 568ec6a64d207c3b545b21f8c1cd24187c3aaa0e..15d6a595f65ecadd8e4dc895e63586de950fe646 100644 (file)
--- a/src/suricata.c
+++ b/src/suricata.c
@@ -514,6 +514,7 @@ void usage(const char *progname)
     printf("\t-U, --unittest-filter=REGEX          : filter unittests with a regex\n");
     printf("\t--list-unittests                     : list unit tests\n");
     printf("\t--fatal-unittests                    : enable fatal failure on unittest error\n");
+    printf("\t--unittests-coverage                 : display unittest coverage report\n");
 #endif /* UNITTESTS */
     printf("\t--list-app-layer-protos              : list supported app layer protocols\n");
     printf("\t--list-keywords[=all|csv|<kword>]    : list keywords implemented by the engine\n");
@@ -703,6 +704,10 @@ void SCPrintBuildInfo(void) {
 #include "build-info.h"
 }
 
+int coverage_unittests;
+int g_ut_modules;
+int g_ut_covered;
+
 int main(int argc, char **argv)
 {
     int opt;
@@ -789,6 +794,12 @@ int main(int argc, char **argv)
     /* Initialize the configuration module. */
     ConfInit();
 
+#ifdef UNITTESTS
+    coverage_unittests = 0;
+    g_ut_modules = 0;
+    g_ut_covered = 0;
+#endif
+
     struct option long_opts[] = {
         {"dump-config", 0, &dump_config, 1},
         {"pfring", optional_argument, 0, 0},
@@ -817,6 +828,7 @@ int main(int argc, char **argv)
         {"pidfile", required_argument, 0, 0},
         {"init-errors-fatal", 0, 0, 0},
         {"fatal-unittests", 0, 0, 0},
+        {"unittests-coverage", 0, &coverage_unittests, 1},
         {"user", required_argument, 0, 0},
         {"group", required_argument, 0, 0},
         {"erf-in", required_argument, 0, 0},
@@ -1612,10 +1624,10 @@ int main(int argc, char **argv)
 
     if (run_mode == RUNMODE_UNITTEST) {
 #ifdef DBG_MEM_ALLOC
-    SCLogInfo("Memory used at startup: %"PRIdMAX, (intmax_t)global_mem);
+        SCLogInfo("Memory used at startup: %"PRIdMAX, (intmax_t)global_mem);
 #endif
         /* test and initialize the unittesting subsystem */
-        if(regex_arg == NULL){
+        if(regex_arg == NULL && !coverage_unittests){
             regex_arg = ".*";
             UtRunSelftest(regex_arg); /* inits and cleans up again */
         }
@@ -1638,15 +1650,9 @@ int main(int argc, char **argv)
         FlowBitRegisterTests();
         FlowAlertSidRegisterTests();
         SCPerfRegisterTests();
+
         DecodePPPRegisterTests();
         DecodeVLANRegisterTests();
-        HTPParserRegisterTests();
-        SSLParserRegisterTests();
-        SSHParserRegisterTests();
-        SMBParserRegisterTests();
-        DCERPCParserRegisterTests();
-        DCERPCUDPParserRegisterTests();
-        FTPParserRegisterTests();
         DecodeRawRegisterTests();
         DecodePPPOERegisterTests();
         DecodeICMPV4RegisterTests();
@@ -1657,6 +1663,7 @@ int main(int argc, char **argv)
         DecodeUDPV4RegisterTests();
         DecodeGRERegisterTests();
         DecodeAsn1RegisterTests();
+
         AlpDetectRegisterTests();
         ConfRegisterTests();
         ConfYamlRegisterTests();
@@ -1701,7 +1708,6 @@ int main(int argc, char **argv)
         DetectEngineHttpHRHRegisterTests();
         DetectEngineRegisterTests();
         SCLogRegisterTests();
-        SMTPParserRegisterTests();
         MagicRegisterTests();
         UtilMiscRegisterTests();
         DetectAddressTests();
@@ -1713,8 +1719,11 @@ int main(int argc, char **argv)
 #endif
         if (list_unittests) {
             UtListTests(regex_arg);
-        }
-        else {
+        } else if (coverage_unittests) {
+            /* nothing */
+            SCLogInfo("%d out of %d code modules have unittests (%.0f%%)",
+                    g_ut_covered, g_ut_modules, (float)((float)g_ut_covered/(float)g_ut_modules)*100);
+        } else {
             uint32_t failed = UtRunTests(regex_arg);
             UtCleanup();
             if (failed) {

diff --git a/src/tm-modules.c b/src/tm-modules.c
index 1aba592431e0458039655668eea60b585421d2d5..89242b9ae7863faddfd482294426d0719d091d31 100644 (file)
--- a/src/tm-modules.c
+++ b/src/tm-modules.c
@@ -202,11 +202,16 @@ void TmModuleRegisterTests(void) {
         if (t->name == NULL)
             continue;
 
+        g_ut_modules++;
+
+
         if (t->RegisterTests == NULL) {
-            SCLogDebug("threading module %s has no unittest "
-                   "registration function.", t->name);
+            if (coverage_unittests)
+                SCLogWarning(SC_WARN_NO_UNITTESTS, "threading module %s has no unittest "
+                        "registration function.", t->name);
         } else {
             t->RegisterTests();
+            g_ut_covered++;
         }
     }
 #endif /* UNITTESTS */

diff --git a/src/util-error.c b/src/util-error.c
index 9990474ab2dd2caa67bffa21b11886dffb7d380f..778d7e79707e1c19880931c76d4eb553e8f4ea7d 100644 (file)
--- a/src/util-error.c
+++ b/src/util-error.c
@@ -274,6 +274,7 @@ const char * SCErrorToString(SCError err)
         CASE_CODE (SC_ERR_CUDA_BUFFER_ERROR);
         CASE_CODE (SC_ERR_DNS_LOG_GENERIC);
         CASE_CODE (SC_WARN_OPTION_OBSOLETE);
+        CASE_CODE (SC_WARN_NO_UNITTESTS);
     }
 
     return "UNKNOWN_ERROR";

diff --git a/src/util-error.h b/src/util-error.h
index 411ee3e970ee83eb29e1f2cb9df0921b5051fc13..706604476b46001edbbc2f32f9e4ddd25b201446 100644 (file)
--- a/src/util-error.h
+++ b/src/util-error.h
@@ -263,6 +263,7 @@ typedef enum {
     SC_ERR_CUDA_BUFFER_ERROR,
     SC_ERR_DNS_LOG_GENERIC,
     SC_WARN_OPTION_OBSOLETE,
+    SC_WARN_NO_UNITTESTS,
 } SCError;
 
 const char *SCErrorToString(SCError);

diff --git a/src/util-mpm.c b/src/util-mpm.c
index 1756f8bfb723c1982a6b7b4aeb2de4da68545802..57a5a3afab9b308ed818134e87b99abcb8381b29 100644 (file)
--- a/src/util-mpm.c
+++ b/src/util-mpm.c
@@ -663,10 +663,15 @@ void MpmRegisterTests(void) {
         if (i == MPM_NOTSET)
             continue;
 
+        g_ut_modules++;
+
         if (mpm_table[i].RegisterUnittests != NULL) {
+            g_ut_covered++;
             mpm_table[i].RegisterUnittests();
         } else {
-            printf("Warning: mpm %s has no unittest registration function...", mpm_table[i].name);
+            if (coverage_unittests)
+                SCLogWarning(SC_WARN_NO_UNITTESTS, "mpm module %s has no "
+                        "unittest registration function.", mpm_table[i].name);
         }
     }