datasets: introduce new IPv6 type

This patch also simplifies IPv6 parsing.

Feature: #5383

diff --git a/src/Makefile.am b/src/Makefile.am
index 73f4d097ff67d8612959d88ebc0f41320c26a3be..3ba7e5e1197d84ee17692c023d4c047957b57260 100755 (executable)
--- a/src/Makefile.am
+++ b/src/Makefile.am
@@ -62,6 +62,7 @@ noinst_HEADERS = \
        counters.h \
        datasets.h \
        datasets-ipv4.h \
+       datasets-ipv6.h \
        datasets-md5.h \
        datasets-reputation.h \
        datasets-sha256.h \
@@ -670,6 +671,7 @@ libsuricata_c_a_SOURCES = \
        counters.c \
        datasets.c \
        datasets-ipv4.c \
+       datasets-ipv6.c \
        datasets-md5.c \
        datasets-sha256.c \
        datasets-string.c \

diff --git a/src/datasets-ipv6.c b/src/datasets-ipv6.c
new file mode 100644 (file)
index 0000000..f907320
--- /dev/null
+++ b/src/datasets-ipv6.c
@@ -0,0 +1,62 @@
+/* Copyright (C) 2022 Open Information Security Foundation
+ *
+ * You can copy, redistribute or modify this Program under the terms of
+ * the GNU General Public License version 2 as published by the Free
+ * Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * version 2 along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
+ * 02110-1301, USA.
+ */
+
+/**
+ * \file
+ *
+ * \author Eric Leblond <el@stamus-networks.com>
+ */
+
+#include "suricata-common.h"
+#include "conf.h"
+#include "datasets.h"
+#include "datasets-ipv6.h"
+#include "util-thash.h"
+#include "util-print.h"
+
+int IPv6Set(void *dst, void *src)
+{
+    IPv6Type *src_s = src;
+    IPv6Type *dst_s = dst;
+    memcpy(dst_s->ipv6, src_s->ipv6, sizeof(dst_s->ipv6));
+    dst_s->rep = src_s->rep;
+    return 0;
+}
+
+bool IPv6Compare(void *a, void *b)
+{
+    const IPv6Type *as = a;
+    const IPv6Type *bs = b;
+
+    return (memcmp(as->ipv6, bs->ipv6, sizeof(as->ipv6)) == 0);
+}
+
+uint32_t IPv6Hash(void *s)
+{
+    const IPv6Type *str = s;
+    uint32_t hash = 5381;
+
+    for (int i = 0; i < (int)sizeof(str->ipv6); i++) {
+        hash = ((hash << 5) + hash) + str->ipv6[i]; /* hash * 33 + c */
+    }
+    return hash;
+}
+
+// data stays in hash
+void IPv6Free(void *s)
+{
+}

diff --git a/src/datasets-ipv6.h b/src/datasets-ipv6.h
new file mode 100644 (file)
index 0000000..a4cabca
--- /dev/null
+++ b/src/datasets-ipv6.h
@@ -0,0 +1,39 @@
+/* Copyright (C) 2022 Open Information Security Foundation
+ *
+ * You can copy, redistribute or modify this Program under the terms of
+ * the GNU General Public License version 2 as published by the Free
+ * Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * version 2 along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
+ * 02110-1301, USA.
+ */
+
+/**
+ * \file
+ *
+ * \author Eric Leblond <el@stamus-networks.com>
+ */
+
+#ifndef __DATASETS_IPV6_H__
+#define __DATASETS_IPV6_H__
+
+#include "datasets-reputation.h"
+
+typedef struct IPv6Type {
+    uint8_t ipv6[16];
+    DataRepType rep;
+} IPv6Type;
+
+int IPv6Set(void *dst, void *src);
+bool IPv6Compare(void *a, void *b);
+uint32_t IPv6Hash(void *s);
+void IPv6Free(void *s);
+
+#endif /* __DATASETS_IPV4_H__ */

diff --git a/src/datasets.c b/src/datasets.c
index 9ad2604f722bd63442443dcb8f14ef03776c1060..6ee1a3e8baf1463fdc2551c083e00f739c04d8b7 100644 (file)
--- a/src/datasets.c
+++ b/src/datasets.c
@@ -26,6 +26,7 @@
 #include "datasets.h"
 #include "datasets-string.h"
 #include "datasets-ipv4.h"
+#include "datasets-ipv6.h"
 #include "datasets-md5.h"
 #include "datasets-sha256.h"
 #include "datasets-reputation.h"
@@ -63,6 +64,8 @@ enum DatasetTypes DatasetGetTypeFromString(const char *s)
         return DATASET_TYPE_STRING;
     if (strcasecmp("ipv4", s) == 0)
         return DATASET_TYPE_IPV4;
+    if (strcasecmp("ip", s) == 0)
+        return DATASET_TYPE_IPV6;
     return DATASET_TYPE_NOTSET;
 }
 
@@ -228,6 +231,105 @@ static int DatasetLoadIPv4(Dataset *set)
     return 0;
 }
 
+static int ParseIpv6String(Dataset *set, char *line, struct in6_addr *in6)
+{
+    /* Checking IPv6 case */
+    char *got_colon = strchr(line, ':');
+    if (got_colon) {
+        uint32_t ip6addr[4];
+        if (inet_pton(AF_INET6, line, in6) != 1) {
+            FatalError(SC_ERR_FATAL, "dataset data parse failed %s/%s: %s", set->name, set->load,
+                    line);
+            return -1;
+        }
+        memcpy(&ip6addr, in6->s6_addr, sizeof(ip6addr));
+        /* IPv4 in IPv6 notation needs transformation to internal Suricata storage */
+        if (ip6addr[0] == 0 && ip6addr[1] == 0 && ip6addr[2] == 0xFFFF0000) {
+            ip6addr[0] = ip6addr[3];
+            ip6addr[2] = 0;
+            ip6addr[3] = 0;
+            memcpy(in6, ip6addr, sizeof(struct in6_addr));
+        }
+    } else {
+        /* IPv4 case */
+        struct in_addr in;
+        if (inet_pton(AF_INET, line, &in) != 1) {
+            FatalError(SC_ERR_FATAL, "dataset data parse failed %s/%s: %s", set->name, set->load,
+                    line);
+            return -1;
+        }
+        memset(in6, 0, sizeof(struct in6_addr));
+        memcpy(in6, &in, sizeof(struct in_addr));
+    }
+    return 0;
+}
+
+static int DatasetLoadIPv6(Dataset *set)
+{
+    if (strlen(set->load) == 0)
+        return 0;
+
+    SCLogConfig("dataset: %s loading from '%s'", set->name, set->load);
+    const char *fopen_mode = "r";
+    if (strlen(set->save) > 0 && strcmp(set->save, set->load) == 0) {
+        fopen_mode = "a+";
+    }
+
+    FILE *fp = fopen(set->load, fopen_mode);
+    if (fp == NULL) {
+        SCLogError(SC_ERR_DATASET, "fopen '%s' failed: %s", set->load, strerror(errno));
+        return -1;
+    }
+
+    uint32_t cnt = 0;
+    char line[1024];
+    while (fgets(line, (int)sizeof(line), fp) != NULL) {
+        char *r = strchr(line, ',');
+        if (r == NULL) {
+            line[strlen(line) - 1] = '\0';
+            SCLogDebug("line: '%s'", line);
+
+            struct in6_addr in6;
+            int ret = ParseIpv6String(set, line, &in6);
+            if (ret < 0)
+                FatalError(SC_ERR_FATAL, "unable to parse IP address");
+
+            if (DatasetAdd(set, (const uint8_t *)&in6.s6_addr, 16) < 0)
+                FatalError(SC_ERR_FATAL, "dataset data add failed %s/%s", set->name, set->load);
+            cnt++;
+
+            /* list with rep data */
+        } else {
+            line[strlen(line) - 1] = '\0';
+            SCLogDebug("IPv6 with REP line: '%s'", line);
+
+            *r = '\0';
+
+            struct in6_addr in6;
+            int ret = ParseIpv6String(set, line, &in6);
+            if (ret < 0)
+                FatalError(SC_ERR_FATAL, "unable to parse IP address");
+
+            r++;
+
+            DataRepType rep = { .value = 0 };
+            if (ParseRepLine(r, strlen(r), &rep) < 0)
+                FatalError(SC_ERR_FATAL, "bad rep for dataset %s/%s", set->name, set->load);
+
+            SCLogDebug("rep v:%u", rep.value);
+            if (DatasetAddwRep(set, (const uint8_t *)&in6.s6_addr, 16, &rep) < 0)
+                FatalError(SC_ERR_FATAL, "dataset data add failed %s/%s", set->name, set->load);
+
+            cnt++;
+        }
+    }
+    THashConsolidateMemcap(set->hash);
+
+    fclose(fp);
+    SCLogConfig("dataset: %s loaded %u records", set->name, cnt);
+    return 0;
+}
+
 static int DatasetLoadMd5(Dataset *set)
 {
     if (strlen(set->load) == 0)
@@ -606,6 +708,15 @@ Dataset *DatasetGet(const char *name, enum DatasetTypes type, const char *save,
             if (DatasetLoadIPv4(set) < 0)
                 goto out_err;
             break;
+        case DATASET_TYPE_IPV6:
+            set->hash = THashInit(cnf_name, sizeof(IPv6Type), IPv6Set, IPv6Free, IPv6Hash,
+                    IPv6Compare, load != NULL ? 1 : 0, memcap > 0 ? memcap : default_memcap,
+                    hashsize > 0 ? hashsize : default_hashsize);
+            if (set->hash == NULL)
+                goto out_err;
+            if (DatasetLoadIPv6(set) < 0)
+                goto out_err;
+            break;
     }
 
     SCLogDebug("set %p/%s type %u save %s load %s",
@@ -878,6 +989,27 @@ static int IPv4AsAscii(const void *s, char *out, size_t out_size)
     return strlen(out);
 }
 
+static int IPv6AsAscii(const void *s, char *out, size_t out_size)
+{
+    const IPv6Type *ip6 = s;
+    char str[256];
+    bool is_ipv4 = true;
+    for (int i = 4; i <= 15; i++) {
+        if (ip6->ipv6[i] != 0) {
+            is_ipv4 = false;
+            break;
+        }
+    }
+    if (is_ipv4) {
+        PrintInet(AF_INET, ip6->ipv6, str, sizeof(str));
+    } else {
+        PrintInet(AF_INET6, ip6->ipv6, str, sizeof(str));
+    }
+    strlcat(out, str, out_size);
+    strlcat(out, "\n", out_size);
+    return strlen(out);
+}
+
 void DatasetsSave(void)
 {
     SCLogDebug("saving datasets: %p", sets);
@@ -906,6 +1038,9 @@ void DatasetsSave(void)
             case DATASET_TYPE_IPV4:
                 THashWalk(set->hash, IPv4AsAscii, SaveCallback, fp);
                 break;
+            case DATASET_TYPE_IPV6:
+                THashWalk(set->hash, IPv6AsAscii, SaveCallback, fp);
+                break;
         }
 
         fclose(fp);
@@ -992,6 +1127,48 @@ static DataRepResultType DatasetLookupIPv4wRep(
     return rrep;
 }
 
+static int DatasetLookupIPv6(Dataset *set, const uint8_t *data, const uint32_t data_len)
+{
+    if (set == NULL)
+        return -1;
+
+    if (data_len != 16 && data_len != 4)
+        return -1;
+
+    IPv6Type lookup = { .rep.value = 0 };
+    memcpy(lookup.ipv6, data, data_len);
+    THashData *rdata = THashLookupFromHash(set->hash, &lookup);
+    if (rdata) {
+        DatasetUnlockData(rdata);
+        return 1;
+    }
+    return 0;
+}
+
+static DataRepResultType DatasetLookupIPv6wRep(
+        Dataset *set, const uint8_t *data, const uint32_t data_len, const DataRepType *rep)
+{
+    DataRepResultType rrep = { .found = false, .rep = { .value = 0 } };
+
+    if (set == NULL)
+        return rrep;
+
+    if (data_len != 16 && data_len != 4)
+        return rrep;
+
+    IPv6Type lookup = { .rep.value = 0 };
+    memcpy(lookup.ipv6, data, data_len);
+    THashData *rdata = THashLookupFromHash(set->hash, &lookup);
+    if (rdata) {
+        IPv6Type *found = rdata->data;
+        rrep.found = true;
+        rrep.rep = found->rep;
+        DatasetUnlockData(rdata);
+        return rrep;
+    }
+    return rrep;
+}
+
 static int DatasetLookupMd5(Dataset *set, const uint8_t *data, const uint32_t data_len)
 {
     if (set == NULL)
@@ -1099,6 +1276,8 @@ int DatasetLookup(Dataset *set, const uint8_t *data, const uint32_t data_len)
             return DatasetLookupSha256(set, data, data_len);
         case DATASET_TYPE_IPV4:
             return DatasetLookupIPv4(set, data, data_len);
+        case DATASET_TYPE_IPV6:
+            return DatasetLookupIPv6(set, data, data_len);
     }
     return -1;
 }
@@ -1119,6 +1298,8 @@ DataRepResultType DatasetLookupwRep(Dataset *set, const uint8_t *data, const uin
             return DatasetLookupSha256wRep(set, data, data_len, rep);
         case DATASET_TYPE_IPV4:
             return DatasetLookupIPv4wRep(set, data, data_len, rep);
+        case DATASET_TYPE_IPV6:
+            return DatasetLookupIPv6wRep(set, data, data_len, rep);
     }
     return rrep;
 }
@@ -1184,6 +1365,26 @@ static int DatasetAddIPv4(Dataset *set, const uint8_t *data, const uint32_t data
     return -1;
 }
 
+static int DatasetAddIPv6(Dataset *set, const uint8_t *data, const uint32_t data_len)
+{
+    if (set == NULL) {
+        return -1;
+    }
+
+    if (data_len != 16) {
+        return -2;
+    }
+
+    IPv6Type lookup = { .rep.value = 0 };
+    memcpy(lookup.ipv6, data, 16);
+    struct THashDataGetResult res = THashGetFromHash(set->hash, &lookup);
+    if (res.data) {
+        DatasetUnlockData(res.data);
+        return res.is_new ? 1 : 0;
+    }
+    return -1;
+}
+
 static int DatasetAddIPv4wRep(
         Dataset *set, const uint8_t *data, const uint32_t data_len, const DataRepType *rep)
 {
@@ -1203,6 +1404,25 @@ static int DatasetAddIPv4wRep(
     return -1;
 }
 
+static int DatasetAddIPv6wRep(
+        Dataset *set, const uint8_t *data, const uint32_t data_len, const DataRepType *rep)
+{
+    if (set == NULL)
+        return -1;
+
+    if (data_len != 16)
+        return -2;
+
+    IPv6Type lookup = { .rep = *rep };
+    memcpy(lookup.ipv6, data, 16);
+    struct THashDataGetResult res = THashGetFromHash(set->hash, &lookup);
+    if (res.data) {
+        DatasetUnlockData(res.data);
+        return res.is_new ? 1 : 0;
+    }
+    return -1;
+}
+
 static int DatasetAddMd5(Dataset *set, const uint8_t *data, const uint32_t data_len)
 {
     if (set == NULL)
@@ -1291,6 +1511,8 @@ int DatasetAdd(Dataset *set, const uint8_t *data, const uint32_t data_len)
             return DatasetAddSha256(set, data, data_len);
         case DATASET_TYPE_IPV4:
             return DatasetAddIPv4(set, data, data_len);
+        case DATASET_TYPE_IPV6:
+            return DatasetAddIPv6(set, data, data_len);
     }
     return -1;
 }
@@ -1310,6 +1532,8 @@ static int DatasetAddwRep(Dataset *set, const uint8_t *data, const uint32_t data
             return DatasetAddSha256wRep(set, data, data_len, rep);
         case DATASET_TYPE_IPV4:
             return DatasetAddIPv4wRep(set, data, data_len, rep);
+        case DATASET_TYPE_IPV6:
+            return DatasetAddIPv6wRep(set, data, data_len, rep);
     }
     return -1;
 }
@@ -1317,7 +1541,8 @@ static int DatasetAddwRep(Dataset *set, const uint8_t *data, const uint32_t data
 typedef int (*DatasetOpFunc)(Dataset *set, const uint8_t *data, const uint32_t data_len);
 
 static int DatasetOpSerialized(Dataset *set, const char *string, DatasetOpFunc DatasetOpString,
-        DatasetOpFunc DatasetOpMd5, DatasetOpFunc DatasetOpSha256, DatasetOpFunc DatasetOpIPv4)
+        DatasetOpFunc DatasetOpMd5, DatasetOpFunc DatasetOpSha256, DatasetOpFunc DatasetOpIPv4,
+        DatasetOpFunc DatasetOpIPv6)
 {
     if (set == NULL)
         return -1;
@@ -1357,6 +1582,12 @@ static int DatasetOpSerialized(Dataset *set, const char *string, DatasetOpFunc D
                 return -2;
             return DatasetOpIPv4(set, (uint8_t *)&in.s_addr, 4);
         }
+        case DATASET_TYPE_IPV6: {
+            struct in_addr in;
+            if (inet_pton(AF_INET6, string, &in) != 1)
+                return -2;
+            return DatasetOpIPv6(set, (uint8_t *)&in.s_addr, 16);
+        }
     }
     return -1;
 }
@@ -1369,8 +1600,8 @@ static int DatasetOpSerialized(Dataset *set, const char *string, DatasetOpFunc D
  */
 int DatasetAddSerialized(Dataset *set, const char *string)
 {
-    return DatasetOpSerialized(
-            set, string, DatasetAddString, DatasetAddMd5, DatasetAddSha256, DatasetAddIPv4);
+    return DatasetOpSerialized(set, string, DatasetAddString, DatasetAddMd5, DatasetAddSha256,
+            DatasetAddIPv4, DatasetAddIPv6);
 }
 
 /** \brief add serialized data to set
@@ -1382,7 +1613,7 @@ int DatasetAddSerialized(Dataset *set, const char *string)
 int DatasetLookupSerialized(Dataset *set, const char *string)
 {
     return DatasetOpSerialized(set, string, DatasetLookupString, DatasetLookupMd5,
-            DatasetLookupSha256, DatasetLookupIPv4);
+            DatasetLookupSha256, DatasetLookupIPv4, DatasetLookupIPv6);
 }
 
 /**
@@ -1413,6 +1644,19 @@ static int DatasetRemoveIPv4(Dataset *set, const uint8_t *data, const uint32_t d
     return THashRemoveFromHash(set->hash, &lookup);
 }
 
+static int DatasetRemoveIPv6(Dataset *set, const uint8_t *data, const uint32_t data_len)
+{
+    if (set == NULL)
+        return -1;
+
+    if (data_len != 16)
+        return -2;
+
+    IPv6Type lookup = { .rep.value = 0 };
+    memcpy(lookup.ipv6, data, 16);
+    return THashRemoveFromHash(set->hash, &lookup);
+}
+
 static int DatasetRemoveMd5(Dataset *set, const uint8_t *data, const uint32_t data_len)
 {
     if (set == NULL)
@@ -1447,5 +1691,5 @@ static int DatasetRemoveSha256(Dataset *set, const uint8_t *data, const uint32_t
 int DatasetRemoveSerialized(Dataset *set, const char *string)
 {
     return DatasetOpSerialized(set, string, DatasetRemoveString, DatasetRemoveMd5,
-            DatasetRemoveSha256, DatasetRemoveIPv4);
+            DatasetRemoveSha256, DatasetRemoveIPv4, DatasetRemoveIPv6);
 }

diff --git a/src/datasets.h b/src/datasets.h
index 9ed8719c8c21ef5d62ca0abec409b587240bc464..af4fc173f1946bab592f2647ebe1b1444db03a78 100644 (file)
--- a/src/datasets.h
+++ b/src/datasets.h
@@ -33,6 +33,7 @@ enum DatasetTypes {
     DATASET_TYPE_MD5,
     DATASET_TYPE_SHA256,
     DATASET_TYPE_IPV4,
+    DATASET_TYPE_IPV6,
 };
 
 #define DATASET_NAME_MAX_LEN 63

diff --git a/src/detect-datarep.c b/src/detect-datarep.c
index 8fae347daa4d3e87e63272376d27580619f0bed5..fac0a131dab798ccabad67856681841d5238c71e 100644 (file)
--- a/src/detect-datarep.c
+++ b/src/detect-datarep.c
@@ -161,6 +161,10 @@ static int DetectDatarepParse(const char *str, char *cmd, int cmd_len, char *nam
                     *type = DATASET_TYPE_STRING;
                 } else if (strcmp(val, "ipv4") == 0) {
                     *type = DATASET_TYPE_IPV4;
+                } else if (strcmp(val, "ip") == 0) {
+                    *type = DATASET_TYPE_IPV6;
+                } else if (strcmp(val, "ipv6") == 0) {
+                    *type = DATASET_TYPE_IPV6;
                 } else {
                     SCLogDebug("bad type %s", val);
                     return -1;

diff --git a/src/detect-dataset.c b/src/detect-dataset.c
index b8f70616898836bc2f77e4f4b89a770519142cbb..ad6a5805895c09fcaf49b3d2ca9ea79afdf88800 100644 (file)
--- a/src/detect-dataset.c
+++ b/src/detect-dataset.c
@@ -158,6 +158,10 @@ static int DetectDatasetParse(const char *str, char *cmd, int cmd_len, char *nam
                     *type = DATASET_TYPE_STRING;
                 } else if (strcmp(val, "ipv4") == 0) {
                     *type = DATASET_TYPE_IPV4;
+                } else if (strcmp(val, "ipv6") == 0) {
+                    *type = DATASET_TYPE_IPV6;
+                } else if (strcmp(val, "ip") == 0) {
+                    *type = DATASET_TYPE_IPV6;
                 } else {
                     SCLogError(SC_ERR_INVALID_SIGNATURE, "bad type %s", val);
                     return -1;