]> git.ipfire.org Git - thirdparty/ulogd2.git/commitdiff
IPFIX: Introduce template record support
authorAnder Juaristi <a@juaristi.eus>
Fri, 26 Apr 2019 07:58:07 +0000 (09:58 +0200)
committerPablo Neira Ayuso <pablo@netfilter.org>
Tue, 30 Apr 2019 12:11:54 +0000 (14:11 +0200)
This commit adds the ability to send template records
to the remote collector.

In addition, it also introduces a new
configuration parameter 'send_template', which tells when template
records should be sent. It accepts the following string values:

 - "once": Send the template record only the first time (might be coalesced
    with data records).
 - "always": Send the template record always, with every data record that is sent
    to the collector (multiple data records might be sent together).
 - "never": Assume the collector knows the schema already. Do not send template records.

If omitted, the default value for 'send_template' is "once".

Signed-off-by: Ander Juaristi <a@juaristi.eus>
Signed-off-by: Pablo Neira Ayuso <pablo@netfilter.org>
include/ulogd/ipfix_protocol.h
output/ipfix/ipfix.c
output/ipfix/ipfix.h
output/ipfix/ulogd_output_IPFIX.c

index aef47f0a37fa6536c802d7a973c204adfd4ae5fd..01dd96abea229b9b373e98e0296b12cc7238800a 100644 (file)
@@ -129,6 +129,7 @@ enum {
        /* reserved */
        IPFIX_fragmentOffsetIPv4        = 88,
        /* reserved */
+       IPFIX_applicationId             = 95,
        IPFIX_bgpNextAdjacentAsNumber   = 128,
        IPFIX_bgpPrevAdjacentAsNumber   = 129,
        IPFIX_exporterIPv4Address       = 130,
index 60a4c7f4da619e3470812fa09070f1b2da9e275d..4bb432a73d14fa1ae4770f9ad9e79c91a9e7ed54 100644 (file)
@@ -2,6 +2,7 @@
  * ipfix.c
  *
  * Holger Eitzenberger, 2009.
+ * Ander Juaristi, 2019
  */
 
 /* These forward declarations are needed since ulogd.h doesn't like to be the first */
 
 #include <ulogd/ulogd.h>
 #include <ulogd/common.h>
+#include <ulogd/ipfix_protocol.h>
+
+struct ipfix_templ_elem {
+       uint16_t id;
+       uint16_t len;
+};
+
+struct ipfix_templ {
+       unsigned int num_templ_elements;
+       struct ipfix_templ_elem templ_elements[];
+};
+
+/* Template fields modeled after vy_ipfix_data */
+static const struct ipfix_templ template = {
+       .num_templ_elements = 10,
+       .templ_elements = {
+               {
+                       .id = IPFIX_sourceIPv4Address,
+                       .len = sizeof(uint32_t)
+               },
+               {
+                       .id = IPFIX_destinationIPv4Address,
+                       .len = sizeof(uint32_t)
+               },
+               {
+                       .id = IPFIX_packetTotalCount,
+                       .len = sizeof(uint32_t)
+               },
+               {
+                       .id = IPFIX_octetTotalCount,
+                       .len = sizeof(uint32_t)
+               },
+               {
+                       .id = IPFIX_flowStartSeconds,
+                       .len = sizeof(uint32_t)
+               },
+               {
+                       .id = IPFIX_flowEndSeconds,
+                       .len = sizeof(uint32_t)
+               },
+               {
+                       .id = IPFIX_sourceTransportPort,
+                       .len = sizeof(uint16_t)
+               },
+               {
+                       .id = IPFIX_destinationTransportPort,
+                       .len = sizeof(uint16_t)
+               },
+               {
+                       .id = IPFIX_protocolIdentifier,
+                       .len = sizeof(uint8_t)
+               },
+               {
+                       .id = IPFIX_applicationId,
+                       .len = sizeof(uint32_t)
+               }
+       }
+};
 
-struct ipfix_msg *ipfix_msg_alloc(size_t len, uint32_t oid)
+struct ipfix_msg *ipfix_msg_alloc(size_t len, uint32_t oid, int tid)
 {
        struct ipfix_msg *msg;
        struct ipfix_hdr *hdr;
+       struct ipfix_templ_hdr *templ_hdr;
+       struct ipfix_templ_elem *elem;
+       unsigned int i = 0;
 
-       if (len < IPFIX_HDRLEN + IPFIX_SET_HDRLEN)
+       if ((tid > 0 && len < IPFIX_HDRLEN + IPFIX_TEMPL_HDRLEN(template.num_templ_elements) + IPFIX_SET_HDRLEN) ||
+           (len < IPFIX_HDRLEN + IPFIX_SET_HDRLEN))
                return NULL;
 
        msg = malloc(sizeof(struct ipfix_msg) + len);
        memset(msg, 0, sizeof(struct ipfix_msg));
-       msg->tail = msg->data + IPFIX_HDRLEN;
+       msg->tid = tid;
        msg->end = msg->data + len;
+       msg->tail = msg->data + IPFIX_HDRLEN;
+       if (tid > 0)
+               msg->tail += IPFIX_TEMPL_HDRLEN(template.num_templ_elements);
 
+       /* Initialize message header */
        hdr = ipfix_msg_hdr(msg);
        memset(hdr, 0, IPFIX_HDRLEN);
        hdr->version = htons(IPFIX_VERSION);
        hdr->oid = htonl(oid);
 
+       if (tid > 0) {
+               /* Initialize template record header */
+               templ_hdr = ipfix_msg_templ_hdr(msg);
+               templ_hdr->sid = htons(2);
+               templ_hdr->tid = htons(tid);
+               templ_hdr->len = htons(IPFIX_TEMPL_HDRLEN(template.num_templ_elements));
+               templ_hdr->cnt = htons(template.num_templ_elements);
+
+               while (i < template.num_templ_elements) {
+                       elem = (struct ipfix_templ_elem *) &templ_hdr->data[i * 4];
+                       elem->id = htons(template.templ_elements[i].id);
+                       elem->len = htons(template.templ_elements[i].len);
+                       i++;
+               }
+       }
+
        return msg;
 }
 
@@ -47,6 +130,14 @@ void ipfix_msg_free(struct ipfix_msg *msg)
        free(msg);
 }
 
+struct ipfix_templ_hdr *ipfix_msg_templ_hdr(const struct ipfix_msg *msg)
+{
+       if (msg->tid > 0)
+               return (struct ipfix_templ_hdr *) (msg->data + IPFIX_HDRLEN);
+
+       return NULL;
+}
+
 struct ipfix_hdr *ipfix_msg_hdr(const struct ipfix_msg *msg)
 {
        return (struct ipfix_hdr *)msg->data;
index cdb5a6fb67f697eadb364e561dd0dfd733119691..93945fbd562bead3167d0b7629696dc7ffb0e065 100644 (file)
@@ -2,6 +2,7 @@
  * ipfix.h
  *
  * Holger Eitzenberger <holger@eitzenberger.org>, 2009.
+ * Ander Juaristi <a@juaristi.eus>, 2019
  */
 #ifndef IPFIX_H
 #define IPFIX_H
@@ -20,17 +21,21 @@ struct ipfix_hdr {
        uint8_t data[];
 } __packed;
 
-#define IPFIX_HDRLEN   sizeof(struct ipfix_hdr)
+#define IPFIX_HDRLEN           sizeof(struct ipfix_hdr)
 
 /*
  * IDs 0-255 are reserved for Template Sets.  IDs of Data Sets are > 255.
  */
 struct ipfix_templ_hdr {
-       uint16_t id;
+       uint16_t sid;
+       uint16_t len;
+       uint16_t tid;
        uint16_t cnt;
        uint8_t data[];
 } __packed;
 
+#define IPFIX_TEMPL_HDRLEN(nfields)    sizeof(struct ipfix_templ_hdr) + (sizeof(uint16_t) * 2 * nfields)
+
 struct ipfix_set_hdr {
 #define IPFIX_SET_TEMPL                        2
 #define IPFIX_SET_OPT_TEMPL            3
@@ -46,6 +51,7 @@ struct ipfix_msg {
        uint8_t *tail;
        uint8_t *end;
        unsigned nrecs;
+       int tid;
        struct ipfix_set_hdr *last_set;
        uint8_t data[];
 };
@@ -53,18 +59,14 @@ struct ipfix_msg {
 struct vy_ipfix_data {
        struct in_addr saddr;
        struct in_addr daddr;
-       uint16_t ifi_in;
-       uint16_t ifi_out;
        uint32_t packets;
        uint32_t bytes;
        uint32_t start;                         /* Unix time */
        uint32_t end;                           /* Unix time */
        uint16_t sport;
        uint16_t dport;
-       uint32_t aid;                           /* Application ID */
        uint8_t l4_proto;
-       uint8_t dscp;
-       uint16_t __padding;
+       uint32_t aid;                           /* Application ID */
 } __packed;
 
 #define VY_IPFIX_SID           256
@@ -73,13 +75,11 @@ struct vy_ipfix_data {
 #define VY_IPFIX_PKT_LEN       (IPFIX_HDRLEN + IPFIX_SET_HDRLEN \
                                                         + VY_IPFIX_FLOWS * sizeof(struct vy_ipfix_data))
 
-/* template management */
-size_t ipfix_rec_len(uint16_t);
-
 /* message handling */
-struct ipfix_msg *ipfix_msg_alloc(size_t, uint32_t);
+struct ipfix_msg *ipfix_msg_alloc(size_t, uint32_t, int);
 void ipfix_msg_free(struct ipfix_msg *);
 struct ipfix_hdr *ipfix_msg_hdr(const struct ipfix_msg *);
+struct ipfix_templ_hdr *ipfix_msg_templ_hdr(const struct ipfix_msg *);
 size_t ipfix_msg_len(const struct ipfix_msg *);
 void *ipfix_msg_data(struct ipfix_msg *);
 struct ipfix_set_hdr *ipfix_msg_add_set(struct ipfix_msg *, uint16_t);
index ec143b1619a80c02868b656cf539a30b54fa4b55..5b59003638531f522021ed960d948dfb58dd4835 100644 (file)
@@ -3,6 +3,9 @@
  *
  * ulogd IPFIX Exporter plugin.
  *
+ * (C) 2009 by Holger Eitzenberger <holger@eitzenberger.org>, Astaro AG
+ * (C) 2019 by Ander Juaristi <a@juaristi.eus>
+ *
  * This program is distributed in the hope that it will be useful,
  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
@@ -11,8 +14,6 @@
  * You should have received a copy of the GNU General Public License
  * along with this program; if not, write to the Free Software
  * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
- *
- * Holger Eitzenberger <holger@eitzenberger.org>  Astaro AG 2009
  */
 #include <unistd.h>
 #include <time.h>
@@ -28,6 +29,7 @@
 #define DEFAULT_MTU            512 /* RFC 5101, 10.3.3 */
 #define DEFAULT_PORT           4739 /* RFC 5101, 10.3.4 */
 #define DEFAULT_SPORT          4740
+#define DEFAULT_SEND_TEMPLATE  "once"
 
 enum {
        OID_CE = 0,
@@ -35,16 +37,18 @@ enum {
        PORT_CE,
        PROTO_CE,
        MTU_CE,
+       SEND_TEMPLATE_CE
 };
 
-#define oid_ce(x)      (x->ces[OID_CE])
-#define host_ce(x)     (x->ces[HOST_CE])
-#define port_ce(x)     (x->ces[PORT_CE])
-#define proto_ce(x)    (x->ces[PROTO_CE])
-#define mtu_ce(x)      (x->ces[MTU_CE])
+#define oid_ce(x)              (x->ces[OID_CE])
+#define host_ce(x)             (x->ces[HOST_CE])
+#define port_ce(x)             (x->ces[PORT_CE])
+#define proto_ce(x)            (x->ces[PROTO_CE])
+#define mtu_ce(x)              (x->ces[MTU_CE])
+#define send_template_ce(x)    (x->ces[SEND_TEMPLATE_CE])
 
 static const struct config_keyset ipfix_kset = {
-       .num_ces = 5,
+       .num_ces = 6,
        .ces = {
                {
                        .key = "oid",
@@ -70,20 +74,21 @@ static const struct config_keyset ipfix_kset = {
                        .key = "mtu",
                        .type = CONFIG_TYPE_INT,
                        .u.value = DEFAULT_MTU
+               },
+               {
+                       .key = "send_template",
+                       .type = CONFIG_TYPE_STRING,
+                       .u.string = DEFAULT_SEND_TEMPLATE
                }
        }
 };
 
-struct ipfix_templ {
-       struct ipfix_templ *next;
-};
-
 struct ipfix_priv {
        struct ulogd_fd ufd;
        uint32_t seqno;
        struct ipfix_msg *msg;          /* current message */
        struct llist_head list;
-       struct ipfix_templ *templates;
+       int tid;
        int proto;
        struct ulogd_timer timer;
        struct sockaddr_in sa;
@@ -258,8 +263,8 @@ static void ipfix_timer_cb(struct ulogd_timer *t, void *data)
 static int ipfix_configure(struct ulogd_pluginstance *pi, struct ulogd_pluginstance_stack *stack)
 {
        struct ipfix_priv *priv = (struct ipfix_priv *) &pi->private;
+       char *host, *proto, *send_template;
        int oid, port, mtu, ret;
-       char *host, *proto;
        char addr[16];
 
        ret = config_parse_file(pi->id, pi->config_kset);
@@ -271,6 +276,7 @@ static int ipfix_configure(struct ulogd_pluginstance *pi, struct ulogd_pluginsta
        port = port_ce(pi->config_kset).u.value;
        proto = proto_ce(pi->config_kset).u.string;
        mtu = mtu_ce(pi->config_kset).u.value;
+       send_template = send_template_ce(pi->config_kset).u.string;
 
        if (!oid) {
                ulogd_log(ULOGD_FATAL, "invalid Observation ID\n");
@@ -303,6 +309,8 @@ static int ipfix_configure(struct ulogd_pluginstance *pi, struct ulogd_pluginsta
 
        ulogd_init_timer(&priv->timer, pi, ipfix_timer_cb);
 
+       priv->tid = (strcmp(send_template, "never") ? VY_IPFIX_SID : -1);
+
        ulogd_log(ULOGD_INFO, "using IPFIX Collector at %s:%d (MTU %d)\n",
                  inet_ntop(AF_INET, &priv->sa.sin_addr, addr, sizeof(addr)),
                  port, mtu);
@@ -410,25 +418,30 @@ static int ipfix_stop(struct ulogd_pluginstance *pi)
 static int ipfix_interp(struct ulogd_pluginstance *pi)
 {
        struct ipfix_priv *priv = (struct ipfix_priv *) &pi->private;
+       char saddr[16], daddr[16], *send_template;
        struct vy_ipfix_data *data;
        int oid, mtu, ret;
-       char addr[16];
 
        if (!(GET_FLAGS(pi->input.keys, InIpSaddr) & ULOGD_RETF_VALID))
                return ULOGD_IRET_OK;
 
        oid = oid_ce(pi->config_kset).u.value;
        mtu = mtu_ce(pi->config_kset).u.value;
+       send_template = send_template_ce(pi->config_kset).u.string;
 
 again:
        if (!priv->msg) {
-               priv->msg = ipfix_msg_alloc(mtu, oid);
+               priv->msg = ipfix_msg_alloc(mtu, oid, priv->tid);
                if (!priv->msg) {
                        /* just drop this flow */
                        ulogd_log(ULOGD_ERROR, "out of memory, dropping flow\n");
                        return ULOGD_IRET_OK;
                }
                ipfix_msg_add_set(priv->msg, VY_IPFIX_SID);
+
+               /* template sent - do not send it again the next time */
+               if (priv->tid == VY_IPFIX_SID && strcmp(send_template, "once") == 0)
+                       priv->tid = -1;
        }
 
        data = ipfix_msg_add_data(priv->msg, sizeof(struct vy_ipfix_data));
@@ -439,8 +452,6 @@ again:
                goto again;
        }
 
-       data->ifi_in = data->ifi_out = 0;
-
        data->saddr.s_addr = ikey_get_u32(&pi->input.keys[InIpSaddr]);
        data->daddr.s_addr = ikey_get_u32(&pi->input.keys[InIpDaddr]);
 
@@ -462,13 +473,12 @@ again:
                data->aid = htonl(ikey_get_u32(&pi->input.keys[InCtMark]));
 
        data->l4_proto = ikey_get_u8(&pi->input.keys[InIpProto]);
-       data->__padding = 0;
 
        ulogd_log(ULOGD_DEBUG, "Got new packet (packets = %u, bytes = %u, flow = (%u, %u), saddr = %s, daddr = %s, sport = %u, dport = %u)\n",
-                       ntohl(data->packets), ntohl(data->bytes), ntohl(data->start), ntohl(data->end),
-                       inet_ntop(AF_INET, &data->saddr.s_addr, addr, sizeof(addr)),
-                       inet_ntop(AF_INET, &data->daddr.s_addr, addr, sizeof(addr)),
-                       ntohs(data->sport), ntohs(data->dport));
+                 ntohl(data->packets), ntohl(data->bytes), ntohl(data->start), ntohl(data->end),
+                 inet_ntop(AF_INET, &data->saddr.s_addr, saddr, sizeof(saddr)),
+                 inet_ntop(AF_INET, &data->daddr.s_addr, daddr, sizeof(daddr)),
+                 ntohs(data->sport), ntohs(data->dport));
 
        if ((ret = send_msgs(pi)) < 0)
                return ret;