From: Matthew Newton Date: Thu, 14 Jan 2016 16:29:02 +0000 (+0000) Subject: Update elasticsearch example files X-Git-Tag: release_3_0_11~22^2 X-Git-Url: http://git.ipfire.org/cgi-bin/gitweb.cgi?a=commitdiff_plain;h=a9d039751c9518cc63dd7dacece84720279f7b86;p=thirdparty%2Ffreeradius-server.git Update elasticsearch example files - update mappings to correctly set common attributes as "long" rather than "string" - logstash now creates useful sub-fields, as well as combining Gigawords/Octets to a single 64-bit value. - add example log-courier configuration - add dashboards for Kibana3 and Kibana4 --- diff --git a/doc/schemas/logstash/README b/doc/schemas/logstash/README index 93b937c866b..e43fe4def27 100644 --- a/doc/schemas/logstash/README +++ b/doc/schemas/logstash/README @@ -2,9 +2,8 @@ Example configuration for logstash/elasticsearch ================================================ So you've got all these RADIUS logs, but how do you analyse them? What is the -easiest way to query the logs, find out when a client connected or -disconnected, or view the top ten clients logging into the system over the last -six hours? +easiest way to query the logs, find out when a client connected or disconnected, +or view the top ten clients logging into the system over the last six hours? The logstash/elasticsearch/kibana stack is designed and built to do just that. elasticsearch is a search engine; logstash is commonly used to feed data in, @@ -15,13 +14,15 @@ in a short amount of time by any competent sysadmin. Then comes getting the logs in. This directory contains the following files as a starting point for feeding -RADIUS logs into elasticsearch via logstash. +RADIUS logs into elasticsearch via logstash, then sample dashboards for Kibana +to explore the data. Files ----- Please note that all files should be reviewed before use to determine if they -are suitable for your configuration/system. +are suitable for your configuration/system, especially if you are integrating +this into an existing logstash/elasticsearch setup. radius-mapping.sh @@ -34,12 +35,12 @@ radius-mapping.sh This shell script (which just runs curl) pushes a template mapping into the elasticsearch cluster. - -radius.conf +logstash-radius.conf A sample configuration file for logstash that parses RADIUS 'detail' files. It processes these by joining each record onto one line, then splitting the - tab-delimited key-value pairs out. + tab-delimited key-value pairs out. Some additional data is then extracted + from certain key attributes. The file will need to be edited at least to set the input method: for experimentation the given input (stdin) may be used. If logstash is running on @@ -47,6 +48,27 @@ radius.conf input such as log-courier or logstash-forwarder may be better to get the data over the network to the logstash server. + It would be best to use an input method that can join the multiple lines of + the detail file together and feed them to logstash as a single entry, rather + than using the logstash multiline filter. + +log-courier.conf + + An example configuration for the log-courier feeder. + +kibana4-dashboard.json + + Basic RADIUS dashboard for Kibana4. + + To import the dashboard first create a new index called "radius-*" in + Settings/Indices. Then go to Kibana's Settings page, "Objects" and "Import". + Once imported open the "RADIUS detail" dashboard. + +kibana3-dashboard.json + + Basic RADIUS dashboard for Kibana3. To import the dashboard go to Load, + Advanced and "Choose File". + Example usage ------------- @@ -55,7 +77,7 @@ Install mapping (only needs to be done once): $ ./radius-mapping.sh Feed a detail file in: -$ /path/to/logstash -f radius.conf < acct-detail +$ /path/to/logstash -f logstash-radius.conf < acct-detail See also @@ -64,7 +86,6 @@ See also elasticsearch web site: http://www.elastic.co/ - Matthew Newton -April 2015 +January 2016 diff --git a/doc/schemas/logstash/kibana3-dashboard.json b/doc/schemas/logstash/kibana3-dashboard.json new file mode 100644 index 00000000000..694105975b0 --- /dev/null +++ b/doc/schemas/logstash/kibana3-dashboard.json @@ -0,0 +1,476 @@ +{ + "title": "RADIUS detail", + "services": { + "query": { + "list": { + "0": { + "query": "*", + "alias": "", + "color": "#584477", + "id": 0, + "pin": false, + "type": "lucene", + "enable": true + }, + "1": { + "id": 1, + "type": "lucene", + "query": "Acct-Status-Type:Start", + "alias": "Accounting Start", + "color": "#629E51", + "pin": false, + "enable": true + }, + "2": { + "id": 2, + "color": "#6ED0E0", + "alias": "", + "pin": false, + "type": "lucene", + "enable": true, + "query": "Acct-Status-Type:Interim-Update" + }, + "3": { + "id": 3, + "color": "#BF1B00", + "alias": "", + "pin": false, + "type": "lucene", + "enable": true, + "query": "Acct-Status-Type:Stop" + } + }, + "ids": [ + 0, + 1, + 2, + 3 + ] + }, + "filter": { + "list": { + "0": { + "type": "time", + "field": "@timestamp", + "from": "now-7d", + "to": "now", + "mandate": "must", + "active": true, + "alias": "", + "id": 0 + } + }, + "ids": [ + 0 + ] + } + }, + "rows": [ + { + "title": "Time series", + "height": "200px", + "editable": true, + "collapse": false, + "collapsable": true, + "panels": [ + { + "span": 9, + "editable": true, + "type": "histogram", + "loadingEditor": false, + "mode": "count", + "time_field": "@timestamp", + "value_field": null, + "x-axis": true, + "y-axis": true, + "scale": 1, + "y_format": "none", + "grid": { + "max": null, + "min": 0 + }, + "queries": { + "mode": "selected", + "ids": [ + 1, + 2, + 3 + ] + }, + "annotate": { + "enable": false, + "query": "*", + "size": 20, + "field": "_type", + "sort": [ + "_score", + "desc" + ] + }, + "auto_int": true, + "resolution": 100, + "interval": "1h", + "intervals": [ + "auto", + "1s", + "1m", + "5m", + "10m", + "30m", + "1h", + "3h", + "12h", + "1d", + "1w", + "1y" + ], + "lines": false, + "fill": 0, + "linewidth": 3, + "points": false, + "pointradius": 5, + "bars": true, + "stack": true, + "spyable": true, + "zoomlinks": true, + "options": true, + "legend": true, + "show_query": true, + "interactive": true, + "legend_counts": true, + "timezone": "browser", + "percentage": false, + "zerofill": true, + "derivative": false, + "tooltip": { + "value_type": "cumulative", + "query_as_alias": true + }, + "title": "RADIUS Accounting data" + }, + { + "error": false, + "span": 3, + "editable": true, + "type": "terms", + "loadingEditor": false, + "field": "NAS-Identifier", + "exclude": [], + "missing": false, + "other": false, + "size": 20, + "order": "count", + "style": { + "font-size": "10pt" + }, + "donut": false, + "tilt": false, + "labels": true, + "arrangement": "horizontal", + "chart": "pie", + "counter_pos": "above", + "spyable": true, + "queries": { + "mode": "selected", + "ids": [ + 1 + ] + }, + "tmode": "terms", + "tstat": "total", + "valuefield": "", + "title": "Sessions by NAS" + } + ], + "notice": false + }, + { + "title": "Graphs", + "height": "200px", + "editable": true, + "collapse": false, + "collapsable": true, + "panels": [ + { + "error": false, + "span": 3, + "editable": true, + "type": "terms", + "loadingEditor": false, + "field": "Calling-Station-Id", + "exclude": [], + "missing": false, + "other": false, + "size": 10, + "order": "count", + "style": { + "font-size": "10pt" + }, + "donut": false, + "tilt": false, + "labels": true, + "arrangement": "horizontal", + "chart": "table", + "counter_pos": "above", + "spyable": true, + "queries": { + "mode": "selected", + "ids": [ + 1 + ] + }, + "tmode": "terms", + "tstat": "total", + "valuefield": "", + "title": "Top Calling-Station-Id" + }, + { + "error": false, + "span": 3, + "editable": true, + "type": "terms", + "loadingEditor": false, + "field": "Called-Station-Id", + "exclude": [], + "missing": false, + "other": false, + "size": 10, + "order": "count", + "style": { + "font-size": "10pt" + }, + "donut": false, + "tilt": false, + "labels": true, + "arrangement": "horizontal", + "chart": "table", + "counter_pos": "above", + "spyable": true, + "queries": { + "mode": "selected", + "ids": [ + 1 + ] + }, + "tmode": "terms", + "tstat": "total", + "valuefield": "", + "title": "TopN Called-Station-Id" + }, + { + "error": false, + "span": 3, + "editable": true, + "type": "terms", + "loadingEditor": false, + "field": "User-Name", + "exclude": [], + "missing": false, + "other": false, + "size": 10, + "order": "max", + "style": { + "font-size": "10pt" + }, + "donut": false, + "tilt": false, + "labels": true, + "arrangement": "horizontal", + "chart": "table", + "counter_pos": "above", + "spyable": true, + "queries": { + "mode": "all", + "ids": [ + 0, + 1, + 2, + 3 + ] + }, + "tmode": "terms_stats", + "tstat": "max", + "valuefield": "Acct-Output-Octets.long", + "title": "TopN data Output" + }, + { + "error": false, + "span": 3, + "editable": true, + "type": "terms", + "loadingEditor": false, + "field": "User-Name", + "exclude": [], + "missing": false, + "other": false, + "size": 10, + "order": "max", + "style": { + "font-size": "10pt" + }, + "donut": false, + "tilt": false, + "labels": true, + "arrangement": "horizontal", + "chart": "table", + "counter_pos": "above", + "spyable": true, + "queries": { + "mode": "all", + "ids": [ + 0, + 1, + 2, + 3 + ] + }, + "tmode": "terms_stats", + "tstat": "max", + "valuefield": "Acct-Input-Octets.long", + "title": "TopN Data Input" + } + ], + "notice": false + }, + { + "title": "Table", + "height": "150px", + "editable": true, + "collapse": false, + "collapsable": true, + "panels": [ + { + "error": false, + "span": 12, + "editable": true, + "type": "table", + "loadingEditor": false, + "size": 100, + "pages": 5, + "offset": 0, + "sort": [ + "@timestamp", + "asc" + ], + "overflow": "min-height", + "fields": [ + "timestamp", + "User-Name", + "Calling-Station-Id", + "Called-Station-Id", + "Framed-IP-Address", + "NAS-Identifier" + ], + "highlight": [], + "sortable": true, + "header": true, + "paging": true, + "field_list": false, + "all_fields": false, + "trimFactor": 500, + "localTime": false, + "timeField": "@timestamp", + "spyable": true, + "queries": { + "mode": "all", + "ids": [ + 0, + 1, + 2, + 3 + ] + }, + "style": { + "font-size": "9pt" + }, + "normTimes": true, + "title": "RADIUS data" + } + ], + "notice": false + } + ], + "editable": true, + "failover": false, + "index": { + "interval": "day", + "pattern": "[radius-]YYYY.MM.DD", + "default": "[radius-]YYYY.MM.DD", + "warm_fields": false + }, + "style": "dark", + "panel_hints": true, + "pulldowns": [ + { + "type": "query", + "collapse": true, + "notice": false, + "enable": true, + "query": "*", + "pinned": true, + "history": [ + "Acct-Status-Type:Stop", + "Acct-Status-Type:Interim-Update", + "Acct-Status-Type:Start", + "*" + ], + "remember": 10 + }, + { + "type": "filtering", + "collapse": true, + "notice": true, + "enable": true + } + ], + "nav": [ + { + "type": "timepicker", + "collapse": false, + "notice": false, + "enable": true, + "status": "Stable", + "time_options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ], + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "timefield": "@timestamp", + "now": true, + "filter_id": 0 + } + ], + "loader": { + "save_gist": false, + "save_elasticsearch": true, + "save_local": true, + "save_default": true, + "save_temp": true, + "save_temp_ttl_enable": true, + "save_temp_ttl": "30d", + "load_gist": false, + "load_elasticsearch": true, + "load_elasticsearch_size": 20, + "load_local": false, + "hide": false + }, + "refresh": false +} \ No newline at end of file diff --git a/doc/schemas/logstash/kibana4-dashboard.json b/doc/schemas/logstash/kibana4-dashboard.json new file mode 100644 index 00000000000..f3cb850a862 --- /dev/null +++ b/doc/schemas/logstash/kibana4-dashboard.json @@ -0,0 +1,123 @@ +[ + { + "_id": "RADIUS-detail", + "_type": "dashboard", + "_source": { + "title": "RADIUS detail", + "hits": 0, + "description": "", + "panelsJSON": "[{\"col\":5,\"id\":\"RADIUS-unique-User-Name-by-day\",\"row\":1,\"size_x\":4,\"size_y\":4,\"type\":\"visualization\"},{\"col\":1,\"columns\":[\"User-Name\",\"Calling-Station-Id\",\"Called-Station-Id\",\"Framed-IP-Address\",\"NAS-Identifier\"],\"id\":\"RADIUS-data\",\"row\":5,\"size_x\":8,\"size_y\":4,\"sort\":[\"@timestamp\",\"desc\"],\"type\":\"search\"},{\"col\":1,\"id\":\"RADIUS-accounting-packets-histogram\",\"row\":1,\"size_x\":4,\"size_y\":4,\"type\":\"visualization\"},{\"col\":9,\"id\":\"RADIUS-table-topN-data-transferred-by-User-Name\",\"row\":1,\"size_x\":4,\"size_y\":4,\"type\":\"visualization\"},{\"id\":\"RADIUS-Sessions-per-NAS\",\"type\":\"visualization\",\"size_x\":4,\"size_y\":4,\"col\":9,\"row\":5}]", + "version": 1, + "timeRestore": true, + "timeTo": "now", + "timeFrom": "now-7d", + "kibanaSavedObjectMeta": { + "searchSourceJSON": "{\"filter\":[{\"query\":{\"query_string\":{\"analyze_wildcard\":true,\"query\":\"*\"}}}]}" + } + } + }, + { + "_id": "RADIUS-Accounting-Start-data", + "_type": "search", + "_source": { + "title": "RADIUS Accounting-Start data", + "description": "", + "hits": 0, + "columns": [ + "User-Name", + "Calling-Station-Id", + "Called-Station-Id", + "Framed-IP-Address", + "NAS-Identifier" + ], + "sort": [ + "@timestamp", + "desc" + ], + "version": 1, + "kibanaSavedObjectMeta": { + "searchSourceJSON": "{\"index\":\"radius-*\",\"query\":{\"query_string\":{\"analyze_wildcard\":true,\"query\":\"*\"}},\"highlight\":{\"pre_tags\":[\"@kibana-highlighted-field@\"],\"post_tags\":[\"@/kibana-highlighted-field@\"],\"fields\":{\"*\":{}},\"fragment_size\":2147483647},\"filter\":[{\"meta\":{\"negate\":false,\"index\":\"radius-*\",\"key\":\"Acct-Status-Type\",\"value\":\"Start\",\"disabled\":false},\"query\":{\"match\":{\"Acct-Status-Type\":{\"query\":\"Start\",\"type\":\"phrase\"}}}}]}" + } + } + }, + { + "_id": "RADIUS-data", + "_type": "search", + "_source": { + "title": "RADIUS data", + "description": "", + "hits": 0, + "columns": [ + "User-Name", + "Calling-Station-Id", + "Called-Station-Id", + "Framed-IP-Address", + "NAS-Identifier" + ], + "sort": [ + "@timestamp", + "desc" + ], + "version": 1, + "kibanaSavedObjectMeta": { + "searchSourceJSON": "{\"index\":\"radius-*\",\"query\":{\"query_string\":{\"analyze_wildcard\":true,\"query\":\"*\"}},\"highlight\":{\"pre_tags\":[\"@kibana-highlighted-field@\"],\"post_tags\":[\"@/kibana-highlighted-field@\"],\"fields\":{\"*\":{}},\"fragment_size\":2147483647},\"filter\":[]}" + } + } + }, + { + "_id": "RADIUS-unique-User-Name-by-day", + "_type": "visualization", + "_source": { + "title": "RADIUS unique User-Name by day", + "visState": "{\n \"type\": \"histogram\",\n \"params\": {\n \"shareYAxis\": true,\n \"addTooltip\": true,\n \"addLegend\": true,\n \"scale\": \"linear\",\n \"mode\": \"stacked\",\n \"times\": [],\n \"addTimeMarker\": false,\n \"defaultYExtents\": false,\n \"setYExtents\": false,\n \"yAxis\": {}\n },\n \"aggs\": [\n {\n \"id\": \"1\",\n \"type\": \"cardinality\",\n \"schema\": \"metric\",\n \"params\": {\n \"field\": \"User-Name\"\n }\n },\n {\n \"id\": \"2\",\n \"type\": \"date_histogram\",\n \"schema\": \"segment\",\n \"params\": {\n \"field\": \"@timestamp\",\n \"interval\": \"d\",\n \"customInterval\": \"2h\",\n \"min_doc_count\": 1,\n \"extended_bounds\": {}\n }\n },\n {\n \"id\": \"3\",\n \"type\": \"terms\",\n \"schema\": \"group\",\n \"params\": {\n \"field\": \"User-Name\",\n \"size\": 50,\n \"order\": \"desc\",\n \"orderBy\": \"1\"\n }\n }\n ],\n \"listeners\": {}\n}", + "description": "", + "version": 1, + "kibanaSavedObjectMeta": { + "searchSourceJSON": "{\n \"filter\": []\n}" + }, + "savedSearchId": "RADIUS-data" + } + }, + { + "_id": "RADIUS-accounting-packets-histogram", + "_type": "visualization", + "_source": { + "title": "RADIUS accounting packets histogram", + "visState": "{\"type\":\"histogram\",\"params\":{\"shareYAxis\":true,\"addTooltip\":true,\"addLegend\":true,\"scale\":\"linear\",\"mode\":\"stacked\",\"times\":[],\"addTimeMarker\":false,\"defaultYExtents\":false,\"setYExtents\":false,\"yAxis\":{}},\"aggs\":[{\"id\":\"1\",\"type\":\"count\",\"schema\":\"metric\",\"params\":{}},{\"id\":\"2\",\"type\":\"date_histogram\",\"schema\":\"segment\",\"params\":{\"field\":\"@timestamp\",\"interval\":\"auto\",\"customInterval\":\"2h\",\"min_doc_count\":1,\"extended_bounds\":{}}},{\"id\":\"3\",\"type\":\"terms\",\"schema\":\"group\",\"params\":{\"field\":\"Acct-Status-Type\",\"size\":5,\"order\":\"desc\",\"orderBy\":\"1\"}}],\"listeners\":{}}", + "description": "", + "savedSearchId": "RADIUS-data", + "version": 1, + "kibanaSavedObjectMeta": { + "searchSourceJSON": "{\"filter\":[]}" + } + } + }, + { + "_id": "RADIUS-table-topN-data-transferred-by-User-Name", + "_type": "visualization", + "_source": { + "title": "RADIUS table topN data transferred by User-Name", + "visState": "{\"type\":\"table\",\"params\":{\"perPage\":10,\"showPartialRows\":false,\"showMeticsAtAllLevels\":false},\"aggs\":[{\"id\":\"1\",\"type\":\"max\",\"schema\":\"metric\",\"params\":{\"field\":\"Acct-Output-Octets.long\"}},{\"id\":\"2\",\"type\":\"terms\",\"schema\":\"bucket\",\"params\":{\"field\":\"User-Name\",\"size\":10,\"order\":\"desc\",\"orderBy\":\"1\"}},{\"id\":\"3\",\"type\":\"max\",\"schema\":\"metric\",\"params\":{\"field\":\"Acct-Input-Octets.long\"}}],\"listeners\":{}}", + "description": "", + "savedSearchId": "RADIUS-data", + "version": 1, + "kibanaSavedObjectMeta": { + "searchSourceJSON": "{\"filter\":[]}" + } + } + }, + { + "_id": "RADIUS-Sessions-per-NAS", + "_type": "visualization", + "_source": { + "title": "RADIUS Sessions per NAS", + "visState": "{\"type\":\"pie\",\"params\":{\"shareYAxis\":true,\"addTooltip\":true,\"addLegend\":true,\"isDonut\":false},\"aggs\":[{\"id\":\"1\",\"type\":\"count\",\"schema\":\"metric\",\"params\":{}},{\"id\":\"2\",\"type\":\"terms\",\"schema\":\"segment\",\"params\":{\"field\":\"NAS-Identifier\",\"size\":20,\"order\":\"desc\",\"orderBy\":\"1\"}}],\"listeners\":{}}", + "description": "", + "savedSearchId": "RADIUS-Accounting-Start-data", + "version": 1, + "kibanaSavedObjectMeta": { + "searchSourceJSON": "{\"filter\":[]}" + } + } + } +] \ No newline at end of file diff --git a/doc/schemas/logstash/log-courier.conf b/doc/schemas/logstash/log-courier.conf new file mode 100644 index 00000000000..95e16769600 --- /dev/null +++ b/doc/schemas/logstash/log-courier.conf @@ -0,0 +1,56 @@ +# Example log-courier configuration file for RADIUS detail files. +# +{ + "general": { + "persist directory": "/var/lib/log-courier", + "log syslog": true, + "log stdout": false, + "admin listen address": "unix:/var/run/log-courier/admin.socket" + }, + + "network": { + "transport": "tcp", + "reconnect": 10, + + # Servers to connect to. + # + "servers": [ + "logstash1.example:5140", + "logstash2.example:5140" + ] + }, + + "files": [ + { + # Match RADIUS detail files, but not anything that has + # been gzipped. + # + "paths": [ "/var/log/radius/radacct/*/detail-????????" ], + + # Add a type:"radiusdetail" field to the data so that + # logstash can tell what type of data this is (in case + # log-courier is being used for other data as well). + # + "fields": { + "type": "radiusdetail" + }, + + # Stop watching a file if nothing has been written in 12h. + # + "dead time": "12h", + + # Process multilines. If this is being used then the + # "multiline" section should be commented out from the + # logstash configuration. Logstash can then also be run + # with multiple workers (using -w). + # + "codec": { + "name": "multiline", + "pattern": "^[A-Z\t]", + "negate": false, + "what": "next" + } + } + ] +} + diff --git a/doc/schemas/logstash/logstash-radius.conf b/doc/schemas/logstash/logstash-radius.conf new file mode 100644 index 00000000000..a1d9f6a7233 --- /dev/null +++ b/doc/schemas/logstash/logstash-radius.conf @@ -0,0 +1,217 @@ +# logstash configuration to process RADIUS detail files +# +# Matthew Newton +# January 2016 +# +# RADIUS "detail" files are textual representations of the RADIUS +# packets, and are written to disk by e.g. FreeRADIUS. They look +# something like the following, with the timestamp on the first +# line then all attributes/values tab-indented. +# +# Tue Mar 10 15:32:24 2015 +# Packet-Type = Access-Request +# User-Name = "test@example.com" +# Calling-Station-Id = "01-02-03-04-05-06" +# Called-Station-Id = "aa-bb-cc-dd-ee-ff:myssid" +# NAS-Port = 10 +# NAS-IP-Address = 10.9.0.4 +# NAS-Identifier = "Wireless-Controller-1" +# Service-Type = Framed-User +# NAS-Port-Type = Wireless-802.11 +# + + + +# Example input - read data from a file. This can be useful for +# testing, but usually not so much for live service. For example, +# to read in a detail file with this input you could use: +# +# /opt/logstash/bin/logstash -v -f logstash-radius.conf < detailfile + +input { + stdin { + type => radiusdetail + } +} + +# Moving into production will likely need something more reliable. +# There are many input methods, an example here using log-courier +# (which supports client-site multiline processing and does not +# lose log events if logstash is restarted). + +# input { +# courier { +# port => 5140 +# transport => "tcp" +# } +# } + + + +# Filter stage. Here we take the raw logs and process them into +# something structured ready to index. Each attribute is stored as +# a separate field in the output document. + +filter { + + if [type] == "radiusdetail" { + + # If you are using a log feeder that can join + # multiple lines together then that is preferrable + # to using multiline here, because this can not be + # used with threaded logstash (i.e. -w at + # startup). + + # In that case you should comment out the following + # section. For example, see the log-courier + # configuration configuration in this directory. + + multiline { + pattern => "^[A-Z\t]" + negate => false + what => "next" + } + + # Pull off the timestamp at the start of the + # detail record. Note there may be additional data + # after it that has been added by the local admin, + # so stop at a newline OR a tab. + + grok { + match => [ "message", "^(?[^\n\t]+)[\n\t]" ] + } + + # Create the @timestamp field. + + date { + match => [ "timestamp", "EEE MMM dd HH:mm:ss yyyy", + "EEE MMM d HH:mm:ss yyyy" ] + } + + # Split the attributes and values into fields. + # This is the bulk of processing that adds all of + # the RADIUS attributes as elasticsearch fields. + + kv { + field_split => "\n" + source => "message" + trim => "\" " + trimkey => "\t " + } + + # Now we try and add some useful additional + # information. If certain fields can be broken + # down into components then do that here and add + # the data as sub-fields. For example, + # Called-Station-Id might be able to be broken + # down to Called-Station-Id.mac and Called-Station-Id.ssid + # on some wireless systems, or to .ip and .port + # with a VPN. + + # Multiple calls to grok otherwise it can stop + # processing once it has matched one field, but + # e.g. you want to pull both IP and port out of + # the same field in two different regex's. + + # Pull out some IP addresses as field.ip: + + grok { + break_on_match => false + tag_on_failure => [] + match => [ + "Framed-IP-Address", "^(?\d+\.\d+\.\d+\.\d+$)", + "NAS-IP-Address", "^(?\d+\.\d+\.\d+\.\d+$)", + "Calling-Station-Id", "^(?\d+\.\d+\.\d+\.\d+)", + "Called-Station-Id", "^(?\d+\.\d+\.\d+\.\d+)" + ] + } + + # Split User-Name, Operator-Name, and pull out + # some IP ports if they are there: + + grok { + break_on_match => false + tag_on_failure => [] + match => [ + "User-Name", "^(?[^@]+)?(?:@(?[^@]+))$", + "Operator-Name", "^(?.)(?.+)$", + + "Calling-Station-Id", "\[(?\d+)\]$", + "Called-Station-Id", "\[(?\d+)\]$" + ] + } + + # Extract MAC addresses (and SSIDs if there). + # MAC address matching here is lazy, but should be + # good enough. + + grok { + break_on_match => false + tag_on_failure => [] + match => [ + "Calling-Station-Id", "^(?[a-fA-F0-9:-]{17})$", + "Calling-Station-Id", "^(?[a-fA-F0-9\.]{14})$", + "Calling-Station-Id", "^(?[a-fA-F0-9]{12})$", + + "Called-Station-Id", "^(?[a-fA-F0-9:-]{17})(?::(?.*))?$", + "Called-Station-Id", "^(?[a-fA-F0-9\.]{14})(?::(?.*))?$", + "Called-Station-Id", "^(?[a-fA-F0-9]{12})(?::(?.*))?$" + ] + } + + # With the optional sanitize_mac plugin, it's + # possible to make sure all MAC addresses look the + # same, which has obvious benefits. + # + # https://github.com/mcnewton/elk/blob/master/logstash-filters/sanitize_mac.rb + + # sanitize_mac { + # match => { + # "Called-Station-Id.mac" => "Called-Station-Id.mac" + # "Calling-Station-Id.mac" => "Calling-Station-Id.mac" + # } + # separator => ":" + # fixcase => "lower" + # } + + + # Gigawords presents an issue because the 64-bit + # value is split across two attributes. Combine + # them both back into a single attribute so that + # the full value is available to use. + + if ([Acct-Input-Octets]) { + ruby { + code => "event['Acct-Input-Octets.long'] = + event['Acct-Input-Octets'].to_i + ( event['Acct-Input-Gigawords'] ? (event['Acct-Input-Gigawords'].to_i * (2**32)) : 0)" + } + } + + if ([Acct-Output-Octets]) { + ruby { + code => "event['Acct-Output-Octets.long'] = + event['Acct-Output-Octets'].to_i + ( event['Acct-Output-Gigawords'] ? (event['Acct-Output-Gigawords'].to_i * (2**32)) : 0)" + } + } + + } +} + + + +# Output data to the local elasticsearch cluster (called +# "elasticsearch") using type "detail" in index "radius-DATE". + +output { + if [type] == "radiusdetail" { + elasticsearch { + host => localhost + protocol => http + cluster => elasticsearch + index_type => "detail" + index => "radius-%{+YYYY.MM.dd}" + flush_size => 1000 + } + } +} + diff --git a/doc/schemas/logstash/radius-mapping.sh b/doc/schemas/logstash/radius-mapping.sh old mode 100644 new mode 100755 index 25f707186f9..8fe8a480dd2 --- a/doc/schemas/logstash/radius-mapping.sh +++ b/doc/schemas/logstash/radius-mapping.sh @@ -4,8 +4,34 @@ # Matthew Newton # April 2015 -# This should be run on an elasticsearch node. Alternatively, adjust -# the curl URI below. +# This should be run on an elasticsearch node. Alternatively, +# adjust the curl URI below. + +# The template will be called "radius", and will apply to all +# indices prefixed with "radius-" that contain data type "detail". +# As not all RADIUS attributes are known to begin with it has the +# following starting point that can be modified to suit the local +# configuration: +# +# Acct-Input- or Acct-Output- attributes are numbers; +# Acct-Session-Time is a number; +# Everything else is a string. + +# Additionally, the supplied logstash config will try and extract +# MAC addresses, IP addresses and ports from the data. These are +# stored as sub-fields under the respective attribute. For +# example, an attribute +# +# Called-Station-Id := "10.0.4.6[4500]" +# +# will be broken down into the following fields in elasticsearch: +# +# Called-Station-Id = "10.0.4.6[4500]" +# Called-Station-Id.ip = "10.0.4.6" +# Called-Station-Id.port = "4500" +# +# This mapping ensures that these have an appropriate data type. + curl -XPUT '127.0.0.1:9200/_template/radius' -d ' { @@ -13,26 +39,56 @@ curl -XPUT '127.0.0.1:9200/_template/radius' -d ' "order":0, "mappings":{ "detail":{ - "dynamic_templates":[ - { "keep_message":{ - "match":"message", - "mapping":{ - "type":"string", - "index":"analyzed" + + "properties": { + "@timestamp": { "format": "dateOptionalTime", "type": "date" }, + "@version": { "type" : "string" }, + "message": { "type" : "string" }, + "Acct-Session-Time": { "type" : "long", "doc_values": true }, + "offset": { "type" : "long", "doc_values": true } + }, + + "dynamic_templates": [ + + { "acct_io_numbers": { + "match_pattern": "regex", + "match": "^Acct-(Input|Output)-.*$", + "mapping": { + "type": "long", + "doc_values": true } } }, - { "no_analyze_strings":{ - "match":"*", - "match_mapping_type":"string", - "mapping":{ - "type":"string", - "index":"not_analyzed" + + { "ipv4_address": { + "path_match": "*.ip", + "mapping": { + "type": "ip", + "doc_values": true + } + } + }, + + { "network_port": { + "path_match": "*.port", + "mapping": { + "type": "integer", + "doc_values": true + } + } + }, + + { "no_analyze_strings": { + "match": "*", + "mapping": { + "type": "string", + "index": "not_analyzed", + "doc_values": true } } } + ] } } }' - diff --git a/doc/schemas/logstash/radius.conf b/doc/schemas/logstash/radius.conf deleted file mode 100644 index a837fa3c909..00000000000 --- a/doc/schemas/logstash/radius.conf +++ /dev/null @@ -1,77 +0,0 @@ -# logstash configuration to process RADIUS detail files -# -# Matthew Newton -# February 2014 -# -# RADIUS "detail" files are textual representations of the RADIUS -# packets, and are written to disk by e.g. FreeRADIUS. They look -# something like the following, with the timestamp on the first -# line then all attributes/values tab-indented. -# -# Tue Mar 10 15:32:24 2015 -# Packet-Type = Access-Request -# User-Name = "test@example.com" -# Calling-Station-Id = "01-02-03-04-05-06" -# Called-Station-Id = "aa-bb-cc-dd-ee-ff:myssid" -# NAS-Port = 10 -# NAS-IP-Address = 10.9.0.4 -# NAS-Identifier = "Wireless-Controller-1" -# Service-Type = Framed-User -# NAS-Port-Type = Wireless-802.11 -# -# This filter processes the detail file such that each attribute -# is stored as a separate field in the output document. - - -input { - stdin { - type => radiusdetail - } -} - - -filter { - - if [type] == "radiusdetail" { - - # join all lines of a record together - multiline { - pattern => "^[^\t]" - negate => true - what => "previous" - } - - # pull off the timestamp - grok { - match => [ "message", "^(?[^\n\t]+)[\n\t]" ] - } - - # create the timestamp field - date { - match => [ "timestamp", "EEE MMM dd HH:mm:ss yyyy", - "EEE MMM d HH:mm:ss yyyy" ] - } - - # split the attributes and values into fields - kv { - field_split => "\n" - source => "message" - trim => "\" " - trimkey => "\t " - } - } -} - -output { - if [type] == "radiusdetail" { - elasticsearch { - host => localhost - protocol => http - cluster => elasticsearch - index_type => "detail" - index => "radius-%{+YYYY.MM.dd}" - flush_size => 1000 - } - } -} -