From: Matthew Newton <matthew-git@newtoncomputing.co.uk>
Date: Thu, 4 Apr 2019 17:24:51 +0000 (+0100)
Subject: Update elasticsearch/logstash examples
X-Git-Url: http://git.ipfire.org/cgi-bin/gitweb.cgi?a=commitdiff_plain;h=892e2cd1b9c2abd40e89ea0ce5603450ea58c8c8;p=thirdparty%2Ffreeradius-server.git

Update elasticsearch/logstash examples

Tested with elk stack 6.7.0
---

diff --git a/doc/schemas/logstash/README.md b/doc/schemas/logstash/README.md
index 39027edda38..2f36eb6931c 100644
--- a/doc/schemas/logstash/README.md
+++ b/doc/schemas/logstash/README.md
@@ -58,7 +58,7 @@ log-courier.conf
 
 kibana4-dashboard.json
 
-  Basic RADIUS dashboard for Kibana 4 and Kibana 5.
+  Basic RADIUS dashboard (for Kibana 4 to Kibana 6).
 
   To import the dashboard first create a new index called "radius-*" in
   Settings/Indices. Then go to Kibana's Settings page, "Objects" and "Import".
@@ -77,6 +77,8 @@ in:
 
     # /usr/share/logstash/bin/logstash --path.settings=/etc/logstash -f logstash-radius.conf
 
+To view debug output, append `--log.level=debug`.
+
 
 See also
 --------
@@ -84,14 +86,10 @@ See also
 elasticsearch web site: http://www.elastic.co/
 
 The configuration examples presented here have been tested with the
-following software versions (note that elasticsearch 2.x may not yet
-work with this config).
+following software versions:
 
-  elasticsearch 5.1.2
-  logstash 5.1.2
+  elasticsearch 6.7.0
+  logstash 6.7.0
+  kibana 6.7.0
   kibana 5.1.2
   kibana 4.1.11
-
-Matthew Newton
-January 2017
-
diff --git a/doc/schemas/logstash/log-courier.conf b/doc/schemas/logstash/log-courier.conf
index 50c0b58d11c..20d106fecbf 100644
--- a/doc/schemas/logstash/log-courier.conf
+++ b/doc/schemas/logstash/log-courier.conf
@@ -54,4 +54,3 @@
 		}
 	]
 }
-
diff --git a/doc/schemas/logstash/logstash-radius.conf b/doc/schemas/logstash/logstash-radius.conf
index e60a57323a1..f473179be25 100644
--- a/doc/schemas/logstash/logstash-radius.conf
+++ b/doc/schemas/logstash/logstash-radius.conf
@@ -1,9 +1,9 @@
 # logstash configuration to process RADIUS detail files
 #
 # Matthew Newton
-# June 2017
+# April 2019
 #
-# This config has been tested with logstash version 5.4.0.
+# This config has been tested with logstash version 6.7.0
 # 
 # RADIUS "detail" files are textual representations of the RADIUS
 # packets, and are written to disk by e.g. FreeRADIUS. They look
@@ -27,12 +27,13 @@
 # Example input - read data from a file. For example, to read in a
 # detail file with this input you could use:
 #
-#  # /usr/share/logstash/bin/logstash --path.settings=/etc/logstash -f logstash-radius.conf
+#  # /usr/share/logstash/bin/logstash --path.settings=/etc/logstash -f logstash-radius.conf --log.level=debug
 #
 
 input {
 	file {
-		path => "/path/to/radius/detail/file"
+		path => "/var/log/radius/radacct/*/detail-*"
+		exclude => "*.gz"
 
 		# Note when testing that logstash will remember where
 		# it got to and continue from there.
@@ -127,7 +128,7 @@ filter {
 		# Note issue https://github.com/logstash-plugins/logstash-filter-kv/issues/10
 		# currently means that all spaces will be stripped
 		# from all fields. If this is a problem, adjust the
-		# trim_value setting.
+		# trim setting.
 
 		kv {
 			field_split => "\n"
@@ -231,6 +232,13 @@ filter {
 			}
 		}
 
+
+		# Remove the original "message" field.
+
+		mutate {
+			remove_field => ["message"]
+		}
+
 	}
 }
 
@@ -242,10 +250,7 @@ filter {
 output {
 	if [type] == "radiusdetail" {
 		elasticsearch {
-			document_type => "detail"
 			index => "radius-%{+YYYY.MM.dd}"
-			flush_size => 1000
 		}
 	}
 }
-
diff --git a/doc/schemas/logstash/radius-mapping.sh b/doc/schemas/logstash/radius-mapping.sh
index 657d502159e..0ee9a3fdd29 100755
--- a/doc/schemas/logstash/radius-mapping.sh
+++ b/doc/schemas/logstash/radius-mapping.sh
@@ -2,15 +2,16 @@
 
 # Create an elasticsearch template mapping for RADIUS data
 # Matthew Newton
-# April 2015
+# April 2019
 
 # This should be run on an elasticsearch node. Alternatively,
 # adjust the curl URI below.
 
-# This version has been tested on elasticsearch 5.1.2
+# This version has been tested on elasticsearch 6.7.0
 
 # The template will be called "radius", and will apply to all
-# indices prefixed with "radius-" that contain data type "detail".
+# indices prefixed with "radius-".
+#
 # As not all RADIUS attributes are known to begin with it has the
 # following starting point that can be modified to suit the local
 # configuration:
@@ -35,12 +36,12 @@
 # This mapping ensures that these have an appropriate data type.
 
 
-curl -XPUT '127.0.0.1:9200/_template/radius' -d '
+curl -s -XPUT -H 'Content-Type: application/json' '127.0.0.1:9200/_template/radius' -d '
 {
   "template":"radius-*",
   "order":0,
   "mappings":{
-    "detail":{
+    "doc":{
 
       "properties": {
         "@timestamp": { "format" : "date_optional_time", "type" : "date" },