Work in progress!

This is the foundation for the distributed T-Pot feature,
highly work in progress, only works with local docker image builds,
will be available for prod for upcoming T-Pot 22xx.

docker/elk/logstash/dist/http_output.conf

@@ -147,6 +147,13 @@ input {
     type => "Ipphoney"
   }
 
+# Log4pot
+  file {
+    path => ["/data/log4pot/log/log4pot.log"]
+    codec => json
+    type => "Log4pot"
+  }
+
 # Mailoney
   file {
     path => ["/data/mailoney/log/commands.log"]
@@ -564,6 +571,20 @@ filter {
     }
   }
 
+# Log4pot
+  if [type] == "Log4pot" {
+    date {
+      match => [ "timestamp", "ISO8601" ]
+    }
+    mutate {
+      rename => {
+        "server_port" => "dest_port"
+        "port" => "src_port"
+        "client" => "src_ip"
+      }
+    }
+  }
+
 # Mailoney
   if [type] == "Mailoney" {
     date {
@@ -649,12 +670,12 @@ if "_jsonparsefailure" in [tags] { drop {} }
     geoip {
       cache_size => 10000
       source => "src_ip"
-      database => "/usr/share/logstash/vendor/bundle/jruby/2.5.0/gems/logstash-filter-geoip-7.2.3-java/vendor/GeoLite2-City.mmdb"
+      database => "/usr/share/logstash/vendor/bundle/jruby/2.5.0/gems/logstash-filter-geoip-7.2.8-java/vendor/GeoLite2-City.mmdb"
     }
     geoip {
       cache_size => 10000
       source => "src_ip"
-      database => "/usr/share/logstash/vendor/bundle/jruby/2.5.0/gems/logstash-filter-geoip-7.2.3-java/vendor/GeoLite2-ASN.mmdb"
+      database => "/usr/share/logstash/vendor/bundle/jruby/2.5.0/gems/logstash-filter-geoip-7.2.8-java/vendor/GeoLite2-ASN.mmdb"
     }
     translate {
       refresh_interval => 86400

docker/elk/logstash/dist/logstash.conf

@@ -730,7 +730,7 @@ output {
     # With templates now being legacy and ILM in place we need to set the daily index with its template manually. Otherwise a new index might be created with differents settings configured through Kibana.
     index => "logstash-%{+YYYY.MM.dd}"
     template => "/etc/logstash/tpot_es_template.json"
-#    document_type => "doc"
+    #document_type => "doc"
   }
 
   #if [type] == "Suricata" {

docker/elk/logstash/dist/logstash.sh

@@ -0,0 +1,55 @@
+#!/bin/bash
+
+# Let's ensure normal operation on exit or if interrupted ...
+function fuCLEANUP {
+  exit 0
+}
+trap fuCLEANUP EXIT
+
+# Check internet availability 
+function fuCHECKINET () {
+mySITES=$1
+error=0
+for i in $mySITES;
+  do
+    curl --connect-timeout 5 -Is $i 2>&1 > /dev/null
+      if [ $? -ne 0 ];
+        then
+          let error+=1
+      fi;
+  done;
+  echo $error
+}
+
+# Check for connectivity and download latest translation maps
+myCHECK=$(fuCHECKINET "listbot.sicherheitstacho.eu")
+if [ "$myCHECK" == "0" ];
+  then
+    echo "Connection to Listbot looks good, now downloading latest translation maps."
+    cd /etc/listbot 
+    aria2c -s16 -x 16 https://listbot.sicherheitstacho.eu/cve.yaml.bz2 && \
+    aria2c -s16 -x 16 https://listbot.sicherheitstacho.eu/iprep.yaml.bz2 && \
+    bunzip2 -f *.bz2
+    cd /
+  else
+    echo "Cannot reach Listbot, starting Logstash without latest translation maps."
+fi
+
+exit
+
+
+# notizen
+
+MY_TPOT_TYPE Standard = SINGLE, Distributed = POT
+
+Wenn POT
+autossh -f -M 0 -4 -l tpot01 -i /data/elk/logstash/tpot01 -p 64295 -N -L64305:127.0.0.1:64305 172.20.254.194 -o "ServerAliveInterval 30" -o "ServerAliveCountMax 3" -o "StrictHostKeyChecking=no" -o "UserKnownHostsFile=/dev/null"
+exec /usr/share/logstash/bin/logstash -f /etc/logstash/conf.d/http_output.conf --config.reload.automatic --java-execution
+
+
+Wenn SINGLE
+exec /usr/share/logstash/bin/logstash --config.reload.automatic --java-execution
+
+Umgebungsvariable holen aus /data/elk/logstash
+m besten über das ELK Environment file, damit es keine probleme gibt
+

docker/elk/logstash/dist/pipelines_pot.yml

@@ -0,0 +1,2 @@
+- pipeline.id: http_output
+  path.config: "/etc/logstash/conf.d/http_output.conf"

docker/elk/logstash/dist/update.sh

@@ -35,6 +35,22 @@ if [ "$myCHECK" == "0" ];
     echo "Cannot reach Listbot, starting Logstash without latest translation maps."
 fi
 
+# Distributed T-Pot installation needs a different pipeline config and autossh tunnel. 
+if [ "$MY_TPOT_TYPE" == "POT" ];
+  then
+    echo
+    echo "Distributed T-Pot setup, sending T-Pot logs to $MY_HIVE_IP."
+    echo
+    echo "T-Pot type: $MY_TPOT_TYPE"
+    echo "Keyfile used: $MY_POT_PRIVATEKEYFILE"
+    echo "Hive username: $MY_HIVE_USERNAME"
+    echo "Hive IP: $MY_HIVE_IP"
+    echo
+    cp /usr/share/logstash/config/pipelines_pot.yml /usr/share/logstash/config/pipelines.yml
+    autossh -f -M 0 -v -4 -l $MY_HIVE_USERNAME -i $MY_POT_PRIVATEKEYFILE -p 64295 -N -L64305:127.0.0.1:64305 $MY_HIVE_IP -o "ServerAliveInterval 30" -o "ServerAliveCountMax 3" -o "StrictHostKeyChecking=no" -o "UserKnownHostsFile=/dev/null"
+    exit 0
+fi
+
 # We do want to enforce our es_template thus we always need to delete the default template, putting our default afterwards
 # This is now done via common_configs.rb => overwrite default logstash template
 echo "Removing logstash template."