diff --git a/docker/elk/docker-compose.yml b/docker/elk/docker-compose.yml index 07d28328..116dc7d1 100644 --- a/docker/elk/docker-compose.yml +++ b/docker/elk/docker-compose.yml @@ -54,7 +54,6 @@ services: image: "dtagdevsec/logstash:1804" volumes: - /data:/data - - /var/log:/data/host/log - /root/tpotce/docker/elk/logstash/dist/logstash.conf:/etc/logstash/conf.d/logstash.conf ## Elasticsearch-head service diff --git a/docker/elk/logstash/dist/logstash.conf b/docker/elk/logstash/dist/logstash.conf index 0dd8bf35..dd3e05b5 100644 --- a/docker/elk/logstash/dist/logstash.conf +++ b/docker/elk/logstash/dist/logstash.conf @@ -22,7 +22,7 @@ input { type => "Ciscoasa" } -# Conpot +# Conpot file { path => ["/data/conpot/log/*.json"] codec => json @@ -82,7 +82,7 @@ input { type => "Honeytrap" } -# Mailoney +# Mailoney file { path => ["/data/mailoney/log/commands.log"] type => "Mailoney" @@ -94,21 +94,14 @@ input { type => "Rdpy" } -# Host Syslog - file { - path => ["/data/host/log/auth.log"] - codec => plain - type => "Syslog" - } - # Host NGINX file { - path => ["/data/host/log/nginx/access.log"] + path => ["/data/nginx/log/access.log"] codec => json type => "NGINX" } -# Tanner +# Tanner file { path => ["/data/tanner/log/tanner_report.json"] codec => json @@ -256,8 +249,8 @@ filter { # Heralding if [type] == "Heralding" { - csv { - columns => ["timestamp","auth_id","session_id","src_ip","src_port","dest_ip","dest_port","proto","username","password"] separator => "," + csv { + columns => ["timestamp","auth_id","session_id","src_ip","src_port","dest_ip","dest_port","proto","username","password"] separator => "," } date { match => [ "timestamp", "yyyy-MM-dd HH:mm:ss.SSSSSS" ] @@ -310,79 +303,6 @@ filter { } } -# Syslog - if [type] == "Syslog" { - grok { - match => { - "message" => ["%{SYSLOGPAMSESSION}", "%{CRONLOG}", "%{SYSLOGLINE}"] - } - overwrite => "message" - } - date { - match => [ "timestamp", "MMM d HH:mm:ss", "MMM dd HH:mm:ss" ] - remove_field => ["timestamp"] - } - date { - match => ["timestamp8601", "ISO8601"] - remove_field => ["timestamp8601"] - } - grok { - match => { "message" => "Connection closed by %{IP:src_ip}" } - add_tag => [ "ssh_connection_closed" ] - tag_on_failure => [] - } - grok { - match => { "message" => "Received disconnect from %{IP:src_ip}" } - add_tag => [ "ssh_connection_disconnect" ] - tag_on_failure => [] - } - grok { - match => { "message" => "Failed password for invalid user %{USERNAME:username} from %{IP:src_ip} port %{BASE10NUM:port} ssh2" } - add_tag => [ "ssh_failed_password" ] - tag_on_failure => [] - } - grok { - match => { "message" => "Did not receive identification string from %{IP:src_ip}" } - add_tag => [ "ssh_no_id" ] - tag_on_failure => [] - } - grok { - match => { "message" => "User %{USERNAME:username} from %{IP:src_ip} not allowed because not listed in AllowUsers" } - add_tag => [ "ssh_user_not_allowed" ] - tag_on_failure => [] - } - grok { - match => { "message" => "authentication failure; logname=%{USERNAME:logname} uid=%{BASE10NUM:uid} euid=%{BASE10NUM:euid} tty=%{TTY:tty} ruser=%{USERNAME:ruser} rhost=(?:%{HOSTNAME:remote_host}|\s*) user=%{USERNAME:user}"} - add_tag => [ "ssh_auth_failure" ] - tag_on_failure => [] - } - grok { - match => { "message" => "pam_unix\(sshd:auth\): authentication failure; logname= uid=0 euid=0 tty=%{NOTSPACE:tty} ruser= rhost=(?:%{HOSTNAME:remote_host}|\s*) user=%{USERNAME:user}"} - add_tag => [ "ssh_auth_failure" ] - tag_on_failure => [] - } - grok { - match => { "message" => "Failed password for %{USERNAME:username} from %{IP:src_ip} port %{BASE10NUM:port} ssh2"} - add_tag => [ "ssh_failed_password" ] - tag_on_failure => [] - } - grok { - match => { "message" => "Accepted password for %{USERNAME:username} from %{IP:src_ip} port %{BASE10NUM:port} ssh2"} - add_tag => [ "ssh_accepted_password" ] - tag_on_failure => [] - } - grok { - match => { "message" => "Accepted publickey for %{USERNAME:username} from %{IP:src_ip} port %{BASE10NUM:port} ssh2"} - add_tag => [ "ssh_accepted_pubkey" ] - tag_on_failure => [] - } - grok { - match => { "message" => "Accepted keyboard-interactive/pam for %{USERNAME:username} from %{IP:src_ip} port %{BASE10NUM:port} ssh2"} - add_tag => [ "ssh_accepted_interactive" ] - tag_on_failure => [] - } - } - # NGINX if [type] == "NGINX" { date { @@ -390,7 +310,7 @@ filter { } } -# Tanner +# Tanner if [type] == "Tanner" { date { match => [ "timestamp", "ISO8601" ] @@ -406,7 +326,7 @@ filter { } } -# Vnclowpot +# Vnclowpot if [type] == "Vnclowpot" { grok { match => [ "message", "\A%{NOTSPACE}%{SPACE}%{TIME}%{SPACE}%{IPV4:src_ip}:%{INT:src_port}%{SPACE}%{NOTSPACE:vnc_handshake}" ] @@ -431,19 +351,19 @@ if "_grokparsefailure" in [tags] { drop {} } cache_size => 10000 source => "src_ip" database => "/usr/share/logstash/vendor/bundle/jruby/2.3.0/gems/logstash-filter-geoip-5.0.3-java/vendor/GeoLite2-City.mmdb" - } + } geoip { cache_size => 10000 source => "src_ip" database => "/usr/share/logstash/vendor/bundle/jruby/2.3.0/gems/logstash-filter-geoip-5.0.3-java/vendor/GeoLite2-ASN.mmdb" - } + } translate { refresh_interval => 86400 field => "src_ip" destination => "ip_rep" dictionary_path => "/etc/listbot/iprep.yaml" } - } + } # In some rare conditions dest_port, src_port is indexed as string, forcing integer for now if [dest_port] { diff --git a/docker/elk/logstash/docker-compose.yml b/docker/elk/logstash/docker-compose.yml index f5eed0dc..18472c60 100644 --- a/docker/elk/logstash/docker-compose.yml +++ b/docker/elk/logstash/docker-compose.yml @@ -15,5 +15,4 @@ services: image: "dtagdevsec/logstash:1804" volumes: - /data:/data - - /var/log:/data/host/log - /root/tpotce/docker/elk/logstash/dist/logstash.conf:/etc/logstash/conf.d/logstash.conf diff --git a/etc/compose/collector.yml b/etc/compose/collector.yml index 36619a18..c2d71a32 100644 --- a/etc/compose/collector.yml +++ b/etc/compose/collector.yml @@ -143,7 +143,6 @@ services: image: "dtagdevsec/logstash:1804" volumes: - /data:/data - - /var/log:/data/host/log ## Elasticsearch-head service head: @@ -203,4 +202,3 @@ services: image: "dtagdevsec/spiderfoot:1804" volumes: - /data/spiderfoot/spiderfoot.db:/home/spiderfoot/spiderfoot.db - diff --git a/etc/compose/experimental.yml b/etc/compose/experimental.yml index b20d1f26..130b7876 100644 --- a/etc/compose/experimental.yml +++ b/etc/compose/experimental.yml @@ -477,7 +477,6 @@ services: image: "dtagdevsec/logstash:1804" volumes: - /data:/data - - /var/log:/data/host/log ## Elasticsearch-head service head: @@ -537,4 +536,3 @@ services: image: "dtagdevsec/spiderfoot:1804" volumes: - /data/spiderfoot/spiderfoot.db:/home/spiderfoot/spiderfoot.db - diff --git a/etc/compose/industrial.yml b/etc/compose/industrial.yml index c653e6af..15229c97 100644 --- a/etc/compose/industrial.yml +++ b/etc/compose/industrial.yml @@ -296,7 +296,6 @@ services: image: "dtagdevsec/logstash:1804" volumes: - /data:/data - - /var/log:/data/host/log ## Elasticsearch-head service head: diff --git a/etc/compose/legacy.yml b/etc/compose/legacy.yml index 0aafb2ea..e77376ac 100644 --- a/etc/compose/legacy.yml +++ b/etc/compose/legacy.yml @@ -262,7 +262,6 @@ services: image: "dtagdevsec/logstash:1804" volumes: - /data:/data - - /var/log:/data/host/log ## Elasticsearch-head service head: @@ -322,4 +321,3 @@ services: image: "dtagdevsec/spiderfoot:1804" volumes: - /data/spiderfoot/spiderfoot.db:/home/spiderfoot/spiderfoot.db - diff --git a/etc/compose/standard.yml b/etc/compose/standard.yml index 6ec62fee..4ed71bbc 100644 --- a/etc/compose/standard.yml +++ b/etc/compose/standard.yml @@ -476,7 +476,6 @@ services: image: "dtagdevsec/logstash:1804" volumes: - /data:/data - - /var/log:/data/host/log ## Elasticsearch-head service head: @@ -536,4 +535,3 @@ services: image: "dtagdevsec/spiderfoot:1804" volumes: - /data/spiderfoot/spiderfoot.db:/home/spiderfoot/spiderfoot.db -