telekom-security/tpotce
1
0
Fork 0
mirror of https://github.com/telekom-security/tpotce.git synced 2025-07-02 01:27:27 -04:00
Code Issues Packages Projects Releases Wiki Activity

elk 7.x dev test

Browse Source
This commit is contained in:
t3chn0m4g3
2020-01-31 14:21:55 +00:00
parent 64907a2eba
commit 5a4724bcba
18 changed files with 302 additions and 136 deletions
Download Patch File Download Diff File Expand all files Collapse all files

10
docker/elk/logstash/Dockerfile
View File

@ -1,5 +1,7 @@
FROM alpine
#
# VARS
ENV LS_VER=7.5.2
# Include dist
ADD dist/ /root/dist/
#
@ -13,7 +15,7 @@ RUN sed -i 's/dl-cdn/dl-2/g' /etc/apk/repositories && \
libc6-compat \
libzmq \
nss \
openjdk8-jre && \
openjdk11-jre && \
#
# Get and install packages
mkdir -p /etc/listbot && \
@ -23,8 +25,8 @@ RUN sed -i 's/dl-cdn/dl-2/g' /etc/apk/repositories && \
bunzip2 *.bz2 && \
cd /root/dist/ && \
mkdir -p /usr/share/logstash/ && \
aria2c -s 16 -x 16 https://artifacts.elastic.co/downloads/logstash/logstash-6.8.6.tar.gz && \
tar xvfz logstash-6.8.6.tar.gz --strip-components=1 -C /usr/share/logstash/ && \
aria2c -s 16 -x 16 https://artifacts.elastic.co/downloads/logstash/logstash-$LS_VER.tar.gz && \
tar xvfz logstash-$LS_VER.tar.gz --strip-components=1 -C /usr/share/logstash/ && \
/usr/share/logstash/bin/logstash-plugin install logstash-filter-translate && \
/usr/share/logstash/bin/logstash-plugin install logstash-output-syslog && \
#
@ -34,7 +36,7 @@ RUN sed -i 's/dl-cdn/dl-2/g' /etc/apk/repositories && \
chmod u+x /usr/bin/update.sh && \
mkdir -p /etc/logstash/conf.d && \
cp logstash.conf /etc/logstash/conf.d/ && \
cp elasticsearch-template-es6x.json /usr/share/logstash/vendor/bundle/jruby/2.5.0/gems/logstash-output-elasticsearch-9.4.0-java/lib/logstash/outputs/elasticsearch/ && \
cp elasticsearch-template-es7x.json /usr/share/logstash/vendor/bundle/jruby/2.5.0/gems/logstash-output-elasticsearch-10.2.3-java/lib/logstash/outputs/elasticsearch/ && \
#
# Setup user, groups and configs
addgroup -g 2000 logstash && \

56
docker/elk/logstash/Dockerfile.old Normal file
View File

@ -0,0 +1,56 @@
FROM alpine
#
# Include dist
ADD dist/ /root/dist/
#
# Setup env and apt
RUN sed -i 's/dl-cdn/dl-2/g' /etc/apk/repositories && \
apk -U --no-cache add \
aria2 \
bash \
bzip2 \
curl \
libc6-compat \
libzmq \
nss \
openjdk8-jre && \
#
# Get and install packages
mkdir -p /etc/listbot && \
cd /etc/listbot && \
aria2c -s16 -x 16 https://raw.githubusercontent.com/dtag-dev-sec/listbot/master/cve.yaml.bz2 && \
aria2c -s16 -x 16 https://raw.githubusercontent.com/dtag-dev-sec/listbot/master/iprep.yaml.bz2 && \
bunzip2 *.bz2 && \
cd /root/dist/ && \
mkdir -p /usr/share/logstash/ && \
aria2c -s 16 -x 16 https://artifacts.elastic.co/downloads/logstash/logstash-6.8.6.tar.gz && \
tar xvfz logstash-6.8.6.tar.gz --strip-components=1 -C /usr/share/logstash/ && \
/usr/share/logstash/bin/logstash-plugin install logstash-filter-translate && \
/usr/share/logstash/bin/logstash-plugin install logstash-output-syslog && \
#
# Add and move files
cd /root/dist/ && \
cp update.sh /usr/bin/ && \
chmod u+x /usr/bin/update.sh && \
mkdir -p /etc/logstash/conf.d && \
cp logstash.conf /etc/logstash/conf.d/ && \
cp elasticsearch-template-es6x.json /usr/share/logstash/vendor/bundle/jruby/2.5.0/gems/logstash-output-elasticsearch-9.4.0-java/lib/logstash/outputs/elasticsearch/ && \
#
# Setup user, groups and configs
addgroup -g 2000 logstash && \
adduser -S -H -s /bin/bash -u 2000 -D -g 2000 logstash && \
chown -R logstash:logstash /usr/share/logstash && \
chown -R logstash:logstash /etc/listbot && \
chmod 755 /usr/bin/update.sh && \
#
# Clean up
rm -rf /root/* && \
rm -rf /tmp/* && \
rm -rf /var/cache/apk/*
#
# Healthcheck
HEALTHCHECK --retries=10 CMD curl -s -XGET 'http://127.0.0.1:9600'
#
# Start logstash
#USER logstash:logstash
CMD update.sh && exec /usr/share/logstash/bin/logstash -f /etc/logstash/conf.d/logstash.conf --config.reload.automatic --java-execution

53
docker/elk/logstash/dist/elasticsearch-template-es5x.json vendored
View File

@ -1,53 +0,0 @@
{
"template" : "logstash-*",
"version" : 50001,
"settings" : {
"index.refresh_interval" : "5s",
"index.number_of_shards" : "1",
"index.number_of_replicas" : "0",
"mapping" : {
"total_fields" : {
"limit" : "2000"
}
}
},
"mappings" : {
"_default_" : {
"_all" : {"enabled" : true, "norms" : false},
"dynamic_templates" : [ {
"message_field" : {
"path_match" : "message",
"match_mapping_type" : "string",
"mapping" : {
"type" : "text",
"norms" : false
}
}
}, {
"string_fields" : {
"match" : "*",
"match_mapping_type" : "string",
"mapping" : {
"type" : "text", "norms" : false,
"fields" : {
"keyword" : { "type": "keyword", "ignore_above": 256 }
}
}
}
} ],
"properties" : {
"@timestamp": { "type": "date", "include_in_all": false },
"@version": { "type": "keyword", "include_in_all": false },
"geoip" : {
"dynamic": true,
"properties" : {
"ip": { "type": "ip" },
"location" : { "type" : "geo_point" },
"latitude" : { "type" : "half_float" },
"longitude" : { "type" : "half_float" }
}
}
}
}
}
}

48
docker/elk/logstash/dist/elasticsearch-template-es6x.json vendored
View File

@ -1,48 +0,0 @@
{
"template" : "logstash-*",
"version" : 60001,
"settings" : {
"index.refresh_interval" : "5s",
"index.number_of_shards" : "1",
"index.number_of_replicas" : "0",
"index.mapping.total_fields.limit": "2000"
},
"mappings" : {
"_default_" : {
"dynamic_templates" : [ {
"message_field" : {
"path_match" : "message",
"match_mapping_type" : "string",
"mapping" : {
"type" : "text",
"norms" : false
}
}
}, {
"string_fields" : {
"match" : "*",
"match_mapping_type" : "string",
"mapping" : {
"type" : "text", "norms" : false,
"fields" : {
"keyword" : { "type": "keyword", "ignore_above": 256 }
}
}
}
} ],
"properties" : {
"@timestamp": { "type": "date"},
"@version": { "type": "keyword"},
"geoip" : {
"dynamic": true,
"properties" : {
"ip": { "type": "ip" },
"location" : { "type" : "geo_point" },
"latitude" : { "type" : "half_float" },
"longitude" : { "type" : "half_float" }
}
}
}
}
}
}

46
docker/elk/logstash/dist/elasticsearch-template-es7x.json vendored Normal file
View File

@ -0,0 +1,46 @@
{
"index_patterns" : "logstash-*",
"version" : 60001,
"settings" : {
"index.refresh_interval" : "5s",
"number_of_shards": 1,
"index.number_of_replicas" : "0",
"index.mapping.total_fields.limit": "2000"
},
"mappings" : {
"dynamic_templates" : [ {
"message_field" : {
"path_match" : "message",
"match_mapping_type" : "string",
"mapping" : {
"type" : "text",
"norms" : false
}
}
}, {
"string_fields" : {
"match" : "*",
"match_mapping_type" : "string",
"mapping" : {
"type" : "text", "norms" : false,
"fields" : {
"keyword" : { "type": "keyword", "ignore_above": 256 }
}
}
}
} ],
"properties" : {
"@timestamp": { "type": "date"},
"@version": { "type": "keyword"},
"geoip" : {
"dynamic": true,
"properties" : {
"ip": { "type": "ip" },
"location" : { "type" : "geo_point" },
"latitude" : { "type" : "half_float" },
"longitude" : { "type" : "half_float" }
}
}
}
}
}

4
docker/elk/logstash/dist/logstash.conf vendored
View File

@ -413,12 +413,12 @@ if "_grokparsefailure" in [tags] { drop {} }
geoip {
cache_size => 10000
source => "src_ip"
database => "/usr/share/logstash/vendor/bundle/jruby/2.5.0/gems/logstash-filter-geoip-5.0.3-java/vendor/GeoLite2-City.mmdb"
database => "/usr/share/logstash/vendor/bundle/jruby/2.5.0/gems/logstash-filter-geoip-6.0.3-java/vendor/GeoLite2-City.mmdb"
}
geoip {
cache_size => 10000
source => "src_ip"
database => "/usr/share/logstash/vendor/bundle/jruby/2.5.0/gems/logstash-filter-geoip-5.0.3-java/vendor/GeoLite2-ASN.mmdb"
database => "/usr/share/logstash/vendor/bundle/jruby/2.5.0/gems/logstash-filter-geoip-6.0.3-java/vendor/GeoLite2-ASN.mmdb"
}
translate {
refresh_interval => 86400

2
docker/elk/logstash/docker-compose.yml
View File

@ -12,7 +12,7 @@ services:
# condition: service_healthy
env_file:
- /opt/tpot/etc/compose/elk_environment
image: "dtagdevsec/logstash:1903"
image: "dtagdevsec/logstash:2006"
volumes:
- /data:/data
- /root/tpotce/docker/elk/logstash/dist/logstash.conf:/etc/logstash/conf.d/logstash.conf