telekom-security/tpotce
1
0
Fork 0
mirror of https://github.com/telekom-security/tpotce.git synced 2025-07-02 01:27:27 -04:00
Code Issues Packages Projects Releases Wiki Activity

Tweaking

Browse Source 
Remove Elasticsearch-Curator in packages, configs and references (BREAKING CHANGE)
Add Index Lifecycle Management in favor of elasticsearch-curator
Point all images to 2203 tags
This commit is contained in:
t3chn0m4g3
2022-01-14 15:52:08 +00:00
parent 86479db8af
commit 0e5986d2df
46 changed files with 117 additions and 205 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
docker/elk/logstash/Dockerfile
View File

@ -67,6 +67,4 @@ HEALTHCHECK --retries=10 CMD curl -s -XGET 'http://127.0.0.1:9600'
#
# Start logstash
#USER logstash:logstash
#CMD update.sh && exec /usr/share/logstash/bin/logstash -f /etc/logstash/conf.d/logstash.conf --config.reload.automatic --java-execution --log.level debug
#CMD update.sh && exec /usr/share/logstash/bin/logstash -f /etc/logstash/conf.d/http_output.conf --config.reload.automatic --java-execution
CMD update.sh && exec /usr/share/logstash/bin/logstash --config.reload.automatic --java-execution

3
docker/elk/logstash/dist/http_input.conf vendored
View File

@ -11,9 +11,10 @@ input {
output {
elasticsearch {
hosts => ["elasticsearch:9200"]
# With templates now being legacy and ILM in place we need to set the daily index with its template manually. Otherwise a new index might be created with differents settings configured through Kibana.
# With templates now being legacy we need to set the daily index with its template manually. Otherwise a new index might be created with differents settings configured through Kibana.
index => "logstash-%{+YYYY.MM.dd}"
template => "/etc/logstash/tpot_es_template.json"
template_overwrite => "true"
}
}

3
docker/elk/logstash/dist/logstash.conf vendored
View File

@ -740,9 +740,10 @@ if "_jsonparsefailure" in [tags] { drop {} }
output {
elasticsearch {
hosts => ["elasticsearch:9200"]
# With templates now being legacy and ILM in place we need to set the daily index with its template manually. Otherwise a new index might be created with differents settings configured through Kibana.
# With templates now being legacy we need to set the daily index with its template manually. Otherwise a new index might be created with differents settings configured through Kibana.
index => "logstash-%{+YYYY.MM.dd}"
template => "/etc/logstash/tpot_es_template.json"
template_overwrite => "true"
#document_type => "doc"
}

1
docker/elk/logstash/dist/tpot_es_template.json vendored
View File

@ -2,6 +2,7 @@
"index_patterns" : "logstash-*",
"version" : 60001,
"settings" : {
"index.lifecycle.name": "tpot",
"index.refresh_interval" : "5s",
"number_of_shards" : 1,
"index.number_of_replicas" : "0",

93
docker/elk/logstash/dist/update.sh vendored
View File

@ -51,72 +51,35 @@ if [ "$MY_TPOT_TYPE" == "POT" ];
exit 0
fi
# We do want to enforce our es_template thus we always need to delete the default template, putting our default afterwards
# This is now done via common_configs.rb => overwrite default logstash template
echo "Removing logstash template."
curl -s -XDELETE http://elasticsearch:9200/_template/logstash
echo
echo "Checking if empty."
curl -s -XGET http://elasticsearch:9200/_template/logstash
echo
echo "Putting default template."
curl -XPUT "http://elasticsearch:9200/_template/logstash" -H 'Content-Type: application/json' -d'
{
"index_patterns" : "logstash-*",
"version" : 60001,
"settings" : {
"index.refresh_interval" : "5s",
"number_of_shards" : 1,
"index.number_of_replicas" : "0",
"index.mapping.total_fields.limit" : "2000",
"index.query": {
"default_field": "*"
}
},
"mappings" : {
"dynamic_templates" : [ {
"message_field" : {
"path_match" : "message",
"match_mapping_type" : "string",
"mapping" : {
"type" : "text",
"norms" : false
}
}
}, {
"string_fields" : {
"match" : "*",
"match_mapping_type" : "string",
"mapping" : {
"type" : "text", "norms" : false,
"fields" : {
"keyword" : { "type": "keyword", "ignore_above": 256 }
# Index Management is happening through ILM, but we need to put T-Pot ILM setting on ES.
myTPOTILM=$(curl -s -XGET "http://elasticsearch:9200/_ilm/policy/tpot" | grep "Lifecycle policy not found: tpot" -c)
if [ "$myTPOTILM" == "1" ];
then
echo "T-Pot ILM template not found on ES, putting it on ES now."
curl -XPUT "http://elasticsearch:9200/_ilm/policy/tpot" -H 'Content-Type: application/json' -d'
{
"policy": {
"phases": {
"hot": {
"min_age": "0ms",
"actions": {}
},
"delete": {
"min_age": "30d",
"actions": {
"delete": {
"delete_searchable_snapshot": true
}
}
}
},
"_meta": {
"managed": true,
"description": "T-Pot ILM policy with a retention of 30 days"
}
}
} ],
"properties" : {
"@timestamp": { "type": "date"},
"@version": { "type": "keyword"},
"geoip" : {
"dynamic": true,
"properties" : {
"ip": { "type": "ip" },
"location" : { "type" : "geo_point" },
"latitude" : { "type" : "half_float" },
"longitude" : { "type" : "half_float" }
}
},
"geoip_ext" : {
"dynamic": true,
"properties" : {
"ip": { "type": "ip" },
"location" : { "type" : "geo_point" },
"latitude" : { "type" : "half_float" },
"longitude" : { "type" : "half_float" }
}
}
}
}
}'
}'
else
echo "T-Pot ILM already configured."
fi
echo

2
docker/elk/logstash/docker-compose.yml
View File

@ -16,7 +16,7 @@ services:
- /opt/tpot/etc/compose/elk_environment
ports:
- "127.0.0.1:64305:80"
image: "dtagdevsec/logstash:2006"
image: "dtagdevsec/logstash:2203"
volumes:
- /data:/data
# - /root/tpotce/docker/elk/logstash/dist/logstash.conf:/etc/logstash/conf.d/logstash.conf