mirror of
https://github.com/telekom-security/tpotce.git
synced 2025-07-02 01:27:27 -04:00
tweaking
where possible kibana visualizations are converted to lens objects (more than 100 objects) all dashboards have been updated fixes #1392 for leaving SentryPeer log tag out add wordpot dashboard after discussion (#1486) and testing iptables-legacy is no longer required include all kibana objects for installation cleaning up some service scripts
This commit is contained in:
t3chn0m4g3
45
docker/tpotinit/dist/bin/deprecated/dump_es.sh
vendored
Executable file
45
docker/tpotinit/dist/bin/deprecated/dump_es.sh
vendored
Executable file
@ -0,0 +1,45 @@
|
||||
#/bin/bash
|
||||
# Dump all ES data
|
||||
# Make sure ES is available
|
||||
myES="http://127.0.0.1:64298/"
|
||||
myESSTATUS=$(curl -s -XGET ''$myES'_cluster/health' | jq '.' | grep -c "green\|yellow")
|
||||
if ! [ "$myESSTATUS" = "1" ]
|
||||
then
|
||||
echo "### Elasticsearch is not available, try starting via 'systemctl start tpot'."
|
||||
exit
|
||||
else
|
||||
echo "### Elasticsearch is available, now continuing."
|
||||
echo
|
||||
fi
|
||||
|
||||
# Let's ensure normal operation on exit or if interrupted ...
|
||||
function fuCLEANUP {
|
||||
rm -rf tmp
|
||||
}
|
||||
trap fuCLEANUP EXIT
|
||||
|
||||
# Set vars
|
||||
myDATE=$(date +%Y%m%d%H%M)
|
||||
myINDICES=$(curl -s -XGET ''$myES'_cat/indices/logstash-*' | awk '{ print $3 }' | sort | grep -v 1970)
|
||||
myINDICES+=" .kibana"
|
||||
myCOL1="[0;34m"
|
||||
myCOL0="[0;0m"
|
||||
|
||||
# Dumping Kibana and Logstash data
|
||||
echo $myCOL1"### The following indices will be dumped: "$myCOL0
|
||||
echo $myINDICES
|
||||
echo
|
||||
|
||||
mkdir tmp
|
||||
for i in $myINDICES;
|
||||
do
|
||||
echo $myCOL1"### Now dumping: "$i $myCOL0
|
||||
elasticdump --input=$myES$i --output="tmp/"$i --limit 7500
|
||||
echo $myCOL1"### Now compressing: tmp/$i" $myCOL0
|
||||
gzip -f "tmp/"$i
|
||||
done;
|
||||
|
||||
# Build tar archive
|
||||
echo $myCOL1"### Now building tar archive: es_dump_"$myDATE".tgz" $myCOL0
|
||||
tar cvf es_dump_$myDATE.tar tmp/.
|
||||
echo $myCOL1"### Done."$myCOL0
|
||||
94
docker/tpotinit/dist/bin/deprecated/export_kibana-objects.sh
vendored
Executable file
94
docker/tpotinit/dist/bin/deprecated/export_kibana-objects.sh
vendored
Executable file
@ -0,0 +1,94 @@
|
||||
#!/bin/bash
|
||||
# Export all Kibana objects through Kibana Saved Objects API
|
||||
# Make sure ES is available
|
||||
myES="http://127.0.0.1:64298/"
|
||||
myKIBANA="http://127.0.0.1:64296/"
|
||||
myESSTATUS=$(curl -s -XGET ''$myES'_cluster/health' | jq '.' | grep -c green)
|
||||
if ! [ "$myESSTATUS" = "1" ]
|
||||
then
|
||||
echo "### Elasticsearch is not available, try starting via 'systemctl start tpot'."
|
||||
exit
|
||||
else
|
||||
echo "### Elasticsearch is available, now continuing."
|
||||
echo
|
||||
fi
|
||||
|
||||
# Set vars
|
||||
myDATE=$(date +%Y%m%d%H%M)
|
||||
myINDEXCOUNT=$(curl -s -XGET ''$myKIBANA'api/saved_objects/_find?type=index-pattern' | jq '.saved_objects[].attributes' | tr '\\' '\n' | grep -E "scripted|url" | wc -w)
|
||||
myINDEXID=$(curl -s -XGET ''$myKIBANA'api/saved_objects/_find?type=index-pattern' | jq '.saved_objects[].id' | tr -d '"')
|
||||
myDASHBOARDS=$(curl -s -XGET ''$myKIBANA'api/saved_objects/_find?type=dashboard&per_page=500' | jq '.saved_objects[].id' | tr -d '"')
|
||||
myVISUALIZATIONS=$(curl -s -XGET ''$myKIBANA'api/saved_objects/_find?type=visualization&per_page=500' | jq '.saved_objects[].id' | tr -d '"')
|
||||
mySEARCHES=$(curl -s -XGET ''$myKIBANA'api/saved_objects/_find?type=search&per_page=500' | jq '.saved_objects[].id' | tr -d '"')
|
||||
myCONFIGS=$(curl -s -XGET ''$myKIBANA'api/saved_objects/_find?type=config&per_page=500' | jq '.saved_objects[].id' | tr -d '"')
|
||||
myCOL1="[0;34m"
|
||||
myCOL0="[0;0m"
|
||||
|
||||
# Let's ensure normal operation on exit or if interrupted ...
|
||||
function fuCLEANUP {
|
||||
rm -rf patterns/ dashboards/ visualizations/ searches/ configs/
|
||||
}
|
||||
trap fuCLEANUP EXIT
|
||||
|
||||
# Export index patterns
|
||||
mkdir -p patterns
|
||||
echo $myCOL1"### Now exporting"$myCOL0 $myINDEXCOUNT $myCOL1"index pattern fields." $myCOL0
|
||||
curl -s -XGET ''$myKIBANA'api/saved_objects/index-pattern/'$myINDEXID'' | jq '. | {attributes, references}' > patterns/$myINDEXID.json &
|
||||
echo
|
||||
|
||||
# Export dashboards
|
||||
mkdir -p dashboards
|
||||
echo $myCOL1"### Now exporting"$myCOL0 $(echo $myDASHBOARDS | wc -w) $myCOL1"dashboards." $myCOL0
|
||||
for i in $myDASHBOARDS;
|
||||
do
|
||||
echo $myCOL1"###### "$i $myCOL0
|
||||
curl -s -XGET ''$myKIBANA'api/saved_objects/dashboard/'$i'' | jq '. | {attributes, references}' > dashboards/$i.json &
|
||||
done;
|
||||
echo
|
||||
|
||||
# Export visualizations
|
||||
mkdir -p visualizations
|
||||
echo $myCOL1"### Now exporting"$myCOL0 $(echo $myVISUALIZATIONS | wc -w) $myCOL1"visualizations." $myCOL0
|
||||
for i in $myVISUALIZATIONS;
|
||||
do
|
||||
echo $myCOL1"###### "$i $myCOL0
|
||||
curl -s -XGET ''$myKIBANA'api/saved_objects/visualization/'$i'' | jq '. | {attributes, references}' > visualizations/$i.json &
|
||||
done;
|
||||
echo
|
||||
|
||||
# Export searches
|
||||
mkdir -p searches
|
||||
echo $myCOL1"### Now exporting"$myCOL0 $(echo $mySEARCHES | wc -w) $myCOL1"searches." $myCOL0
|
||||
for i in $mySEARCHES;
|
||||
do
|
||||
echo $myCOL1"###### "$i $myCOL0
|
||||
curl -s -XGET ''$myKIBANA'api/saved_objects/search/'$i'' | jq '. | {attributes, references}' > searches/$i.json &
|
||||
done;
|
||||
echo
|
||||
|
||||
# Export configs
|
||||
mkdir -p configs
|
||||
echo $myCOL1"### Now exporting"$myCOL0 $(echo $myCONFIGS | wc -w) $myCOL1"configs." $myCOL0
|
||||
for i in $myCONFIGS;
|
||||
do
|
||||
echo $myCOL1"###### "$i $myCOL0
|
||||
curl -s -XGET ''$myKIBANA'api/saved_objects/config/'$i'' | jq '. | {attributes, references}' > configs/$i.json &
|
||||
done;
|
||||
echo
|
||||
|
||||
# Wait for background exports to finish
|
||||
wait
|
||||
|
||||
# Building tar archive
|
||||
echo $myCOL1"### Now building archive"$myCOL0 "kibana-objects_"$myDATE".tgz"
|
||||
tar cvfz kibana-objects_$myDATE.tgz patterns dashboards visualizations searches configs > /dev/null
|
||||
|
||||
# Stats
|
||||
echo
|
||||
echo $myCOL1"### Statistics"
|
||||
echo $myCOL1"###### Exported"$myCOL0 $myINDEXCOUNT $myCOL1"index patterns." $myCOL0
|
||||
echo $myCOL1"###### Exported"$myCOL0 $(echo $myDASHBOARDS | wc -w) $myCOL1"dashboards." $myCOL0
|
||||
echo $myCOL1"###### Exported"$myCOL0 $(echo $myVISUALIZATIONS | wc -w) $myCOL1"visualizations." $myCOL0
|
||||
echo $myCOL1"###### Exported"$myCOL0 $(echo $mySEARCHES | wc -w) $myCOL1"searches." $myCOL0
|
||||
echo $myCOL1"###### Exported"$myCOL0 $(echo $myCONFIGS | wc -w) $myCOL1"configs." $myCOL0
|
||||
echo
|
||||
126
docker/tpotinit/dist/bin/deprecated/import_kibana-objects.sh
vendored
Executable file
126
docker/tpotinit/dist/bin/deprecated/import_kibana-objects.sh
vendored
Executable file
@ -0,0 +1,126 @@
|
||||
#!/bin/bash
|
||||
# Import Kibana objects
|
||||
# Make sure ES is available
|
||||
myES="http://127.0.0.1:64298/"
|
||||
myKIBANA="http://127.0.0.1:64296/"
|
||||
myESSTATUS=$(curl -s -XGET ''$myES'_cluster/health' | jq '.' | grep -c green)
|
||||
if ! [ "$myESSTATUS" = "1" ]
|
||||
then
|
||||
echo "### Elasticsearch is not available, try starting via 'systemctl start tpot'."
|
||||
exit
|
||||
else
|
||||
echo "### Elasticsearch is available, now continuing."
|
||||
echo
|
||||
fi
|
||||
|
||||
# Set vars
|
||||
myDUMP=$1
|
||||
myCOL1="[0;34m"
|
||||
myCOL0="[0;0m"
|
||||
|
||||
# Let's ensure normal operation on exit or if interrupted ...
|
||||
function fuCLEANUP {
|
||||
rm -rf patterns/ dashboards/ visualizations/ searches/ configs/
|
||||
}
|
||||
trap fuCLEANUP EXIT
|
||||
|
||||
# Check if parameter is given and file exists
|
||||
if [ "$myDUMP" = "" ];
|
||||
then
|
||||
echo $myCOL1"### Please provide a backup file name."$myCOL0
|
||||
echo $myCOL1"### import_kibana-objects.sh <kibana-objects.tgz>"$myCOL0
|
||||
echo
|
||||
exit
|
||||
fi
|
||||
if ! [ -a $myDUMP ];
|
||||
then
|
||||
echo $myCOL1"### File not found."$myCOL0
|
||||
exit
|
||||
fi
|
||||
|
||||
# Unpack tar
|
||||
tar xvfz $myDUMP > /dev/null
|
||||
|
||||
# Restore index patterns
|
||||
myINDEXID=$(ls patterns/*.json | cut -c 10- | rev | cut -c 6- | rev)
|
||||
myINDEXCOUNT=$(cat patterns/$myINDEXID.json | tr '\\' '\n' | grep -E "scripted|url" | wc -w)
|
||||
echo $myCOL1"### Now importing"$myCOL0 $myINDEXCOUNT $myCOL1"index pattern fields." $myCOL0
|
||||
curl -s -XDELETE ''$myKIBANA'api/saved_objects/index-pattern/logstash-*' -H "Content-Type: application/json" -H "kbn-xsrf: true" > /dev/null
|
||||
curl -s -XDELETE ''$myKIBANA'api/saved_objects/index-pattern/'$myINDEXID'' -H "Content-Type: application/json" -H "kbn-xsrf: true" > /dev/null
|
||||
curl -s -XPOST ''$myKIBANA'api/saved_objects/index-pattern/'$myINDEXID'' -H "Content-Type: application/json" -H "kbn-xsrf: true" -d @patterns/$myINDEXID.json > /dev/null &
|
||||
echo
|
||||
|
||||
# Restore dashboards
|
||||
myDASHBOARDS=$(ls dashboards/*.json | cut -c 12- | rev | cut -c 6- | rev)
|
||||
echo $myCOL1"### Now importing "$myCOL0$(echo $myDASHBOARDS | wc -w)$myCOL1 "dashboards." $myCOL0
|
||||
for i in $myDASHBOARDS;
|
||||
do
|
||||
curl -s -XDELETE ''$myKIBANA'api/saved_objects/dashboard/'$i'' -H "Content-Type: application/json" -H "kbn-xsrf: true" > /dev/null &
|
||||
done;
|
||||
wait
|
||||
for i in $myDASHBOARDS;
|
||||
do
|
||||
echo $myCOL1"###### "$i $myCOL0
|
||||
curl -s -XPOST ''$myKIBANA'api/saved_objects/dashboard/'$i'' -H "Content-Type: application/json" -H "kbn-xsrf: true" -d @dashboards/$i.json > /dev/null &
|
||||
done;
|
||||
wait
|
||||
echo
|
||||
|
||||
# Restore visualizations
|
||||
myVISUALIZATIONS=$(ls visualizations/*.json | cut -c 16- | rev | cut -c 6- | rev)
|
||||
echo $myCOL1"### Now importing "$myCOL0$(echo $myVISUALIZATIONS | wc -w)$myCOL1 "visualizations." $myCOL0
|
||||
for i in $myVISUALIZATIONS;
|
||||
do
|
||||
curl -s -XDELETE ''$myKIBANA'api/saved_objects/visualization/'$i'' -H "Content-Type: application/json" -H "kbn-xsrf: true" > /dev/null &
|
||||
done;
|
||||
wait
|
||||
for i in $myVISUALIZATIONS;
|
||||
do
|
||||
echo $myCOL1"###### "$i $myCOL0
|
||||
curl -s -XPOST ''$myKIBANA'api/saved_objects/visualization/'$i'' -H "Content-Type: application/json" -H "kbn-xsrf: true" -d @visualizations/$i.json > /dev/null &
|
||||
done;
|
||||
wait
|
||||
echo
|
||||
|
||||
# Restore searches
|
||||
mySEARCHES=$(ls searches/*.json | cut -c 10- | rev | cut -c 6- | rev)
|
||||
echo $myCOL1"### Now importing "$myCOL0$(echo $mySEARCHES | wc -w)$myCOL1 "searches." $myCOL0
|
||||
for i in $mySEARCHES;
|
||||
do
|
||||
curl -s -XDELETE ''$myKIBANA'api/saved_objects/search/'$i'' -H "Content-Type: application/json" -H "kbn-xsrf: true" > /dev/null &
|
||||
done;
|
||||
wait
|
||||
for i in $mySEARCHES;
|
||||
do
|
||||
echo $myCOL1"###### "$i $myCOL0
|
||||
curl -s -XPOST ''$myKIBANA'api/saved_objects/search/'$i'' -H "Content-Type: application/json" -H "kbn-xsrf: true" -d @searches/$i.json > /dev/null &
|
||||
done;
|
||||
echo
|
||||
wait
|
||||
|
||||
# Restore configs
|
||||
myCONFIGS=$(ls configs/*.json | cut -c 9- | rev | cut -c 6- | rev)
|
||||
echo $myCOL1"### Now importing "$myCOL0$(echo $myCONFIGS | wc -w)$myCOL1 "configs." $myCOL0
|
||||
for i in $myCONFIGS;
|
||||
do
|
||||
curl -s -XDELETE ''$myKIBANA'api/saved_objects/configs/'$i'' -H "Content-Type: application/json" -H "kbn-xsrf: true" > /dev/null &
|
||||
done;
|
||||
wait
|
||||
for i in $myCONFIGS;
|
||||
do
|
||||
echo $myCOL1"###### "$i $myCOL0
|
||||
curl -s -XPOST ''$myKIBANA'api/saved_objects/configs/'$i'' -H "Content-Type: application/json" -H "kbn-xsrf: true" -d @configs/$i.json > /dev/null &
|
||||
done;
|
||||
echo
|
||||
wait
|
||||
|
||||
# Stats
|
||||
echo
|
||||
echo $myCOL1"### Statistics"
|
||||
echo $myCOL1"###### Imported"$myCOL0 $myINDEXCOUNT $myCOL1"index patterns." $myCOL0
|
||||
echo $myCOL1"###### Imported"$myCOL0 $(echo $myDASHBOARDS | wc -w) $myCOL1"dashboards." $myCOL0
|
||||
echo $myCOL1"###### Imported"$myCOL0 $(echo $myVISUALIZATIONS | wc -w) $myCOL1"visualizations." $myCOL0
|
||||
echo $myCOL1"###### Imported"$myCOL0 $(echo $mySEARCHES | wc -w) $myCOL1"searches." $myCOL0
|
||||
echo $myCOL1"###### Imported"$myCOL0 $(echo $myCONFIGS | wc -w) $myCOL1"configs." $myCOL0
|
||||
echo
|
||||
|
||||
95
docker/tpotinit/dist/bin/deprecated/restore_es.sh
vendored
Executable file
95
docker/tpotinit/dist/bin/deprecated/restore_es.sh
vendored
Executable file
@ -0,0 +1,95 @@
|
||||
#/bin/bash
|
||||
# Restore folder based ES backup
|
||||
# Make sure ES is available
|
||||
myES="http://127.0.0.1:64298/"
|
||||
myESSTATUS=$(curl -s -XGET ''$myES'_cluster/health' | jq '.' | grep -c "green\|yellow")
|
||||
if ! [ "$myESSTATUS" = "1" ]
|
||||
then
|
||||
echo "### Elasticsearch is not available, try starting via 'systemctl start tpot'."
|
||||
exit
|
||||
else
|
||||
echo "### Elasticsearch is available, now continuing."
|
||||
fi
|
||||
|
||||
# Let's ensure normal operation on exit or if interrupted ...
|
||||
function fuCLEANUP {
|
||||
rm -rf tmp
|
||||
}
|
||||
trap fuCLEANUP EXIT
|
||||
|
||||
# Set vars
|
||||
myDUMP=$1
|
||||
myCOL1="[0;34m"
|
||||
myCOL0="[0;0m"
|
||||
|
||||
# Check if parameter is given and file exists
|
||||
if [ "$myDUMP" = "" ];
|
||||
then
|
||||
echo $myCOL1"### Please provide a backup file name."$myCOL0
|
||||
echo $myCOL1"### restore-elk.sh <es_dump.tar>"$myCOL0
|
||||
echo
|
||||
exit
|
||||
fi
|
||||
if ! [ -a $myDUMP ];
|
||||
then
|
||||
echo $myCOL1"### File not found."$myCOL0
|
||||
exit
|
||||
fi
|
||||
|
||||
# Unpack tar archive
|
||||
echo $myCOL1"### Now unpacking tar archive: "$myDUMP $myCOL0
|
||||
tar xvf $myDUMP
|
||||
|
||||
# Build indices list
|
||||
myINDICES="$(ls tmp/logstash*.gz | cut -c 5- | rev | cut -c 4- | rev)"
|
||||
myINDICES+=" .kibana"
|
||||
echo $myCOL1"### The following indices will be restored: "$myCOL0
|
||||
echo $myINDICES
|
||||
echo
|
||||
|
||||
# Force single seat template for everything
|
||||
echo -n $myCOL1"### Forcing single seat template: "$myCOL0
|
||||
curl -s XPUT ''$myES'_template/.*' -H 'Content-Type: application/json' -d'
|
||||
{ "index_patterns": ".*",
|
||||
"order": 1,
|
||||
"settings":
|
||||
{
|
||||
"number_of_shards": 1,
|
||||
"number_of_replicas": 0
|
||||
}
|
||||
}'
|
||||
echo
|
||||
|
||||
# Set logstash template
|
||||
echo -n $myCOL1"### Setting up logstash template: "$myCOL0
|
||||
curl -s XPUT ''$myES'_template/logstash' -H 'Content-Type: application/json' -d'
|
||||
{
|
||||
"index_patterns": "logstash-*",
|
||||
"settings" : {
|
||||
"index" : {
|
||||
"number_of_shards": 1,
|
||||
"number_of_replicas": 0,
|
||||
"mapping" : {
|
||||
"total_fields" : {
|
||||
"limit" : "2000"
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}'
|
||||
echo
|
||||
|
||||
# Restore indices
|
||||
curl -s -X DELETE ''$myES'.kibana*' > /dev/null
|
||||
for i in $myINDICES;
|
||||
do
|
||||
# Delete index if it already exists
|
||||
curl -s -X DELETE $myES$i > /dev/null
|
||||
echo $myCOL1"### Now uncompressing: tmp/$i.gz" $myCOL0
|
||||
gunzip -f tmp/$i.gz
|
||||
# Restore index to ES
|
||||
echo $myCOL1"### Now restoring: "$i $myCOL0
|
||||
elasticdump --input=tmp/$i --output=$myES$i --limit 7500
|
||||
rm tmp/$i
|
||||
done;
|
||||
echo $myCOL1"### Done."$myCOL0
|
||||
19
docker/tpotinit/dist/bin/deprecated/unlock_es.sh
vendored
Executable file
19
docker/tpotinit/dist/bin/deprecated/unlock_es.sh
vendored
Executable file
@ -0,0 +1,19 @@
|
||||
#/bin/bash
|
||||
# Unlock all ES indices for read / write mode
|
||||
# Useful in cases where ES locked all indices after disk quota has been reached
|
||||
# Make sure ES is available
|
||||
myES="http://127.0.0.1:64298/"
|
||||
myESSTATUS=$(curl -s -XGET ''$myES'_cluster/health' | jq '.' | grep -c "green\|yellow")
|
||||
if ! [ "$myESSTATUS" = "1" ]
|
||||
then
|
||||
echo "### Elasticsearch is not available, try starting via 'systemctl start tpot'."
|
||||
exit
|
||||
else
|
||||
echo "### Elasticsearch is available, now continuing."
|
||||
echo
|
||||
fi
|
||||
|
||||
echo "### Trying to unlock all ES indices for read / write operation: "
|
||||
curl -XPUT -H "Content-Type: application/json" ''$myES'_all/_settings' -d '{"index.blocks.read_only_allow_delete": null}'
|
||||
echo
|
||||
|
||||
Reference in New Issue
Block a user