mirror of
https://github.com/telekom-security/tpotce.git
synced 2025-07-02 01:27:27 -04:00
update scripts to new file layout
iso will not include tpot repo tpot repo will be cloned during install as all the other tools
This commit is contained in:
Marco Ochse
38
bin/backup_es_folders.sh
Executable file
38
bin/backup_es_folders.sh
Executable file
@ -0,0 +1,38 @@
|
||||
#!/bin/bash
|
||||
# Backup all ES relevant folders
|
||||
# Make sure ES is available
|
||||
myES="http://127.0.0.1:64298/"
|
||||
myESSTATUS=$(curl -s -XGET ''$myES'_cluster/health' | jq '.' | grep -c green)
|
||||
if ! [ "$myESSTATUS" = "1" ]
|
||||
then
|
||||
echo "### Elasticsearch is not available, try starting via 'systemctl start elk'."
|
||||
exit
|
||||
else
|
||||
echo "### Elasticsearch is available, now continuing."
|
||||
echo
|
||||
fi
|
||||
|
||||
# Set vars
|
||||
myCOUNT=1
|
||||
myDATE=$(date +%Y%m%d%H%M)
|
||||
myELKPATH="/data/elk/data"
|
||||
myKIBANAINDEXNAME=$(curl -s -XGET ''$myES'_cat/indices/' | grep .kibana | awk '{ print $4 }')
|
||||
myKIBANAINDEXPATH=$myELKPATH/nodes/0/indices/$myKIBANAINDEXNAME
|
||||
|
||||
# Let's ensure normal operation on exit or if interrupted ...
|
||||
function fuCLEANUP {
|
||||
### Start ELK
|
||||
systemctl start tpot
|
||||
echo "### Now starting T-Pot ..."
|
||||
}
|
||||
trap fuCLEANUP EXIT
|
||||
|
||||
# Stop T-Pot to lift db lock
|
||||
echo "### Now stopping T-Pot"
|
||||
systemctl stop tpot
|
||||
sleep 2
|
||||
|
||||
# Backup DB in 2 flavors
|
||||
echo "### Now backing up Elasticsearch folders ..."
|
||||
tar cvfz "elkall_"$myDATE".tgz" $myELKPATH
|
||||
tar cvfz "elkbase_"$myDATE".tgz" $myKIBANAINDEXPATH
|
||||
219
bin/clean.sh
Executable file
219
bin/clean.sh
Executable file
@ -0,0 +1,219 @@
|
||||
#!/bin/bash
|
||||
# T-Pot Container Data Cleaner & Log Rotator
|
||||
|
||||
# Set colors
|
||||
myRED="[0;31m"
|
||||
myGREEN="[0;32m"
|
||||
myWHITE="[0;0m"
|
||||
|
||||
# Set persistence
|
||||
myPERSISTENCE=$1
|
||||
|
||||
# Let's create a function to check if folder is empty
|
||||
fuEMPTY () {
|
||||
local myFOLDER=$1
|
||||
|
||||
echo $(ls $myFOLDER | wc -l)
|
||||
}
|
||||
|
||||
# Let's create a function to rotate and compress logs
|
||||
fuLOGROTATE () {
|
||||
local mySTATUS="/opt/tpot/etc/logrotate/status"
|
||||
local myCONF="/opt/tpot/etc/logrotate/logrotate.conf"
|
||||
local myCOWRIETTYLOGS="/data/cowrie/log/tty/"
|
||||
local myCOWRIETTYTGZ="/data/cowrie/log/ttylogs.tgz"
|
||||
local myCOWRIEDL="/data/cowrie/downloads/"
|
||||
local myCOWRIEDLTGZ="/data/cowrie/downloads.tgz"
|
||||
local myDIONAEABI="/data/dionaea/bistreams/"
|
||||
local myDIONAEABITGZ="/data/dionaea/bistreams.tgz"
|
||||
local myDIONAEABIN="/data/dionaea/binaries/"
|
||||
local myDIONAEABINTGZ="/data/dionaea/binaries.tgz"
|
||||
local myHONEYTRAPATTACKS="/data/honeytrap/attacks/"
|
||||
local myHONEYTRAPATTACKSTGZ="/data/honeytrap/attacks.tgz"
|
||||
local myHONEYTRAPDL="/data/honeytrap/downloads/"
|
||||
local myHONEYTRAPDLTGZ="/data/honeytrap/downloads.tgz"
|
||||
|
||||
# Ensure correct permissions and ownerships for logrotate to run without issues
|
||||
chmod 760 /data/ -R
|
||||
chown tpot:tpot /data -R
|
||||
|
||||
# Run logrotate with force (-f) first, so the status file can be written and race conditions (with tar) be avoided
|
||||
logrotate -f -s $mySTATUS $myCONF
|
||||
|
||||
# Compressing some folders first and rotate them later
|
||||
if [ "$(fuEMPTY $myCOWRIETTYLOGS)" != "0" ]; then tar cvfz $myCOWRIETTYTGZ $myCOWRIETTYLOGS; fi
|
||||
if [ "$(fuEMPTY $myCOWRIEDL)" != "0" ]; then tar cvfz $myCOWRIEDLTGZ $myCOWRIEDL; fi
|
||||
if [ "$(fuEMPTY $myDIONAEABI)" != "0" ]; then tar cvfz $myDIONAEABITGZ $myDIONAEABI; fi
|
||||
if [ "$(fuEMPTY $myDIONAEABIN)" != "0" ]; then tar cvfz $myDIONAEABINTGZ $myDIONAEABIN; fi
|
||||
if [ "$(fuEMPTY $myHONEYTRAPATTACKS)" != "0" ]; then tar cvfz $myHONEYTRAPATTACKSTGZ $myHONEYTRAPATTACKS; fi
|
||||
if [ "$(fuEMPTY $myHONEYTRAPDL)" != "0" ]; then tar cvfz $myHONEYTRAPDLTGZ $myHONEYTRAPDL; fi
|
||||
|
||||
# Ensure correct permissions and ownership for previously created archives
|
||||
chmod 760 $myCOWRIETTYTGZ $myCOWRIEDLTGZ $myDIONAEABITGZ $myDIONAEABINTGZ $myHONEYTRAPATTACKSTGZ $myHONEYTRAPDLTGZ
|
||||
chown tpot:tpot $myCOWRIETTYTGZ $myCOWRIEDLTGZ $myDIONAEABITGZ $myDIONAEABINTGZ $myHONEYTRAPATTACKSTGZ $myHONEYTRAPDLTGZ
|
||||
|
||||
# Need to remove subfolders since too many files cause rm to exit with errors
|
||||
rm -rf $myCOWRIETTYLOGS $myCOWRIEDL $myDIONAEABI $myDIONAEABIN $myHONEYTRAPATTACKS $myHONEYTRAPDL
|
||||
|
||||
# Recreate subfolders with correct permissions and ownership
|
||||
mkdir -p $myCOWRIETTYLOGS $myCOWRIEDL $myDIONAEABI $myDIONAEABIN $myHONEYTRAPATTACKS $myHONEYTRAPDL
|
||||
chmod 760 $myCOWRIETTYLOGS $myCOWRIEDL $myDIONAEABI $myDIONAEABIN $myHONEYTRAPATTACKS $myHONEYTRAPDL
|
||||
chown tpot:tpot $myCOWRIETTYLOGS $myCOWRIEDL $myDIONAEABI $myDIONAEABIN $myHONEYTRAPATTACKS $myHONEYTRAPDL
|
||||
|
||||
# Run logrotate again to account for previously created archives - DO NOT FORCE HERE!
|
||||
logrotate -s $mySTATUS $myCONF
|
||||
}
|
||||
|
||||
# Let's create a function to clean up and prepare conpot data
|
||||
fuCONPOT () {
|
||||
if [ "$myPERSISTENCE" != "on" ]; then rm -rf /data/conpot/*; fi
|
||||
mkdir -p /data/conpot/log
|
||||
chmod 760 /data/conpot -R
|
||||
chown tpot:tpot /data/conpot -R
|
||||
}
|
||||
|
||||
# Let's create a function to clean up and prepare cowrie data
|
||||
fuCOWRIE () {
|
||||
if [ "$myPERSISTENCE" != "on" ]; then rm -rf /data/cowrie/*; fi
|
||||
mkdir -p /data/cowrie/log/tty/ /data/cowrie/downloads/ /data/cowrie/keys/ /data/cowrie/misc/
|
||||
chmod 760 /data/cowrie -R
|
||||
chown tpot:tpot /data/cowrie -R
|
||||
}
|
||||
|
||||
# Let's create a function to clean up and prepare dionaea data
|
||||
fuDIONAEA () {
|
||||
if [ "$myPERSISTENCE" != "on" ]; then rm -rf /data/dionaea/*; fi
|
||||
mkdir -p /data/dionaea/log /data/dionaea/bistreams /data/dionaea/binaries /data/dionaea/rtp /data/dionaea/roots/ftp /data/dionaea/roots/tftp /data/dionaea/roots/www /data/dionaea/roots/upnp
|
||||
chmod 760 /data/dionaea -R
|
||||
chown tpot:tpot /data/dionaea -R
|
||||
}
|
||||
|
||||
# Let's create a function to clean up and prepare elasticpot data
|
||||
fuELASTICPOT () {
|
||||
if [ "$myPERSISTENCE" != "on" ]; then rm -rf /data/elasticpot/*; fi
|
||||
mkdir -p /data/elasticpot/log
|
||||
chmod 760 /data/elasticpot -R
|
||||
chown tpot:tpot /data/elasticpot -R
|
||||
}
|
||||
|
||||
# Let's create a function to clean up and prepare elk data
|
||||
fuELK () {
|
||||
# ELK data will be kept for <= 90 days, check /etc/crontab for curator modification
|
||||
# ELK daemon log files will be removed
|
||||
if [ "$myPERSISTENCE" != "on" ]; then rm -rf /data/elk/log/*; fi
|
||||
mkdir -p /data/elk
|
||||
chmod 760 /data/elk -R
|
||||
chown tpot:tpot /data/elk -R
|
||||
}
|
||||
|
||||
# Let's create a function to clean up and prepare emobility data
|
||||
fuEMOBILITY () {
|
||||
if [ "$myPERSISTENCE" != "on" ]; then rm -rf /data/emobility/*; fi
|
||||
mkdir -p /data/emobility/log
|
||||
chmod 760 /data/emobility -R
|
||||
chown tpot:tpot /data/emobility -R
|
||||
}
|
||||
|
||||
# Let's create a function to clean up and prepare glastopf data
|
||||
fuGLASTOPF () {
|
||||
if [ "$myPERSISTENCE" != "on" ]; then rm -rf /data/glastopf/*; fi
|
||||
mkdir -p /data/glastopf
|
||||
chmod 760 /data/glastopf -R
|
||||
chown tpot:tpot /data/glastopf -R
|
||||
}
|
||||
|
||||
# Let's create a function to clean up and prepare honeytrap data
|
||||
fuHONEYTRAP () {
|
||||
if [ "$myPERSISTENCE" != "on" ]; then rm -rf /data/honeytrap/*; fi
|
||||
mkdir -p /data/honeytrap/log/ /data/honeytrap/attacks/ /data/honeytrap/downloads/
|
||||
chmod 760 /data/honeytrap/ -R
|
||||
chown tpot:tpot /data/honeytrap/ -R
|
||||
}
|
||||
|
||||
# Let's create a function to clean up and prepare mailoney data
|
||||
fuMAILONEY () {
|
||||
if [ "$myPERSISTENCE" != "on" ]; then rm -rf /data/mailoney/*; fi
|
||||
mkdir -p /data/mailoney/log/
|
||||
chmod 760 /data/mailoney/ -R
|
||||
chown tpot:tpot /data/mailoney/ -R
|
||||
}
|
||||
|
||||
# Let's create a function to clean up and prepare rdpy data
|
||||
fuRDPY () {
|
||||
if [ "$myPERSISTENCE" != "on" ]; then rm -rf /data/rdpy/*; fi
|
||||
mkdir -p /data/rdpy/log/
|
||||
chmod 760 /data/rdpy/ -R
|
||||
chown tpot:tpot /data/rdpy/ -R
|
||||
}
|
||||
|
||||
# Let's create a function to prepare spiderfoot db
|
||||
fuSPIDERFOOT () {
|
||||
mkdir -p /data/spiderfoot
|
||||
touch /data/spiderfoot/spiderfoot.db
|
||||
chmod 760 -R /data/spiderfoot
|
||||
chown tpot:tpot -R /data/spiderfoot
|
||||
}
|
||||
|
||||
# Let's create a function to clean up and prepare suricata data
|
||||
fuSURICATA () {
|
||||
if [ "$myPERSISTENCE" != "on" ]; then rm -rf /data/suricata/*; fi
|
||||
mkdir -p /data/suricata/log
|
||||
chmod 760 -R /data/suricata
|
||||
chown tpot:tpot -R /data/suricata
|
||||
}
|
||||
|
||||
# Let's create a function to clean up and prepare p0f data
|
||||
fuP0F () {
|
||||
if [ "$myPERSISTENCE" != "on" ]; then rm -rf /data/p0f/*; fi
|
||||
mkdir -p /data/p0f/log
|
||||
chmod 760 -R /data/p0f
|
||||
chown tpot:tpot -R /data/p0f
|
||||
}
|
||||
|
||||
# Let's create a function to clean up and prepare vnclowpot data
|
||||
fuVNCLOWPOT () {
|
||||
if [ "$myPERSISTENCE" != "on" ]; then rm -rf /data/vnclowpot/*; fi
|
||||
mkdir -p /data/vnclowpot/log/
|
||||
chmod 760 /data/vnclowpot/ -R
|
||||
chown tpot:tpot /data/vnclowpot/ -R
|
||||
}
|
||||
|
||||
|
||||
# Avoid unwanted cleaning
|
||||
if [ "$myPERSISTENCE" = "" ];
|
||||
then
|
||||
echo $myRED"!!! WARNING !!! - This will delete ALL honeypot logs. "$myWHITE
|
||||
while [ "$myQST" != "y" ] && [ "$myQST" != "n" ];
|
||||
do
|
||||
read -p "Continue? (y/n) " myQST
|
||||
done
|
||||
if [ "$myQST" = "n" ];
|
||||
then
|
||||
echo $myGREEN"Puuh! That was close! Aborting!"$myWHITE
|
||||
exit
|
||||
fi
|
||||
fi
|
||||
|
||||
# Check persistence, if enabled compress and rotate logs
|
||||
if [ "$myPERSISTENCE" = "on" ];
|
||||
then
|
||||
echo "Persistence enabled, now rotating and compressing logs."
|
||||
fuLOGROTATE
|
||||
else
|
||||
echo "Cleaning up and preparing data folders."
|
||||
fuCONPOT
|
||||
fuCOWRIE
|
||||
fuDIONAEA
|
||||
fuELASTICPOT
|
||||
fuELK
|
||||
fuEMOBILITY
|
||||
fuGLASTOPF
|
||||
fuHONEYTRAP
|
||||
fuMAILONEY
|
||||
fuRDPY
|
||||
fuSPIDERFOOT
|
||||
fuSURICATA
|
||||
fuP0F
|
||||
fuVNCLOWPOT
|
||||
fi
|
||||
|
||||
71
bin/dps.sh
Executable file
71
bin/dps.sh
Executable file
@ -0,0 +1,71 @@
|
||||
#/bin/bash
|
||||
# Show current status of all running containers
|
||||
myPARAM="$1"
|
||||
myIMAGES="$(cat /opt/tpot/etc/tpot.yml | grep -v '#' | grep container_name | cut -d: -f2)"
|
||||
myRED="[1;31m"
|
||||
myGREEN="[1;32m"
|
||||
myBLUE="[1;34m"
|
||||
myWHITE="[0;0m"
|
||||
myMAGENTA="[1;35m"
|
||||
|
||||
function fuCONTAINERSTATUS {
|
||||
local myNAME="$1"
|
||||
local mySTATUS="$(/usr/bin/docker ps -f name=$myNAME --format "table {{.Status}}" -f status=running -f status=exited | tail -n 1)"
|
||||
myDOWN="$(echo "$mySTATUS" | grep -o -E "(STATUS|NAMES|Exited)")"
|
||||
|
||||
case "$myDOWN" in
|
||||
STATUS)
|
||||
mySTATUS="$myRED"DOWN"$myWHITE"
|
||||
;;
|
||||
NAMES)
|
||||
mySTATUS="$myRED"DOWN"$myWHITE"
|
||||
;;
|
||||
Exited)
|
||||
mySTATUS="$myRED$mySTATUS$myWHITE"
|
||||
;;
|
||||
*)
|
||||
mySTATUS="$myGREEN$mySTATUS$myWHITE"
|
||||
;;
|
||||
esac
|
||||
|
||||
printf "$mySTATUS"
|
||||
}
|
||||
|
||||
function fuCONTAINERPORTS {
|
||||
local myNAME="$1"
|
||||
local myPORTS="$(/usr/bin/docker ps -f name=$myNAME --format "table {{.Ports}}" -f status=running -f status=exited | tail -n 1 | sed s/","/",\n\t\t\t\t\t\t\t"/g)"
|
||||
|
||||
if [ "$myPORTS" != "PORTS" ];
|
||||
then
|
||||
printf "$myBLUE$myPORTS$myWHITE"
|
||||
fi
|
||||
}
|
||||
|
||||
function fuGETSYS {
|
||||
printf "========| System |========\n"
|
||||
printf "%+10s %-20s\n" "Date: " "$(date)"
|
||||
printf "%+10s %-20s\n" "Uptime: " "$(uptime | cut -b 2-)"
|
||||
printf "%+10s %-20s\n" "CPU temp: " "$(sensors | grep 'Physical' | awk '{ print $4" " }' | tr -d [:cntrl:])"
|
||||
echo
|
||||
}
|
||||
|
||||
while true
|
||||
do
|
||||
fuGETSYS
|
||||
printf "%-19s %-36s %s\n" "NAME" "STATUS" "PORTS"
|
||||
for i in $myIMAGES; do
|
||||
myNAME="$myMAGENTA$i$myWHITE"
|
||||
printf "%-32s %-49s %s" "$myNAME" "$(fuCONTAINERSTATUS $i)" "$(fuCONTAINERPORTS $i)"
|
||||
echo
|
||||
if [ "$myPARAM" = "vv" ];
|
||||
then
|
||||
/usr/bin/docker exec -t "$i" /bin/ps awfuwfxwf | egrep -v -E "awfuwfxwf|/bin/ps"
|
||||
fi
|
||||
done
|
||||
if [[ $myPARAM =~ ^([1-9]|[1-9][0-9]|[1-9][0-9][0-9])$ ]];
|
||||
then
|
||||
sleep "$myPARAM"
|
||||
else
|
||||
break
|
||||
fi
|
||||
done
|
||||
45
bin/dump_es.sh
Executable file
45
bin/dump_es.sh
Executable file
@ -0,0 +1,45 @@
|
||||
#/bin/bash
|
||||
# Dump all ES data
|
||||
# Make sure ES is available
|
||||
myES="http://127.0.0.1:64298/"
|
||||
myESSTATUS=$(curl -s -XGET ''$myES'_cluster/health' | jq '.' | grep -c green)
|
||||
if ! [ "$myESSTATUS" = "1" ]
|
||||
then
|
||||
echo "### Elasticsearch is not available, try starting via 'systemctl start elk'."
|
||||
exit
|
||||
else
|
||||
echo "### Elasticsearch is available, now continuing."
|
||||
echo
|
||||
fi
|
||||
|
||||
# Let's ensure normal operation on exit or if interrupted ...
|
||||
function fuCLEANUP {
|
||||
rm -rf tmp
|
||||
}
|
||||
trap fuCLEANUP EXIT
|
||||
|
||||
# Set vars
|
||||
myDATE=$(date +%Y%m%d%H%M)
|
||||
myINDICES=$(curl -s -XGET ''$myES'_cat/indices/' | grep logstash | awk '{ print $3 }' | sort | grep -v 1970)
|
||||
myES="http://127.0.0.1:64298/"
|
||||
myCOL1="[0;34m"
|
||||
myCOL0="[0;0m"
|
||||
|
||||
# Dumping all ES data
|
||||
echo $myCOL1"### The following indices will be dumped: "$myCOL0
|
||||
echo $myINDICES
|
||||
echo
|
||||
|
||||
mkdir tmp
|
||||
for i in $myINDICES;
|
||||
do
|
||||
echo $myCOL1"### Now dumping: "$i $myCOL0
|
||||
elasticdump --input=$myES$i --output="tmp/"$i --limit 7500
|
||||
echo $myCOL1"### Now compressing: tmp/$i" $myCOL0
|
||||
gzip -f "tmp/"$i
|
||||
done;
|
||||
|
||||
# Build tar archive
|
||||
echo $myCOL1"### Now building tar archive: es_dump_"$myDATE".tgz" $myCOL0
|
||||
tar cvf es_dump_$myDATE.tar tmp/*
|
||||
echo $myCOL1"### Done."$myCOL0
|
||||
77
bin/export_kibana-objects.sh
Executable file
77
bin/export_kibana-objects.sh
Executable file
@ -0,0 +1,77 @@
|
||||
#!/bin/bash
|
||||
# Export all Kibana objects
|
||||
# Make sure ES is available
|
||||
myES="http://127.0.0.1:64298/"
|
||||
myESSTATUS=$(curl -s -XGET ''$myES'_cluster/health' | jq '.' | grep -c green)
|
||||
if ! [ "$myESSTATUS" = "1" ]
|
||||
then
|
||||
echo "### Elasticsearch is not available, try starting via 'systemctl start elk'."
|
||||
exit
|
||||
else
|
||||
echo "### Elasticsearch is available, now continuing."
|
||||
echo
|
||||
fi
|
||||
|
||||
# Set vars
|
||||
myDATE=$(date +%Y%m%d%H%M)
|
||||
myINDEXCOUNT=$(curl -s -XGET ''$myES'.kibana/index-pattern/logstash-*' | tr '\\' '\n' | grep "scripted" | wc -w)
|
||||
myDASHBOARDS=$(curl -s -XGET ''$myES'.kibana/dashboard/_search?filter_path=hits.hits._id&pretty&size=10000' | jq '.hits.hits[] | {_id}' | jq -r '._id')
|
||||
myVISUALIZATIONS=$(curl -s -XGET ''$myES'.kibana/visualization/_search?filter_path=hits.hits._id&pretty&size=10000' | jq '.hits.hits[] | {_id}' | jq -r '._id')
|
||||
mySEARCHES=$(curl -s -XGET ''$myES'.kibana/search/_search?filter_path=hits.hits._id&pretty&size=10000' | jq '.hits.hits[] | {_id}' | jq -r '._id')
|
||||
myCOL1="[0;34m"
|
||||
myCOL0="[0;0m"
|
||||
|
||||
# Let's ensure normal operation on exit or if interrupted ...
|
||||
function fuCLEANUP {
|
||||
rm -rf patterns/ dashboards/ visualizations/ searches/
|
||||
}
|
||||
trap fuCLEANUP EXIT
|
||||
|
||||
# Export index patterns
|
||||
mkdir -p patterns
|
||||
echo $myCOL1"### Now exporting"$myCOL0 $myINDEXCOUNT $myCOL1"index patterns." $myCOL0
|
||||
curl -s -XGET ''$myES'.kibana/index-pattern/logstash-*?' | jq '._source' > patterns/index-patterns.json
|
||||
echo
|
||||
|
||||
# Export dashboards
|
||||
mkdir -p dashboards
|
||||
echo $myCOL1"### Now exporting"$myCOL0 $(echo $myDASHBOARDS | wc -w) $myCOL1"dashboards." $myCOL0
|
||||
for i in $myDASHBOARDS;
|
||||
do
|
||||
echo $myCOL1"###### "$i $myCOL0
|
||||
curl -s -XGET ''$myES'.kibana/dashboard/'$i'' | jq '._source' > dashboards/$i.json
|
||||
done;
|
||||
echo
|
||||
|
||||
# Export visualizations
|
||||
mkdir -p visualizations
|
||||
echo $myCOL1"### Now exporting"$myCOL0 $(echo $myVISUALIZATIONS | wc -w) $myCOL1"visualizations." $myCOL0
|
||||
for i in $myVISUALIZATIONS;
|
||||
do
|
||||
echo $myCOL1"###### "$i $myCOL0
|
||||
curl -s -XGET ''$myES'.kibana/visualization/'$i'' | jq '._source' > visualizations/$i.json
|
||||
done;
|
||||
echo
|
||||
|
||||
# Export searches
|
||||
mkdir -p searches
|
||||
echo $myCOL1"### Now exporting"$myCOL0 $(echo $mySEARCHES | wc -w) $myCOL1"searches." $myCOL0
|
||||
for i in $mySEARCHES;
|
||||
do
|
||||
echo $myCOL1"###### "$i $myCOL0
|
||||
curl -s -XGET ''$myES'.kibana/search/'$i'' | jq '._source' > searches/$i.json
|
||||
done;
|
||||
echo
|
||||
|
||||
# Building tar archive
|
||||
echo $myCOL1"### Now building archive"$myCOL0 "kibana-objects_"$myDATE".tgz"
|
||||
tar cvfz kibana-objects_$myDATE.tgz patterns dashboards visualizations searches > /dev/null
|
||||
|
||||
# Stats
|
||||
echo
|
||||
echo $myCOL1"### Statistics"
|
||||
echo $myCOL1"###### Exported"$myCOL0 $myINDEXCOUNT $myCOL1"index patterns." $myCOL0
|
||||
echo $myCOL1"###### Exported"$myCOL0 $(echo $myDASHBOARDS | wc -w) $myCOL1"dashboards." $myCOL0
|
||||
echo $myCOL1"###### Exported"$myCOL0 $(echo $myVISUALIZATIONS | wc -w) $myCOL1"visualizations." $myCOL0
|
||||
echo $myCOL1"###### Exported"$myCOL0 $(echo $mySEARCHES | wc -w) $myCOL1"searches." $myCOL0
|
||||
echo
|
||||
91
bin/import_kibana-objects.sh
Executable file
91
bin/import_kibana-objects.sh
Executable file
@ -0,0 +1,91 @@
|
||||
#!/bin/bash
|
||||
# Import Kibana objects
|
||||
# Make sure ES is available
|
||||
myES="http://127.0.0.1:64298/"
|
||||
myESSTATUS=$(curl -s -XGET ''$myES'_cluster/health' | jq '.' | grep -c green)
|
||||
if ! [ "$myESSTATUS" = "1" ]
|
||||
then
|
||||
echo "### Elasticsearch is not available, try starting via 'systemctl start elk'."
|
||||
exit
|
||||
else
|
||||
echo "### Elasticsearch is available, now continuing."
|
||||
echo
|
||||
fi
|
||||
|
||||
# Set vars
|
||||
myDUMP=$1
|
||||
myCOL1="[0;34m"
|
||||
myCOL0="[0;0m"
|
||||
|
||||
# Let's ensure normal operation on exit or if interrupted ...
|
||||
function fuCLEANUP {
|
||||
rm -rf patterns/ dashboards/ visualizations/ searches/
|
||||
}
|
||||
trap fuCLEANUP EXIT
|
||||
|
||||
# Check if parameter is given and file exists
|
||||
if [ "$myDUMP" = "" ];
|
||||
then
|
||||
echo $myCOL1"### Please provide a backup file name."$myCOL0
|
||||
echo $myCOL1"### restore-kibana-objects.sh <kibana-objects.tgz>"$myCOL0
|
||||
echo
|
||||
exit
|
||||
fi
|
||||
if ! [ -a $myDUMP ];
|
||||
then
|
||||
echo $myCOL1"### File not found."$myCOL0
|
||||
exit
|
||||
fi
|
||||
|
||||
# Unpack tar
|
||||
tar xvfz $myDUMP > /dev/null
|
||||
|
||||
# Restore index patterns
|
||||
myINDEXCOUNT=$(cat patterns/index-patterns.json | tr '\\' '\n' | grep "scripted" | wc -w)
|
||||
echo $myCOL1"### Now importing"$myCOL0 $myINDEXCOUNT $myCOL1"index patterns." $myCOL0
|
||||
curl -s -XDELETE ''$myES'.kibana/index-pattern/logstash-*' > /dev/null
|
||||
curl -s -XPUT ''$myES'.kibana/index-pattern/logstash-*' -T patterns/index-patterns.json > /dev/null
|
||||
echo
|
||||
|
||||
# Restore dashboards
|
||||
myDASHBOARDS=$(ls dashboards/*.json | cut -c 12- | rev | cut -c 6- | rev)
|
||||
echo $myCOL1"### Now importing "$myCOL0$(echo $myDASHBOARDS | wc -w)$myCOL1 "dashboards." $myCOL0
|
||||
for i in $myDASHBOARDS;
|
||||
do
|
||||
echo $myCOL1"###### "$i $myCOL0
|
||||
curl -s -XDELETE ''$myES'.kibana/dashboard/'$i'' > /dev/null
|
||||
curl -s -XPUT ''$myES'.kibana/dashboard/'$i'' -T dashboards/$i.json > /dev/null
|
||||
done;
|
||||
echo
|
||||
|
||||
# Restore visualizations
|
||||
myVISUALIZATIONS=$(ls visualizations/*.json | cut -c 16- | rev | cut -c 6- | rev)
|
||||
echo $myCOL1"### Now importing "$myCOL0$(echo $myVISUALIZATIONS | wc -w)$myCOL1 "visualizations." $myCOL0
|
||||
for i in $myVISUALIZATIONS;
|
||||
do
|
||||
echo $myCOL1"###### "$i $myCOL0
|
||||
curl -s -XDELETE ''$myES'.kibana/visualization/'$i'' > /dev/null
|
||||
curl -s -XPUT ''$myES'.kibana/visualization/'$i'' -T visualizations/$i.json > /dev/null
|
||||
done;
|
||||
echo
|
||||
|
||||
# Restore searches
|
||||
mySEARCHES=$(ls searches/*.json | cut -c 10- | rev | cut -c 6- | rev)
|
||||
echo $myCOL1"### Now importing "$myCOL0$(echo $mySEARCHES | wc -w)$myCOL1 "searches." $myCOL0
|
||||
for i in $mySEARCHES;
|
||||
do
|
||||
echo $myCOL1"###### "$i $myCOL0
|
||||
curl -s -XDELETE ''$myES'.kibana/search/'$i'' > /dev/null
|
||||
curl -s -XPUT ''$myES'.kibana/search/'$i'' -T searches/$i.json > /dev/null
|
||||
done;
|
||||
echo
|
||||
|
||||
# Stats
|
||||
echo
|
||||
echo $myCOL1"### Statistics"
|
||||
echo $myCOL1"###### Imported"$myCOL0 $myINDEXCOUNT $myCOL1"index patterns." $myCOL0
|
||||
echo $myCOL1"###### Imported"$myCOL0 $(echo $myDASHBOARDS | wc -w) $myCOL1"dashboards." $myCOL0
|
||||
echo $myCOL1"###### Imported"$myCOL0 $(echo $myVISUALIZATIONS | wc -w) $myCOL1"visualizations." $myCOL0
|
||||
echo $myCOL1"###### Imported"$myCOL0 $(echo $mySEARCHES | wc -w) $myCOL1"searches." $myCOL0
|
||||
echo
|
||||
|
||||
88
bin/myip.sh
Executable file
88
bin/myip.sh
Executable file
@ -0,0 +1,88 @@
|
||||
#!/bin/bash
|
||||
|
||||
## Get my external IP
|
||||
|
||||
timeout=2 # seconds to wait for a reply before trying next server
|
||||
verbose=1 # prints which server was used to STDERR
|
||||
|
||||
dnslist=(
|
||||
"dig +short myip.opendns.com @resolver1.opendns.com"
|
||||
"dig +short myip.opendns.com @resolver2.opendns.com"
|
||||
"dig +short myip.opendns.com @resolver3.opendns.com"
|
||||
"dig +short myip.opendns.com @resolver4.opendns.com"
|
||||
"dig +short -4 -t a whoami.akamai.net @ns1-1.akamaitech.net"
|
||||
"dig +short whoami.akamai.net @ns1-1.akamaitech.net"
|
||||
)
|
||||
|
||||
httplist=(
|
||||
alma.ch/myip.cgi
|
||||
api.infoip.io/ip
|
||||
api.ipify.org
|
||||
bot.whatismyipaddress.com
|
||||
canhazip.com
|
||||
checkip.amazonaws.com
|
||||
eth0.me
|
||||
icanhazip.com
|
||||
ident.me
|
||||
ipecho.net/plain
|
||||
ipinfo.io/ip
|
||||
ipof.in/txt
|
||||
ip.tyk.nu
|
||||
l2.io/ip
|
||||
smart-ip.net/myip
|
||||
wgetip.com
|
||||
whatismyip.akamai.com
|
||||
)
|
||||
|
||||
# function to shuffle the global array "array"
|
||||
shuffle() {
|
||||
local i tmp size max rand
|
||||
size=${#array[*]}
|
||||
max=$(( 32768 / size * size ))
|
||||
for ((i=size-1; i>0; i--)); do
|
||||
while (( (rand=$RANDOM) >= max )); do :; done
|
||||
rand=$(( rand % (i+1) ))
|
||||
tmp=${array[i]} array[i]=${array[rand]} array[rand]=$tmp
|
||||
done
|
||||
}
|
||||
|
||||
# if we have dig and a list of dns methods, try that first
|
||||
if hash dig 2>/dev/null && [ ${#dnslist[*]} -gt 0 ]; then
|
||||
eval array=( \"\${dnslist[@]}\" )
|
||||
shuffle
|
||||
|
||||
for cmd in "${array[@]}"; do
|
||||
[ "$verbose" == 1 ] && echo Trying: $cmd 1>&2
|
||||
ip=$(timeout $timeout $cmd)
|
||||
if [ -n "$ip" ]; then
|
||||
echo $ip
|
||||
exit
|
||||
fi
|
||||
done
|
||||
fi
|
||||
|
||||
# if we haven't succeeded with DNS, try HTTP
|
||||
if [ ${#httplist[*]} == 0 ]; then
|
||||
echo "No hosts in httplist array!" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# use curl or wget, depending on which one we find
|
||||
curl_or_wget=$(if hash curl 2>/dev/null; then echo curl; elif hash wget 2>/dev/null; then echo "wget -qO-"; fi);
|
||||
|
||||
if [ -z "$curl_or_wget" ]; then
|
||||
echo "Neither curl nor wget found. Cannot use http method." >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
eval array=( \"\${httplist[@]}\" )
|
||||
shuffle
|
||||
|
||||
for url in "${array[@]}"; do
|
||||
[ "$verbose" == 1 ] && echo Trying: $curl_or_wget -s "$url" 1>&2
|
||||
ip=$(timeout $timeout $curl_or_wget -s "$url")
|
||||
if [ -n "$ip" ]; then
|
||||
echo $ip
|
||||
exit
|
||||
fi
|
||||
done
|
||||
61
bin/restore_es.sh
Executable file
61
bin/restore_es.sh
Executable file
@ -0,0 +1,61 @@
|
||||
#/bin/bash
|
||||
# Restore folder based ES backup
|
||||
# Make sure ES is available
|
||||
myES="http://127.0.0.1:64298/"
|
||||
myESSTATUS=$(curl -s -XGET ''$myES'_cluster/health' | jq '.' | grep -c green)
|
||||
if ! [ "$myESSTATUS" = "1" ]
|
||||
then
|
||||
echo "### Elasticsearch is not available, try starting via 'systemctl start elk'."
|
||||
exit
|
||||
else
|
||||
echo "### Elasticsearch is available, now continuing."
|
||||
fi
|
||||
|
||||
# Let's ensure normal operation on exit or if interrupted ...
|
||||
function fuCLEANUP {
|
||||
rm -rf tmp
|
||||
}
|
||||
trap fuCLEANUP EXIT
|
||||
|
||||
# Set vars
|
||||
myDUMP=$1
|
||||
myCOL1="[0;34m"
|
||||
myCOL0="[0;0m"
|
||||
|
||||
# Check if parameter is given and file exists
|
||||
if [ "$myDUMP" = "" ];
|
||||
then
|
||||
echo $myCOL1"### Please provide a backup file name."$myCOL0
|
||||
echo $myCOL1"### restore-elk.sh <es_dump.tar>"$myCOL0
|
||||
echo
|
||||
exit
|
||||
fi
|
||||
if ! [ -a $myDUMP ];
|
||||
then
|
||||
echo $myCOL1"### File not found."$myCOL0
|
||||
exit
|
||||
fi
|
||||
|
||||
# Unpack tar archive
|
||||
echo $myCOL1"### Now unpacking tar archive: "$myDUMP $myCOL0
|
||||
tar xvf $myDUMP
|
||||
|
||||
# Build indices list
|
||||
myINDICES=$(ls tmp/logstash*.gz | cut -c 5- | rev | cut -c 4- | rev)
|
||||
echo $myCOL1"### The following indices will be restored: "$myCOL0
|
||||
echo $myINDICES
|
||||
echo
|
||||
|
||||
# Restore indices
|
||||
for i in $myINDICES;
|
||||
do
|
||||
# Delete index if it already exists
|
||||
curl -s -XDELETE $myES$i > /dev/null
|
||||
echo $myCOL1"### Now uncompressing: tmp/$i.gz" $myCOL0
|
||||
gunzip -f tmp/$i.gz
|
||||
# Restore index to ES
|
||||
echo $myCOL1"### Now restoring: "$i $myCOL0
|
||||
elasticdump --input=tmp/$i --output=$myES$i --limit 7500
|
||||
rm tmp/$i
|
||||
done;
|
||||
echo $myCOL1"### Done."$myCOL0
|
||||
24
bin/updateip.sh
Executable file
24
bin/updateip.sh
Executable file
@ -0,0 +1,24 @@
|
||||
#!/bin/bash
|
||||
# Let's add the first local ip to the /etc/issue and external ip to ews.ip file
|
||||
# If the external IP cannot be detected, the internal IP will be inherited.
|
||||
source /etc/environment
|
||||
myLOCALIP=$(hostname -I | awk '{ print $1 }')
|
||||
myEXTIP=$(/opt/tpot/bin/myip.sh)
|
||||
if [ "$myEXTIP" = "" ];
|
||||
then
|
||||
myEXTIP=$myLOCALIP
|
||||
fi
|
||||
sed -i "s#IP:.*#IP: $myLOCALIP ($myEXTIP)[0m#" /etc/issue
|
||||
sed -i "s#SSH:.*#SSH: ssh -l tsec -p 64295 $myLOCALIP[0m#" /etc/issue
|
||||
sed -i "s#WEB:.*#WEB: https://$myLOCALIP:64297[0m#" /etc/issue
|
||||
tee /data/ews/conf/ews.ip << EOF
|
||||
[MAIN]
|
||||
ip = $myEXTIP
|
||||
EOF
|
||||
tee /opt/tpot/etc/compose/elk_environment << EOF
|
||||
MY_EXTIP=$myEXTIP
|
||||
MY_INTIP=$myLOCALIP
|
||||
MY_HOSTNAME=$HOSTNAME
|
||||
EOF
|
||||
chown tpot:tpot /data/ews/conf/ews.ip
|
||||
chmod 760 /data/ews/conf/ews.ip
|
||||
Reference in New Issue
Block a user