update scripts to new file layout

iso will not include tpot repo
tpot repo will be cloned during install as all the other tools
This commit is contained in:
Marco Ochse
2017-09-26 15:15:17 +00:00
parent b5a4ef948f
commit 1347eac88f
43 changed files with 12923 additions and 13 deletions

38
bin/backup_es_folders.sh Executable file
View File

@ -0,0 +1,38 @@
#!/bin/bash
# Backup all ES relevant folders
# Make sure ES is available
myES="http://127.0.0.1:64298/"
myESSTATUS=$(curl -s -XGET ''$myES'_cluster/health' | jq '.' | grep -c green)
if ! [ "$myESSTATUS" = "1" ]
then
echo "### Elasticsearch is not available, try starting via 'systemctl start elk'."
exit
else
echo "### Elasticsearch is available, now continuing."
echo
fi
# Set vars
myCOUNT=1
myDATE=$(date +%Y%m%d%H%M)
myELKPATH="/data/elk/data"
myKIBANAINDEXNAME=$(curl -s -XGET ''$myES'_cat/indices/' | grep .kibana | awk '{ print $4 }')
myKIBANAINDEXPATH=$myELKPATH/nodes/0/indices/$myKIBANAINDEXNAME
# Let's ensure normal operation on exit or if interrupted ...
function fuCLEANUP {
### Start ELK
systemctl start tpot
echo "### Now starting T-Pot ..."
}
trap fuCLEANUP EXIT
# Stop T-Pot to lift db lock
echo "### Now stopping T-Pot"
systemctl stop tpot
sleep 2
# Backup DB in 2 flavors
echo "### Now backing up Elasticsearch folders ..."
tar cvfz "elkall_"$myDATE".tgz" $myELKPATH
tar cvfz "elkbase_"$myDATE".tgz" $myKIBANAINDEXPATH

219
bin/clean.sh Executable file
View File

@ -0,0 +1,219 @@
#!/bin/bash
# T-Pot Container Data Cleaner & Log Rotator
# Set colors
myRED=""
myGREEN=""
myWHITE=""
# Set persistence
myPERSISTENCE=$1
# Let's create a function to check if folder is empty
fuEMPTY () {
local myFOLDER=$1
echo $(ls $myFOLDER | wc -l)
}
# Let's create a function to rotate and compress logs
fuLOGROTATE () {
local mySTATUS="/opt/tpot/etc/logrotate/status"
local myCONF="/opt/tpot/etc/logrotate/logrotate.conf"
local myCOWRIETTYLOGS="/data/cowrie/log/tty/"
local myCOWRIETTYTGZ="/data/cowrie/log/ttylogs.tgz"
local myCOWRIEDL="/data/cowrie/downloads/"
local myCOWRIEDLTGZ="/data/cowrie/downloads.tgz"
local myDIONAEABI="/data/dionaea/bistreams/"
local myDIONAEABITGZ="/data/dionaea/bistreams.tgz"
local myDIONAEABIN="/data/dionaea/binaries/"
local myDIONAEABINTGZ="/data/dionaea/binaries.tgz"
local myHONEYTRAPATTACKS="/data/honeytrap/attacks/"
local myHONEYTRAPATTACKSTGZ="/data/honeytrap/attacks.tgz"
local myHONEYTRAPDL="/data/honeytrap/downloads/"
local myHONEYTRAPDLTGZ="/data/honeytrap/downloads.tgz"
# Ensure correct permissions and ownerships for logrotate to run without issues
chmod 760 /data/ -R
chown tpot:tpot /data -R
# Run logrotate with force (-f) first, so the status file can be written and race conditions (with tar) be avoided
logrotate -f -s $mySTATUS $myCONF
# Compressing some folders first and rotate them later
if [ "$(fuEMPTY $myCOWRIETTYLOGS)" != "0" ]; then tar cvfz $myCOWRIETTYTGZ $myCOWRIETTYLOGS; fi
if [ "$(fuEMPTY $myCOWRIEDL)" != "0" ]; then tar cvfz $myCOWRIEDLTGZ $myCOWRIEDL; fi
if [ "$(fuEMPTY $myDIONAEABI)" != "0" ]; then tar cvfz $myDIONAEABITGZ $myDIONAEABI; fi
if [ "$(fuEMPTY $myDIONAEABIN)" != "0" ]; then tar cvfz $myDIONAEABINTGZ $myDIONAEABIN; fi
if [ "$(fuEMPTY $myHONEYTRAPATTACKS)" != "0" ]; then tar cvfz $myHONEYTRAPATTACKSTGZ $myHONEYTRAPATTACKS; fi
if [ "$(fuEMPTY $myHONEYTRAPDL)" != "0" ]; then tar cvfz $myHONEYTRAPDLTGZ $myHONEYTRAPDL; fi
# Ensure correct permissions and ownership for previously created archives
chmod 760 $myCOWRIETTYTGZ $myCOWRIEDLTGZ $myDIONAEABITGZ $myDIONAEABINTGZ $myHONEYTRAPATTACKSTGZ $myHONEYTRAPDLTGZ
chown tpot:tpot $myCOWRIETTYTGZ $myCOWRIEDLTGZ $myDIONAEABITGZ $myDIONAEABINTGZ $myHONEYTRAPATTACKSTGZ $myHONEYTRAPDLTGZ
# Need to remove subfolders since too many files cause rm to exit with errors
rm -rf $myCOWRIETTYLOGS $myCOWRIEDL $myDIONAEABI $myDIONAEABIN $myHONEYTRAPATTACKS $myHONEYTRAPDL
# Recreate subfolders with correct permissions and ownership
mkdir -p $myCOWRIETTYLOGS $myCOWRIEDL $myDIONAEABI $myDIONAEABIN $myHONEYTRAPATTACKS $myHONEYTRAPDL
chmod 760 $myCOWRIETTYLOGS $myCOWRIEDL $myDIONAEABI $myDIONAEABIN $myHONEYTRAPATTACKS $myHONEYTRAPDL
chown tpot:tpot $myCOWRIETTYLOGS $myCOWRIEDL $myDIONAEABI $myDIONAEABIN $myHONEYTRAPATTACKS $myHONEYTRAPDL
# Run logrotate again to account for previously created archives - DO NOT FORCE HERE!
logrotate -s $mySTATUS $myCONF
}
# Let's create a function to clean up and prepare conpot data
fuCONPOT () {
if [ "$myPERSISTENCE" != "on" ]; then rm -rf /data/conpot/*; fi
mkdir -p /data/conpot/log
chmod 760 /data/conpot -R
chown tpot:tpot /data/conpot -R
}
# Let's create a function to clean up and prepare cowrie data
fuCOWRIE () {
if [ "$myPERSISTENCE" != "on" ]; then rm -rf /data/cowrie/*; fi
mkdir -p /data/cowrie/log/tty/ /data/cowrie/downloads/ /data/cowrie/keys/ /data/cowrie/misc/
chmod 760 /data/cowrie -R
chown tpot:tpot /data/cowrie -R
}
# Let's create a function to clean up and prepare dionaea data
fuDIONAEA () {
if [ "$myPERSISTENCE" != "on" ]; then rm -rf /data/dionaea/*; fi
mkdir -p /data/dionaea/log /data/dionaea/bistreams /data/dionaea/binaries /data/dionaea/rtp /data/dionaea/roots/ftp /data/dionaea/roots/tftp /data/dionaea/roots/www /data/dionaea/roots/upnp
chmod 760 /data/dionaea -R
chown tpot:tpot /data/dionaea -R
}
# Let's create a function to clean up and prepare elasticpot data
fuELASTICPOT () {
if [ "$myPERSISTENCE" != "on" ]; then rm -rf /data/elasticpot/*; fi
mkdir -p /data/elasticpot/log
chmod 760 /data/elasticpot -R
chown tpot:tpot /data/elasticpot -R
}
# Let's create a function to clean up and prepare elk data
fuELK () {
# ELK data will be kept for <= 90 days, check /etc/crontab for curator modification
# ELK daemon log files will be removed
if [ "$myPERSISTENCE" != "on" ]; then rm -rf /data/elk/log/*; fi
mkdir -p /data/elk
chmod 760 /data/elk -R
chown tpot:tpot /data/elk -R
}
# Let's create a function to clean up and prepare emobility data
fuEMOBILITY () {
if [ "$myPERSISTENCE" != "on" ]; then rm -rf /data/emobility/*; fi
mkdir -p /data/emobility/log
chmod 760 /data/emobility -R
chown tpot:tpot /data/emobility -R
}
# Let's create a function to clean up and prepare glastopf data
fuGLASTOPF () {
if [ "$myPERSISTENCE" != "on" ]; then rm -rf /data/glastopf/*; fi
mkdir -p /data/glastopf
chmod 760 /data/glastopf -R
chown tpot:tpot /data/glastopf -R
}
# Let's create a function to clean up and prepare honeytrap data
fuHONEYTRAP () {
if [ "$myPERSISTENCE" != "on" ]; then rm -rf /data/honeytrap/*; fi
mkdir -p /data/honeytrap/log/ /data/honeytrap/attacks/ /data/honeytrap/downloads/
chmod 760 /data/honeytrap/ -R
chown tpot:tpot /data/honeytrap/ -R
}
# Let's create a function to clean up and prepare mailoney data
fuMAILONEY () {
if [ "$myPERSISTENCE" != "on" ]; then rm -rf /data/mailoney/*; fi
mkdir -p /data/mailoney/log/
chmod 760 /data/mailoney/ -R
chown tpot:tpot /data/mailoney/ -R
}
# Let's create a function to clean up and prepare rdpy data
fuRDPY () {
if [ "$myPERSISTENCE" != "on" ]; then rm -rf /data/rdpy/*; fi
mkdir -p /data/rdpy/log/
chmod 760 /data/rdpy/ -R
chown tpot:tpot /data/rdpy/ -R
}
# Let's create a function to prepare spiderfoot db
fuSPIDERFOOT () {
mkdir -p /data/spiderfoot
touch /data/spiderfoot/spiderfoot.db
chmod 760 -R /data/spiderfoot
chown tpot:tpot -R /data/spiderfoot
}
# Let's create a function to clean up and prepare suricata data
fuSURICATA () {
if [ "$myPERSISTENCE" != "on" ]; then rm -rf /data/suricata/*; fi
mkdir -p /data/suricata/log
chmod 760 -R /data/suricata
chown tpot:tpot -R /data/suricata
}
# Let's create a function to clean up and prepare p0f data
fuP0F () {
if [ "$myPERSISTENCE" != "on" ]; then rm -rf /data/p0f/*; fi
mkdir -p /data/p0f/log
chmod 760 -R /data/p0f
chown tpot:tpot -R /data/p0f
}
# Let's create a function to clean up and prepare vnclowpot data
fuVNCLOWPOT () {
if [ "$myPERSISTENCE" != "on" ]; then rm -rf /data/vnclowpot/*; fi
mkdir -p /data/vnclowpot/log/
chmod 760 /data/vnclowpot/ -R
chown tpot:tpot /data/vnclowpot/ -R
}
# Avoid unwanted cleaning
if [ "$myPERSISTENCE" = "" ];
then
echo $myRED"!!! WARNING !!! - This will delete ALL honeypot logs. "$myWHITE
while [ "$myQST" != "y" ] && [ "$myQST" != "n" ];
do
read -p "Continue? (y/n) " myQST
done
if [ "$myQST" = "n" ];
then
echo $myGREEN"Puuh! That was close! Aborting!"$myWHITE
exit
fi
fi
# Check persistence, if enabled compress and rotate logs
if [ "$myPERSISTENCE" = "on" ];
then
echo "Persistence enabled, now rotating and compressing logs."
fuLOGROTATE
else
echo "Cleaning up and preparing data folders."
fuCONPOT
fuCOWRIE
fuDIONAEA
fuELASTICPOT
fuELK
fuEMOBILITY
fuGLASTOPF
fuHONEYTRAP
fuMAILONEY
fuRDPY
fuSPIDERFOOT
fuSURICATA
fuP0F
fuVNCLOWPOT
fi

71
bin/dps.sh Executable file
View File

@ -0,0 +1,71 @@
#/bin/bash
# Show current status of all running containers
myPARAM="$1"
myIMAGES="$(cat /opt/tpot/etc/tpot.yml | grep -v '#' | grep container_name | cut -d: -f2)"
myRED=""
myGREEN=""
myBLUE=""
myWHITE=""
myMAGENTA=""
function fuCONTAINERSTATUS {
local myNAME="$1"
local mySTATUS="$(/usr/bin/docker ps -f name=$myNAME --format "table {{.Status}}" -f status=running -f status=exited | tail -n 1)"
myDOWN="$(echo "$mySTATUS" | grep -o -E "(STATUS|NAMES|Exited)")"
case "$myDOWN" in
STATUS)
mySTATUS="$myRED"DOWN"$myWHITE"
;;
NAMES)
mySTATUS="$myRED"DOWN"$myWHITE"
;;
Exited)
mySTATUS="$myRED$mySTATUS$myWHITE"
;;
*)
mySTATUS="$myGREEN$mySTATUS$myWHITE"
;;
esac
printf "$mySTATUS"
}
function fuCONTAINERPORTS {
local myNAME="$1"
local myPORTS="$(/usr/bin/docker ps -f name=$myNAME --format "table {{.Ports}}" -f status=running -f status=exited | tail -n 1 | sed s/","/",\n\t\t\t\t\t\t\t"/g)"
if [ "$myPORTS" != "PORTS" ];
then
printf "$myBLUE$myPORTS$myWHITE"
fi
}
function fuGETSYS {
printf "========| System |========\n"
printf "%+10s %-20s\n" "Date: " "$(date)"
printf "%+10s %-20s\n" "Uptime: " "$(uptime | cut -b 2-)"
printf "%+10s %-20s\n" "CPU temp: " "$(sensors | grep 'Physical' | awk '{ print $4" " }' | tr -d [:cntrl:])"
echo
}
while true
do
fuGETSYS
printf "%-19s %-36s %s\n" "NAME" "STATUS" "PORTS"
for i in $myIMAGES; do
myNAME="$myMAGENTA$i$myWHITE"
printf "%-32s %-49s %s" "$myNAME" "$(fuCONTAINERSTATUS $i)" "$(fuCONTAINERPORTS $i)"
echo
if [ "$myPARAM" = "vv" ];
then
/usr/bin/docker exec -t "$i" /bin/ps awfuwfxwf | egrep -v -E "awfuwfxwf|/bin/ps"
fi
done
if [[ $myPARAM =~ ^([1-9]|[1-9][0-9]|[1-9][0-9][0-9])$ ]];
then
sleep "$myPARAM"
else
break
fi
done

45
bin/dump_es.sh Executable file
View File

@ -0,0 +1,45 @@
#/bin/bash
# Dump all ES data
# Make sure ES is available
myES="http://127.0.0.1:64298/"
myESSTATUS=$(curl -s -XGET ''$myES'_cluster/health' | jq '.' | grep -c green)
if ! [ "$myESSTATUS" = "1" ]
then
echo "### Elasticsearch is not available, try starting via 'systemctl start elk'."
exit
else
echo "### Elasticsearch is available, now continuing."
echo
fi
# Let's ensure normal operation on exit or if interrupted ...
function fuCLEANUP {
rm -rf tmp
}
trap fuCLEANUP EXIT
# Set vars
myDATE=$(date +%Y%m%d%H%M)
myINDICES=$(curl -s -XGET ''$myES'_cat/indices/' | grep logstash | awk '{ print $3 }' | sort | grep -v 1970)
myES="http://127.0.0.1:64298/"
myCOL1=""
myCOL0=""
# Dumping all ES data
echo $myCOL1"### The following indices will be dumped: "$myCOL0
echo $myINDICES
echo
mkdir tmp
for i in $myINDICES;
do
echo $myCOL1"### Now dumping: "$i $myCOL0
elasticdump --input=$myES$i --output="tmp/"$i --limit 7500
echo $myCOL1"### Now compressing: tmp/$i" $myCOL0
gzip -f "tmp/"$i
done;
# Build tar archive
echo $myCOL1"### Now building tar archive: es_dump_"$myDATE".tgz" $myCOL0
tar cvf es_dump_$myDATE.tar tmp/*
echo $myCOL1"### Done."$myCOL0

77
bin/export_kibana-objects.sh Executable file
View File

@ -0,0 +1,77 @@
#!/bin/bash
# Export all Kibana objects
# Make sure ES is available
myES="http://127.0.0.1:64298/"
myESSTATUS=$(curl -s -XGET ''$myES'_cluster/health' | jq '.' | grep -c green)
if ! [ "$myESSTATUS" = "1" ]
then
echo "### Elasticsearch is not available, try starting via 'systemctl start elk'."
exit
else
echo "### Elasticsearch is available, now continuing."
echo
fi
# Set vars
myDATE=$(date +%Y%m%d%H%M)
myINDEXCOUNT=$(curl -s -XGET ''$myES'.kibana/index-pattern/logstash-*' | tr '\\' '\n' | grep "scripted" | wc -w)
myDASHBOARDS=$(curl -s -XGET ''$myES'.kibana/dashboard/_search?filter_path=hits.hits._id&pretty&size=10000' | jq '.hits.hits[] | {_id}' | jq -r '._id')
myVISUALIZATIONS=$(curl -s -XGET ''$myES'.kibana/visualization/_search?filter_path=hits.hits._id&pretty&size=10000' | jq '.hits.hits[] | {_id}' | jq -r '._id')
mySEARCHES=$(curl -s -XGET ''$myES'.kibana/search/_search?filter_path=hits.hits._id&pretty&size=10000' | jq '.hits.hits[] | {_id}' | jq -r '._id')
myCOL1=""
myCOL0=""
# Let's ensure normal operation on exit or if interrupted ...
function fuCLEANUP {
rm -rf patterns/ dashboards/ visualizations/ searches/
}
trap fuCLEANUP EXIT
# Export index patterns
mkdir -p patterns
echo $myCOL1"### Now exporting"$myCOL0 $myINDEXCOUNT $myCOL1"index patterns." $myCOL0
curl -s -XGET ''$myES'.kibana/index-pattern/logstash-*?' | jq '._source' > patterns/index-patterns.json
echo
# Export dashboards
mkdir -p dashboards
echo $myCOL1"### Now exporting"$myCOL0 $(echo $myDASHBOARDS | wc -w) $myCOL1"dashboards." $myCOL0
for i in $myDASHBOARDS;
do
echo $myCOL1"###### "$i $myCOL0
curl -s -XGET ''$myES'.kibana/dashboard/'$i'' | jq '._source' > dashboards/$i.json
done;
echo
# Export visualizations
mkdir -p visualizations
echo $myCOL1"### Now exporting"$myCOL0 $(echo $myVISUALIZATIONS | wc -w) $myCOL1"visualizations." $myCOL0
for i in $myVISUALIZATIONS;
do
echo $myCOL1"###### "$i $myCOL0
curl -s -XGET ''$myES'.kibana/visualization/'$i'' | jq '._source' > visualizations/$i.json
done;
echo
# Export searches
mkdir -p searches
echo $myCOL1"### Now exporting"$myCOL0 $(echo $mySEARCHES | wc -w) $myCOL1"searches." $myCOL0
for i in $mySEARCHES;
do
echo $myCOL1"###### "$i $myCOL0
curl -s -XGET ''$myES'.kibana/search/'$i'' | jq '._source' > searches/$i.json
done;
echo
# Building tar archive
echo $myCOL1"### Now building archive"$myCOL0 "kibana-objects_"$myDATE".tgz"
tar cvfz kibana-objects_$myDATE.tgz patterns dashboards visualizations searches > /dev/null
# Stats
echo
echo $myCOL1"### Statistics"
echo $myCOL1"###### Exported"$myCOL0 $myINDEXCOUNT $myCOL1"index patterns." $myCOL0
echo $myCOL1"###### Exported"$myCOL0 $(echo $myDASHBOARDS | wc -w) $myCOL1"dashboards." $myCOL0
echo $myCOL1"###### Exported"$myCOL0 $(echo $myVISUALIZATIONS | wc -w) $myCOL1"visualizations." $myCOL0
echo $myCOL1"###### Exported"$myCOL0 $(echo $mySEARCHES | wc -w) $myCOL1"searches." $myCOL0
echo

91
bin/import_kibana-objects.sh Executable file
View File

@ -0,0 +1,91 @@
#!/bin/bash
# Import Kibana objects
# Make sure ES is available
myES="http://127.0.0.1:64298/"
myESSTATUS=$(curl -s -XGET ''$myES'_cluster/health' | jq '.' | grep -c green)
if ! [ "$myESSTATUS" = "1" ]
then
echo "### Elasticsearch is not available, try starting via 'systemctl start elk'."
exit
else
echo "### Elasticsearch is available, now continuing."
echo
fi
# Set vars
myDUMP=$1
myCOL1=""
myCOL0=""
# Let's ensure normal operation on exit or if interrupted ...
function fuCLEANUP {
rm -rf patterns/ dashboards/ visualizations/ searches/
}
trap fuCLEANUP EXIT
# Check if parameter is given and file exists
if [ "$myDUMP" = "" ];
then
echo $myCOL1"### Please provide a backup file name."$myCOL0
echo $myCOL1"### restore-kibana-objects.sh <kibana-objects.tgz>"$myCOL0
echo
exit
fi
if ! [ -a $myDUMP ];
then
echo $myCOL1"### File not found."$myCOL0
exit
fi
# Unpack tar
tar xvfz $myDUMP > /dev/null
# Restore index patterns
myINDEXCOUNT=$(cat patterns/index-patterns.json | tr '\\' '\n' | grep "scripted" | wc -w)
echo $myCOL1"### Now importing"$myCOL0 $myINDEXCOUNT $myCOL1"index patterns." $myCOL0
curl -s -XDELETE ''$myES'.kibana/index-pattern/logstash-*' > /dev/null
curl -s -XPUT ''$myES'.kibana/index-pattern/logstash-*' -T patterns/index-patterns.json > /dev/null
echo
# Restore dashboards
myDASHBOARDS=$(ls dashboards/*.json | cut -c 12- | rev | cut -c 6- | rev)
echo $myCOL1"### Now importing "$myCOL0$(echo $myDASHBOARDS | wc -w)$myCOL1 "dashboards." $myCOL0
for i in $myDASHBOARDS;
do
echo $myCOL1"###### "$i $myCOL0
curl -s -XDELETE ''$myES'.kibana/dashboard/'$i'' > /dev/null
curl -s -XPUT ''$myES'.kibana/dashboard/'$i'' -T dashboards/$i.json > /dev/null
done;
echo
# Restore visualizations
myVISUALIZATIONS=$(ls visualizations/*.json | cut -c 16- | rev | cut -c 6- | rev)
echo $myCOL1"### Now importing "$myCOL0$(echo $myVISUALIZATIONS | wc -w)$myCOL1 "visualizations." $myCOL0
for i in $myVISUALIZATIONS;
do
echo $myCOL1"###### "$i $myCOL0
curl -s -XDELETE ''$myES'.kibana/visualization/'$i'' > /dev/null
curl -s -XPUT ''$myES'.kibana/visualization/'$i'' -T visualizations/$i.json > /dev/null
done;
echo
# Restore searches
mySEARCHES=$(ls searches/*.json | cut -c 10- | rev | cut -c 6- | rev)
echo $myCOL1"### Now importing "$myCOL0$(echo $mySEARCHES | wc -w)$myCOL1 "searches." $myCOL0
for i in $mySEARCHES;
do
echo $myCOL1"###### "$i $myCOL0
curl -s -XDELETE ''$myES'.kibana/search/'$i'' > /dev/null
curl -s -XPUT ''$myES'.kibana/search/'$i'' -T searches/$i.json > /dev/null
done;
echo
# Stats
echo
echo $myCOL1"### Statistics"
echo $myCOL1"###### Imported"$myCOL0 $myINDEXCOUNT $myCOL1"index patterns." $myCOL0
echo $myCOL1"###### Imported"$myCOL0 $(echo $myDASHBOARDS | wc -w) $myCOL1"dashboards." $myCOL0
echo $myCOL1"###### Imported"$myCOL0 $(echo $myVISUALIZATIONS | wc -w) $myCOL1"visualizations." $myCOL0
echo $myCOL1"###### Imported"$myCOL0 $(echo $mySEARCHES | wc -w) $myCOL1"searches." $myCOL0
echo

88
bin/myip.sh Executable file
View File

@ -0,0 +1,88 @@
#!/bin/bash
## Get my external IP
timeout=2 # seconds to wait for a reply before trying next server
verbose=1 # prints which server was used to STDERR
dnslist=(
"dig +short myip.opendns.com @resolver1.opendns.com"
"dig +short myip.opendns.com @resolver2.opendns.com"
"dig +short myip.opendns.com @resolver3.opendns.com"
"dig +short myip.opendns.com @resolver4.opendns.com"
"dig +short -4 -t a whoami.akamai.net @ns1-1.akamaitech.net"
"dig +short whoami.akamai.net @ns1-1.akamaitech.net"
)
httplist=(
alma.ch/myip.cgi
api.infoip.io/ip
api.ipify.org
bot.whatismyipaddress.com
canhazip.com
checkip.amazonaws.com
eth0.me
icanhazip.com
ident.me
ipecho.net/plain
ipinfo.io/ip
ipof.in/txt
ip.tyk.nu
l2.io/ip
smart-ip.net/myip
wgetip.com
whatismyip.akamai.com
)
# function to shuffle the global array "array"
shuffle() {
local i tmp size max rand
size=${#array[*]}
max=$(( 32768 / size * size ))
for ((i=size-1; i>0; i--)); do
while (( (rand=$RANDOM) >= max )); do :; done
rand=$(( rand % (i+1) ))
tmp=${array[i]} array[i]=${array[rand]} array[rand]=$tmp
done
}
# if we have dig and a list of dns methods, try that first
if hash dig 2>/dev/null && [ ${#dnslist[*]} -gt 0 ]; then
eval array=( \"\${dnslist[@]}\" )
shuffle
for cmd in "${array[@]}"; do
[ "$verbose" == 1 ] && echo Trying: $cmd 1>&2
ip=$(timeout $timeout $cmd)
if [ -n "$ip" ]; then
echo $ip
exit
fi
done
fi
# if we haven't succeeded with DNS, try HTTP
if [ ${#httplist[*]} == 0 ]; then
echo "No hosts in httplist array!" >&2
exit 1
fi
# use curl or wget, depending on which one we find
curl_or_wget=$(if hash curl 2>/dev/null; then echo curl; elif hash wget 2>/dev/null; then echo "wget -qO-"; fi);
if [ -z "$curl_or_wget" ]; then
echo "Neither curl nor wget found. Cannot use http method." >&2
exit 1
fi
eval array=( \"\${httplist[@]}\" )
shuffle
for url in "${array[@]}"; do
[ "$verbose" == 1 ] && echo Trying: $curl_or_wget -s "$url" 1>&2
ip=$(timeout $timeout $curl_or_wget -s "$url")
if [ -n "$ip" ]; then
echo $ip
exit
fi
done

61
bin/restore_es.sh Executable file
View File

@ -0,0 +1,61 @@
#/bin/bash
# Restore folder based ES backup
# Make sure ES is available
myES="http://127.0.0.1:64298/"
myESSTATUS=$(curl -s -XGET ''$myES'_cluster/health' | jq '.' | grep -c green)
if ! [ "$myESSTATUS" = "1" ]
then
echo "### Elasticsearch is not available, try starting via 'systemctl start elk'."
exit
else
echo "### Elasticsearch is available, now continuing."
fi
# Let's ensure normal operation on exit or if interrupted ...
function fuCLEANUP {
rm -rf tmp
}
trap fuCLEANUP EXIT
# Set vars
myDUMP=$1
myCOL1=""
myCOL0=""
# Check if parameter is given and file exists
if [ "$myDUMP" = "" ];
then
echo $myCOL1"### Please provide a backup file name."$myCOL0
echo $myCOL1"### restore-elk.sh <es_dump.tar>"$myCOL0
echo
exit
fi
if ! [ -a $myDUMP ];
then
echo $myCOL1"### File not found."$myCOL0
exit
fi
# Unpack tar archive
echo $myCOL1"### Now unpacking tar archive: "$myDUMP $myCOL0
tar xvf $myDUMP
# Build indices list
myINDICES=$(ls tmp/logstash*.gz | cut -c 5- | rev | cut -c 4- | rev)
echo $myCOL1"### The following indices will be restored: "$myCOL0
echo $myINDICES
echo
# Restore indices
for i in $myINDICES;
do
# Delete index if it already exists
curl -s -XDELETE $myES$i > /dev/null
echo $myCOL1"### Now uncompressing: tmp/$i.gz" $myCOL0
gunzip -f tmp/$i.gz
# Restore index to ES
echo $myCOL1"### Now restoring: "$i $myCOL0
elasticdump --input=tmp/$i --output=$myES$i --limit 7500
rm tmp/$i
done;
echo $myCOL1"### Done."$myCOL0

24
bin/updateip.sh Executable file
View File

@ -0,0 +1,24 @@
#!/bin/bash
# Let's add the first local ip to the /etc/issue and external ip to ews.ip file
# If the external IP cannot be detected, the internal IP will be inherited.
source /etc/environment
myLOCALIP=$(hostname -I | awk '{ print $1 }')
myEXTIP=$(/opt/tpot/bin/myip.sh)
if [ "$myEXTIP" = "" ];
then
myEXTIP=$myLOCALIP
fi
sed -i "s#IP:.*#IP: $myLOCALIP ($myEXTIP)#" /etc/issue
sed -i "s#SSH:.*#SSH: ssh -l tsec -p 64295 $myLOCALIP#" /etc/issue
sed -i "s#WEB:.*#WEB: https://$myLOCALIP:64297#" /etc/issue
tee /data/ews/conf/ews.ip << EOF
[MAIN]
ip = $myEXTIP
EOF
tee /opt/tpot/etc/compose/elk_environment << EOF
MY_EXTIP=$myEXTIP
MY_INTIP=$myLOCALIP
MY_HOSTNAME=$HOSTNAME
EOF
chown tpot:tpot /data/ews/conf/ews.ip
chmod 760 /data/ews/conf/ews.ip