re-implement distributed feature, without ssh

add sensor compose file
add distributed option to tpot config
housekeeping / cleanup

docker/elk/logstash/Dockerfile

@@ -9,12 +9,9 @@ COPY dist/ /root/dist/
 RUN apt-get update -y && \
     apt-get install -y \
              aria2 \
-	     autossh \
              bash \
              bzip2 \
-	     curl \
-#             openjdk-11-jre \
-             openssh-client && \
+             curl && \
 #
 # Determine arch, get and install packages
     ARCH=$(arch) && \

docker/elk/logstash/dist/entrypoint.sh

@@ -42,25 +42,27 @@ if [ "$myCHECK" == "0" ];
     echo "Cannot reach Listbot, starting Logstash without latest translation maps."
 fi
 
-# Distributed T-Pot installation needs a different pipeline config and autossh tunnel. 
-if [ "$MY_TPOT_TYPE" == "SENSOR" ];
+# Distributed T-Pot installation needs a different pipeline config 
+if [ "$TPOT_TYPE" == "SENSOR" ];
   then
     echo
-    echo "Distributed T-Pot setup, sending T-Pot logs to $MY_HIVE_IP."
+    echo "Distributed T-Pot setup, sending T-Pot logs to $TPOT_HIVE_IP."
     echo
-    echo "T-Pot type: $MY_TPOT_TYPE"
-    echo "Keyfile used: $MY_SENSOR_PRIVATEKEYFILE"
-    echo "Hive username: $MY_HIVE_USERNAME"
-    echo "Hive IP: $MY_HIVE_IP"
+    echo "T-Pot type: $TPOT_TYPE"
+    echo "Hive IP: $TPOT_HIVE_IP"
     echo
-    # Ensure correct file permissions for private keyfile or SSH will ask for password
-    chmod 600 $MY_SENSOR_PRIVATEKEYFILE
+   # Ensure correct file permissions for private keyfile or SSH will ask for password
     cp /usr/share/logstash/config/pipelines_sensor.yml /usr/share/logstash/config/pipelines.yml
-    autossh -f -M 0 -4 -l $MY_HIVE_USERNAME -i $MY_SENSOR_PRIVATEKEYFILE -p 64295 -N -L64305:127.0.0.1:64305 $MY_HIVE_IP -o "ServerAliveInterval 30" -o "ServerAliveCountMax 3" -o "StrictHostKeyChecking=no" -o "UserKnownHostsFile=/dev/null"
 fi
 
-if [ "$MY_TPOT_TYPE" != "SENSOR" ];
+if [ "$TPOT_TYPE" != "SENSOR" ];
   then
+    echo
+    echo "This is a T-Pot STANDARD / HIVE installation."
+    echo
+    echo "T-Pot type: $TPOT_TYPE"
+    echo
+
     # Index Management is happening through ILM, but we need to put T-Pot ILM setting on ES.
     myTPOTILM=$(curl -s -XGET "http://elasticsearch:9200/_ilm/policy/tpot" | grep "Lifecycle policy not found: tpot" -c)
     if [ "$myTPOTILM" == "1" ];

docker/elk/logstash/dist/http_output.conf

@@ -708,7 +708,10 @@ output {
       id => "${MY_HOSTNAME}"
       codec => "json"
       format => "json_batch"
-      url => "http://127.0.0.1:64305"
+      url => "https://${TPOT_HIVE_IP}:64294"
+      cacert => "/data/hive.crt"
+      headers => {
+        "Authorization" => "Basic ${TPOT_HIVE_USER}"
+      }
     }
-
 }

docker/tpotinit/dist/bin/2fa.sh

@@ -1,77 +0,0 @@
-#!/bin/bash
-
-# Make sure script is started as non-root.
-myWHOAMI=$(whoami)
-if [ "$myWHOAMI" = "root" ]
-  then
-    echo "Need to run as non-root ..."
-    echo ""
-    exit
-fi
-
-# set vars, check deps
-myPAM_COCKPIT_FILE="/etc/pam.d/cockpit"
-if ! [ -s "$myPAM_COCKPIT_FILE" ];
-  then
-    echo "### Cockpit PAM module config does not exist. Something went wrong."
-    echo ""
-    exit 1
-fi
-myPAM_COCKPIT_GA="
-
-# google authenticator for two-factor
-auth required pam_google_authenticator.so
-"
-myAUTHENTICATOR=$(which google-authenticator)
-if [ "$myAUTHENTICATOR" == "" ];
-  then
-    echo "### Could not locate google-authenticator, trying to install (if asked provide root password)."
-    echo ""
-    sudo apt-get update
-    sudo apt-get install -y libpam-google-authenticator
-    exec "$1" "$2"    
-    exit 1
-fi
-
-
-# write PAM changes 
-function fuWRITE_PAM_CHANGES {
-  myCHECK=$(cat $myPAM_COCKPIT_FILE | grep -c "google")
-  if ! [ "$myCHECK" == "0" ];
-    then
-      echo "### PAM config already enabled. Skipped."
-      echo ""
-    else
-      echo "### Updating PAM config for Cockpit (if asked provide root password)."
-      echo "$myPAM_COCKPIT_GA" | sudo tee -a $myPAM_COCKPIT_FILE
-      sudo systemctl restart cockpit
-  fi
-}
-
-# create 2fa
-function fuGEN_TOKEN {
-  echo "### Now generating token for Google Authenticator."
-  echo ""
-  google-authenticator -t -d -r 3 -R 30 -w 17
-}
-
-
-# main
-echo "### This script will enable Two Factor Authentication for Cockpit."
-echo ""
-echo "### Please download one of the many authenticator apps from the appstore of your choice."
-echo ""
-while true;
-  do
-    read -p "### Ready to start (y/n)? " myANSWER
-    case $myANSWER in
-      [Yy]* ) echo "### OK. Starting ..."; break;;
-      [Nn]* ) echo "### Exiting."; exit;;
-    esac
-done
-
-fuWRITE_PAM_CHANGES
-fuGEN_TOKEN
-
-echo "Done. Re-run this script by every user who needs Cockpit access."
-echo ""

docker/tpotinit/dist/bin/dps.sh

@@ -1,73 +0,0 @@
-#!/bin/bash
-
-# Run as root only.
-myWHOAMI=$(whoami)
-if [ "$myWHOAMI" != "root" ]
-  then
-    echo "Need to run as root ..."
-    exit
-fi
-
-myPARAM="$1"
-if [[ $myPARAM =~ ^([1-9]|[1-9][0-9]|[1-9][0-9][0-9])$ ]];
-  then
-    watch --color -n $myPARAM "$0"
-    exit
-fi
-
-# Show current status of T-Pot containers
-myCONTAINERS="$(cat /opt/tpot/etc/tpot.yml | grep -v '#' | grep container_name | cut -d: -f2 | sort | tr -d " ")"
-myRED=""
-myGREEN=""
-myBLUE=""
-myWHITE=""
-myMAGENTA=""
-
-# Blackhole Status
-myBLACKHOLE_STATUS=$(ip r | grep "blackhole" -c)
-if [ "$myBLACKHOLE_STATUS" -gt "500" ];
-  then
-    myBLACKHOLE_STATUS="${myGREEN}ENABLED"
-  else
-    myBLACKHOLE_STATUS="${myRED}DISABLED"
-fi
-
-function fuGETTPOT_STATUS {
-# T-Pot Status
-myTPOT_STATUS=$(systemctl status tpot | grep "Active" | awk '{ print $2 }')
-if [ "$myTPOT_STATUS" == "active" ];
-  then
-    echo "${myGREEN}ACTIVE"
-  else
-    echo "${myRED}INACTIVE"
-fi
-}
-
-function fuGETSTATUS {
-grc --colour=on docker ps -f status=running -f status=exited --format "table {{.Names}}\t{{.Status}}\t{{.Ports}}" | grep -v "NAME" | sort
-}
-
-function fuGETSYS {
-printf "[ ========| System |======== ]\n"
-printf "${myBLUE}%+11s ${myWHITE}%-20s\n" "DATE: " "$(date)"
-printf "${myBLUE}%+11s ${myWHITE}%-20s\n" "UPTIME: " "$(grc --colour=on uptime)"
-printf "${myMAGENTA}%+11s %-20s\n" "T-POT: " "$(fuGETTPOT_STATUS)"
-printf "${myMAGENTA}%+11s %-20s\n" "BLACKHOLE: " "$myBLACKHOLE_STATUS${myWHITE}"
-echo
-}
-
-    myDPS=$(fuGETSTATUS)
-    myDPSNAMES=$(echo "$myDPS" | awk '{ print $1 }' | sort)
-    fuGETSYS
-    printf "%-21s %-28s %s\n" "NAME" "STATUS" "PORTS"
-    if [ "$myDPS" != "" ];
-      then
-        echo "$myDPS"
-    fi
-    for i in $myCONTAINERS; do
-      myAVAIL=$(echo "$myDPSNAMES" | grep -o "$i" | uniq | wc -l)      	    
-      if [ "$myAVAIL" = "0" ];
-	then
-	  printf "%-28s %-28s\n" "$myRED$i" "DOWN$myWHITE"
-      fi
-    done

docker/tpotinit/dist/bin/setup_builder.sh

@@ -1,45 +0,0 @@
-#!/bin/bash
-
-# Got root?
-myWHOAMI=$(whoami)
-if [ "$myWHOAMI" != "root" ]
-  then
-    echo "Need to run as root ..."
-    exit
-fi
-
-# Only run with command switch
-if [ "$1" != "-y" ]; then
-  echo "### Setting up docker for Multi Arch Builds."
-  echo "### Use on x64 only!"
-  echo "### Run with -y to install!"
-  echo
-  exit
-fi
-
-# Main
-mkdir -p /root/.docker/cli-plugins/
-cd /root/.docker/cli-plugins/
-wget https://github.com/docker/buildx/releases/download/v0.10.0/buildx-v0.10.0.linux-amd64 -O docker-buildx
-chmod +x docker-buildx
-
-docker buildx ls
-
-# We need to create a new builder as the default one cannot handle multi-arch builds
-# https://docs.docker.com/desktop/multi-arch/
-docker buildx create --name mybuilder
-
-# Set as default
-docker buildx use mybuilder
-
-# We need to install emulators, arm64 should be fine for now
-# https://github.com/tonistiigi/binfmt/
-docker run --privileged --rm tonistiigi/binfmt --install arm64
-
-# Check if everything is setup correctly
-docker buildx inspect --bootstrap
-echo
-echo "### Done."
-echo
-echo "Example: docker buildx build --platform linux/amd64,linux/arm64 -t username/demo:latest --push ."
-echo "Docs: https://docs.docker.com/desktop/multi-arch/"

docker/tpotinit/dist/bin/updateip.sh

@@ -1,7 +1,6 @@
 #!/bin/bash
 # Let's add the first local ip to the /tmp/etc/issue and external ip to ews.ip file
 # If the external IP cannot be detected, the internal IP will be inherited.
-#myCHECKIFSENSOR=$(head -n 1 /opt/tpot/etc/tpot.yml | grep "Sensor" | wc -l)
 myUUID=$(cat /data/uuid)
 myLOCALIP=$(ip address show | awk '/inet .*brd/{split($2,a,"/"); print a[1]; exit}')
 myEXTIP=$(/opt/tpot/bin/myip.sh)
@@ -10,14 +9,6 @@ if [ "$myEXTIP" = "" ];
     myEXTIP=$myLOCALIP
 fi
 
-# Load Blackhole routes if enabled 
-#myBLACKHOLE_FILE1="/etc/blackhole/mass_scanner.txt"
-#myBLACKHOLE_FILE2="/etc/blackhole/mass_scanner_cidr.txt"
-#if [ -f "$myBLACKHOLE_FILE1" ] || [ -f "$myBLACKHOLE_FILE2" ];
-#  then
-#    /opt/tpot/bin/blackhole.sh add
-#fi
-
 myBLACKHOLE_STATUS=$(ip r | grep "blackhole" -c)
 if [ "$myBLACKHOLE_STATUS" -gt "500" ];
   then
@@ -54,16 +45,5 @@ MY_INTIP=$myLOCALIP
 MY_HOSTNAME=$HOSTNAME
 EOF
 
-#if [ -s "/data/elk/logstash/ls_environment" ];
-#  then
-#    source /data/elk/logstash/ls_environment
-#    tee -a /data/tpot/etc/compose/elk_environment << EOF
-#MY_TPOT_TYPE=$MY_TPOT_TYPE
-#MY_SENSOR_PRIVATEKEYFILE=$MY_SENSOR_PRIVATEKEYFILE
-#MY_HIVE_USERNAME=$MY_HIVE_USERNAME
-#MY_HIVE_IP=$MY_HIVE_IP
-#EOF
-#fi
-
 chown tpot:tpot /data/ews/conf/ews.ip
 chmod 770 /data/ews/conf/ews.ip