mirror of
https://github.com/telekom-security/tpotce.git
synced 2025-07-02 01:27:27 -04:00
Finish work on new builder, tweaking
This commit is contained in:
t3chn0m4g3
@ -21,7 +21,3 @@ TPOT_VERSION=testing
|
||||
# Most docker features are available on linux
|
||||
TPOT_AMD64=linux/amd64
|
||||
TPOT_ARM64=linux/arm64
|
||||
|
||||
# Proxy
|
||||
# Set Proxy (i.e. "http://proxy:3128") to improve speed (while caching)
|
||||
PROXY="http://proxy:3128"
|
||||
|
||||
@ -1,13 +1,16 @@
|
||||
#!/bin/bash
|
||||
#!/usr/bin/env bash
|
||||
|
||||
# ANSI color codes for green (OK) and red (FAIL)
|
||||
GREEN='\033[0;32m'
|
||||
RED='\033[0;31m'
|
||||
NC='\033[0m' # No Color
|
||||
|
||||
# Default flags
|
||||
# Default settings
|
||||
PUSH_IMAGES=false
|
||||
NO_CACHE=false
|
||||
PARALLELBUILDS=8
|
||||
UPLOAD_BANDWIDTH=40mbit # Set this to max 90% of available upload bandwidth
|
||||
INTERFACE=$(/sbin/ip address show | /usr/bin/awk '/inet.*brd/{ print $NF; exit }')
|
||||
|
||||
# Help message
|
||||
usage() {
|
||||
@ -37,6 +40,45 @@ while getopts ":pnh" opt; do
|
||||
esac
|
||||
done
|
||||
|
||||
# Function to apply upload bandwidth limit using tc
|
||||
apply_bandwidth_limit() {
|
||||
echo -n "Applying upload bandwidth limit of $UPLOAD_BANDWIDTH on interface $INTERFACE..."
|
||||
if sudo tc qdisc add dev $INTERFACE root tbf rate $UPLOAD_BANDWIDTH burst 32kbit latency 400ms >/dev/null 2>&1; then
|
||||
echo -e " [${GREEN}OK${NC}]"
|
||||
else
|
||||
echo -e " [${RED}FAIL${NC}]"
|
||||
remove_bandwidth_limit
|
||||
|
||||
# Try to reapply the limit
|
||||
echo -n "Reapplying upload bandwidth limit of $UPLOAD_BANDWIDTH on interface $INTERFACE..."
|
||||
if sudo tc qdisc add dev $INTERFACE root tbf rate $UPLOAD_BANDWIDTH burst 32kbit latency 400ms >/dev/null 2>&1; then
|
||||
echo -e " [${GREEN}OK${NC}]"
|
||||
else
|
||||
echo -e " [${RED}FAIL${NC}]"
|
||||
echo "Failed to apply bandwidth limit on $INTERFACE. Exiting."
|
||||
echo
|
||||
exit 1
|
||||
fi
|
||||
fi
|
||||
}
|
||||
|
||||
# Function to check if the bandwidth limit is set
|
||||
is_bandwidth_limit_set() {
|
||||
sudo tc qdisc show dev $INTERFACE | grep -q 'tbf'
|
||||
}
|
||||
|
||||
# Function to remove the bandwidth limit using tc if it is set
|
||||
remove_bandwidth_limit() {
|
||||
if is_bandwidth_limit_set; then
|
||||
echo -n "Removing upload bandwidth limit on interface $INTERFACE..."
|
||||
if sudo tc qdisc del dev $INTERFACE root; then
|
||||
echo -e " [${GREEN}OK${NC}]"
|
||||
else
|
||||
echo -e " [${RED}FAIL${NC}]"
|
||||
fi
|
||||
fi
|
||||
}
|
||||
|
||||
echo "###########################"
|
||||
echo "# T-Pot Image Builder"
|
||||
echo "###########################"
|
||||
@ -86,6 +128,24 @@ else
|
||||
echo -e " [${RED}FAIL${NC}]"
|
||||
fi
|
||||
|
||||
# Apply bandwidth limit only if pushing images
|
||||
if $PUSH_IMAGES; then
|
||||
echo
|
||||
echo "########################################"
|
||||
echo "# Setting Upload Bandwidth limit ..."
|
||||
echo "########################################"
|
||||
echo
|
||||
apply_bandwidth_limit
|
||||
fi
|
||||
|
||||
# Trap to ensure bandwidth limit is removed on script error, exit
|
||||
trap_cleanup() {
|
||||
if is_bandwidth_limit_set; then
|
||||
remove_bandwidth_limit
|
||||
fi
|
||||
}
|
||||
trap trap_cleanup INT ERR EXIT
|
||||
|
||||
echo
|
||||
echo "################################"
|
||||
echo "# Now building images ..."
|
||||
@ -95,11 +155,10 @@ echo
|
||||
mkdir -p log
|
||||
|
||||
# List of services to build
|
||||
#services=$(docker compose config --services)
|
||||
services="tpotinit beelzebub nginx p0f"
|
||||
services=$(docker compose config --services | sort)
|
||||
|
||||
# Loop through each service
|
||||
echo $services | tr ' ' '\n' | xargs -I {} -P 3 bash -c '
|
||||
# Loop through each service to build
|
||||
echo $services | tr ' ' '\n' | xargs -I {} -P $PARALLELBUILDS bash -c '
|
||||
echo "Building image: {}" && \
|
||||
build_cmd="docker compose build {}" && \
|
||||
if '$PUSH_IMAGES'; then \
|
||||
@ -109,10 +168,20 @@ echo $services | tr ' ' '\n' | xargs -I {} -P 3 bash -c '
|
||||
build_cmd="$build_cmd --no-cache"; \
|
||||
fi && \
|
||||
eval "$build_cmd 2>&1 > log/{}.log" && \
|
||||
echo -e "Service {}: ['$GREEN'OK'$NC']" || \
|
||||
echo -e "Service {}: ['$RED'FAIL'$NC']"
|
||||
echo -e "Image {}: ['$GREEN'OK'$NC']" || \
|
||||
echo -e "Image {}: ['$RED'FAIL'$NC']"
|
||||
'
|
||||
|
||||
# Remove bandwidth limit if it was applied
|
||||
if is_bandwidth_limit_set; then
|
||||
echo
|
||||
echo "########################################"
|
||||
echo "# Removiong Upload Bandwidth limit ..."
|
||||
echo "########################################"
|
||||
echo
|
||||
remove_bandwidth_limit
|
||||
fi
|
||||
|
||||
echo
|
||||
echo "#######################################################"
|
||||
echo "# Done."
|
||||
|
||||
@ -7,8 +7,6 @@
|
||||
|
||||
# Common build config
|
||||
x-common-build: &common-build
|
||||
args:
|
||||
PROXY: ${PROXY}
|
||||
dockerfile: ./Dockerfile
|
||||
platforms:
|
||||
- ${TPOT_AMD64}
|
||||
|
||||
@ -1,57 +1,99 @@
|
||||
#!/usr/bin/env bash
|
||||
|
||||
# Got root?
|
||||
myWHOAMI=$(whoami)
|
||||
if [ "$myWHOAMI" != "root" ]
|
||||
then
|
||||
echo "Need to run as root ..."
|
||||
exit
|
||||
# ANSI color codes for green (OK) and red (FAIL)
|
||||
BLUE='\033[0;34m'
|
||||
GREEN='\033[0;32m'
|
||||
RED='\033[0;31m'
|
||||
NC='\033[0m' # No Color
|
||||
|
||||
# Check if the user is in the docker group
|
||||
if ! groups $(whoami) | grep &>/dev/null '\bdocker\b'; then
|
||||
echo -e "${RED}You need to be in the docker group to run this script without root privileges.${NC}"
|
||||
echo "Please run the following command to add yourself to the docker group:"
|
||||
echo " sudo usermod -aG docker $(whoami)"
|
||||
echo "Then log out and log back in or run the script with sudo."
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# Only run with command switch
|
||||
# Command-line switch check
|
||||
if [ "$1" != "-y" ]; then
|
||||
echo "### Setting up docker for Multi Arch Builds."
|
||||
echo "### Requires Docker packages from https://get.docker.com/"
|
||||
echo "### Use on x64 only!"
|
||||
echo "### Run with -y if you fit the requirements!"
|
||||
echo
|
||||
exit
|
||||
echo "### Setting up Docker for Multi-Arch Builds."
|
||||
echo "### Requires Docker packages from https://get.docker.com/"
|
||||
echo "### Use on x64 only!"
|
||||
echo "### Run with -y if you fit the requirements!"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
# We need to create a new builder as the default one cannot handle multi-arch builds
|
||||
# https://docs.docker.com/desktop/multi-arch/
|
||||
docker buildx create --name mybuilder
|
||||
# Check if the mybuilder exists and is running
|
||||
echo -n "Checking if buildx builder 'mybuilder' exists and is running..."
|
||||
if ! docker buildx inspect mybuilder --bootstrap >/dev/null 2>&1; then
|
||||
echo
|
||||
echo -n " Creating and starting buildx builder 'mybuilder'..."
|
||||
if docker buildx create --name mybuilder --driver docker-container --use >/dev/null 2>&1 && \
|
||||
docker buildx inspect mybuilder --bootstrap >/dev/null 2>&1; then
|
||||
echo -e " [${GREEN}OK${NC}]"
|
||||
else
|
||||
echo -e " [${RED}FAIL${NC}]"
|
||||
exit 1
|
||||
fi
|
||||
else
|
||||
echo -e " [${GREEN}OK${NC}]"
|
||||
fi
|
||||
|
||||
# Set as default
|
||||
docker buildx use mybuilder
|
||||
# Ensure QEMU is set up for cross-platform builds
|
||||
echo -n "Ensuring QEMU is configured for cross-platform builds..."
|
||||
if docker run --rm --privileged multiarch/qemu-user-static --reset -p yes >/dev/null 2>&1; then
|
||||
echo -e " [${GREEN}OK${NC}]"
|
||||
else
|
||||
echo -e " [${RED}FAIL${NC}]"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# We need to install emulators, arm64 should be fine for now
|
||||
# https://github.com/tonistiigi/binfmt/
|
||||
docker run --privileged --rm tonistiigi/binfmt --install arm64
|
||||
# Ensure arm64 and amd64 platforms are active
|
||||
echo -n "Ensuring 'mybuilder' supports linux/arm64 and linux/amd64..."
|
||||
active_platforms=$(docker buildx inspect mybuilder --bootstrap | grep -oP '(?<=Platforms: ).*')
|
||||
|
||||
# Check if everything is setup correctly
|
||||
docker buildx inspect --bootstrap
|
||||
echo
|
||||
echo "### Done."
|
||||
echo
|
||||
echo "Example (manual build): docker buildx build --platform linux/amd64,linux/arm64 -t username/demo:latest --push ."
|
||||
echo "Docs: https://docs.docker.com/desktop/multi-arch/"
|
||||
echo
|
||||
echo "Example (build release): docker compose build"
|
||||
echo
|
||||
echo "Example (build and push release): docker compose build --push"
|
||||
echo
|
||||
echo "Example (build single image): docker compose build tpotinit"
|
||||
echo
|
||||
echo "Example (build and push single image): docker compose build tpotinit --push"
|
||||
echo
|
||||
echo "Resolve problems running buildx:"
|
||||
echo "docker buildx create --use --name mybuilder"
|
||||
echo "docker buildx inspect mybuilder --bootstrap"
|
||||
echo "docker login -u <username>"
|
||||
echo "docker login ghcr.io - <username>"
|
||||
echo
|
||||
echo "Resolve segmentation faults when building arm64 images in qemu on amd64:"
|
||||
echo "docker run --rm --privileged multiarch/qemu-user-static --reset -p yes"
|
||||
echo
|
||||
if [[ "$active_platforms" == *"linux/arm64"* && "$active_platforms" == *"linux/amd64"* ]]; then
|
||||
echo -e " [${GREEN}OK${NC}]"
|
||||
else
|
||||
echo
|
||||
echo -n " Enabling platforms linux/arm64 and linux/amd64..."
|
||||
if docker buildx create --name mybuilder --driver docker-container --use --platform linux/amd64,linux/arm64 >/dev/null 2>&1 && \
|
||||
docker buildx inspect mybuilder --bootstrap >/dev/null 2>&1; then
|
||||
echo -e " [${GREEN}OK${NC}]"
|
||||
else
|
||||
echo -e " [${RED}FAIL${NC}]"
|
||||
exit 1
|
||||
fi
|
||||
fi
|
||||
|
||||
echo
|
||||
echo -e "${BLUE}### Done.${NC}"
|
||||
echo
|
||||
echo -e "${BLUE}Examples:${NC}"
|
||||
echo -e " ${BLUE}Manual multi-arch build:${NC}"
|
||||
echo " docker buildx build --platform linux/amd64,linux/arm64 -t username/demo:latest --push ."
|
||||
echo
|
||||
echo -e " ${BLUE}Documentation:${NC} https://docs.docker.com/desktop/multi-arch/"
|
||||
echo
|
||||
echo -e " ${BLUE}Build release with Docker Compose:${NC}"
|
||||
echo " docker compose build"
|
||||
echo
|
||||
echo -e " ${BLUE}Build and push release with Docker Compose:${NC}"
|
||||
echo " docker compose build --push"
|
||||
echo
|
||||
echo -e " ${BLUE}Build a single image with Docker Compose:${NC}"
|
||||
echo " docker compose build tpotinit"
|
||||
echo
|
||||
echo -e " ${BLUE}Build and push a single image with Docker Compose:${NC}"
|
||||
echo " docker compose build tpotinit --push"
|
||||
echo
|
||||
echo -e "${BLUE}Resolve buildx issues:${NC}"
|
||||
echo " docker buildx create --use --name mybuilder"
|
||||
echo " docker buildx inspect mybuilder --bootstrap"
|
||||
echo " docker login -u <username>"
|
||||
echo " docker login ghcr.io -u <username>"
|
||||
echo
|
||||
echo -e "${BLUE}Fix segmentation faults when building arm64 images:${NC}"
|
||||
echo " docker run --rm --privileged multiarch/qemu-user-static --reset -p yes"
|
||||
echo
|
||||
|
||||
Reference in New Issue
Block a user