From a507bc5f391645ddd91cb2bd662b12ca3ee556ea Mon Sep 17 00:00:00 2001 From: t3chn0m4g3 Date: Sun, 23 Jan 2022 14:49:07 +0000 Subject: [PATCH] logstash cleanup, prep for multiarch, move to ubuntu log4pot tweaking --- docker/elk/logstash/Dockerfile | 62 ++++++++--------- docker/elk/logstash/Dockerfile.new | 68 ------------------- .../dist/{update.sh => entrypoint.sh} | 2 +- docker/elk/logstash/dist/pipelines.yml | 2 + ...pipelines_pot.yml => pipelines_sensor.yml} | 1 + docker/log4pot/Dockerfile | 3 +- 6 files changed, 33 insertions(+), 105 deletions(-) delete mode 100644 docker/elk/logstash/Dockerfile.new rename docker/elk/logstash/dist/{update.sh => entrypoint.sh} (96%) rename docker/elk/logstash/dist/{pipelines_pot.yml => pipelines_sensor.yml} (67%) diff --git a/docker/elk/logstash/Dockerfile b/docker/elk/logstash/Dockerfile index 3cde66a8..fc758371 100644 --- a/docker/elk/logstash/Dockerfile +++ b/docker/elk/logstash/Dockerfile @@ -1,4 +1,4 @@ -FROM alpine:3.15 +FROM ubuntu:20.04 # # VARS ENV LS_VER=7.16.3 @@ -6,65 +6,59 @@ ENV LS_VER=7.16.3 ADD dist/ /root/dist/ # # Setup env and apt -#RUN sed -i 's/dl-cdn/dl-2/g' /etc/apk/repositories && \ -RUN apk -U --no-cache add \ +RUN apt-get update -y && \ + apt-get dist-upgrade -y && \ + apt-get install -y \ aria2 \ autossh \ bash \ bzip2 \ curl \ - libc6-compat \ - libzmq \ - nss \ - openssh && \ - apk add --no-cache -X http://dl-cdn.alpinelinux.org/alpine/edge/community openjdk16-jre && \ + openssh-client && \ # -# Get and install packages +# Determine arch, get and install packages + ARCH=$(arch) && \ + if [ "$ARCH" = "x86_64" ]; then LS_ARCH="amd64"; fi && \ + if [ "$ARCH" = "aarch64" ]; then LS_ARCH="arm64"; fi && \ + echo "$ARCH" && \ mkdir -p /etc/listbot && \ cd /etc/listbot && \ aria2c -s16 -x 16 https://listbot.sicherheitstacho.eu/cve.yaml.bz2 && \ aria2c -s16 -x 16 https://listbot.sicherheitstacho.eu/iprep.yaml.bz2 && \ bunzip2 *.bz2 && \ cd /root/dist/ && \ - mkdir -p /usr/share/logstash/ && \ - aria2c -s 16 -x 16 https://artifacts.elastic.co/downloads/logstash/logstash-$LS_VER-linux-x86_64.tar.gz && \ - tar xvfz logstash-$LS_VER-linux-x86_64.tar.gz --strip-components=1 -C /usr/share/logstash/ && \ - rm -rf /usr/share/logstash/jdk && \ - # For some reason Alpine 3.14 does not report the -x flag correctly and thus elasticsearch does not find java - sed -i 's/! -x/! -e/g' /usr/share/logstash/bin/logstash.lib.sh && \ - /usr/share/logstash/bin/logstash-plugin install logstash-filter-translate && \ - /usr/share/logstash/bin/logstash-plugin install logstash-input-http && \ - /usr/share/logstash/bin/logstash-plugin install logstash-output-gelf && \ - /usr/share/logstash/bin/logstash-plugin install logstash-output-http && \ - /usr/share/logstash/bin/logstash-plugin install logstash-output-syslog && \ + aria2c -s 16 -x 16 https://artifacts.elastic.co/downloads/logstash/logstash-$LS_VER-$LS_ARCH.deb && \ + dpkg -i logstash-$LS_VER-$LS_ARCH.deb && \ +# /usr/share/logstash/bin/logstash-plugin install logstash-output-gelf logstash-output-syslog && \ # # Add and move files cd /root/dist/ && \ - cp update.sh /usr/bin/ && \ - chmod u+x /usr/bin/update.sh && \ - mkdir -p /etc/logstash/conf.d && \ + cp entrypoint.sh /usr/bin/ && \ + chmod u+x /usr/bin/entrypoint.sh && \ + mkdir -p /etc/logstash/conf.d /usr/share/logstash/config && \ cp logstash.conf /etc/logstash/conf.d/ && \ cp http_input.conf /etc/logstash/conf.d/ && \ cp http_output.conf /etc/logstash/conf.d/ && \ cp pipelines.yml /usr/share/logstash/config/pipelines.yml && \ - cp pipelines_pot.yml /usr/share/logstash/config/pipelines_pot.yml && \ + cp pipelines_sensor.yml /usr/share/logstash/config/pipelines_sensor.yml && \ cp tpot_es_template.json /etc/logstash/ && \ # # Setup user, groups and configs - addgroup -g 2000 logstash && \ - adduser -S -H -s /bin/bash -u 2000 -D -g 2000 logstash && \ - chown -R logstash:logstash /usr/share/logstash && \ - chown -R logstash:logstash /etc/listbot && \ - chmod 755 /usr/bin/update.sh && \ + groupmod -g 2000 logstash && \ + usermod -u 2000 logstash && \ + chown -R logstash:logstash /etc/listbot \ + /var/log/logstash/ \ + /var/lib/logstash \ + /usr/share/logstash/data && \ + chmod 755 /usr/bin/entrypoint.sh && \ # # Clean up - rm -rf /root/* && \ - rm -rf /tmp/* && \ - rm -rf /var/cache/apk/* + apt-get autoremove -y --purge && \ + apt-get clean && rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* /root/dist # # Healthcheck HEALTHCHECK --retries=10 CMD curl -s -XGET 'http://127.0.0.1:9600' # # Start logstash -#USER logstash:logstash -CMD update.sh && exec /usr/share/logstash/bin/logstash --config.reload.automatic --java-execution +USER logstash:logstash +CMD entrypoint.sh && exec /usr/share/logstash/bin/logstash --config.reload.automatic --java-execution diff --git a/docker/elk/logstash/Dockerfile.new b/docker/elk/logstash/Dockerfile.new deleted file mode 100644 index 72cf3fd2..00000000 --- a/docker/elk/logstash/Dockerfile.new +++ /dev/null @@ -1,68 +0,0 @@ -FROM alpine:3.14 -# -# VARS -ENV LS_VER=7.15.1 -# Include dist -ADD dist/ /root/dist/ -# -# Setup env and apt -#RUN sed -i 's/dl-cdn/dl-2/g' /etc/apk/repositories && \ -RUN apk -U --no-cache add \ - aria2 \ - bash \ - bzip2 \ - curl \ - libc6-compat \ - libzmq \ - nss && \ - apk add --no-cache -X http://dl-cdn.alpinelinux.org/alpine/edge/community openjdk16-jre && \ -# -# Get and install packages - mkdir -p /etc/listbot && \ - cd /etc/listbot && \ - aria2c -s16 -x 16 https://listbot.sicherheitstacho.eu/cve.yaml.bz2 && \ - aria2c -s16 -x 16 https://listbot.sicherheitstacho.eu/iprep.yaml.bz2 && \ - bunzip2 *.bz2 && \ - cd /root/dist/ && \ - mkdir -p /usr/share/logstash/ && \ - aria2c -s 16 -x 16 https://artifacts.elastic.co/downloads/logstash/logstash-$LS_VER-linux-x86_64.tar.gz && \ - tar xvfz logstash-$LS_VER-linux-x86_64.tar.gz --strip-components=1 -C /usr/share/logstash/ && \ - rm -rf /usr/share/logstash/jdk && \ - # For some reason Alpine 3.14 does not report the -x flag correctly and thus elasticsearch does not find java - sed -i 's/! -x/! -e/g' /usr/share/logstash/bin/logstash.lib.sh && \ - /usr/share/logstash/bin/logstash-plugin install logstash-filter-translate && \ - /usr/share/logstash/bin/logstash-plugin install logstash-input-http && \ - /usr/share/logstash/bin/logstash-plugin install logstash-output-gelf && \ - /usr/share/logstash/bin/logstash-plugin install logstash-output-http && \ - /usr/share/logstash/bin/logstash-plugin install logstash-output-syslog && \ -# -# Add and move files - cd /root/dist/ && \ - cp update.sh /usr/bin/ && \ - chmod u+x /usr/bin/update.sh && \ - mkdir -p /etc/logstash/conf.d && \ - cp logstash.conf /etc/logstash/conf.d/ && \ - cp http.conf /etc/logstash/conf.d/ && \ - cp pipelines.yml /usr/share/logstash/config/pipelines.yml && \ - cp tpot_es_template.json /etc/logstash/ && \ -# -# Setup user, groups and configs - addgroup -g 2000 logstash && \ - adduser -S -H -s /bin/bash -u 2000 -D -g 2000 logstash && \ - chown -R logstash:logstash /usr/share/logstash && \ - chown -R logstash:logstash /etc/listbot && \ - chmod 755 /usr/bin/update.sh && \ -# -# Clean up - rm -rf /root/* && \ - rm -rf /tmp/* && \ - rm -rf /var/cache/apk/* -# -# Healthcheck -HEALTHCHECK --retries=10 CMD curl -s -XGET 'http://127.0.0.1:9600' -# -# Start logstash -#USER logstash:logstash -#CMD update.sh && exec /usr/share/logstash/bin/logstash -f /etc/logstash/conf.d/logstash.conf --config.reload.automatic --java-execution --log.level debug -#CMD update.sh && exec /usr/share/logstash/bin/logstash -f /etc/logstash/conf.d/logstash.conf --config.reload.automatic --java-execution -CMD update.sh && exec /usr/share/logstash/bin/logstash --config.reload.automatic --java-execution diff --git a/docker/elk/logstash/dist/update.sh b/docker/elk/logstash/dist/entrypoint.sh similarity index 96% rename from docker/elk/logstash/dist/update.sh rename to docker/elk/logstash/dist/entrypoint.sh index 19f2c155..aaa962e7 100644 --- a/docker/elk/logstash/dist/update.sh +++ b/docker/elk/logstash/dist/entrypoint.sh @@ -46,7 +46,7 @@ if [ "$MY_TPOT_TYPE" == "SENSOR" ]; echo "Hive username: $MY_HIVE_USERNAME" echo "Hive IP: $MY_HIVE_IP" echo - cp /usr/share/logstash/config/pipelines_pot.yml /usr/share/logstash/config/pipelines.yml + cp /usr/share/logstash/config/pipelines_sensor.yml /usr/share/logstash/config/pipelines.yml autossh -f -M 0 -4 -l $MY_HIVE_USERNAME -i $MY_SENSOR_PRIVATEKEYFILE -p 64295 -N -L64305:127.0.0.1:64305 $MY_HIVE_IP -o "ServerAliveInterval 30" -o "ServerAliveCountMax 3" -o "StrictHostKeyChecking=no" -o "UserKnownHostsFile=/dev/null" exit 0 fi diff --git a/docker/elk/logstash/dist/pipelines.yml b/docker/elk/logstash/dist/pipelines.yml index 41883e78..e7d53bfe 100644 --- a/docker/elk/logstash/dist/pipelines.yml +++ b/docker/elk/logstash/dist/pipelines.yml @@ -1,4 +1,6 @@ - pipeline.id: logstash path.config: "/etc/logstash/conf.d/logstash.conf" + pipeline.ecs_compatibility: disabled - pipeline.id: http_input path.config: "/etc/logstash/conf.d/http_input.conf" + pipeline.ecs_compatibility: disabled diff --git a/docker/elk/logstash/dist/pipelines_pot.yml b/docker/elk/logstash/dist/pipelines_sensor.yml similarity index 67% rename from docker/elk/logstash/dist/pipelines_pot.yml rename to docker/elk/logstash/dist/pipelines_sensor.yml index cf6201a1..a3bf7619 100644 --- a/docker/elk/logstash/dist/pipelines_pot.yml +++ b/docker/elk/logstash/dist/pipelines_sensor.yml @@ -1,2 +1,3 @@ - pipeline.id: http_output path.config: "/etc/logstash/conf.d/http_output.conf" + pipeline.ecs_compatibility: disabled diff --git a/docker/log4pot/Dockerfile b/docker/log4pot/Dockerfile index 408b15db..46efdac4 100644 --- a/docker/log4pot/Dockerfile +++ b/docker/log4pot/Dockerfile @@ -2,8 +2,7 @@ FROM ubuntu:20.04 ENV DEBIAN_FRONTEND noninteractive # # Install packages -RUN apt-get update && \ - apt-get update -y && \ +RUN apt-get update -y && \ apt-get dist-upgrade -y && \ apt-get install -y \ build-essential \