tweaking

ELK 7.6.0 is not ready for production, however it works if APM is enabled (disabled in config, so image wont build as precaution) Remove SISSDEN from ewsposter, suricata Bump suricata to 5.0.1 Alpine now support suricata incl. enabled JA3 support, move back to Alpine install
2025-07-02 01:27:27 -04:00 · 2020-02-14 15:28:06 +00:00
parent a49d560809
commit f11ad6b523
13 changed files with 161 additions and 211 deletions
--- a/docker/elk/elasticsearch/Dockerfile
+++ b/docker/elk/elasticsearch/Dockerfile
@ -1,7 +1,7 @@
 FROM alpine
 #
 # VARS
-ENV ES_VER=7.5.2 \
+ENV ES_VER=7.6.0 \
    JAVA_HOME=/usr/lib/jvm/java-11-openjdk
 # Include dist
 ADD dist/ /root/dist/
--- a/docker/elk/elasticsearch/dist/elasticsearch.yml
+++ b/docker/elk/elasticsearch/dist/elasticsearch.yml
@ -9,6 +9,7 @@ path:
 http.host: 0.0.0.0
 http.cors.enabled: true
 http.cors.allow-origin: "*"
+indices.query.bool.max_clause_count: 2000
 cluster.initial_master_nodes:
 - "tpotcluster-node-01"
 discovery.zen.ping.unicast.hosts:
--- a/docker/elk/kibana/Dockerfile
+++ b/docker/elk/kibana/Dockerfile
@ -1,7 +1,7 @@
-FROM node:10.15.2-alpine
+FROM node:10.18.0-alpine
 #
 # VARS
-ENV KB_VER=7.5.2
+ENV KB_VER=7.6.0
 # 
 # Include dist
 ADD dist/ /root/dist/
@ -47,6 +47,8 @@ RUN sed -i 's/dl-cdn/dl-2/g' /etc/apk/repositories && \
    echo "xpack.security.enabled: false" >> /usr/share/kibana/config/kibana.yml && \
    echo "xpack.uptime.enabled: false" >> /usr/share/kibana/config/kibana.yml && \
    echo "xpack.siem.enabled: false" >> /usr/share/kibana/config/kibana.yml && \
+    echo "elasticsearch.requestTimeout: 60000" >> /usr/share/kibana/config/kibana.yml && \
+    echo "elasticsearch.shardTimeout: 60000" >> /usr/share/kibana/config/kibana.yml && \
    rm -rf /usr/share/kibana/optimize/bundles/* && \
    /usr/share/kibana/bin/kibana --optimize --allow-root && \
    addgroup -g 2000 kibana && \
--- a/docker/elk/logstash/Dockerfile
+++ b/docker/elk/logstash/Dockerfile
@ -1,7 +1,7 @@
 FROM alpine
 #
 # VARS
-ENV LS_VER=7.5.2
+ENV LS_VER=7.6.0
 # Include dist
 ADD dist/ /root/dist/
 #
@ -36,7 +36,7 @@ RUN sed -i 's/dl-cdn/dl-2/g' /etc/apk/repositories && \
    chmod u+x /usr/bin/update.sh && \
    mkdir -p /etc/logstash/conf.d && \
    cp logstash.conf /etc/logstash/conf.d/ && \
-    cp elasticsearch-template-es7x.json /usr/share/logstash/vendor/bundle/jruby/2.5.0/gems/logstash-output-elasticsearch-10.2.3-java/lib/logstash/outputs/elasticsearch/ && \
+    cp elasticsearch-template-es7x.json /usr/share/logstash/vendor/bundle/jruby/2.5.0/gems/logstash-output-elasticsearch-10.3.0-java/lib/logstash/outputs/elasticsearch/ && \
 #
 # Setup user, groups and configs
    addgroup -g 2000 logstash && \
--- a/docker/ews/Dockerfile
+++ b/docker/ews/Dockerfile
@ -32,7 +32,7 @@ RUN apk -U --no-cache add \
 #
 # Supply configs
    mv /root/dist/ews.cfg /opt/ewsposter/ && \
-    mv /root/dist/*.pem /opt/ewsposter/ && \
+#    mv /root/dist/*.pem /opt/ewsposter/ && \
 #
 # Clean up
    apk del build-base \
--- a/docker/ews/dist/sissden.pem
+++ b/docker/ews/dist/sissden.pem
@ -1,70 +0,0 @@
-----BEGIN CERTIFICATE-----
-MIIGBDCCA+ygAwIBAgIBATANBgkqhkiG9w0BAQsFADCBnTEYMBYGA1UEAwwPU0lT
-U0RFTiBSb290IENBMQswCQYDVQQGEwJQTDERMA8GA1UEBwwIV2Fyc3phd2ExLjAs
-BgNVBAoMJU5hdWtvd2EgaSBBa2FkZW1pY2thIFNpZWMgS29tcHV0ZXJvd2ExEDAO
-BgNVBAsMB1NJU1NERU4xHzAdBgkqhkiG9w0BCQEWEGFkbWluQHNpc3NkZW4uZXUw
-HhcNMTcwNDExMTMxNDE2WhcNMjcwNDA5MTMxNDE2WjCBjTEbMBkGA1UEAwwSU0lT
-U0RFTiBTZXJ2aWNlIENBMQswCQYDVQQGEwJQTDEfMB0GCSqGSIb3DQEJARYQYWRt
-aW5Ac2lzc2Rlbi5ldTEuMCwGA1UECgwlTmF1a293YSBpIEFrYWRlbWlja2EgU2ll
-YyBLb21wdXRlcm93YTEQMA4GA1UECwwHU0lTU0RFTjCCAiIwDQYJKoZIhvcNAQEB
-BQADggIPADCCAgoCggIBAPFLjU6cLQoGz1s73QMPiRxYISCMUh3CXFe52Uim9a60
-nkBDLfjMFW87MNhFCcE2xmxwdPPTz4+f5+DsEV3eZf0y63NxWx+RFV+UpODuEW5n
-tWPFUDxmgKx6iAR/tyeLVNqmgtCnWzSthE0cg71dlil6onWvkMc+Wn5Kv6aXoz4e
-5YVVhNsymhhrR0BntospY8EvtPm70hHAzOty957/zixOQ/MM+4SHRsWXTlKqv0K2
-udWpkUy1Ihs3bpea2KAvn9bBWejFwy7K4q3LyhSyqwpVCYjNi+s+9z4ipSMfvAlT
-FvHrMrODv/Iz/TQOfypYSlpX2gBP9WKLgOQj3wulJnMDQlvG1XNgOAqKfEF52YGF
-eUu21UraRgDAguIIhWxRwgXenmRo8ngWjfk9Q8734PzzXt8cwzbxJWiJLMew1SiW
-I+Kg8uYNGNT4mdBeUMo92S17ZNMXVnkt1TYfxT0A0ZlTCrhXPiWITtsVZXAdqFtl
-j5hASmEcRYNgXEUQHBn13O9IinEmks2PEcqbbbKbs2Je0DS/JvxBkqES51UdsaVQ
-zITKw3deCk0pISG8WDWZ97LEeDCvAKA5l/ooKjDwfS5vWw11mTUCOdhCoF0m8Lao
-TwE1fzzNbSaqMsT6JF/n0ACabfuvF2aqCmWsZC/Hpw8LQQS62zOouCLdcqizL9+z
-AgMBAAGjXTBbMAwGA1UdEwQFMAMBAf8wCwYDVR0PBAQDAgHuMB0GA1UdDgQWBBQ4
-nurxBppBA5PTNvFFU/vhDr/NFzAfBgNVHSMEGDAWgBSDpRyQSgaBD5XvyFOA8YHH
-tbUAbzANBgkqhkiG9w0BAQsFAAOCAgEAIvA2gkYsIVH7FGuoIo9RIxgwy7G/SHNC
-Xllz6hyTx10UwbttJ+o4gdNt8WPuGnkmywFgsjL1//bFw2+fUO5IRvWKSmXzwx9N
-faRJAjQT4JNx2uOW0ctw4USngPrLjXr3UrIQQlJFtZnEyT9u5VJXX8zkhfNJudyJ
-N88YVrPEf6Gh1Q0P+yCX0rDEb3PlP2jsYyXZtcYA5kDQ6Qq7jpLT/zrjJdaPTmzh
-2NUe7jJOBfZxPCoeev7meafY2vVOgqRqMz1+DZRoOgwq+ysczzRaXmd5a2p9Tabc
-L1w5FXKNJQ4apszA0cEScI+4mBIIQ7VFT3GO098GOcYsC2MelRkgONAIyamm66AP
-tvLQAKoiK/xz3sEHN4zaZvN/YVHaSYZEXUP0QHdyL62P62a92aCNyrHpzKURhEDA
-n8cs6icxKrS4xuVa517m53zun0brjrfeltfbO7z1A2TstFYu9BHKzRuhwV9cGRHP
-EDcb7PkfA/08sDHsyfsWtzIysNo3hwCmQ6gtOW5xlrGplFfwSsXmPG4SR3ByW379
-RA5h3zzrO0g7iCvbLclqHoqLTJTMS+6U43qXjnQ7DJ+mcbhRGcMHcZVKqO3QmLm+
-mmkDNzNYfTgY52D5mXJqUK50750mQ8dwMSkD2TufSAPmAPUp90LdQ8u9CIv6gQ+x
-A08hDHJ1cdY=
-----END CERTIFICATE-----
-----BEGIN CERTIFICATE-----
-MIIGHDCCBASgAwIBAgIJAPZqsOOroxaHMA0GCSqGSIb3DQEBCwUAMIGdMRgwFgYD
-VQQDDA9TSVNTREVOIFJvb3QgQ0ExCzAJBgNVBAYTAlBMMREwDwYDVQQHDAhXYXJz
-emF3YTEuMCwGA1UECgwlTmF1a293YSBpIEFrYWRlbWlja2EgU2llYyBLb21wdXRl
-cm93YTEQMA4GA1UECwwHU0lTU0RFTjEfMB0GCSqGSIb3DQEJARYQYWRtaW5Ac2lz
-c2Rlbi5ldTAeFw0xNzA0MTExMzA3NTZaFw0yNzA0MDkxMzA3NTZaMIGdMRgwFgYD
-VQQDDA9TSVNTREVOIFJvb3QgQ0ExCzAJBgNVBAYTAlBMMREwDwYDVQQHDAhXYXJz
-emF3YTEuMCwGA1UECgwlTmF1a293YSBpIEFrYWRlbWlja2EgU2llYyBLb21wdXRl
-cm93YTEQMA4GA1UECwwHU0lTU0RFTjEfMB0GCSqGSIb3DQEJARYQYWRtaW5Ac2lz
-c2Rlbi5ldTCCAiIwDQYJKoZIhvcNAQEBBQADggIPADCCAgoCggIBANKT77EYYEhV
-tJUnfnvQtGttfgqIzKIV2W6nPK9aDsKRTX5BVDHF6P5ZAF1u/52ATwdyTK7+LD66
-Q/nCzyyA2kqTgdruX6VGucpD2DVVSVF6nZhV9PcISNaMXytoG2HHlqrim53E/rVa
-rskColfs7oCxama6lPKZ/rqrJlVjA1Pl5ZtxR0IORjpOyZjSbSzKQwLp/JxHPMCU
-2cVirS7aEu5UGj+Q7Ibg0AEyoAu5tnHBKun4hmIoo7LtKWNEe1TdboxOSboGJ5wd
-UTEmNH+7izZ5FAogTUINjubkf2zZ65xEnN7DT/zFS30vYU1EclqCTp96EKPANogV
-ZeBKntEN6M5azM6Q6+nFI56TV5DWHTIXm85zzeDj5JM7TQlIGTh8A5APHpr0YyUP
-AiIUrixV2lqSDrjewey5qQcWV6WbjMS72OFKh/x7+UJICJhoUw+KwnPmWSq1WAlt
-n7C+W0raSQzt7puI30LUkInKL6iEQebMoYg0eDRI5vsRIpbo+PzflIuk/Vea/D1Y
-twgRc8ujoKI9GpPJyP4yO4nY7BkShLqKJ251lEJZnxq8LiFVi8aN6ZHt//OGEtVs
-6L97cPzqFx7qx8vnyLBFk23lb8pilHK1G0nqxCCjakTruT/JgkLXnZcLu/IDSqd3
-QLjJL0rmU9q6+RTH8A782pcBUNzeLKnlAgMBAAGjXTBbMAwGA1UdEwQFMAMBAf8w
-CwYDVR0PBAQDAgHuMB0GA1UdDgQWBBSDpRyQSgaBD5XvyFOA8YHHtbUAbzAfBgNV
-HSMEGDAWgBSDpRyQSgaBD5XvyFOA8YHHtbUAbzANBgkqhkiG9w0BAQsFAAOCAgEA
-IA0U6znfPykr5PoQlXb/Wr4L5mY/ZtNAJsvJ8jwNMsj3ZlqLOJfnHHoG5LHkb2b/
-xfM1Ee2ojmYBt4VDARqrHLLbup38Ivqt0aEco3Qx/WqbIR4IlvZBF+/qKF/wIUuc
-CuBYNIy12PcLzafT+SJosj1BJ+XiUCj/RsVXIT5CxsdXIABWC+5b3T3/PrAtKk+C
-sVjA/ck1KAHDd+3VUyRjLAAekYWA9C/hek3YwWQ3OvmyHos5gxifqMMDj6bx5qgv
-AuIs4mYJlBlHE19GxRmo2TDwE0eZiUoUdavdRBbl9v7dex+AF2GegmnC1ouYc9kv
-9moNBcuPFXuJMCOCU44aTpgEKRm3QTZTvVcUza251T+4kgT2wlFyzPqQ8hcpih4t
-knlqHhNc9ibL3/qzWr093AgC9uNaNRqmqu1WAu3vs9g3DVb/RSMrUG/V0YS1GgPq
-E+nVJ1AIJoee8YaxHztRfjPsmu1R3pp633lfcRPUKCkz52dZDFRPuQP36DuJzl2M
-itTra0MtDUuRCsuJfVGe1op2wFprswLI0qy7O9N21D4Ab8g0ik+lhmpOf5DpYxmx
-C2Xpe4d/5Xlg3wIYhEs5MnfeEy4lSMA4cxwJs11gVYHba62L7/5lqzpPmHdRYHu3
-Vf0pM/6zniQpy58Pf9+9CNU15I3iWF5K3zmevFArd6s=
-----END CERTIFICATE-----
--- a/docker/ews/docker-compose.yml
+++ b/docker/ews/docker-compose.yml
@ -23,7 +23,7 @@ services:
     - EWS_HPFEEDS_FORMAT=json
    env_file:
     - /opt/tpot/etc/compose/elk_environment
-    image: "dtagdevsec/ewsposter:1903"
+    image: "dtagdevsec/ewsposter:2006"
    volumes:
     - /data:/data
     - /data/ews/conf/ews.ip:/opt/ewsposter/ews.ip
--- a/docker/suricata/Dockerfile
+++ b/docker/suricata/Dockerfile
@ -1,90 +1,17 @@
-FROM alpine
+FROM alpine:latest
 #
 # Include dist
 ADD dist/ /root/dist/
 #
 # Install packages
-#RUN sed -i 's/dl-cdn/dl-2/g' /etc/apk/repositories && \
-RUN    apk -U add \
+RUN apk -U --no-cache add \
                 ca-certificates \
                 curl \
                 file \
-                 geoip \
-                 hiredis \
-                 jansson \
-                 libcap-ng \
-                 libmagic \
-		 libmaxminddb \
-                 libnet \
-                 libnetfilter_queue \
-                 libnfnetlink \
-                 libpcap \
-                 luajit \
-                 lz4-libs \
-                 musl \
-                 nspr \
-                 nss \
-                 pcre \
-                 yaml \
-                 wget \
-                 automake \
-                 autoconf \
-                 build-base \
-                 cargo \
-                 file-dev \
-                 geoip-dev \
-                 hiredis-dev \
-                 jansson-dev \
-                 libtool \
-                 libcap-ng-dev \
-                 luajit-dev \
-		 libmaxminddb-dev \
-                 libpcap-dev \
-                 libnet-dev \
-                 libnetfilter_queue-dev \
-                 libnfnetlink-dev \
-                 lz4-dev \
-                 nss-dev \
-                 nspr-dev \
-                 pcre-dev \
-                 python3 \
-                 rust \
-                 yaml-dev && \
-#
-# We need latest libhtp[-dev] which is only available in community
-    apk -U add --repository=http://dl-cdn.alpinelinux.org/alpine/edge/community \
-               libhtp \
-               libhtp-dev && \
-#
-# Upgrade pip, install suricata-update to meet deps, however we will not be using it 
-# to reduce image (no python needed) and use the update script.
-    pip3 install --no-cache-dir --upgrade pip && \
-    pip3 install --no-cache-dir suricata-update && \
-#
-# Get and build Suricata
-    mkdir -p /opt/builder/ && \
-    wget https://www.openinfosecfoundation.org/download/suricata-5.0.0.tar.gz && \
-    tar xvfz suricata-5.0.0.tar.gz --strip-components=1 -C /opt/builder/ && \
-    rm suricata-5.0.0.tar.gz && \
-    cd /opt/builder && \
-    ./configure \
-	--prefix=/usr \
-	--sysconfdir=/etc \
-	--mandir=/usr/share/man \
-	--localstatedir=/var \
-	--enable-non-bundled-htp \
-	--enable-nfqueue \
-        --enable-rust \
-	--disable-gccmarch-native \
-	--enable-hiredis \
-	--enable-geoip \
-	--enable-gccprotect \
-	--enable-pie \
-	--enable-luajit && \
-    make && \
-    make check && \
-    make install && \
-    make install-full && \
+                 libcap \
+                 wget && \
+    apk -U add --repository http://dl-cdn.alpinelinux.org/alpine/edge/community \
+                 suricata && \
 #
 # Setup user, groups and configs
    addgroup -g 2000 suri && \
@ -92,8 +19,6 @@ RUN    apk -U add \
    chmod 644 /etc/suricata/*.config && \
    cp /root/dist/suricata.yaml /etc/suricata/suricata.yaml && \
    cp /root/dist/*.bpf /etc/suricata/ && \
-    mkdir -p /etc/suricata/rules && \
-    cp /opt/builder/rules/* /etc/suricata/rules/ && \
 #
 # Download the latest EmergingThreats ruleset, replace rulebase and enable all rules
    cp /root/dist/update.sh /usr/bin/ && \
@ -101,32 +26,6 @@ RUN    apk -U add \
    update.sh OPEN && \
 #
 # Clean up
-    apk del --purge \
-                 automake \
-                 autoconf \
-                 build-base \
-                 cargo \
-                 file-dev \
-                 geoip-dev \
-                 hiredis-dev \
-                 jansson-dev \
-                 libtool \
-                 libhtp-dev \
-                 libcap-ng-dev \
-                 luajit-dev \
-                 libpcap-dev \
-		 libmaxminddb-dev \
-                 libnet-dev \
-                 libnetfilter_queue-dev \
-                 libnfnetlink-dev \
-                 lz4-dev \
-                 nss-dev \
-                 nspr-dev \
-                 pcre-dev \
-		 python3 \
-                 rust \
-                 yaml-dev && \
-    rm -rf /opt/builder && \
    rm -rf /root/* && \
    rm -rf /tmp/* && \
    rm -rf /var/cache/apk/*
--- a/docker/suricata/Dockerfile.old
+++ b/docker/suricata/Dockerfile.old
@ -0,0 +1,139 @@
+FROM alpine
+#
+# VARS
+ENV VER=5.0.2
+#
+# Include dist
+ADD dist/ /root/dist/
+#
+# Install packages
+#RUN sed -i 's/dl-cdn/dl-2/g' /etc/apk/repositories && \
+RUN    apk -U add \
+                 ca-certificates \
+                 curl \
+                 file \
+                 geoip \
+                 hiredis \
+                 jansson \
+                 libcap-ng \
+                 libmagic \
+		 libmaxminddb \
+                 libnet \
+                 libnetfilter_queue \
+                 libnfnetlink \
+                 libpcap \
+                 luajit \
+                 lz4-libs \
+                 musl \
+                 nspr \
+                 nss \
+                 pcre \
+                 yaml \
+                 wget \
+                 automake \
+                 autoconf \
+                 build-base \
+                 cargo \
+                 file-dev \
+                 geoip-dev \
+                 hiredis-dev \
+                 jansson-dev \
+                 libtool \
+                 libcap-ng-dev \
+                 luajit-dev \
+		 libmaxminddb-dev \
+                 libpcap-dev \
+                 libnet-dev \
+                 libnetfilter_queue-dev \
+                 libnfnetlink-dev \
+                 lz4-dev \
+                 nss-dev \
+                 nspr-dev \
+                 pcre-dev \
+                 python3 \
+                 rust \
+                 yaml-dev && \
+#
+# We need latest libhtp[-dev] which is only available in community
+    apk -U add --repository=http://dl-cdn.alpinelinux.org/alpine/edge/community \
+               libhtp \
+               libhtp-dev && \
+#
+# Upgrade pip, install suricata-update to meet deps, however we will not be using it 
+# to reduce image (no python needed) and use the update script.
+    pip3 install --no-cache-dir --upgrade pip && \
+    pip3 install --no-cache-dir suricata-update && \
+#
+# Get and build Suricata
+    mkdir -p /opt/builder/ && \
+    wget https://www.openinfosecfoundation.org/download/suricata-$VER.tar.gz && \
+    tar xvfz suricata-$VER.tar.gz --strip-components=1 -C /opt/builder/ && \
+    rm suricata-$VER.tar.gz && \
+    cd /opt/builder && \
+    ./configure \
+	--prefix=/usr \
+	--sysconfdir=/etc \
+	--mandir=/usr/share/man \
+	--localstatedir=/var \
+	--enable-non-bundled-htp \
+	--enable-nfqueue \
+        --enable-rust \
+	--disable-gccmarch-native \
+	--enable-hiredis \
+	--enable-geoip \
+	--enable-gccprotect \
+	--enable-pie \
+	--enable-luajit && \
+    make && \
+    make check && \
+    make install && \
+    make install-full && \
+#
+# Setup user, groups and configs
+    addgroup -g 2000 suri && \
+    adduser -S -H -u 2000 -D -g 2000 suri && \
+    chmod 644 /etc/suricata/*.config && \
+    cp /root/dist/suricata.yaml /etc/suricata/suricata.yaml && \
+    cp /root/dist/*.bpf /etc/suricata/ && \
+    mkdir -p /etc/suricata/rules && \
+    cp /opt/builder/rules/* /etc/suricata/rules/ && \
+#
+# Download the latest EmergingThreats ruleset, replace rulebase and enable all rules
+    cp /root/dist/update.sh /usr/bin/ && \
+    chmod 755 /usr/bin/update.sh && \
+    update.sh OPEN && \
+#
+# Clean up
+    apk del --purge \
+                 automake \
+                 autoconf \
+                 build-base \
+                 cargo \
+                 file-dev \
+                 geoip-dev \
+                 hiredis-dev \
+                 jansson-dev \
+                 libtool \
+                 libhtp-dev \
+                 libcap-ng-dev \
+                 luajit-dev \
+                 libpcap-dev \
+		 libmaxminddb-dev \
+                 libnet-dev \
+                 libnetfilter_queue-dev \
+                 libnfnetlink-dev \
+                 lz4-dev \
+                 nss-dev \
+                 nspr-dev \
+                 pcre-dev \
+		 python3 \
+                 rust \
+                 yaml-dev && \
+    rm -rf /opt/builder && \
+    rm -rf /root/* && \
+    rm -rf /tmp/* && \
+    rm -rf /var/cache/apk/*
+#
+# Start suricata
+STOPSIGNAL SIGINT
+CMD SURICATA_CAPTURE_FILTER=$(update.sh $OINKCODE) && exec suricata -v -F $SURICATA_CAPTURE_FILTER -i $(/sbin/ip address | grep '^2: ' | awk '{ print $2 }' | tr -d [:punct:])
--- a/docker/suricata/dist/capture-filter.bpf
+++ b/docker/suricata/dist/capture-filter.bpf
@ -1,4 +1,3 @@
 not (host sicherheitstacho.eu or community.sicherheitstacho.eu) and
 not (host deb.debian.org) and
-not (host index.docker.io or docker.io) and
-not (host hpfeeds.sissden.eu)
+not (host index.docker.io or docker.io)
--- a/docker/suricata/docker-compose.yml
+++ b/docker/suricata/docker-compose.yml
@ -15,6 +15,6 @@ services:
     - NET_ADMIN
     - SYS_NICE
     - NET_RAW
-    image: "dtagdevsec/suricata:1903"
+    image: "dtagdevsec/suricata:2006"
    volumes:
     - /data/suricata/log:/var/log/suricata