From 14970bdf39d9e2af8bce60181af3efea79eccd8a Mon Sep 17 00:00:00 2001 From: Seth Grover Date: Mon, 4 Dec 2023 11:28:03 -0700 Subject: [PATCH 1/5] issue with K8s with zeek custom/intel --- kubernetes/10-zeek.yml | 21 ++++++++++----------- kubernetes/21-zeek-live.yml | 21 ++++++++++----------- scripts/control.py | 1 + scripts/malcolm_kubernetes.py | 12 ++++++++++++ 4 files changed, 33 insertions(+), 22 deletions(-) diff --git a/kubernetes/10-zeek.yml b/kubernetes/10-zeek.yml index 809b3171c..ab3b403f1 100644 --- a/kubernetes/10-zeek.yml +++ b/kubernetes/10-zeek.yml @@ -63,12 +63,10 @@ spec: - mountPath: "/zeek/upload" name: zeek-offline-zeek-volume subPath: "upload" - - mountPath: "/opt/zeek/share/zeek/site/intel" - name: zeek-offline-zeek-intel-and-config - subPath: "zeek/intel" - mountPath: "/opt/zeek/share/zeek/site/custom" - name: zeek-offline-zeek-intel-and-config - subPath: "zeek/custom" + name: zeek-offline-custom-volume + - mountPath: "/opt/zeek/share/zeek/site/intel/configmap" + name: zeek-offline-intel-volume initContainers: - name: zeek-offline-dirinit-container image: ghcr.io/idaholab/malcolm/dirinit:23.12.0 @@ -80,10 +78,8 @@ spec: name: process-env env: - name: PUSER_MKDIR - value: "/data/config:zeek/intel/MISP,zeek/intel/STIX,zeek/custom;/data/pcap:processed;/data/zeek-logs:current,extract_files/preserved,extract_files/quarantine,live,processed,upload" + value: "/data/pcap:processed;/data/zeek-logs:current,extract_files/preserved,extract_files/quarantine,live,processed,upload" volumeMounts: - - name: zeek-offline-zeek-intel-and-config - mountPath: "/data/config" - name: zeek-offline-pcap-volume mountPath: "/data/pcap" - name: zeek-offline-zeek-volume @@ -98,6 +94,9 @@ spec: - name: zeek-offline-zeek-volume persistentVolumeClaim: claimName: zeek-claim - - name: zeek-offline-zeek-intel-and-config - persistentVolumeClaim: - claimName: config-claim + - name: zeek-offline-custom-volume + configMap: + name: zeek-custom + - name: zeek-offline-intel-volume + configMap: + name: zeek-intel diff --git a/kubernetes/21-zeek-live.yml b/kubernetes/21-zeek-live.yml index a31997b6e..f97e71a62 100644 --- a/kubernetes/21-zeek-live.yml +++ b/kubernetes/21-zeek-live.yml @@ -55,12 +55,10 @@ spec: - mountPath: "/zeek/upload" name: zeek-live-zeek-volume subPath: "upload" - - mountPath: "/opt/zeek/share/zeek/site/intel" - name: zeek-live-zeek-intel-and-config - subPath: "zeek/intel" - mountPath: "/opt/zeek/share/zeek/site/custom" - name: zeek-live-zeek-intel-and-config - subPath: "zeek/custom" + name: zeek-live-custom-volume + - mountPath: "/opt/zeek/share/zeek/site/intel/configmap" + name: zeek-live-intel-volume initContainers: - name: zeek-live-dirinit-container image: ghcr.io/idaholab/malcolm/dirinit:23.12.0 @@ -72,10 +70,8 @@ spec: name: process-env env: - name: PUSER_MKDIR - value: "/data/config:zeek/intel/MISP,zeek/intel/STIX,zeek/custom;/data/zeek-logs:current,extract_files/preserved,extract_files/quarantine,live,processed,upload" + value: "/data/zeek-logs:current,extract_files/preserved,extract_files/quarantine,live,processed,upload" volumeMounts: - - name: zeek-live-zeek-intel-and-config - mountPath: "/data/config" - name: zeek-live-zeek-volume mountPath: "/data/zeek-logs" volumes: @@ -85,6 +81,9 @@ spec: - name: zeek-live-zeek-volume persistentVolumeClaim: claimName: zeek-claim - - name: zeek-live-zeek-intel-and-config - persistentVolumeClaim: - claimName: config-claim + - name: zeek-live-custom-volume + configMap: + name: zeek-custom + - name: zeek-live-intel-volume + configMap: + name: zeek-intel \ No newline at end of file diff --git a/scripts/control.py b/scripts/control.py index f74d9e914..c6b39d6c9 100755 --- a/scripts/control.py +++ b/scripts/control.py @@ -1015,6 +1015,7 @@ def start(): ), BoundPath("zeek", "/zeek/extract_files", False, None, None), BoundPath("zeek", "/zeek/upload", False, None, None), + BoundPath("zeek", "/opt/zeek/share/zeek/site/custom", False, None, None), BoundPath("zeek", "/opt/zeek/share/zeek/site/intel", False, ["MISP", "STIX"], None), BoundPath("zeek-live", "/zeek/live", False, ["spool"], None), BoundPath("filebeat", "/zeek", False, ["processed", "current", "live", "extract_files", "upload"], None), diff --git a/scripts/malcolm_kubernetes.py b/scripts/malcolm_kubernetes.py index 6945a50f9..d83c5e178 100644 --- a/scripts/malcolm_kubernetes.py +++ b/scripts/malcolm_kubernetes.py @@ -159,6 +159,18 @@ 'path': os.path.join(MalcolmPath, os.path.join('htadmin', 'metadata')), }, ], + 'zeek-custom': [ + { + 'secret': False, + 'path': os.path.join(MalcolmPath, os.path.join('zeek', 'custom')), + }, + ], + 'zeek-intel': [ + { + 'secret': False, + 'path': os.path.join(MalcolmPath, os.path.join('zeek', 'intel')), + }, + ], } # the PersistentVolumes themselves aren't used directly, From 7cc9e105a69cb32351cc5be3a5d1b2c958263646 Mon Sep 17 00:00:00 2001 From: Seth Grover Date: Mon, 4 Dec 2023 11:56:35 -0700 Subject: [PATCH 2/5] fix issue loading zeek intel on startup --- shared/bin/zeek_intel_setup.sh | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/shared/bin/zeek_intel_setup.sh b/shared/bin/zeek_intel_setup.sh index 186b95538..eb2b12a6a 100755 --- a/shared/bin/zeek_intel_setup.sh +++ b/shared/bin/zeek_intel_setup.sh @@ -48,7 +48,8 @@ EOF THREAT_JSON_FILES=() # process subdirectories under INTEL_DIR - for DIR in $(find . -mindepth 1 -maxdepth 1 -type d 2>/dev/null); do + for DIR in $(find . -mindepth 1 -maxdepth 1 -type d 2>/dev/null | grep -v -P "$(echo "${CONFIG_MAP_DIR:-configmap;secretmap}" | sed 's/\(.*\)/^.\/(\1)$/' | tr ';' '|')"); do + if [[ "${DIR}" == "./STIX" ]]; then # this directory contains STIX JSON files we'll need to convert to zeek intel files then load while IFS= read -r line; do @@ -73,7 +74,7 @@ EOF done # process STIX and MISP inputs by converting them to Zeek intel format - if ( (( ${#THREAT_JSON_FILES[@]} )) || [[ -r ./STIX/.stix_input.txt ]] || [[ -r ./STIX/.misp_input.txt ]] ) && [[ -x "${THREAT_FEED_TO_ZEEK_SCRIPT}" ]]; then + if ( (( ${#THREAT_JSON_FILES[@]} )) || [[ -r ./STIX/.stix_input.txt ]] || [[ -r ./MISP/.misp_input.txt ]] ) && [[ -x "${THREAT_FEED_TO_ZEEK_SCRIPT}" ]]; then "${THREAT_FEED_TO_ZEEK_SCRIPT}" \ --since "${ZEEK_INTEL_FEED_SINCE}" \ --threads ${ZEEK_INTEL_REFRESH_THREADS} \ From acbe2d37fb30f6f4b3c12ebb7bd000cddc4c9ed2 Mon Sep 17 00:00:00 2001 From: Seth Grover Date: Mon, 4 Dec 2023 12:23:09 -0700 Subject: [PATCH 3/5] fix issue loading zeek intel on startup --- kubernetes/10-zeek.yml | 16 ++++++++++++---- kubernetes/21-zeek-live.yml | 16 ++++++++++++---- scripts/malcolm_kubernetes.py | 2 +- shared/bin/zeek_intel_setup.sh | 7 +++++++ 4 files changed, 32 insertions(+), 9 deletions(-) diff --git a/kubernetes/10-zeek.yml b/kubernetes/10-zeek.yml index ab3b403f1..3e5c25046 100644 --- a/kubernetes/10-zeek.yml +++ b/kubernetes/10-zeek.yml @@ -65,8 +65,11 @@ spec: subPath: "upload" - mountPath: "/opt/zeek/share/zeek/site/custom" name: zeek-offline-custom-volume - - mountPath: "/opt/zeek/share/zeek/site/intel/configmap" + - mountPath: "/opt/zeek/share/zeek/site/intel-preseed" + name: zeek-offline-intel-preseed-volume + - mountPath: "/opt/zeek/share/zeek/site/intel" name: zeek-offline-intel-volume + subPath: "zeek/intel" initContainers: - name: zeek-offline-dirinit-container image: ghcr.io/idaholab/malcolm/dirinit:23.12.0 @@ -78,8 +81,10 @@ spec: name: process-env env: - name: PUSER_MKDIR - value: "/data/pcap:processed;/data/zeek-logs:current,extract_files/preserved,extract_files/quarantine,live,processed,upload" + value: "/data/config:zeek/intel/MISP,zeek/intel/STIX;/data/pcap:processed;/data/zeek-logs:current,extract_files/preserved,extract_files/quarantine,live,processed,upload" volumeMounts: + - name: zeek-offline-intel-volume + mountPath: "/data/config" - name: zeek-offline-pcap-volume mountPath: "/data/pcap" - name: zeek-offline-zeek-volume @@ -97,6 +102,9 @@ spec: - name: zeek-offline-custom-volume configMap: name: zeek-custom - - name: zeek-offline-intel-volume + - name: zeek-offline-intel-preseed-volume configMap: - name: zeek-intel + name: zeek-intel-preseed + - name: zeek-offline-intel-volume + persistentVolumeClaim: + claimName: config-claim diff --git a/kubernetes/21-zeek-live.yml b/kubernetes/21-zeek-live.yml index f97e71a62..725a21b10 100644 --- a/kubernetes/21-zeek-live.yml +++ b/kubernetes/21-zeek-live.yml @@ -57,8 +57,11 @@ spec: subPath: "upload" - mountPath: "/opt/zeek/share/zeek/site/custom" name: zeek-live-custom-volume - - mountPath: "/opt/zeek/share/zeek/site/intel/configmap" + - mountPath: "/opt/zeek/share/zeek/site/intel-preseed" + name: zeek-live-intel-preseed-volume + - mountPath: "/opt/zeek/share/zeek/site/intel" name: zeek-live-intel-volume + subPath: "zeek/intel" initContainers: - name: zeek-live-dirinit-container image: ghcr.io/idaholab/malcolm/dirinit:23.12.0 @@ -70,8 +73,10 @@ spec: name: process-env env: - name: PUSER_MKDIR - value: "/data/zeek-logs:current,extract_files/preserved,extract_files/quarantine,live,processed,upload" + value: "/data/config:zeek/intel/MISP,zeek/intel/STIX;/data/zeek-logs:current,extract_files/preserved,extract_files/quarantine,live,processed,upload" volumeMounts: + - name: zeek-live-intel-volume + mountPath: "/data/config" - name: zeek-live-zeek-volume mountPath: "/data/zeek-logs" volumes: @@ -84,6 +89,9 @@ spec: - name: zeek-live-custom-volume configMap: name: zeek-custom - - name: zeek-live-intel-volume + - name: zeek-live-intel-preseed-volume configMap: - name: zeek-intel \ No newline at end of file + name: zeek-intel-preseed + - name: zeek-live-intel-volume + persistentVolumeClaim: + claimName: config-claim diff --git a/scripts/malcolm_kubernetes.py b/scripts/malcolm_kubernetes.py index d83c5e178..4bb7bc47c 100644 --- a/scripts/malcolm_kubernetes.py +++ b/scripts/malcolm_kubernetes.py @@ -165,7 +165,7 @@ 'path': os.path.join(MalcolmPath, os.path.join('zeek', 'custom')), }, ], - 'zeek-intel': [ + 'zeek-intel-preseed': [ { 'secret': False, 'path': os.path.join(MalcolmPath, os.path.join('zeek', 'intel')), diff --git a/shared/bin/zeek_intel_setup.sh b/shared/bin/zeek_intel_setup.sh index eb2b12a6a..7a868d828 100755 --- a/shared/bin/zeek_intel_setup.sh +++ b/shared/bin/zeek_intel_setup.sh @@ -17,6 +17,7 @@ ZEEK_INTEL_ITEM_EXPIRATION=${ZEEK_INTEL_ITEM_EXPIRATION:-"-1min"} ZEEK_INTEL_FEED_SINCE=${ZEEK_INTEL_FEED_SINCE:-""} ZEEK_INTEL_REFRESH_THREADS=${ZEEK_INTEL_REFRESH_THREADS:-"2"} INTEL_DIR=${INTEL_DIR:-"${ZEEK_DIR}/share/zeek/site/intel"} +INTEL_PRESEED_DIR=${INTEL_PRESEED_DIR:-"${ZEEK_DIR}/share/zeek/site/intel-preseed"} THREAT_FEED_TO_ZEEK_SCRIPT=${THREAT_FEED_TO_ZEEK_SCRIPT:-"${ZEEK_DIR}/bin/zeek_intel_from_threat_feed.py"} LOCK_DIR="${INTEL_DIR}/lock" @@ -29,6 +30,12 @@ mkdir -p -- "$(dirname "$LOCK_DIR")" if mkdir -- "$LOCK_DIR" 2>/dev/null; then trap finish EXIT + # if we have a directory to seed the intel config for the first time, start from a blank slate with just its contents + if [[ -d "${INTEL_DIR}" ]] && [[ -d "${INTEL_PRESEED_DIR}" ]]; then + rsync -av --delete "${INTEL_PRESEED_DIR}"/ "${INTEL_DIR}"/ + mkdir -p "${INTEL_DIR}"/MISP "${INTEL_DIR}"/STIX || true + fi + # create directive to @load every subdirectory in /opt/zeek/share/zeek/site/intel if [[ -d "${INTEL_DIR}" ]] && (( $(find "${INTEL_DIR}" -mindepth 1 -maxdepth 1 -type d 2>/dev/null | wc -l) > 0 )); then pushd "${INTEL_DIR}" >/dev/null 2>&1 From 39bdbd5474445d4fab9778c20b8acb12675b6bbc Mon Sep 17 00:00:00 2001 From: Seth Grover Date: Mon, 4 Dec 2023 13:49:53 -0700 Subject: [PATCH 4/5] fix issue loading zeek intel on startup --- shared/bin/zeek_intel_setup.sh | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/shared/bin/zeek_intel_setup.sh b/shared/bin/zeek_intel_setup.sh index 7a868d828..077dade07 100755 --- a/shared/bin/zeek_intel_setup.sh +++ b/shared/bin/zeek_intel_setup.sh @@ -32,7 +32,16 @@ if mkdir -- "$LOCK_DIR" 2>/dev/null; then # if we have a directory to seed the intel config for the first time, start from a blank slate with just its contents if [[ -d "${INTEL_DIR}" ]] && [[ -d "${INTEL_PRESEED_DIR}" ]]; then - rsync -av --delete "${INTEL_PRESEED_DIR}"/ "${INTEL_DIR}"/ + + EXCLUDES=() + EXCLUDES+=( --exclude='..*' ) + EXCLUDES+=( --exclude='.dockerignore' ) + EXCLUDES+=( --exclude='.gitignore' ) + while read MAP_DIR; do + EXCLUDES+=( --exclude="${MAP_DIR}/" ) + done < <(echo "${CONFIG_MAP_DIR:-configmap;secretmap}" | tr ';' '\n') + + rsync --recursive --delete --delete-excluded "${EXCLUDES[@]}" "${INTEL_PRESEED_DIR}"/ "${INTEL_DIR}"/ mkdir -p "${INTEL_DIR}"/MISP "${INTEL_DIR}"/STIX || true fi From eca0c868974dc74ad8bf063648a3bea61889703d Mon Sep 17 00:00:00 2001 From: Seth Grover Date: Mon, 4 Dec 2023 15:29:26 -0700 Subject: [PATCH 5/5] sha1sum update --- docs/download.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/download.md b/docs/download.md index 632d3c035..1aca19bca 100644 --- a/docs/download.md +++ b/docs/download.md @@ -16,7 +16,7 @@ While official downloads of the Malcolm installer ISO are not provided, an **uno | ISO | SHA256 | |---|---| -| [malcolm-23.12.0.iso](/iso/malcolm-23.12.0.iso) (5.1GiB) | [`da22b4bfab2ca8cb2a3ea6266b6cea603a9ae119de83009f3d133f0f202b566f`](/iso/malcolm-23.12.0.iso.sha256.txt) | +| [malcolm-23.12.0.iso](/iso/malcolm-23.12.0.iso) (5.1GiB) | [`3e836d09cd79a4e3f54c6fc365b032385312ad885b8483a0df156b59175d4909`](/iso/malcolm-23.12.0.iso.sha256.txt) | ## Hedgehog Linux