From c98e24e8ed8146b8013adb73fecbeb0109991318 Mon Sep 17 00:00:00 2001 From: wULLSnpAXbWZGYDYyhWTKKspEQoaYxXyhoisqHf Date: Tue, 29 Oct 2024 09:34:05 +0100 Subject: [PATCH 001/128] fix a readme typo --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 9c9e8ee7b..a70de22d7 100644 --- a/README.md +++ b/README.md @@ -136,7 +136,7 @@ Then navigate to ```http://localhost:55000/``` from your browser. For more info about the web interface, check the docs: https://stratospherelinuxips.readthedocs.io/en/develop/usage.html#the-web-interface -##### Kalispo (CLI-Interface) +##### Kalipso (CLI-Interface) ./kalipso.sh From ef5a29ed805bddd4f0aecb654465514667acf5e6 Mon Sep 17 00:00:00 2001 From: alya Date: Wed, 30 Oct 2024 16:01:19 +0300 Subject: [PATCH 002/128] Add a light dockerfile --- docker/light/Dockerfile | 58 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 58 insertions(+) create mode 100644 docker/light/Dockerfile diff --git a/docker/light/Dockerfile b/docker/light/Dockerfile new file mode 100644 index 000000000..a09c74d28 --- /dev/null +++ b/docker/light/Dockerfile @@ -0,0 +1,58 @@ +FROM ubuntu:22.04 +# To avoid user interaction when installing libraries +ENV DEBIAN_FRONTEND=noninteractive +# Blocking module requirement to avoid using sudo +ENV IS_IN_A_DOCKER_CONTAINER=True +# destionation dir for slips inside the container +ENV SLIPS_DIR=/StratosphereLinuxIPs + +# use bash instead of sh +SHELL ["/bin/bash", "-c"] + +# Install wget and add Zeek and redis repositories to our sources. +RUN apt update && apt install -y --no-install-recommends \ + wget \ + ca-certificates \ + git \ + curl \ + gnupg \ + lsb-release \ + software-properties-common \ + build-essential \ + file \ + lsof \ + iproute2 \ + tshark \ + whois \ + yara \ + net-tools \ + less \ + unzip \ + python3-certifi \ + python3-dev \ + python3-tzlocal \ + python3-pip \ + && echo 'deb http://download.opensuse.org/repositories/security:/zeek/xUbuntu_22.04/ /' | tee /etc/apt/sources.list.d/security:zeek.list \ + && curl -fsSL https://download.opensuse.org/repositories/security:zeek/xUbuntu_22.04/Release.key | gpg --dearmor | tee /etc/apt/trusted.gpg.d/security_zeek.gpg > /dev/null \ + && curl -fsSL https://packages.redis.io/gpg | gpg --dearmor -o /usr/share/keyrings/redis-archive-keyring.gpg \ + && echo "deb [signed-by=/usr/share/keyrings/redis-archive-keyring.gpg] https://packages.redis.io/deb $(lsb_release -cs) main" > /etc/apt/sources.list.d/redis.list \ + && apt update \ + && apt install -y --no-install-recommends --fix-missing \ + zeek \ + redis \ + && ln -s /opt/zeek/bin/zeek /usr/local/bin/bro \ + && apt clean \ + && rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* + +COPY . ${SLIPS_DIR} + +WORKDIR ${SLIPS_DIR} + +RUN pip3 install --no-cache-dir --upgrade pip \ + && egrep -v "tensorflow|Keras|scipy|pytest|flask|tqdm|coverage|pre-commit|ruff|black|scikit-learn|slackclient|matplotlib|stix2|cabby|pandas|numpy" install/requirements.txt | xargs -n 1 pip install \ + && pip freeze \ + && chmod 774 slips.py \ + && git init \ + && git remote add origin https://github.com/stratosphereips/StratosphereLinuxIPS.git + +CMD /bin/bash From 701888c8333b158371e07fae26c024d3d7fbba44 Mon Sep 17 00:00:00 2001 From: alya Date: Wed, 30 Oct 2024 16:22:35 +0300 Subject: [PATCH 003/128] dont install scikit learn in the light dockerfile --- docker/light/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/light/Dockerfile b/docker/light/Dockerfile index a09c74d28..1e126f389 100644 --- a/docker/light/Dockerfile +++ b/docker/light/Dockerfile @@ -49,7 +49,7 @@ COPY . ${SLIPS_DIR} WORKDIR ${SLIPS_DIR} RUN pip3 install --no-cache-dir --upgrade pip \ - && egrep -v "tensorflow|Keras|scipy|pytest|flask|tqdm|coverage|pre-commit|ruff|black|scikit-learn|slackclient|matplotlib|stix2|cabby|pandas|numpy" install/requirements.txt | xargs -n 1 pip install \ + && egrep -v "tensorflow|Keras|scipy|pytest|flask|tqdm|coverage|pre-commit|ruff|black|scikit-learn|scikit_learn|slackclient|matplotlib|stix2|cabby|pandas|numpy" install/requirements.txt | xargs -n 1 pip install \ && pip freeze \ && chmod 774 slips.py \ && git init \ From 6f09f83ff873b45b158f13c29d28d6110c12268f Mon Sep 17 00:00:00 2001 From: alya Date: Wed, 30 Oct 2024 16:23:41 +0300 Subject: [PATCH 004/128] dont install yappi in the light dockerfile --- docker/light/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/light/Dockerfile b/docker/light/Dockerfile index 1e126f389..b8c5e85fd 100644 --- a/docker/light/Dockerfile +++ b/docker/light/Dockerfile @@ -49,7 +49,7 @@ COPY . ${SLIPS_DIR} WORKDIR ${SLIPS_DIR} RUN pip3 install --no-cache-dir --upgrade pip \ - && egrep -v "tensorflow|Keras|scipy|pytest|flask|tqdm|coverage|pre-commit|ruff|black|scikit-learn|scikit_learn|slackclient|matplotlib|stix2|cabby|pandas|numpy" install/requirements.txt | xargs -n 1 pip install \ + && egrep -v "tensorflow|Keras|scipy|pytest|flask|tqdm|coverage|yappi|pre-commit|ruff|black|scikit-learn|scikit_learn|slackclient|matplotlib|stix2|cabby|pandas|numpy" install/requirements.txt | xargs -n 1 pip install \ && pip freeze \ && chmod 774 slips.py \ && git init \ From 8c4d6967edf1d6ee580f448b977ef714fecf1647 Mon Sep 17 00:00:00 2001 From: alya Date: Wed, 30 Oct 2024 16:26:13 +0300 Subject: [PATCH 005/128] dont install recommonmark in the light dockerfile --- docker/light/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/light/Dockerfile b/docker/light/Dockerfile index b8c5e85fd..ef7036a0d 100644 --- a/docker/light/Dockerfile +++ b/docker/light/Dockerfile @@ -49,7 +49,7 @@ COPY . ${SLIPS_DIR} WORKDIR ${SLIPS_DIR} RUN pip3 install --no-cache-dir --upgrade pip \ - && egrep -v "tensorflow|Keras|scipy|pytest|flask|tqdm|coverage|yappi|pre-commit|ruff|black|scikit-learn|scikit_learn|slackclient|matplotlib|stix2|cabby|pandas|numpy" install/requirements.txt | xargs -n 1 pip install \ + && egrep -v "tensorflow|recommonmark|Keras|scipy|pytest|flask|tqdm|coverage|yappi|pre-commit|ruff|black|scikit-learn|scikit_learn|slackclient|matplotlib|stix2|cabby|pandas|numpy" install/requirements.txt | xargs -n 1 pip install \ && pip freeze \ && chmod 774 slips.py \ && git init \ From b2465299bcf836f06d534836fc59c2bbe36e566f Mon Sep 17 00:00:00 2001 From: alya Date: Wed, 30 Oct 2024 16:32:56 +0300 Subject: [PATCH 006/128] dont install setuptools and wheel in the light dockerfile --- docker/light/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/light/Dockerfile b/docker/light/Dockerfile index ef7036a0d..e10469d05 100644 --- a/docker/light/Dockerfile +++ b/docker/light/Dockerfile @@ -49,7 +49,7 @@ COPY . ${SLIPS_DIR} WORKDIR ${SLIPS_DIR} RUN pip3 install --no-cache-dir --upgrade pip \ - && egrep -v "tensorflow|recommonmark|Keras|scipy|pytest|flask|tqdm|coverage|yappi|pre-commit|ruff|black|scikit-learn|scikit_learn|slackclient|matplotlib|stix2|cabby|pandas|numpy" install/requirements.txt | xargs -n 1 pip install \ + && egrep -v "tensorflow|recommonmark|Keras|scipy|pytest|flask|tqdm|coverage|yappi|pre-commit|ruff|protobuf|black|wheel|scikit-learn|scikit_learn|slackclient|matplotlib|stix2|cabby|pandas|setuptools|numpy|certifi" install/requirements.txt | xargs -n 1 pip install \ && pip freeze \ && chmod 774 slips.py \ && git init \ From 70b3536ef06c2fcc97cd37e9cb2e80031943bff0 Mon Sep 17 00:00:00 2001 From: alya Date: Wed, 30 Oct 2024 17:05:35 +0300 Subject: [PATCH 007/128] move all the profiling logic from main to managers/profiler_manager.py --- managers/profilers_manager.py | 146 ++++++++++++++++++++++++++++++++++ slips/main.py | 126 +---------------------------- 2 files changed, 150 insertions(+), 122 deletions(-) create mode 100644 managers/profilers_manager.py diff --git a/managers/profilers_manager.py b/managers/profilers_manager.py new file mode 100644 index 000000000..35c2a0772 --- /dev/null +++ b/managers/profilers_manager.py @@ -0,0 +1,146 @@ +import multiprocessing +import os +import subprocess +import sys +import time + +class ProfilersManager: + def __init__(self, main): + self.main = main + self.read_configurations() + + def read_configurations(self): + self.cpu_profiler_enabled = self.main.conf.get_cpu_profiler_enable() + self.cpu_profiler_mode = self.main.conf.get_cpu_profiler_mode() + self.cpu_profiler_multiprocess = ( + self.main.conf.get_cpu_profiler_multiprocess() + ) + self.cpu_profiler_dev_mode_entries = ( + self.main.conf.get_cpu_profiler_dev_mode_entries() + ) + self.cpu_profiler_output_limit \ + = self.main.conf.get_cpu_profiler_output_limit(), + self.cpu_profiler_sampling_interval = ( + self.main.conf.get_cpu_profiler_sampling_interval() + ) + + self.memory_profiler_mode = self.main.conf.get_memory_profiler_mode() + self.memory_profiler_enabled = self.main.conf.get_memory_profiler_enable() + self.memory_profiler_multiprocess = ( + self.main.conf.get_memory_profiler_multiprocess() + ) + def cpu_profiler_init(self): + from slips_files.common.performance_profilers.cpu_profiler import CPUProfiler + if not self.cpu_profiler_enabled: + return + try: + if ( + self.cpu_profiler_multiprocess + and self.cpu_profiler_mode == "dev" + ): + args = sys.argv + if args[-1] != "--no-recurse": + tracer_entries = str( + self.cpu_profiler_dev_mode_entries + ) + viz_args = [ + "viztracer", + "--tracer_entries", + tracer_entries, + "--max_stack_depth", + "10", + "-o", + str( + os.path.join( + self.args.output, + "cpu_profiling_result.json", + ) + ), + ] + viz_args.extend(args) + viz_args.append("--no-recurse") + print( + "Starting multiprocess profiling recursive subprocess" + ) + subprocess.run(viz_args) + exit(0) + else: + self.cpu_profiler = CPUProfiler( + db=self.main.db, + output=self.args.output, + mode=self.cpu_profiler_mode, + limit=self.cpu_profiler_output_limit, + interval=self.cpu_profiler_sampling_interval, + ) + self.cpu_profiler.start() + except Exception as e: + print(e) + self.cpu_profiler_enabled = False + + def cpu_profiler_release(self): + if hasattr(self, "cpuProfilerEnabled"): + if self.cpu_profiler_enabled and not self.cpu_profiler_multiprocess: + self.cpu_profiler.stop() + self.cpu_profiler.print() + + def memory_profiler_init(self): + from slips_files.common.performance_profilers.memory_profiler import ( + MemoryProfiler, + ) + + if not self.memory_profiler_enabled: + return + + output_dir = os.path.join(self.args.output, "memoryprofile/") + if not os.path.exists(output_dir): + os.makedirs(output_dir) + output_file = os.path.join(output_dir, "memory_profile.bin") + self.memory_profiler = MemoryProfiler( + output_file, + db=self.main.db, + mode=self.memory_profiler_mode, + multiprocess=self.memory_profiler_multiprocess, + ) + self.memory_profiler.start() + + + def memory_profiler_release(self): + if ( + hasattr(self, "memoryProfilerEnabled") + and self.memory_profiler_enabled + ): + self.memory_profiler.stop() + + def memory_profiler_multiproc_test(self): + def target_function(): + print("Target function started") + time.sleep(5) + + def mem_function(): + print("Mem function started") + while True: + time.sleep(1) + array = [] + for i in range(1000000): + array.append(i) + + processes = [] + num_processes = 3 + + for _ in range(num_processes): + process = multiprocessing.Process( + target=target_function if _ % 2 else mem_function + ) + process.start() + processes.append(process) + + # Message passing + self.main.db.publish("memory_profile", processes[1].pid) # successful + # target_function will timeout and tracker will be cleared + time.sleep(5) + # end but maybe don't start + self.main.db.publish("memory_profile", processes[0].pid) + time.sleep(5) # mem_function will get tracker started + # start successfully + self.main.db.publish("memory_profile", processes[0].pid) + input() \ No newline at end of file diff --git a/slips/main.py b/slips/main.py index 8e28358d4..4276769fe 100644 --- a/slips/main.py +++ b/slips/main.py @@ -15,13 +15,10 @@ from managers.host_ip_manager import HostIPManager from managers.metadata_manager import MetadataManager from managers.process_manager import ProcessManager +from managers.profilers_manager import ProfilersManager from managers.redis_manager import RedisManager from managers.ui_manager import UIManager from slips_files.common.parsers.config_parser import ConfigParser -from slips_files.common.performance_profilers.cpu_profiler import CPUProfiler -from slips_files.common.performance_profilers.memory_profiler import ( - MemoryProfiler, -) from slips_files.common.printer import Printer from slips_files.common.slips_utils import utils from slips_files.common.style import green @@ -42,6 +39,7 @@ def __init__(self, testing=False): self.metadata_man = MetadataManager(self) self.conf = ConfigParser() self.ui_man = UIManager(self) + self.profilers_manager = ProfilersManager(self) self.version = utils.get_slips_version() # will be filled later @@ -73,122 +71,6 @@ def __init__(self, testing=False): # should be initialised after self.input_type is set self.host_ip_man = HostIPManager(self) - def cpu_profiler_init(self): - self.cpuProfilerEnabled = self.conf.get_cpu_profiler_enable() - self.cpuProfilerMode = self.conf.get_cpu_profiler_mode() - self.cpuProfilerMultiprocess = ( - self.conf.get_cpu_profiler_multiprocess() - ) - if self.cpuProfilerEnabled: - try: - if ( - self.cpuProfilerMultiprocess - and self.cpuProfilerMode == "dev" - ): - args = sys.argv - if args[-1] != "--no-recurse": - tracer_entries = str( - self.conf.get_cpu_profiler_dev_mode_entries() - ) - viz_args = [ - "viztracer", - "--tracer_entries", - tracer_entries, - "--max_stack_depth", - "10", - "-o", - str( - os.path.join( - self.args.output, - "cpu_profiling_result.json", - ) - ), - ] - viz_args.extend(args) - viz_args.append("--no-recurse") - print( - "Starting multiprocess profiling recursive subprocess" - ) - subprocess.run(viz_args) - exit(0) - else: - self.cpuProfiler = CPUProfiler( - db=self.db, - output=self.args.output, - mode=self.conf.get_cpu_profiler_mode(), - limit=self.conf.get_cpu_profiler_output_limit(), - interval=self.conf.get_cpu_profiler_sampling_interval(), - ) - self.cpuProfiler.start() - except Exception as e: - print(e) - self.cpuProfilerEnabled = False - - def cpu_profiler_release(self): - if hasattr(self, "cpuProfilerEnabled"): - if self.cpuProfilerEnabled and not self.cpuProfilerMultiprocess: - self.cpuProfiler.stop() - self.cpuProfiler.print() - - def memory_profiler_init(self): - self.memoryProfilerEnabled = self.conf.get_memory_profiler_enable() - memoryProfilerMode = self.conf.get_memory_profiler_mode() - memoryProfilerMultiprocess = ( - self.conf.get_memory_profiler_multiprocess() - ) - if self.memoryProfilerEnabled: - output_dir = os.path.join(self.args.output, "memoryprofile/") - if not os.path.exists(output_dir): - os.makedirs(output_dir) - output_file = os.path.join(output_dir, "memory_profile.bin") - self.memoryProfiler = MemoryProfiler( - output_file, - db=self.db, - mode=memoryProfilerMode, - multiprocess=memoryProfilerMultiprocess, - ) - self.memoryProfiler.start() - - def memory_profiler_release(self): - if ( - hasattr(self, "memoryProfilerEnabled") - and self.memoryProfilerEnabled - ): - self.memoryProfiler.stop() - - def memory_profiler_multiproc_test(self): - def target_function(): - print("Target function started") - time.sleep(5) - - def mem_function(): - print("Mem function started") - while True: - time.sleep(1) - array = [] - for i in range(1000000): - array.append(i) - - processes = [] - num_processes = 3 - - for _ in range(num_processes): - process = multiprocessing.Process( - target=target_function if _ % 2 else mem_function - ) - process.start() - processes.append(process) - - # Message passing - self.db.publish("memory_profile", processes[1].pid) # successful - # target_function will timeout and tracker will be cleared - time.sleep(5) - # end but maybe don't start - self.db.publish("memory_profile", processes[0].pid) - time.sleep(5) # mem_function will get tracker started - # start successfully - self.db.publish("memory_profile", processes[0].pid) - input() def check_zeek_or_bro(self): """ @@ -617,8 +499,8 @@ def start(self): 1, 0, ) - self.cpu_profiler_init() - self.memory_profiler_init() + self.profilers_manager.cpu_profiler_init() + self.profilers_manager.memory_profiler_init() if self.args.growing: if self.input_type != "zeek_folder": From 7a9d7f9d2c7c84e7567c800290f2d6e6ae5359c3 Mon Sep 17 00:00:00 2001 From: alya Date: Wed, 30 Oct 2024 17:06:34 +0300 Subject: [PATCH 008/128] profiler_manager.py: delete the memory_profiler_multiproc_test() --- managers/profilers_manager.py | 35 ++--------------------------------- 1 file changed, 2 insertions(+), 33 deletions(-) diff --git a/managers/profilers_manager.py b/managers/profilers_manager.py index 35c2a0772..4f6e1a8c9 100644 --- a/managers/profilers_manager.py +++ b/managers/profilers_manager.py @@ -29,6 +29,8 @@ def read_configurations(self): self.memory_profiler_multiprocess = ( self.main.conf.get_memory_profiler_multiprocess() ) + + def cpu_profiler_init(self): from slips_files.common.performance_profilers.cpu_profiler import CPUProfiler if not self.cpu_profiler_enabled: @@ -111,36 +113,3 @@ def memory_profiler_release(self): ): self.memory_profiler.stop() - def memory_profiler_multiproc_test(self): - def target_function(): - print("Target function started") - time.sleep(5) - - def mem_function(): - print("Mem function started") - while True: - time.sleep(1) - array = [] - for i in range(1000000): - array.append(i) - - processes = [] - num_processes = 3 - - for _ in range(num_processes): - process = multiprocessing.Process( - target=target_function if _ % 2 else mem_function - ) - process.start() - processes.append(process) - - # Message passing - self.main.db.publish("memory_profile", processes[1].pid) # successful - # target_function will timeout and tracker will be cleared - time.sleep(5) - # end but maybe don't start - self.main.db.publish("memory_profile", processes[0].pid) - time.sleep(5) # mem_function will get tracker started - # start successfully - self.main.db.publish("memory_profile", processes[0].pid) - input() \ No newline at end of file From 3c05fab878fc96249158f92f2f18e8e95f369fb0 Mon Sep 17 00:00:00 2001 From: alya Date: Wed, 30 Oct 2024 17:11:59 +0300 Subject: [PATCH 009/128] profiler_manager.py: import viztracer and memray on demand --- managers/profilers_manager.py | 38 +++++++++++++++-------------- slips/main.py | 1 - slips_files/core/evidencehandler.py | 2 -- 3 files changed, 20 insertions(+), 21 deletions(-) diff --git a/managers/profilers_manager.py b/managers/profilers_manager.py index 4f6e1a8c9..f1005c168 100644 --- a/managers/profilers_manager.py +++ b/managers/profilers_manager.py @@ -13,28 +13,28 @@ def read_configurations(self): self.cpu_profiler_enabled = self.main.conf.get_cpu_profiler_enable() self.cpu_profiler_mode = self.main.conf.get_cpu_profiler_mode() self.cpu_profiler_multiprocess = ( - self.main.conf.get_cpu_profiler_multiprocess() - ) + self.main.conf.get_cpu_profiler_multiprocess()) self.cpu_profiler_dev_mode_entries = ( - self.main.conf.get_cpu_profiler_dev_mode_entries() - ) + self.main.conf.get_cpu_profiler_dev_mode_entries()) self.cpu_profiler_output_limit \ = self.main.conf.get_cpu_profiler_output_limit(), self.cpu_profiler_sampling_interval = ( - self.main.conf.get_cpu_profiler_sampling_interval() - ) + self.main.conf.get_cpu_profiler_sampling_interval()) self.memory_profiler_mode = self.main.conf.get_memory_profiler_mode() - self.memory_profiler_enabled = self.main.conf.get_memory_profiler_enable() + self.memory_profiler_enabled = ( + self.main.conf.get_memory_profiler_enable()) self.memory_profiler_multiprocess = ( - self.main.conf.get_memory_profiler_multiprocess() - ) + self.main.conf.get_memory_profiler_multiprocess()) def cpu_profiler_init(self): - from slips_files.common.performance_profilers.cpu_profiler import CPUProfiler if not self.cpu_profiler_enabled: return + + from slips_files.common.performance_profilers.cpu_profiler import ( + CPUProfiler) + try: if ( self.cpu_profiler_multiprocess @@ -80,19 +80,21 @@ def cpu_profiler_init(self): self.cpu_profiler_enabled = False def cpu_profiler_release(self): - if hasattr(self, "cpuProfilerEnabled"): - if self.cpu_profiler_enabled and not self.cpu_profiler_multiprocess: - self.cpu_profiler.stop() - self.cpu_profiler.print() + if not hasattr(self, "cpuProfilerEnabled"): + return + + if (self.cpu_profiler_enabled + and not self.cpu_profiler_multiprocess): + self.cpu_profiler.stop() + self.cpu_profiler.print() def memory_profiler_init(self): + if not self.memory_profiler_enabled: + return + from slips_files.common.performance_profilers.memory_profiler import ( MemoryProfiler, ) - - if not self.memory_profiler_enabled: - return - output_dir = os.path.join(self.args.output, "memoryprofile/") if not os.path.exists(output_dir): os.makedirs(output_dir) diff --git a/slips/main.py b/slips/main.py index 4276769fe..04e3aa3e0 100644 --- a/slips/main.py +++ b/slips/main.py @@ -40,7 +40,6 @@ def __init__(self, testing=False): self.conf = ConfigParser() self.ui_man = UIManager(self) self.profilers_manager = ProfilersManager(self) - self.version = utils.get_slips_version() # will be filled later self.commit = "None" diff --git a/slips_files/core/evidencehandler.py b/slips_files/core/evidencehandler.py index 71a721a6f..d7b5b487a 100644 --- a/slips_files/core/evidencehandler.py +++ b/slips_files/core/evidencehandler.py @@ -21,8 +21,6 @@ import json from typing import List, Dict, Optional from datetime import datetime - -# from colorama import Fore, Style from os import path import sys import os From eed5fd7f40b4c413dd20ad026745f495df9cc4c5 Mon Sep 17 00:00:00 2001 From: alya Date: Wed, 30 Oct 2024 17:12:23 +0300 Subject: [PATCH 010/128] move the logic of stopping the profilers to process_manager's shutdown_gracefully() --- managers/process_manager.py | 4 ++++ slips.py | 2 -- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/managers/process_manager.py b/managers/process_manager.py index fc1521149..c3a007717 100644 --- a/managers/process_manager.py +++ b/managers/process_manager.py @@ -711,6 +711,10 @@ def shutdown_gracefully(self): if self.main.conf.export_labeled_flows(): format_ = self.main.conf.export_labeled_flows_to().lower() self.main.db.export_labeled_flows(format_) + + self.main.profilers_manager.cpu_profiler_release() + self.main.profilers_manager.memory_profiler_release() + # if store_a_copy_of_zeek_files is set to yes in slips.yaml # copy the whole zeek_files dir to the output dir diff --git a/slips.py b/slips.py index 8110106d3..62a8efbe2 100755 --- a/slips.py +++ b/slips.py @@ -61,5 +61,3 @@ # interactive mode slips.start() - slips.cpu_profiler_release() - slips.memory_profiler_release() From 07d7cdc3d697796e28e9e81678315c4134cd1c47 Mon Sep 17 00:00:00 2001 From: alya Date: Wed, 30 Oct 2024 17:33:52 +0300 Subject: [PATCH 011/128] remove vistrace, memray, tqdm, setuptools and wheel from requirements.txt --- install/requirements.txt | 6 ------ 1 file changed, 6 deletions(-) diff --git a/install/requirements.txt b/install/requirements.txt index 9fa899129..ef2301e7f 100644 --- a/install/requirements.txt +++ b/install/requirements.txt @@ -9,7 +9,6 @@ cabby==0.1.23 stix2==3.0.1 certifi==2024.7.4 tensorflow==2.16.1 -colorama==0.4.6 Keras validators==0.34.0 ipwhois==1.2.0 @@ -27,18 +26,13 @@ scikit-learn==1.5.1 GitPython==3.1.43 protobuf==4.25.3 exclusiveprocess==0.9.4 -setuptools pytest-dependency==0.6.0 whois==1.20240129.2 -wheel flask tldextract==5.1.2 -tqdm==4.66.5 termcolor==2.4.0 -viztracer==0.16.3 yappi==1.6.0 pytest-sugar==1.0.0 -memray==1.13.4 aid_hash black==24.8.0 ruff==0.6.8 From 558a081b2acf974deecc0b041e1a3e96c4181b56 Mon Sep 17 00:00:00 2001 From: alya Date: Thu, 31 Oct 2024 16:49:32 +0300 Subject: [PATCH 012/128] update profiler unit tests --- managers/profilers_manager.py | 51 ++++++++++++++++++----------------- tests/test_main.py | 45 ++++++++++++++++--------------- 2 files changed, 50 insertions(+), 46 deletions(-) diff --git a/managers/profilers_manager.py b/managers/profilers_manager.py index f1005c168..b184ebd67 100644 --- a/managers/profilers_manager.py +++ b/managers/profilers_manager.py @@ -1,39 +1,44 @@ -import multiprocessing import os import subprocess import sys -import time + class ProfilersManager: def __init__(self, main): self.main = main self.read_configurations() - + def read_configurations(self): self.cpu_profiler_enabled = self.main.conf.get_cpu_profiler_enable() self.cpu_profiler_mode = self.main.conf.get_cpu_profiler_mode() self.cpu_profiler_multiprocess = ( - self.main.conf.get_cpu_profiler_multiprocess()) + self.main.conf.get_cpu_profiler_multiprocess() + ) self.cpu_profiler_dev_mode_entries = ( - self.main.conf.get_cpu_profiler_dev_mode_entries()) - self.cpu_profiler_output_limit \ - = self.main.conf.get_cpu_profiler_output_limit(), + self.main.conf.get_cpu_profiler_dev_mode_entries() + ) + self.cpu_profiler_output_limit = ( + self.main.conf.get_cpu_profiler_output_limit(), + ) self.cpu_profiler_sampling_interval = ( - self.main.conf.get_cpu_profiler_sampling_interval()) - + self.main.conf.get_cpu_profiler_sampling_interval() + ) + self.memory_profiler_mode = self.main.conf.get_memory_profiler_mode() self.memory_profiler_enabled = ( - self.main.conf.get_memory_profiler_enable()) + self.main.conf.get_memory_profiler_enable() + ) self.memory_profiler_multiprocess = ( - self.main.conf.get_memory_profiler_multiprocess()) - - + self.main.conf.get_memory_profiler_multiprocess() + ) + def cpu_profiler_init(self): if not self.cpu_profiler_enabled: return from slips_files.common.performance_profilers.cpu_profiler import ( - CPUProfiler) + CPUProfiler, + ) try: if ( @@ -42,9 +47,7 @@ def cpu_profiler_init(self): ): args = sys.argv if args[-1] != "--no-recurse": - tracer_entries = str( - self.cpu_profiler_dev_mode_entries - ) + tracer_entries = str(self.cpu_profiler_dev_mode_entries) viz_args = [ "viztracer", "--tracer_entries", @@ -78,13 +81,12 @@ def cpu_profiler_init(self): except Exception as e: print(e) self.cpu_profiler_enabled = False - + def cpu_profiler_release(self): - if not hasattr(self, "cpuProfilerEnabled"): + if not hasattr(self, "cpu_profiler_enabled"): return - - if (self.cpu_profiler_enabled - and not self.cpu_profiler_multiprocess): + + if self.cpu_profiler_enabled and not self.cpu_profiler_multiprocess: self.cpu_profiler.stop() self.cpu_profiler.print() @@ -95,6 +97,7 @@ def memory_profiler_init(self): from slips_files.common.performance_profilers.memory_profiler import ( MemoryProfiler, ) + output_dir = os.path.join(self.args.output, "memoryprofile/") if not os.path.exists(output_dir): os.makedirs(output_dir) @@ -107,11 +110,9 @@ def memory_profiler_init(self): ) self.memory_profiler.start() - def memory_profiler_release(self): if ( - hasattr(self, "memoryProfilerEnabled") + hasattr(self, "memory_profiler_enabled") and self.memory_profiler_enabled ): self.memory_profiler.stop() - diff --git a/tests/test_main.py b/tests/test_main.py index ecc8233a1..d2615efea 100644 --- a/tests/test_main.py +++ b/tests/test_main.py @@ -72,41 +72,44 @@ def test_cpu_profiler_release_enabled( expected_print_calls, ): main = ModuleFactory().create_main_obj() - main.cpuProfilerEnabled = True - main.cpuProfilerMultiprocess = cpu_profiler_multiprocess - - main.cpuProfiler = MagicMock() - - main.cpu_profiler_release() + main.profilers_manager.cpu_profiler_enabled = True + main.profilers_manager.cpu_profiler_multiprocess = ( + cpu_profiler_multiprocess + ) + main.profilers_manager.cpu_profiler = MagicMock() + main.profilers_manager.cpu_profiler_release() - assert main.cpuProfiler.stop.call_count == expected_stop_calls - assert main.cpuProfiler.print.call_count == expected_print_calls + assert ( + main.profilers_manager.cpu_profiler.stop.call_count + == expected_stop_calls + ) + assert ( + main.profilers_manager.cpu_profiler.print.call_count + == expected_print_calls + ) def test_cpu_profiler_release_disabled(): main = ModuleFactory().create_main_obj() - main.cpuProfilerEnabled = False - main.cpu_profiler_release() - assert not hasattr(main, "memoryProfiler") + main.profilers_manager.cpu_profiler_enabled = False + main.profilers_manager.cpu_profiler_release() + assert not hasattr(main.profilers_manager, "memory_profiler") def test_memory_profiler_release_enabled(): main = ModuleFactory().create_main_obj() - main.memoryProfilerEnabled = True - main.memoryProfiler = MagicMock() - - main.memory_profiler_release() - - main.memoryProfiler.stop.assert_called_once() + main.profilers_manager.memory_profiler_enabled = True + main.profilers_manager.memory_profiler = MagicMock() + main.profilers_manager.memory_profiler_release() + main.profilers_manager.memory_profiler.stop.assert_called_once() def test_memory_profiler_release_disabled(): main = ModuleFactory().create_main_obj() - main.memoryProfilerEnabled = False - - main.memory_profiler_release() + main.profilers_manager.memory_profiler_enabled = False + main.profilers_manager.memory_profiler_release() - assert not hasattr(main, "memoryProfiler") + assert not hasattr(main.profilers_manager, "memory_profiler") @pytest.mark.parametrize( From ccff6ddc98a6f9a730f213e65ffa328f492d5343 Mon Sep 17 00:00:00 2001 From: alya Date: Thu, 31 Oct 2024 18:18:20 +0300 Subject: [PATCH 013/128] CI: cache zeek builds --- .github/workflows/unit-tests.yml | 46 +++++++++++++++++++++++--------- 1 file changed, 34 insertions(+), 12 deletions(-) diff --git a/.github/workflows/unit-tests.yml b/.github/workflows/unit-tests.yml index 346b51be0..1bed8cbe3 100644 --- a/.github/workflows/unit-tests.yml +++ b/.github/workflows/unit-tests.yml @@ -7,9 +7,38 @@ on: - 'develop' jobs: + build-zeek: + runs-on: ubuntu-22.04 + steps: + - uses: actions/checkout@v4 + with: + ref: ${{ github.ref }} + fetch-depth: '' + + - name: Enable memory overcommit (for redis) + run: sysctl vm.overcommit_memory=1 + + - name: Install dependencies + run: sudo apt-get update --fix-missing && sudo apt-get -y --no-install-recommends install python3 redis-server python3-pip python3-certifi python3-dev build-essential file lsof net-tools iproute2 iptables python3-tzlocal nfdump tshark git whois golang nodejs notify-osd yara libnotify-bin + + - name: Install Zeek + run: | + echo 'deb http://download.opensuse.org/repositories/security:/zeek/xUbuntu_22.04/ /' | sudo tee /etc/apt/sources.list.d/security:zeek.list + curl -fsSL https://download.opensuse.org/repositories/security:zeek/xUbuntu_22.04/Release.key | gpg --dearmor | sudo tee /etc/apt/trusted.gpg.d/security_zeek.gpg > /dev/null + sudo apt update && sudo apt install -y --no-install-recommends --fix-missing zeek + sudo ln -s /opt/zeek/bin/zeek /usr/local/bin/bro + + - name: Cache Zeek Build + uses: actions/cache@v3 + with: + path: /opt/zeek + key: ${{ runner.os }}-zeek + tests: runs-on: ubuntu-22.04 timeout-minutes: 1800 + # make this one depend on the first job + needs: build-zeek strategy: matrix: @@ -71,18 +100,11 @@ jobs: ref: ${{ github.ref }} fetch-depth: '' - - name: Enable memory overcommit (for redis) - run: sysctl vm.overcommit_memory=1 - - - name: Install slips dependencies - run: sudo apt-get update --fix-missing && sudo apt-get -y --no-install-recommends install python3 redis-server python3-pip python3-certifi python3-dev build-essential file lsof net-tools iproute2 iptables python3-tzlocal nfdump tshark git whois golang nodejs notify-osd yara libnotify-bin - - - name: Install Zeek - run: | - sudo echo 'deb http://download.opensuse.org/repositories/security:/zeek/xUbuntu_22.04/ /' | sudo tee /etc/apt/sources.list.d/security:zeek.list - curl -fsSL https://download.opensuse.org/repositories/security:zeek/xUbuntu_22.04/Release.key | gpg --dearmor | sudo tee /etc/apt/trusted.gpg.d/security_zeek.gpg > /dev/null - sudo apt update && sudo apt install -y --no-install-recommends --fix-missing zeek - sudo ln -s /opt/zeek/bin/zeek /usr/local/bin/bro + - name: Restore Zeek Build from Cache + uses: actions/cache@v3 + with: + path: /opt/zeek + key: ${{ runner.os }}-zeek - name: Set up Python 3.10.12 uses: actions/setup-python@v5 From 9c578334d065f23539b74389a0c0d8f5e5a44ec1 Mon Sep 17 00:00:00 2001 From: alya Date: Fri, 1 Nov 2024 17:39:45 +0200 Subject: [PATCH 014/128] unit-tests: cache apt and pip dependencies --- .github/workflows/unit-tests.yml | 55 +++++++++++++++++++++++++------- 1 file changed, 44 insertions(+), 11 deletions(-) diff --git a/.github/workflows/unit-tests.yml b/.github/workflows/unit-tests.yml index 1bed8cbe3..07f15b9de 100644 --- a/.github/workflows/unit-tests.yml +++ b/.github/workflows/unit-tests.yml @@ -7,7 +7,7 @@ on: - 'develop' jobs: - build-zeek: + install-dependencies: runs-on: ubuntu-22.04 steps: - uses: actions/checkout@v4 @@ -18,9 +18,17 @@ jobs: - name: Enable memory overcommit (for redis) run: sysctl vm.overcommit_memory=1 - - name: Install dependencies + - name: Install APT dependencies run: sudo apt-get update --fix-missing && sudo apt-get -y --no-install-recommends install python3 redis-server python3-pip python3-certifi python3-dev build-essential file lsof net-tools iproute2 iptables python3-tzlocal nfdump tshark git whois golang nodejs notify-osd yara libnotify-bin + - name: Cache APT packages + uses: actions/cache@v3 + with: + path: /var/cache/apt + key: ${{ runner.os }}-apt-${{ hashFiles('**/apt-packages.txt') }} + restore-keys: | + ${{ runner.os }}-apt- + - name: Install Zeek run: | echo 'deb http://download.opensuse.org/repositories/security:/zeek/xUbuntu_22.04/ /' | sudo tee /etc/apt/sources.list.d/security:zeek.list @@ -34,11 +42,29 @@ jobs: path: /opt/zeek key: ${{ runner.os }}-zeek + - name: Set up Python 3.10.12 + uses: actions/setup-python@v5 + with: + python-version: "3.10.12" + + - name: Install Python dependencies + run: | + python -m pip install --upgrade pip + python3 -m pip install --no-cache-dir -r install/requirements.txt + + - name: Cache pip dependencies + uses: actions/cache@v3 + with: + path: ~/.cache/pip + key: ${{ runner.os }}-pip-${{ hashFiles('**/requirements.txt') }} + restore-keys: | + ${{ runner.os }}-pip- + tests: runs-on: ubuntu-22.04 timeout-minutes: 1800 - # make this one depend on the first job - needs: build-zeek + # make this job depend on the first job + needs: install-dependencies strategy: matrix: @@ -106,15 +132,22 @@ jobs: path: /opt/zeek key: ${{ runner.os }}-zeek - - name: Set up Python 3.10.12 - uses: actions/setup-python@v5 + - name: Restore APT packages + uses: actions/cache@v3 with: - python-version: "3.10.12" + path: /var/cache/apt + key: ${{ runner.os }}-apt-${{ hashFiles('**/apt-packages.txt') }} + restore-keys: | + ${{ runner.os }}-apt- + + - name: Restore pip dependencies + uses: actions/cache@v3 + with: + path: ~/.cache/pip + key: ${{ runner.os }}-pip-${{ hashFiles('**/requirements.txt') }} + restore-keys: | + ${{ runner.os }}-pip- - - name: Install Python dependencies - run: | - python -m pip install --upgrade pip - python3 -m pip install --no-cache-dir -r install/requirements.txt - name: Start redis server run: redis-server --daemonize yes From 4c9e7f1a7cd99efac7f1b2fb8b3b882b7122ab13 Mon Sep 17 00:00:00 2001 From: alya Date: Fri, 1 Nov 2024 17:50:02 +0200 Subject: [PATCH 015/128] unit-tests: hardcode cache keys to avoid cache miss --- .github/workflows/unit-tests.yml | 23 ++++++++--------------- 1 file changed, 8 insertions(+), 15 deletions(-) diff --git a/.github/workflows/unit-tests.yml b/.github/workflows/unit-tests.yml index 07f15b9de..81dfb7636 100644 --- a/.github/workflows/unit-tests.yml +++ b/.github/workflows/unit-tests.yml @@ -25,9 +25,7 @@ jobs: uses: actions/cache@v3 with: path: /var/cache/apt - key: ${{ runner.os }}-apt-${{ hashFiles('**/apt-packages.txt') }} - restore-keys: | - ${{ runner.os }}-apt- + key: apt-cache - name: Install Zeek run: | @@ -40,7 +38,7 @@ jobs: uses: actions/cache@v3 with: path: /opt/zeek - key: ${{ runner.os }}-zeek + key: zeek-cache - name: Set up Python 3.10.12 uses: actions/setup-python@v5 @@ -56,9 +54,9 @@ jobs: uses: actions/cache@v3 with: path: ~/.cache/pip - key: ${{ runner.os }}-pip-${{ hashFiles('**/requirements.txt') }} - restore-keys: | - ${{ runner.os }}-pip- + key: pip-cache + + tests: runs-on: ubuntu-22.04 @@ -130,24 +128,19 @@ jobs: uses: actions/cache@v3 with: path: /opt/zeek - key: ${{ runner.os }}-zeek + key: zeek-cache - name: Restore APT packages uses: actions/cache@v3 with: path: /var/cache/apt - key: ${{ runner.os }}-apt-${{ hashFiles('**/apt-packages.txt') }} - restore-keys: | - ${{ runner.os }}-apt- + key: apt-cache - name: Restore pip dependencies uses: actions/cache@v3 with: path: ~/.cache/pip - key: ${{ runner.os }}-pip-${{ hashFiles('**/requirements.txt') }} - restore-keys: | - ${{ runner.os }}-pip- - + key: pip-cache - name: Start redis server run: redis-server --daemonize yes From 91b34de0f38d702c4a5d5b53cc2c966578dc4053 Mon Sep 17 00:00:00 2001 From: alya Date: Fri, 1 Nov 2024 17:59:58 +0200 Subject: [PATCH 016/128] unit-tests:use actions/setup-python@v5 for caching python dependencies --- .github/workflows/unit-tests.yml | 21 +++------------------ 1 file changed, 3 insertions(+), 18 deletions(-) diff --git a/.github/workflows/unit-tests.yml b/.github/workflows/unit-tests.yml index 81dfb7636..ca1098933 100644 --- a/.github/workflows/unit-tests.yml +++ b/.github/workflows/unit-tests.yml @@ -40,22 +40,13 @@ jobs: path: /opt/zeek key: zeek-cache + - name: Set up Python 3.10.12 uses: actions/setup-python@v5 with: python-version: "3.10.12" - - - name: Install Python dependencies - run: | - python -m pip install --upgrade pip - python3 -m pip install --no-cache-dir -r install/requirements.txt - - - name: Cache pip dependencies - uses: actions/cache@v3 - with: - path: ~/.cache/pip - key: pip-cache - + cache: 'pip' # caching pip dependencies + run: pip install -r install/requirements.txt tests: @@ -136,12 +127,6 @@ jobs: path: /var/cache/apt key: apt-cache - - name: Restore pip dependencies - uses: actions/cache@v3 - with: - path: ~/.cache/pip - key: pip-cache - - name: Start redis server run: redis-server --daemonize yes From a0fa9b2330f1b020bd93faad602461b349e64fa3 Mon Sep 17 00:00:00 2001 From: alya Date: Fri, 1 Nov 2024 18:21:28 +0200 Subject: [PATCH 017/128] add apt_Dependencies file in install/ to avoid repetition of dependencies in dockerfiles and github runners --- install/apt_dependencies.txt | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) create mode 100644 install/apt_dependencies.txt diff --git a/install/apt_dependencies.txt b/install/apt_dependencies.txt new file mode 100644 index 000000000..b174121fe --- /dev/null +++ b/install/apt_dependencies.txt @@ -0,0 +1,21 @@ +python3 +redis-server +python3-pip +python3-certifi +python3-dev +build-essential +file +lsof +net-tools +iproute2 +iptables +python3-tzlocal +nfdump +tshark +git +whois +golang +nodejs +notify-osd +yara +libnotify-bin From 42b1b1477d176b7727e3375d15ff15b8fd9b2a70 Mon Sep 17 00:00:00 2001 From: alya Date: Fri, 1 Nov 2024 18:22:28 +0200 Subject: [PATCH 018/128] CI: try caching /var/cache/ and /var/lib --- .github/workflows/unit-tests.yml | 24 ++++++++++++++++++------ 1 file changed, 18 insertions(+), 6 deletions(-) diff --git a/.github/workflows/unit-tests.yml b/.github/workflows/unit-tests.yml index ca1098933..ba80060b4 100644 --- a/.github/workflows/unit-tests.yml +++ b/.github/workflows/unit-tests.yml @@ -19,12 +19,14 @@ jobs: run: sysctl vm.overcommit_memory=1 - name: Install APT dependencies - run: sudo apt-get update --fix-missing && sudo apt-get -y --no-install-recommends install python3 redis-server python3-pip python3-certifi python3-dev build-essential file lsof net-tools iproute2 iptables python3-tzlocal nfdump tshark git whois golang nodejs notify-osd yara libnotify-bin + run: sudo apt-get update --fix-missing && sudo apt-get -y --no-install-recommends install $(cat install/apt_dependencies.txt) - - name: Cache APT packages + - name: Save APT Cache uses: actions/cache@v3 with: - path: /var/cache/apt + path: | + /var/cache/apt/archives + /var/lib/apt/lists key: apt-cache - name: Install Zeek @@ -34,13 +36,13 @@ jobs: sudo apt update && sudo apt install -y --no-install-recommends --fix-missing zeek sudo ln -s /opt/zeek/bin/zeek /usr/local/bin/bro + - name: Cache Zeek Build uses: actions/cache@v3 with: path: /opt/zeek key: zeek-cache - - name: Set up Python 3.10.12 uses: actions/setup-python@v5 with: @@ -121,12 +123,22 @@ jobs: path: /opt/zeek key: zeek-cache - - name: Restore APT packages + - name: Restore APT cache uses: actions/cache@v3 with: - path: /var/cache/apt + path: | + /var/cache/apt/archives + /var/lib/apt/lists key: apt-cache + + - name: Install apt dependencies (from cache if possible) + run: | + sudo apt-get update + sudo apt-get install -y $(cat install/apt_dependencies.txt) + + + - name: Start redis server run: redis-server --daemonize yes From 24be296f2ca0105c5231d9141ca15a3bf706f4ed Mon Sep 17 00:00:00 2001 From: alya Date: Fri, 1 Nov 2024 21:53:45 +0200 Subject: [PATCH 019/128] CI: install python dependencies/restore from cache before running unit tests --- .github/workflows/unit-tests.yml | 17 ++++++++++++++++- 1 file changed, 16 insertions(+), 1 deletion(-) diff --git a/.github/workflows/unit-tests.yml b/.github/workflows/unit-tests.yml index ba80060b4..86ae5d092 100644 --- a/.github/workflows/unit-tests.yml +++ b/.github/workflows/unit-tests.yml @@ -118,12 +118,19 @@ jobs: fetch-depth: '' - name: Restore Zeek Build from Cache + id: zeek-cache uses: actions/cache@v3 with: path: /opt/zeek key: zeek-cache + - if: ${{ steps.zeek-cache.outputs.cache-hit == 'true' }} + name: Echo restored from cache + continue-on-error: true + run: echo "Restored Zeek dependencies from cache successfully" + - name: Restore APT cache + id: apt-cache uses: actions/cache@v3 with: path: | @@ -131,6 +138,15 @@ jobs: /var/lib/apt/lists key: apt-cache + - if: ${{ steps.apt-cache.outputs.cache-hit == 'true' }} + name: Echo restored from cache + continue-on-error: true + run: echo "Restored APT dependencies from cache successfully" + + - name: Install Python dependencies (from cache if possible) + run: | + python3 -m pip install --upgrade pip + python3 -m pip install -r install/requirements.txt - name: Install apt dependencies (from cache if possible) run: | @@ -138,7 +154,6 @@ jobs: sudo apt-get install -y $(cat install/apt_dependencies.txt) - - name: Start redis server run: redis-server --daemonize yes From c73f7f609139815cd2d70baa474c8034c61d7f7d Mon Sep 17 00:00:00 2001 From: alya Date: Fri, 1 Nov 2024 22:13:04 +0200 Subject: [PATCH 020/128] CI: cache pip, zeek and apt dependencies in integration tests workflow to avoid building slips for each test --- .github/workflows/integration-tests.yml | 99 ++++++++++++++++++----- .github/workflows/unit-tests.yml | 102 ++++++++++++------------ 2 files changed, 132 insertions(+), 69 deletions(-) diff --git a/.github/workflows/integration-tests.yml b/.github/workflows/integration-tests.yml index b5705f400..e9bd64817 100644 --- a/.github/workflows/integration-tests.yml +++ b/.github/workflows/integration-tests.yml @@ -7,49 +7,112 @@ on: - 'develop' jobs: - tests: + install-dependencies: runs-on: ubuntu-22.04 - timeout-minutes: 7200 - - strategy: - matrix: - test_file: - - tests/integration_tests/test_config_files.py - - tests/integration_tests/test_portscans.py - - tests/integration_tests/test_dataset.py - steps: - uses: actions/checkout@v4 with: ref: ${{ github.ref }} fetch-depth: '' - - name: Install slips dependencies - run: sudo apt-get update --fix-missing && sudo apt-get -y --no-install-recommends install python3 redis-server python3-pip python3-certifi python3-dev build-essential file lsof net-tools iproute2 iptables python3-tzlocal nfdump tshark git whois golang nodejs notify-osd yara libnotify-bin + - name: Enable memory overcommit (for redis) + run: sysctl vm.overcommit_memory=1 + + - name: Install APT dependencies + run: sudo apt-get update --fix-missing && sudo apt-get -y --no-install-recommends install $(cat install/apt_dependencies.txt) + + - name: Save APT Cache + uses: actions/cache@v3 + with: + path: | + /var/cache/apt/archives + /var/lib/apt/lists + key: apt-cache - name: Install Zeek run: | - sudo echo 'deb http://download.opensuse.org/repositories/security:/zeek/xUbuntu_22.04/ /' | sudo tee /etc/apt/sources.list.d/security:zeek.list + echo 'deb http://download.opensuse.org/repositories/security:/zeek/xUbuntu_22.04/ /' | sudo tee /etc/apt/sources.list.d/security:zeek.list curl -fsSL https://download.opensuse.org/repositories/security:zeek/xUbuntu_22.04/Release.key | gpg --dearmor | sudo tee /etc/apt/trusted.gpg.d/security_zeek.gpg > /dev/null sudo apt update && sudo apt install -y --no-install-recommends --fix-missing zeek sudo ln -s /opt/zeek/bin/zeek /usr/local/bin/bro + + - name: Cache Zeek Build + uses: actions/cache@v3 + with: + path: /opt/zeek + key: zeek-cache + - name: Set up Python 3.10.12 uses: actions/setup-python@v5 with: python-version: "3.10.12" + cache: 'pip' # caching pip dependencies + run: pip install -r install/requirements.txt + + + tests: + runs-on: ubuntu-22.04 + timeout-minutes: 1800 + # make this job depend on the first job + needs: install-dependencies + + strategy: + matrix: + test_file: + - test_config_files.py + - test_portscans.py + - test_dataset.py + + steps: + - uses: actions/checkout@v4 + with: + ref: ${{ github.ref }} + fetch-depth: '' + + - name: Restore Zeek Build from Cache + id: zeek-cache + uses: actions/cache@v3 + with: + path: /opt/zeek + key: zeek-cache - - name: Install Python dependencies + - if: ${{ steps.zeek-cache.outputs.cache-hit == 'true' }} + name: Echo restored from cache + continue-on-error: true + run: echo "Restored Zeek dependencies from cache successfully" + + - name: Restore APT cache + id: apt-cache + uses: actions/cache@v3 + with: + path: | + /var/cache/apt/archives + /var/lib/apt/lists + key: apt-cache + + - if: ${{ steps.apt-cache.outputs.cache-hit == 'true' }} + name: Echo restored from cache + continue-on-error: true + run: echo "Restored APT dependencies from cache successfully" + + - name: Install Python dependencies (from cache if possible) run: | - python -m pip install --upgrade pip - python3 -m pip install --no-cache-dir -r install/requirements.txt - python3 -m pip install pytest-timeout + python3 -m pip install --upgrade pip + python3 -m pip install -r install/requirements.txt + + - name: Install apt dependencies (from cache if possible) + run: | + sudo apt-get update + sudo apt-get install -y $(cat install/apt_dependencies.txt) + - name: Start redis server run: redis-server --daemonize yes - name: Run Integration Tests for ${{ matrix.test_file }} - run: python3 -m pytest ${{ matrix.test_file }} -vvv -s + run: | + python3 -m pytest tests/integration_tests/${{ matrix.test_file }} -p no:warnings -vv -s -n 5 - name: Upload Artifacts if: success() || failure() diff --git a/.github/workflows/unit-tests.yml b/.github/workflows/unit-tests.yml index 86ae5d092..7266267b3 100644 --- a/.github/workflows/unit-tests.yml +++ b/.github/workflows/unit-tests.yml @@ -60,56 +60,56 @@ jobs: strategy: matrix: test_file: - - tests/test_inputProc.py - - tests/test_main.py - - tests/test_conn.py - - tests/test_downloaded_file.py - - tests/test_ssl.py - - tests/test_tunnel.py - - tests/test_ssh.py - - tests/test_dns.py - - tests/test_notice.py - - tests/test_software.py - - tests/test_smtp.py - - tests/test_whitelist.py - - tests/test_arp.py - - tests/test_blocking.py - - tests/test_flow_handler.py - - tests/test_horizontal_portscans.py - - tests/test_http_analyzer.py - - tests/test_vertical_portscans.py - - tests/test_network_discovery.py - - tests/test_virustotal.py - - tests/test_update_file_manager.py - - tests/test_threat_intelligence.py - - tests/test_slips_utils.py - - tests/test_slips.py - - tests/test_profiler.py - - tests/test_leak_detector.py - - tests/test_ip_info.py - - tests/test_evidence.py - - tests/test_asn_info.py - - tests/test_urlhaus.py - - tests/test_markov_chain.py - - tests/test_daemon.py - - tests/test_go_director.py - - tests/test_notify.py - - tests/test_checker.py - - tests/test_base_model.py - - tests/test_set_evidence.py - - tests/test_trustdb.py - - tests/test_cesnet.py - - tests/test_output.py - - tests/test_riskiq.py - - tests/test_spamhaus.py - - tests/test_circllu.py - - tests/test_evidence_handler.py - - tests/test_alert_handler.py - - tests/test_redis_manager.py - - tests/test_ioc_handler.py - - tests/test_timeline.py - - tests/test_database.py - - tests/test_symbols_handler.py + - test_inputProc.py + - test_main.py + - test_conn.py + - test_downloaded_file.py + - test_ssl.py + - test_tunnel.py + - test_ssh.py + - test_dns.py + - test_notice.py + - test_software.py + - test_smtp.py + - test_whitelist.py + - test_arp.py + - test_blocking.py + - test_flow_handler.py + - test_horizontal_portscans.py + - test_http_analyzer.py + - test_vertical_portscans.py + - test_network_discovery.py + - test_virustotal.py + - test_update_file_manager.py + - test_threat_intelligence.py + - test_slips_utils.py + - test_slips.py + - test_profiler.py + - test_leak_detector.py + - test_ip_info.py + - test_evidence.py + - test_asn_info.py + - test_urlhaus.py + - test_markov_chain.py + - test_daemon.py + - test_go_director.py + - test_notify.py + - test_checker.py + - test_base_model.py + - test_set_evidence.py + - test_trustdb.py + - test_cesnet.py + - test_output.py + - test_riskiq.py + - test_spamhaus.py + - test_circllu.py + - test_evidence_handler.py + - test_alert_handler.py + - test_redis_manager.py + - test_ioc_handler.py + - test_timeline.py + - test_database.py + - test_symbols_handler.py steps: - uses: actions/checkout@v4 @@ -159,7 +159,7 @@ jobs: - name: Run Unit Tests for ${{ matrix.test_file }} run: | - python3 -m pytest ${{ matrix.test_file }} -p no:warnings -vv -s -n 5 + python3 -m pytest tests/${{ matrix.test_file }} -p no:warnings -vv -s -n 5 - name: Upload Artifacts if: success() || failure() From f35b258bc751ca66ceb64418bc5a420817a9ebe4 Mon Sep 17 00:00:00 2001 From: alya Date: Fri, 1 Nov 2024 22:20:01 +0200 Subject: [PATCH 021/128] CI: unit-tests: run unit tests without installing zeek --- .github/workflows/unit-tests.yml | 25 ------------------------- 1 file changed, 25 deletions(-) diff --git a/.github/workflows/unit-tests.yml b/.github/workflows/unit-tests.yml index 7266267b3..5212d20f5 100644 --- a/.github/workflows/unit-tests.yml +++ b/.github/workflows/unit-tests.yml @@ -29,20 +29,6 @@ jobs: /var/lib/apt/lists key: apt-cache - - name: Install Zeek - run: | - echo 'deb http://download.opensuse.org/repositories/security:/zeek/xUbuntu_22.04/ /' | sudo tee /etc/apt/sources.list.d/security:zeek.list - curl -fsSL https://download.opensuse.org/repositories/security:zeek/xUbuntu_22.04/Release.key | gpg --dearmor | sudo tee /etc/apt/trusted.gpg.d/security_zeek.gpg > /dev/null - sudo apt update && sudo apt install -y --no-install-recommends --fix-missing zeek - sudo ln -s /opt/zeek/bin/zeek /usr/local/bin/bro - - - - name: Cache Zeek Build - uses: actions/cache@v3 - with: - path: /opt/zeek - key: zeek-cache - - name: Set up Python 3.10.12 uses: actions/setup-python@v5 with: @@ -117,17 +103,6 @@ jobs: ref: ${{ github.ref }} fetch-depth: '' - - name: Restore Zeek Build from Cache - id: zeek-cache - uses: actions/cache@v3 - with: - path: /opt/zeek - key: zeek-cache - - - if: ${{ steps.zeek-cache.outputs.cache-hit == 'true' }} - name: Echo restored from cache - continue-on-error: true - run: echo "Restored Zeek dependencies from cache successfully" - name: Restore APT cache id: apt-cache From 1988fcd6bb375221757442cf4e07fd40220f1208 Mon Sep 17 00:00:00 2001 From: alya Date: Fri, 1 Nov 2024 22:40:37 +0200 Subject: [PATCH 022/128] CI: make installing slips a reusable workflow --- .../workflows/install-slips-dependencies.yml | 57 +++++++++++++++++++ .github/workflows/integration-tests.yml | 47 ++------------- .github/workflows/unit-tests.yml | 37 ++---------- 3 files changed, 67 insertions(+), 74 deletions(-) create mode 100644 .github/workflows/install-slips-dependencies.yml diff --git a/.github/workflows/install-slips-dependencies.yml b/.github/workflows/install-slips-dependencies.yml new file mode 100644 index 000000000..18e1c2eb0 --- /dev/null +++ b/.github/workflows/install-slips-dependencies.yml @@ -0,0 +1,57 @@ +name: Install Slips Dependencies + +on: + # workflow_call make this workflow re-usable + workflow_call: + # these are like variables to make the workflow more clean + # we can pass these variable from another workflows if we want + inputs: + zeek-repo-url: + description: 'Zeek repository URL' + required: true + default: 'http://download.opensuse.org/repositories/security:/zeek/xUbuntu_22.04/' + zeek-key-url: + description: 'Zeek key URL' + required: true + default: 'https://download.opensuse.org/repositories/security:zeek/xUbuntu_22.04/Release.key' + python-version: + description: 'Python version to set up' + required: false + default: '3.10.12' + +jobs: + install-dependencies: + runs-on: ubuntu-22.04 + steps: + - uses: actions/checkout@v4 + with: + ref: ${{ github.ref }} + fetch-depth: '' + + - name: Enable memory overcommit (for Redis) + run: sysctl vm.overcommit_memory=1 + + - name: Install APT dependencies + run: sudo apt-get update --fix-missing && sudo apt-get -y --no-install-recommends install $(cat install/apt_dependencies.txt) + + - name: Save APT Cache + uses: actions/cache@v3 + with: + path: | + /var/cache/apt/archives + /var/lib/apt/lists + key: apt-cache + + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: ${{ inputs.python-version }} + cache: 'pip' + run: pip install -r install/requirements.txt + + - name: Install Zeek + run: | + echo "deb ${{ inputs.zeek-repo-url }} /" | sudo tee /etc/apt/sources.list.d/security:zeek.list + curl -fsSL ${{ inputs.zeek-key-url }} | sudo gpg --dearmor -o /etc/apt/trusted.gpg.d/security_zeek.gpg + sudo apt update && sudo apt install -y --no-install-recommends zeek + sudo ln -s /opt/zeek/bin/zeek /usr/local/bin/bro diff --git a/.github/workflows/integration-tests.yml b/.github/workflows/integration-tests.yml index e9bd64817..3d1475a6b 100644 --- a/.github/workflows/integration-tests.yml +++ b/.github/workflows/integration-tests.yml @@ -7,51 +7,12 @@ on: - 'develop' jobs: - install-dependencies: - runs-on: ubuntu-22.04 - steps: - - uses: actions/checkout@v4 - with: - ref: ${{ github.ref }} - fetch-depth: '' - - - name: Enable memory overcommit (for redis) - run: sysctl vm.overcommit_memory=1 - - - name: Install APT dependencies - run: sudo apt-get update --fix-missing && sudo apt-get -y --no-install-recommends install $(cat install/apt_dependencies.txt) - - - name: Save APT Cache - uses: actions/cache@v3 - with: - path: | - /var/cache/apt/archives - /var/lib/apt/lists - key: apt-cache - - - name: Install Zeek - run: | - echo 'deb http://download.opensuse.org/repositories/security:/zeek/xUbuntu_22.04/ /' | sudo tee /etc/apt/sources.list.d/security:zeek.list - curl -fsSL https://download.opensuse.org/repositories/security:zeek/xUbuntu_22.04/Release.key | gpg --dearmor | sudo tee /etc/apt/trusted.gpg.d/security_zeek.gpg > /dev/null - sudo apt update && sudo apt install -y --no-install-recommends --fix-missing zeek - sudo ln -s /opt/zeek/bin/zeek /usr/local/bin/bro - - - - name: Cache Zeek Build - uses: actions/cache@v3 - with: - path: /opt/zeek - key: zeek-cache - - - name: Set up Python 3.10.12 - uses: actions/setup-python@v5 - with: - python-version: "3.10.12" - cache: 'pip' # caching pip dependencies - run: pip install -r install/requirements.txt + # uses the common workflow that builds slips + install-dependencies-using-reusable-workflow: + uses: ./.github/workflows/install-slips-dependencies.yaml - tests: + integration-tests: runs-on: ubuntu-22.04 timeout-minutes: 1800 # make this job depend on the first job diff --git a/.github/workflows/unit-tests.yml b/.github/workflows/unit-tests.yml index 5212d20f5..6c7c0929e 100644 --- a/.github/workflows/unit-tests.yml +++ b/.github/workflows/unit-tests.yml @@ -6,42 +6,17 @@ on: - 'master' - 'develop' -jobs: - install-dependencies: - runs-on: ubuntu-22.04 - steps: - - uses: actions/checkout@v4 - with: - ref: ${{ github.ref }} - fetch-depth: '' - - - name: Enable memory overcommit (for redis) - run: sysctl vm.overcommit_memory=1 - - - name: Install APT dependencies - run: sudo apt-get update --fix-missing && sudo apt-get -y --no-install-recommends install $(cat install/apt_dependencies.txt) - - - name: Save APT Cache - uses: actions/cache@v3 - with: - path: | - /var/cache/apt/archives - /var/lib/apt/lists - key: apt-cache - - - name: Set up Python 3.10.12 - uses: actions/setup-python@v5 - with: - python-version: "3.10.12" - cache: 'pip' # caching pip dependencies - run: pip install -r install/requirements.txt +jobs: + # uses the common workflow that builds slips + install-dependencies-using-reusable-workflow: + uses: ./.github/workflows/install-slips-dependencies.yaml - tests: + unit-tests: runs-on: ubuntu-22.04 timeout-minutes: 1800 # make this job depend on the first job - needs: install-dependencies + needs: install-dependencies-using-reusable-workflow strategy: matrix: From cf2a88d6fb1e426b76690bb3503d68e19c480d1a Mon Sep 17 00:00:00 2001 From: alya Date: Fri, 1 Nov 2024 22:42:44 +0200 Subject: [PATCH 023/128] CI: fix a typo in install-slips-dependencies.yml --- .github/workflows/integration-tests.yml | 2 +- .github/workflows/unit-tests.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/integration-tests.yml b/.github/workflows/integration-tests.yml index 3d1475a6b..748ae6e94 100644 --- a/.github/workflows/integration-tests.yml +++ b/.github/workflows/integration-tests.yml @@ -9,7 +9,7 @@ on: jobs: # uses the common workflow that builds slips install-dependencies-using-reusable-workflow: - uses: ./.github/workflows/install-slips-dependencies.yaml + uses: ./.github/workflows/install-slips-dependencies.yml integration-tests: diff --git a/.github/workflows/unit-tests.yml b/.github/workflows/unit-tests.yml index 6c7c0929e..d42089ca1 100644 --- a/.github/workflows/unit-tests.yml +++ b/.github/workflows/unit-tests.yml @@ -10,7 +10,7 @@ on: jobs: # uses the common workflow that builds slips install-dependencies-using-reusable-workflow: - uses: ./.github/workflows/install-slips-dependencies.yaml + uses: ./.github/workflows/install-slips-dependencies.yml unit-tests: runs-on: ubuntu-22.04 From 2a967167b67f5b94b778f325116ef1bc1be8809a Mon Sep 17 00:00:00 2001 From: alya Date: Fri, 1 Nov 2024 22:47:56 +0200 Subject: [PATCH 024/128] CI: make hardcoded zeek repos optional variables --- .github/workflows/install-slips-dependencies.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/install-slips-dependencies.yml b/.github/workflows/install-slips-dependencies.yml index 18e1c2eb0..f84bff89b 100644 --- a/.github/workflows/install-slips-dependencies.yml +++ b/.github/workflows/install-slips-dependencies.yml @@ -8,11 +8,11 @@ on: inputs: zeek-repo-url: description: 'Zeek repository URL' - required: true + required: false default: 'http://download.opensuse.org/repositories/security:/zeek/xUbuntu_22.04/' zeek-key-url: description: 'Zeek key URL' - required: true + required: false default: 'https://download.opensuse.org/repositories/security:zeek/xUbuntu_22.04/Release.key' python-version: description: 'Python version to set up' From 4f30789be7a7e9cfe19a8b8bb001395784414fb0 Mon Sep 17 00:00:00 2001 From: alya Date: Fri, 1 Nov 2024 22:57:22 +0200 Subject: [PATCH 025/128] CI: fix installing python dependencies --- .github/workflows/install-slips-dependencies.yml | 6 ++++-- install/install.sh | 8 -------- 2 files changed, 4 insertions(+), 10 deletions(-) diff --git a/.github/workflows/install-slips-dependencies.yml b/.github/workflows/install-slips-dependencies.yml index f84bff89b..d35d8e05a 100644 --- a/.github/workflows/install-slips-dependencies.yml +++ b/.github/workflows/install-slips-dependencies.yml @@ -42,12 +42,14 @@ jobs: /var/lib/apt/lists key: apt-cache - - name: Set up Python + - name: Set up Python with caching enabled uses: actions/setup-python@v5 with: python-version: ${{ inputs.python-version }} cache: 'pip' - run: pip install -r install/requirements.txt + + - name: Install Python dependencies + run: python3 -m pip install -r install/requirements.txt - name: Install Zeek run: | diff --git a/install/install.sh b/install/install.sh index caaf1f651..0ed8aab37 100755 --- a/install/install.sh +++ b/install/install.sh @@ -137,14 +137,6 @@ curl -o- https://raw.githubusercontent.com/nvm-sh/nvm/v0.39.7/install.sh | sudo exit_on_cmd_failure -print_green "Installing IDMEFv2 python library" -git clone https://github.com/IDMEFv2/python-idmefv2 \ - && cd python-idmefv2 \ - && git submodule update --init --recursive \ - && python3 setup.py install && cd .. - -exit_on_cmd_failure - print_green "Installing p2p4slips" # build the pigeon and Add pigeon to path git submodule init && git submodule update && cd p2p4slips && go build && export PATH=$PATH:$(pwd) >> ~/.bashrc && cd .. From 054b7c2660ec35553cc7cee2a6900ab21ff04cdb Mon Sep 17 00:00:00 2001 From: alya Date: Fri, 1 Nov 2024 23:00:17 +0200 Subject: [PATCH 026/128] CI: integration-tests: fix using reusable workflow --- .github/workflows/integration-tests.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/integration-tests.yml b/.github/workflows/integration-tests.yml index 748ae6e94..92ef8fe1b 100644 --- a/.github/workflows/integration-tests.yml +++ b/.github/workflows/integration-tests.yml @@ -16,7 +16,7 @@ jobs: runs-on: ubuntu-22.04 timeout-minutes: 1800 # make this job depend on the first job - needs: install-dependencies + needs: install-dependencies-using-reusable-workflow strategy: matrix: From f5fbface88b714621e5f8b9a7a33275b629d8fd4 Mon Sep 17 00:00:00 2001 From: alya Date: Fri, 1 Nov 2024 23:02:13 +0200 Subject: [PATCH 027/128] CI: dependencies: add missing "type" property to workflow_call --- .github/workflows/install-slips-dependencies.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/install-slips-dependencies.yml b/.github/workflows/install-slips-dependencies.yml index d35d8e05a..eda20f8a6 100644 --- a/.github/workflows/install-slips-dependencies.yml +++ b/.github/workflows/install-slips-dependencies.yml @@ -10,14 +10,17 @@ on: description: 'Zeek repository URL' required: false default: 'http://download.opensuse.org/repositories/security:/zeek/xUbuntu_22.04/' + type: string zeek-key-url: description: 'Zeek key URL' required: false default: 'https://download.opensuse.org/repositories/security:zeek/xUbuntu_22.04/Release.key' + type: string python-version: description: 'Python version to set up' required: false default: '3.10.12' + type: string jobs: install-dependencies: From 4a06d6c82d03a1c628a89f0f7c315c0359de30e1 Mon Sep 17 00:00:00 2001 From: alya Date: Mon, 4 Nov 2024 12:20:09 +0200 Subject: [PATCH 028/128] CI: make sure integration-tests can access Zeek from the cache if it was successfully installed by install-dependencies-using-reusable-workflo --- .github/workflows/integration-tests.yml | 11 ++--------- tests/module_factory.py | 2 +- tests/test_database.py | 4 +++- 3 files changed, 6 insertions(+), 11 deletions(-) diff --git a/.github/workflows/integration-tests.yml b/.github/workflows/integration-tests.yml index 92ef8fe1b..25e59996f 100644 --- a/.github/workflows/integration-tests.yml +++ b/.github/workflows/integration-tests.yml @@ -38,10 +38,8 @@ jobs: path: /opt/zeek key: zeek-cache - - if: ${{ steps.zeek-cache.outputs.cache-hit == 'true' }} - name: Echo restored from cache - continue-on-error: true - run: echo "Restored Zeek dependencies from cache successfully" + - name: Create Zeek symlink + run: sudo ln -s /opt/zeek/bin/zeek /usr/local/bin/bro - name: Restore APT cache id: apt-cache @@ -52,11 +50,6 @@ jobs: /var/lib/apt/lists key: apt-cache - - if: ${{ steps.apt-cache.outputs.cache-hit == 'true' }} - name: Echo restored from cache - continue-on-error: true - run: echo "Restored APT dependencies from cache successfully" - - name: Install Python dependencies (from cache if possible) run: | python3 -m pip install --upgrade pip diff --git a/tests/module_factory.py b/tests/module_factory.py index bd9189e96..5cc644197 100644 --- a/tests/module_factory.py +++ b/tests/module_factory.py @@ -115,7 +115,7 @@ def create_db_manager_obj( start_redis_server=True, ): """ - flush_db is False by default because we use this funtion to check + flush_db is False by default because we use this function to check the db after integration tests to make sure everything's going fine """ # to prevent config/redis.conf from being overwritten diff --git a/tests/test_database.py b/tests/test_database.py index b546bd88d..84a5b0391 100644 --- a/tests/test_database.py +++ b/tests/test_database.py @@ -179,7 +179,9 @@ def test_profile_moddule_labels(): def test_add_mac_addr_to_profile(): - db = ModuleFactory().create_db_manager_obj(1234, flush_db=True) + db = ModuleFactory().create_db_manager_obj( + get_random_port(), flush_db=True + ) ipv4 = "192.168.1.5" profileid_ipv4 = f"profile_{ipv4}" mac_addr = "00:00:5e:00:53:af" From 1c88582b8936f6aab6a535f9ceb964057a4bc88d Mon Sep 17 00:00:00 2001 From: alya Date: Wed, 30 Oct 2024 17:05:35 +0300 Subject: [PATCH 029/128] move all the profiling logic from main to managers/profiler_manager.py --- managers/profilers_manager.py | 146 ++++++++++++++++++++++++++++++++++ slips/main.py | 126 +---------------------------- 2 files changed, 150 insertions(+), 122 deletions(-) create mode 100644 managers/profilers_manager.py diff --git a/managers/profilers_manager.py b/managers/profilers_manager.py new file mode 100644 index 000000000..35c2a0772 --- /dev/null +++ b/managers/profilers_manager.py @@ -0,0 +1,146 @@ +import multiprocessing +import os +import subprocess +import sys +import time + +class ProfilersManager: + def __init__(self, main): + self.main = main + self.read_configurations() + + def read_configurations(self): + self.cpu_profiler_enabled = self.main.conf.get_cpu_profiler_enable() + self.cpu_profiler_mode = self.main.conf.get_cpu_profiler_mode() + self.cpu_profiler_multiprocess = ( + self.main.conf.get_cpu_profiler_multiprocess() + ) + self.cpu_profiler_dev_mode_entries = ( + self.main.conf.get_cpu_profiler_dev_mode_entries() + ) + self.cpu_profiler_output_limit \ + = self.main.conf.get_cpu_profiler_output_limit(), + self.cpu_profiler_sampling_interval = ( + self.main.conf.get_cpu_profiler_sampling_interval() + ) + + self.memory_profiler_mode = self.main.conf.get_memory_profiler_mode() + self.memory_profiler_enabled = self.main.conf.get_memory_profiler_enable() + self.memory_profiler_multiprocess = ( + self.main.conf.get_memory_profiler_multiprocess() + ) + def cpu_profiler_init(self): + from slips_files.common.performance_profilers.cpu_profiler import CPUProfiler + if not self.cpu_profiler_enabled: + return + try: + if ( + self.cpu_profiler_multiprocess + and self.cpu_profiler_mode == "dev" + ): + args = sys.argv + if args[-1] != "--no-recurse": + tracer_entries = str( + self.cpu_profiler_dev_mode_entries + ) + viz_args = [ + "viztracer", + "--tracer_entries", + tracer_entries, + "--max_stack_depth", + "10", + "-o", + str( + os.path.join( + self.args.output, + "cpu_profiling_result.json", + ) + ), + ] + viz_args.extend(args) + viz_args.append("--no-recurse") + print( + "Starting multiprocess profiling recursive subprocess" + ) + subprocess.run(viz_args) + exit(0) + else: + self.cpu_profiler = CPUProfiler( + db=self.main.db, + output=self.args.output, + mode=self.cpu_profiler_mode, + limit=self.cpu_profiler_output_limit, + interval=self.cpu_profiler_sampling_interval, + ) + self.cpu_profiler.start() + except Exception as e: + print(e) + self.cpu_profiler_enabled = False + + def cpu_profiler_release(self): + if hasattr(self, "cpuProfilerEnabled"): + if self.cpu_profiler_enabled and not self.cpu_profiler_multiprocess: + self.cpu_profiler.stop() + self.cpu_profiler.print() + + def memory_profiler_init(self): + from slips_files.common.performance_profilers.memory_profiler import ( + MemoryProfiler, + ) + + if not self.memory_profiler_enabled: + return + + output_dir = os.path.join(self.args.output, "memoryprofile/") + if not os.path.exists(output_dir): + os.makedirs(output_dir) + output_file = os.path.join(output_dir, "memory_profile.bin") + self.memory_profiler = MemoryProfiler( + output_file, + db=self.main.db, + mode=self.memory_profiler_mode, + multiprocess=self.memory_profiler_multiprocess, + ) + self.memory_profiler.start() + + + def memory_profiler_release(self): + if ( + hasattr(self, "memoryProfilerEnabled") + and self.memory_profiler_enabled + ): + self.memory_profiler.stop() + + def memory_profiler_multiproc_test(self): + def target_function(): + print("Target function started") + time.sleep(5) + + def mem_function(): + print("Mem function started") + while True: + time.sleep(1) + array = [] + for i in range(1000000): + array.append(i) + + processes = [] + num_processes = 3 + + for _ in range(num_processes): + process = multiprocessing.Process( + target=target_function if _ % 2 else mem_function + ) + process.start() + processes.append(process) + + # Message passing + self.main.db.publish("memory_profile", processes[1].pid) # successful + # target_function will timeout and tracker will be cleared + time.sleep(5) + # end but maybe don't start + self.main.db.publish("memory_profile", processes[0].pid) + time.sleep(5) # mem_function will get tracker started + # start successfully + self.main.db.publish("memory_profile", processes[0].pid) + input() \ No newline at end of file diff --git a/slips/main.py b/slips/main.py index 8e28358d4..4276769fe 100644 --- a/slips/main.py +++ b/slips/main.py @@ -15,13 +15,10 @@ from managers.host_ip_manager import HostIPManager from managers.metadata_manager import MetadataManager from managers.process_manager import ProcessManager +from managers.profilers_manager import ProfilersManager from managers.redis_manager import RedisManager from managers.ui_manager import UIManager from slips_files.common.parsers.config_parser import ConfigParser -from slips_files.common.performance_profilers.cpu_profiler import CPUProfiler -from slips_files.common.performance_profilers.memory_profiler import ( - MemoryProfiler, -) from slips_files.common.printer import Printer from slips_files.common.slips_utils import utils from slips_files.common.style import green @@ -42,6 +39,7 @@ def __init__(self, testing=False): self.metadata_man = MetadataManager(self) self.conf = ConfigParser() self.ui_man = UIManager(self) + self.profilers_manager = ProfilersManager(self) self.version = utils.get_slips_version() # will be filled later @@ -73,122 +71,6 @@ def __init__(self, testing=False): # should be initialised after self.input_type is set self.host_ip_man = HostIPManager(self) - def cpu_profiler_init(self): - self.cpuProfilerEnabled = self.conf.get_cpu_profiler_enable() - self.cpuProfilerMode = self.conf.get_cpu_profiler_mode() - self.cpuProfilerMultiprocess = ( - self.conf.get_cpu_profiler_multiprocess() - ) - if self.cpuProfilerEnabled: - try: - if ( - self.cpuProfilerMultiprocess - and self.cpuProfilerMode == "dev" - ): - args = sys.argv - if args[-1] != "--no-recurse": - tracer_entries = str( - self.conf.get_cpu_profiler_dev_mode_entries() - ) - viz_args = [ - "viztracer", - "--tracer_entries", - tracer_entries, - "--max_stack_depth", - "10", - "-o", - str( - os.path.join( - self.args.output, - "cpu_profiling_result.json", - ) - ), - ] - viz_args.extend(args) - viz_args.append("--no-recurse") - print( - "Starting multiprocess profiling recursive subprocess" - ) - subprocess.run(viz_args) - exit(0) - else: - self.cpuProfiler = CPUProfiler( - db=self.db, - output=self.args.output, - mode=self.conf.get_cpu_profiler_mode(), - limit=self.conf.get_cpu_profiler_output_limit(), - interval=self.conf.get_cpu_profiler_sampling_interval(), - ) - self.cpuProfiler.start() - except Exception as e: - print(e) - self.cpuProfilerEnabled = False - - def cpu_profiler_release(self): - if hasattr(self, "cpuProfilerEnabled"): - if self.cpuProfilerEnabled and not self.cpuProfilerMultiprocess: - self.cpuProfiler.stop() - self.cpuProfiler.print() - - def memory_profiler_init(self): - self.memoryProfilerEnabled = self.conf.get_memory_profiler_enable() - memoryProfilerMode = self.conf.get_memory_profiler_mode() - memoryProfilerMultiprocess = ( - self.conf.get_memory_profiler_multiprocess() - ) - if self.memoryProfilerEnabled: - output_dir = os.path.join(self.args.output, "memoryprofile/") - if not os.path.exists(output_dir): - os.makedirs(output_dir) - output_file = os.path.join(output_dir, "memory_profile.bin") - self.memoryProfiler = MemoryProfiler( - output_file, - db=self.db, - mode=memoryProfilerMode, - multiprocess=memoryProfilerMultiprocess, - ) - self.memoryProfiler.start() - - def memory_profiler_release(self): - if ( - hasattr(self, "memoryProfilerEnabled") - and self.memoryProfilerEnabled - ): - self.memoryProfiler.stop() - - def memory_profiler_multiproc_test(self): - def target_function(): - print("Target function started") - time.sleep(5) - - def mem_function(): - print("Mem function started") - while True: - time.sleep(1) - array = [] - for i in range(1000000): - array.append(i) - - processes = [] - num_processes = 3 - - for _ in range(num_processes): - process = multiprocessing.Process( - target=target_function if _ % 2 else mem_function - ) - process.start() - processes.append(process) - - # Message passing - self.db.publish("memory_profile", processes[1].pid) # successful - # target_function will timeout and tracker will be cleared - time.sleep(5) - # end but maybe don't start - self.db.publish("memory_profile", processes[0].pid) - time.sleep(5) # mem_function will get tracker started - # start successfully - self.db.publish("memory_profile", processes[0].pid) - input() def check_zeek_or_bro(self): """ @@ -617,8 +499,8 @@ def start(self): 1, 0, ) - self.cpu_profiler_init() - self.memory_profiler_init() + self.profilers_manager.cpu_profiler_init() + self.profilers_manager.memory_profiler_init() if self.args.growing: if self.input_type != "zeek_folder": From 79c08166fa48fea6e0b339c222c492a24624ec6f Mon Sep 17 00:00:00 2001 From: alya Date: Wed, 30 Oct 2024 17:06:34 +0300 Subject: [PATCH 030/128] profiler_manager.py: delete the memory_profiler_multiproc_test() --- managers/profilers_manager.py | 35 ++--------------------------------- 1 file changed, 2 insertions(+), 33 deletions(-) diff --git a/managers/profilers_manager.py b/managers/profilers_manager.py index 35c2a0772..4f6e1a8c9 100644 --- a/managers/profilers_manager.py +++ b/managers/profilers_manager.py @@ -29,6 +29,8 @@ def read_configurations(self): self.memory_profiler_multiprocess = ( self.main.conf.get_memory_profiler_multiprocess() ) + + def cpu_profiler_init(self): from slips_files.common.performance_profilers.cpu_profiler import CPUProfiler if not self.cpu_profiler_enabled: @@ -111,36 +113,3 @@ def memory_profiler_release(self): ): self.memory_profiler.stop() - def memory_profiler_multiproc_test(self): - def target_function(): - print("Target function started") - time.sleep(5) - - def mem_function(): - print("Mem function started") - while True: - time.sleep(1) - array = [] - for i in range(1000000): - array.append(i) - - processes = [] - num_processes = 3 - - for _ in range(num_processes): - process = multiprocessing.Process( - target=target_function if _ % 2 else mem_function - ) - process.start() - processes.append(process) - - # Message passing - self.main.db.publish("memory_profile", processes[1].pid) # successful - # target_function will timeout and tracker will be cleared - time.sleep(5) - # end but maybe don't start - self.main.db.publish("memory_profile", processes[0].pid) - time.sleep(5) # mem_function will get tracker started - # start successfully - self.main.db.publish("memory_profile", processes[0].pid) - input() \ No newline at end of file From d5f4ae18de2bfc6abe3d50185f54aca9d1586884 Mon Sep 17 00:00:00 2001 From: alya Date: Wed, 30 Oct 2024 17:11:59 +0300 Subject: [PATCH 031/128] profiler_manager.py: import viztracer and memray on demand --- managers/profilers_manager.py | 38 +++++++++++++++-------------- slips/main.py | 1 - slips_files/core/evidencehandler.py | 2 -- 3 files changed, 20 insertions(+), 21 deletions(-) diff --git a/managers/profilers_manager.py b/managers/profilers_manager.py index 4f6e1a8c9..f1005c168 100644 --- a/managers/profilers_manager.py +++ b/managers/profilers_manager.py @@ -13,28 +13,28 @@ def read_configurations(self): self.cpu_profiler_enabled = self.main.conf.get_cpu_profiler_enable() self.cpu_profiler_mode = self.main.conf.get_cpu_profiler_mode() self.cpu_profiler_multiprocess = ( - self.main.conf.get_cpu_profiler_multiprocess() - ) + self.main.conf.get_cpu_profiler_multiprocess()) self.cpu_profiler_dev_mode_entries = ( - self.main.conf.get_cpu_profiler_dev_mode_entries() - ) + self.main.conf.get_cpu_profiler_dev_mode_entries()) self.cpu_profiler_output_limit \ = self.main.conf.get_cpu_profiler_output_limit(), self.cpu_profiler_sampling_interval = ( - self.main.conf.get_cpu_profiler_sampling_interval() - ) + self.main.conf.get_cpu_profiler_sampling_interval()) self.memory_profiler_mode = self.main.conf.get_memory_profiler_mode() - self.memory_profiler_enabled = self.main.conf.get_memory_profiler_enable() + self.memory_profiler_enabled = ( + self.main.conf.get_memory_profiler_enable()) self.memory_profiler_multiprocess = ( - self.main.conf.get_memory_profiler_multiprocess() - ) + self.main.conf.get_memory_profiler_multiprocess()) def cpu_profiler_init(self): - from slips_files.common.performance_profilers.cpu_profiler import CPUProfiler if not self.cpu_profiler_enabled: return + + from slips_files.common.performance_profilers.cpu_profiler import ( + CPUProfiler) + try: if ( self.cpu_profiler_multiprocess @@ -80,19 +80,21 @@ def cpu_profiler_init(self): self.cpu_profiler_enabled = False def cpu_profiler_release(self): - if hasattr(self, "cpuProfilerEnabled"): - if self.cpu_profiler_enabled and not self.cpu_profiler_multiprocess: - self.cpu_profiler.stop() - self.cpu_profiler.print() + if not hasattr(self, "cpuProfilerEnabled"): + return + + if (self.cpu_profiler_enabled + and not self.cpu_profiler_multiprocess): + self.cpu_profiler.stop() + self.cpu_profiler.print() def memory_profiler_init(self): + if not self.memory_profiler_enabled: + return + from slips_files.common.performance_profilers.memory_profiler import ( MemoryProfiler, ) - - if not self.memory_profiler_enabled: - return - output_dir = os.path.join(self.args.output, "memoryprofile/") if not os.path.exists(output_dir): os.makedirs(output_dir) diff --git a/slips/main.py b/slips/main.py index 4276769fe..04e3aa3e0 100644 --- a/slips/main.py +++ b/slips/main.py @@ -40,7 +40,6 @@ def __init__(self, testing=False): self.conf = ConfigParser() self.ui_man = UIManager(self) self.profilers_manager = ProfilersManager(self) - self.version = utils.get_slips_version() # will be filled later self.commit = "None" diff --git a/slips_files/core/evidencehandler.py b/slips_files/core/evidencehandler.py index 71a721a6f..d7b5b487a 100644 --- a/slips_files/core/evidencehandler.py +++ b/slips_files/core/evidencehandler.py @@ -21,8 +21,6 @@ import json from typing import List, Dict, Optional from datetime import datetime - -# from colorama import Fore, Style from os import path import sys import os From 2bb024c5e628d23890b2642c73c1b838d5feeafb Mon Sep 17 00:00:00 2001 From: alya Date: Wed, 30 Oct 2024 17:12:23 +0300 Subject: [PATCH 032/128] move the logic of stopping the profilers to process_manager's shutdown_gracefully() --- managers/process_manager.py | 4 ++++ slips.py | 2 -- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/managers/process_manager.py b/managers/process_manager.py index fc1521149..c3a007717 100644 --- a/managers/process_manager.py +++ b/managers/process_manager.py @@ -711,6 +711,10 @@ def shutdown_gracefully(self): if self.main.conf.export_labeled_flows(): format_ = self.main.conf.export_labeled_flows_to().lower() self.main.db.export_labeled_flows(format_) + + self.main.profilers_manager.cpu_profiler_release() + self.main.profilers_manager.memory_profiler_release() + # if store_a_copy_of_zeek_files is set to yes in slips.yaml # copy the whole zeek_files dir to the output dir diff --git a/slips.py b/slips.py index 8110106d3..62a8efbe2 100755 --- a/slips.py +++ b/slips.py @@ -61,5 +61,3 @@ # interactive mode slips.start() - slips.cpu_profiler_release() - slips.memory_profiler_release() From e8f2a47efaca95a12de1b5c267c346670f6a4bd0 Mon Sep 17 00:00:00 2001 From: alya Date: Wed, 30 Oct 2024 17:33:52 +0300 Subject: [PATCH 033/128] remove vistrace, memray, tqdm, setuptools and wheel from requirements.txt --- install/requirements.txt | 6 ------ 1 file changed, 6 deletions(-) diff --git a/install/requirements.txt b/install/requirements.txt index 9fa899129..ef2301e7f 100644 --- a/install/requirements.txt +++ b/install/requirements.txt @@ -9,7 +9,6 @@ cabby==0.1.23 stix2==3.0.1 certifi==2024.7.4 tensorflow==2.16.1 -colorama==0.4.6 Keras validators==0.34.0 ipwhois==1.2.0 @@ -27,18 +26,13 @@ scikit-learn==1.5.1 GitPython==3.1.43 protobuf==4.25.3 exclusiveprocess==0.9.4 -setuptools pytest-dependency==0.6.0 whois==1.20240129.2 -wheel flask tldextract==5.1.2 -tqdm==4.66.5 termcolor==2.4.0 -viztracer==0.16.3 yappi==1.6.0 pytest-sugar==1.0.0 -memray==1.13.4 aid_hash black==24.8.0 ruff==0.6.8 From 27dada8b4f4d7337d2d974b6b6b4ad12d1e1c103 Mon Sep 17 00:00:00 2001 From: alya Date: Thu, 31 Oct 2024 16:49:32 +0300 Subject: [PATCH 034/128] update profiler unit tests --- managers/profilers_manager.py | 51 ++++++++++++++++++----------------- tests/test_main.py | 45 ++++++++++++++++--------------- 2 files changed, 50 insertions(+), 46 deletions(-) diff --git a/managers/profilers_manager.py b/managers/profilers_manager.py index f1005c168..b184ebd67 100644 --- a/managers/profilers_manager.py +++ b/managers/profilers_manager.py @@ -1,39 +1,44 @@ -import multiprocessing import os import subprocess import sys -import time + class ProfilersManager: def __init__(self, main): self.main = main self.read_configurations() - + def read_configurations(self): self.cpu_profiler_enabled = self.main.conf.get_cpu_profiler_enable() self.cpu_profiler_mode = self.main.conf.get_cpu_profiler_mode() self.cpu_profiler_multiprocess = ( - self.main.conf.get_cpu_profiler_multiprocess()) + self.main.conf.get_cpu_profiler_multiprocess() + ) self.cpu_profiler_dev_mode_entries = ( - self.main.conf.get_cpu_profiler_dev_mode_entries()) - self.cpu_profiler_output_limit \ - = self.main.conf.get_cpu_profiler_output_limit(), + self.main.conf.get_cpu_profiler_dev_mode_entries() + ) + self.cpu_profiler_output_limit = ( + self.main.conf.get_cpu_profiler_output_limit(), + ) self.cpu_profiler_sampling_interval = ( - self.main.conf.get_cpu_profiler_sampling_interval()) - + self.main.conf.get_cpu_profiler_sampling_interval() + ) + self.memory_profiler_mode = self.main.conf.get_memory_profiler_mode() self.memory_profiler_enabled = ( - self.main.conf.get_memory_profiler_enable()) + self.main.conf.get_memory_profiler_enable() + ) self.memory_profiler_multiprocess = ( - self.main.conf.get_memory_profiler_multiprocess()) - - + self.main.conf.get_memory_profiler_multiprocess() + ) + def cpu_profiler_init(self): if not self.cpu_profiler_enabled: return from slips_files.common.performance_profilers.cpu_profiler import ( - CPUProfiler) + CPUProfiler, + ) try: if ( @@ -42,9 +47,7 @@ def cpu_profiler_init(self): ): args = sys.argv if args[-1] != "--no-recurse": - tracer_entries = str( - self.cpu_profiler_dev_mode_entries - ) + tracer_entries = str(self.cpu_profiler_dev_mode_entries) viz_args = [ "viztracer", "--tracer_entries", @@ -78,13 +81,12 @@ def cpu_profiler_init(self): except Exception as e: print(e) self.cpu_profiler_enabled = False - + def cpu_profiler_release(self): - if not hasattr(self, "cpuProfilerEnabled"): + if not hasattr(self, "cpu_profiler_enabled"): return - - if (self.cpu_profiler_enabled - and not self.cpu_profiler_multiprocess): + + if self.cpu_profiler_enabled and not self.cpu_profiler_multiprocess: self.cpu_profiler.stop() self.cpu_profiler.print() @@ -95,6 +97,7 @@ def memory_profiler_init(self): from slips_files.common.performance_profilers.memory_profiler import ( MemoryProfiler, ) + output_dir = os.path.join(self.args.output, "memoryprofile/") if not os.path.exists(output_dir): os.makedirs(output_dir) @@ -107,11 +110,9 @@ def memory_profiler_init(self): ) self.memory_profiler.start() - def memory_profiler_release(self): if ( - hasattr(self, "memoryProfilerEnabled") + hasattr(self, "memory_profiler_enabled") and self.memory_profiler_enabled ): self.memory_profiler.stop() - diff --git a/tests/test_main.py b/tests/test_main.py index ecc8233a1..d2615efea 100644 --- a/tests/test_main.py +++ b/tests/test_main.py @@ -72,41 +72,44 @@ def test_cpu_profiler_release_enabled( expected_print_calls, ): main = ModuleFactory().create_main_obj() - main.cpuProfilerEnabled = True - main.cpuProfilerMultiprocess = cpu_profiler_multiprocess - - main.cpuProfiler = MagicMock() - - main.cpu_profiler_release() + main.profilers_manager.cpu_profiler_enabled = True + main.profilers_manager.cpu_profiler_multiprocess = ( + cpu_profiler_multiprocess + ) + main.profilers_manager.cpu_profiler = MagicMock() + main.profilers_manager.cpu_profiler_release() - assert main.cpuProfiler.stop.call_count == expected_stop_calls - assert main.cpuProfiler.print.call_count == expected_print_calls + assert ( + main.profilers_manager.cpu_profiler.stop.call_count + == expected_stop_calls + ) + assert ( + main.profilers_manager.cpu_profiler.print.call_count + == expected_print_calls + ) def test_cpu_profiler_release_disabled(): main = ModuleFactory().create_main_obj() - main.cpuProfilerEnabled = False - main.cpu_profiler_release() - assert not hasattr(main, "memoryProfiler") + main.profilers_manager.cpu_profiler_enabled = False + main.profilers_manager.cpu_profiler_release() + assert not hasattr(main.profilers_manager, "memory_profiler") def test_memory_profiler_release_enabled(): main = ModuleFactory().create_main_obj() - main.memoryProfilerEnabled = True - main.memoryProfiler = MagicMock() - - main.memory_profiler_release() - - main.memoryProfiler.stop.assert_called_once() + main.profilers_manager.memory_profiler_enabled = True + main.profilers_manager.memory_profiler = MagicMock() + main.profilers_manager.memory_profiler_release() + main.profilers_manager.memory_profiler.stop.assert_called_once() def test_memory_profiler_release_disabled(): main = ModuleFactory().create_main_obj() - main.memoryProfilerEnabled = False - - main.memory_profiler_release() + main.profilers_manager.memory_profiler_enabled = False + main.profilers_manager.memory_profiler_release() - assert not hasattr(main, "memoryProfiler") + assert not hasattr(main.profilers_manager, "memory_profiler") @pytest.mark.parametrize( From 6971edd4502b2edb708c794c9206f0d1ee7035ba Mon Sep 17 00:00:00 2001 From: alya Date: Mon, 4 Nov 2024 12:54:57 +0200 Subject: [PATCH 035/128] CI: integration tests: attempt to re-install zeek to be able to restore it from cache --- .github/workflows/integration-tests.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/integration-tests.yml b/.github/workflows/integration-tests.yml index 25e59996f..3c4bc8c21 100644 --- a/.github/workflows/integration-tests.yml +++ b/.github/workflows/integration-tests.yml @@ -39,7 +39,9 @@ jobs: key: zeek-cache - name: Create Zeek symlink - run: sudo ln -s /opt/zeek/bin/zeek /usr/local/bin/bro + run: | + sudo apt install -y --no-install-recommends zeek + sudo ln -s /opt/zeek/bin/zeek /usr/local/bin/bro - name: Restore APT cache id: apt-cache From 68ac8cd1c5ddf5a14b2336b820b452bd6a6f4ccf Mon Sep 17 00:00:00 2001 From: alya Date: Mon, 4 Nov 2024 13:00:38 +0200 Subject: [PATCH 036/128] CI: integration tests: restore apt cache before attempting to restore zeek from cache --- .github/workflows/integration-tests.yml | 20 ++++++++++++-------- 1 file changed, 12 insertions(+), 8 deletions(-) diff --git a/.github/workflows/integration-tests.yml b/.github/workflows/integration-tests.yml index 3c4bc8c21..d5dfce7f4 100644 --- a/.github/workflows/integration-tests.yml +++ b/.github/workflows/integration-tests.yml @@ -38,11 +38,6 @@ jobs: path: /opt/zeek key: zeek-cache - - name: Create Zeek symlink - run: | - sudo apt install -y --no-install-recommends zeek - sudo ln -s /opt/zeek/bin/zeek /usr/local/bin/bro - - name: Restore APT cache id: apt-cache uses: actions/cache@v3 @@ -52,16 +47,25 @@ jobs: /var/lib/apt/lists key: apt-cache - - name: Install Python dependencies (from cache if possible) + - name: Install Zeek run: | - python3 -m pip install --upgrade pip - python3 -m pip install -r install/requirements.txt +# sudo echo 'deb http://download.opensuse.org/repositories/security:/zeek/xUbuntu_22.04/ /' | sudo tee /etc/apt/sources.list.d/security:zeek.list +# curl -fsSL https://download.opensuse.org/repositories/security:zeek/xUbuntu_22.04/Release.key | gpg --dearmor | sudo tee /etc/apt/trusted.gpg.d/security_zeek.gpg > /dev/null +# sudo apt update && sudo apt install -y --no-install-recommends --fix-missing zeek + sudo ln -s /opt/zeek/bin/zeek /usr/local/bin/bro + - name: Install apt dependencies (from cache if possible) run: | sudo apt-get update sudo apt-get install -y $(cat install/apt_dependencies.txt) + - name: Install Python dependencies (from cache if possible) + run: | + python3 -m pip install --upgrade pip + python3 -m pip install -r install/requirements.txt + + - name: Start redis server run: redis-server --daemonize yes From 49a8ca9ef917693c1ca738708d7baa6a1964190f Mon Sep 17 00:00:00 2001 From: alya Date: Mon, 4 Nov 2024 13:01:29 +0200 Subject: [PATCH 037/128] CI: integration tests: fix typo --- .github/workflows/integration-tests.yml | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/.github/workflows/integration-tests.yml b/.github/workflows/integration-tests.yml index d5dfce7f4..6deb4d29f 100644 --- a/.github/workflows/integration-tests.yml +++ b/.github/workflows/integration-tests.yml @@ -48,11 +48,7 @@ jobs: key: apt-cache - name: Install Zeek - run: | -# sudo echo 'deb http://download.opensuse.org/repositories/security:/zeek/xUbuntu_22.04/ /' | sudo tee /etc/apt/sources.list.d/security:zeek.list -# curl -fsSL https://download.opensuse.org/repositories/security:zeek/xUbuntu_22.04/Release.key | gpg --dearmor | sudo tee /etc/apt/trusted.gpg.d/security_zeek.gpg > /dev/null -# sudo apt update && sudo apt install -y --no-install-recommends --fix-missing zeek - sudo ln -s /opt/zeek/bin/zeek /usr/local/bin/bro + run: sudo ln -s /opt/zeek/bin/zeek /usr/local/bin/bro - name: Install apt dependencies (from cache if possible) From 397fc7302108fa6b1e257ec394a077528062da62 Mon Sep 17 00:00:00 2001 From: alya Date: Mon, 4 Nov 2024 13:07:04 +0200 Subject: [PATCH 038/128] CI: integration tests: install zeek 3 times in parallel due to cache miss --- .github/workflows/integration-tests.yml | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/.github/workflows/integration-tests.yml b/.github/workflows/integration-tests.yml index 6deb4d29f..f9e79ed26 100644 --- a/.github/workflows/integration-tests.yml +++ b/.github/workflows/integration-tests.yml @@ -48,7 +48,11 @@ jobs: key: apt-cache - name: Install Zeek - run: sudo ln -s /opt/zeek/bin/zeek /usr/local/bin/bro + run: | + sudo echo 'deb http://download.opensuse.org/repositories/security:/zeek/xUbuntu_22.04/ /' | sudo tee /etc/apt/sources.list.d/security:zeek.list + curl -fsSL https://download.opensuse.org/repositories/security:zeek/xUbuntu_22.04/Release.key | gpg --dearmor | sudo tee /etc/apt/trusted.gpg.d/security_zeek.gpg > /dev/null + sudo apt update && sudo apt install -y --no-install-recommends --fix-missing zeek + sudo ln -s /opt/zeek/bin/zeek /usr/local/bin/bro - name: Install apt dependencies (from cache if possible) From 9c202adfb0f77e053df9cc522f8179da406ff23d Mon Sep 17 00:00:00 2001 From: alya Date: Wed, 30 Oct 2024 16:01:19 +0300 Subject: [PATCH 039/128] Add a light dockerfile --- docker/light/Dockerfile | 58 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 58 insertions(+) create mode 100644 docker/light/Dockerfile diff --git a/docker/light/Dockerfile b/docker/light/Dockerfile new file mode 100644 index 000000000..a09c74d28 --- /dev/null +++ b/docker/light/Dockerfile @@ -0,0 +1,58 @@ +FROM ubuntu:22.04 +# To avoid user interaction when installing libraries +ENV DEBIAN_FRONTEND=noninteractive +# Blocking module requirement to avoid using sudo +ENV IS_IN_A_DOCKER_CONTAINER=True +# destionation dir for slips inside the container +ENV SLIPS_DIR=/StratosphereLinuxIPs + +# use bash instead of sh +SHELL ["/bin/bash", "-c"] + +# Install wget and add Zeek and redis repositories to our sources. +RUN apt update && apt install -y --no-install-recommends \ + wget \ + ca-certificates \ + git \ + curl \ + gnupg \ + lsb-release \ + software-properties-common \ + build-essential \ + file \ + lsof \ + iproute2 \ + tshark \ + whois \ + yara \ + net-tools \ + less \ + unzip \ + python3-certifi \ + python3-dev \ + python3-tzlocal \ + python3-pip \ + && echo 'deb http://download.opensuse.org/repositories/security:/zeek/xUbuntu_22.04/ /' | tee /etc/apt/sources.list.d/security:zeek.list \ + && curl -fsSL https://download.opensuse.org/repositories/security:zeek/xUbuntu_22.04/Release.key | gpg --dearmor | tee /etc/apt/trusted.gpg.d/security_zeek.gpg > /dev/null \ + && curl -fsSL https://packages.redis.io/gpg | gpg --dearmor -o /usr/share/keyrings/redis-archive-keyring.gpg \ + && echo "deb [signed-by=/usr/share/keyrings/redis-archive-keyring.gpg] https://packages.redis.io/deb $(lsb_release -cs) main" > /etc/apt/sources.list.d/redis.list \ + && apt update \ + && apt install -y --no-install-recommends --fix-missing \ + zeek \ + redis \ + && ln -s /opt/zeek/bin/zeek /usr/local/bin/bro \ + && apt clean \ + && rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* + +COPY . ${SLIPS_DIR} + +WORKDIR ${SLIPS_DIR} + +RUN pip3 install --no-cache-dir --upgrade pip \ + && egrep -v "tensorflow|Keras|scipy|pytest|flask|tqdm|coverage|pre-commit|ruff|black|scikit-learn|slackclient|matplotlib|stix2|cabby|pandas|numpy" install/requirements.txt | xargs -n 1 pip install \ + && pip freeze \ + && chmod 774 slips.py \ + && git init \ + && git remote add origin https://github.com/stratosphereips/StratosphereLinuxIPS.git + +CMD /bin/bash From 341ddf62d2f605365aa3c835cd58140229d2469c Mon Sep 17 00:00:00 2001 From: alya Date: Wed, 30 Oct 2024 16:22:35 +0300 Subject: [PATCH 040/128] dont install scikit learn in the light dockerfile --- docker/light/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/light/Dockerfile b/docker/light/Dockerfile index a09c74d28..1e126f389 100644 --- a/docker/light/Dockerfile +++ b/docker/light/Dockerfile @@ -49,7 +49,7 @@ COPY . ${SLIPS_DIR} WORKDIR ${SLIPS_DIR} RUN pip3 install --no-cache-dir --upgrade pip \ - && egrep -v "tensorflow|Keras|scipy|pytest|flask|tqdm|coverage|pre-commit|ruff|black|scikit-learn|slackclient|matplotlib|stix2|cabby|pandas|numpy" install/requirements.txt | xargs -n 1 pip install \ + && egrep -v "tensorflow|Keras|scipy|pytest|flask|tqdm|coverage|pre-commit|ruff|black|scikit-learn|scikit_learn|slackclient|matplotlib|stix2|cabby|pandas|numpy" install/requirements.txt | xargs -n 1 pip install \ && pip freeze \ && chmod 774 slips.py \ && git init \ From d5ea84181060c85aaf448e5281ab94d8a2525a2b Mon Sep 17 00:00:00 2001 From: alya Date: Wed, 30 Oct 2024 16:23:41 +0300 Subject: [PATCH 041/128] dont install yappi in the light dockerfile --- docker/light/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/light/Dockerfile b/docker/light/Dockerfile index 1e126f389..b8c5e85fd 100644 --- a/docker/light/Dockerfile +++ b/docker/light/Dockerfile @@ -49,7 +49,7 @@ COPY . ${SLIPS_DIR} WORKDIR ${SLIPS_DIR} RUN pip3 install --no-cache-dir --upgrade pip \ - && egrep -v "tensorflow|Keras|scipy|pytest|flask|tqdm|coverage|pre-commit|ruff|black|scikit-learn|scikit_learn|slackclient|matplotlib|stix2|cabby|pandas|numpy" install/requirements.txt | xargs -n 1 pip install \ + && egrep -v "tensorflow|Keras|scipy|pytest|flask|tqdm|coverage|yappi|pre-commit|ruff|black|scikit-learn|scikit_learn|slackclient|matplotlib|stix2|cabby|pandas|numpy" install/requirements.txt | xargs -n 1 pip install \ && pip freeze \ && chmod 774 slips.py \ && git init \ From 1021aa406c742772a8744b3a2bf409a90958cfe6 Mon Sep 17 00:00:00 2001 From: alya Date: Wed, 30 Oct 2024 16:26:13 +0300 Subject: [PATCH 042/128] dont install recommonmark in the light dockerfile --- docker/light/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/light/Dockerfile b/docker/light/Dockerfile index b8c5e85fd..ef7036a0d 100644 --- a/docker/light/Dockerfile +++ b/docker/light/Dockerfile @@ -49,7 +49,7 @@ COPY . ${SLIPS_DIR} WORKDIR ${SLIPS_DIR} RUN pip3 install --no-cache-dir --upgrade pip \ - && egrep -v "tensorflow|Keras|scipy|pytest|flask|tqdm|coverage|yappi|pre-commit|ruff|black|scikit-learn|scikit_learn|slackclient|matplotlib|stix2|cabby|pandas|numpy" install/requirements.txt | xargs -n 1 pip install \ + && egrep -v "tensorflow|recommonmark|Keras|scipy|pytest|flask|tqdm|coverage|yappi|pre-commit|ruff|black|scikit-learn|scikit_learn|slackclient|matplotlib|stix2|cabby|pandas|numpy" install/requirements.txt | xargs -n 1 pip install \ && pip freeze \ && chmod 774 slips.py \ && git init \ From f8685d3c98c7c0e1334cdac3e65cca27acff5ac7 Mon Sep 17 00:00:00 2001 From: alya Date: Wed, 30 Oct 2024 16:32:56 +0300 Subject: [PATCH 043/128] dont install setuptools and wheel in the light dockerfile --- docker/light/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/light/Dockerfile b/docker/light/Dockerfile index ef7036a0d..e10469d05 100644 --- a/docker/light/Dockerfile +++ b/docker/light/Dockerfile @@ -49,7 +49,7 @@ COPY . ${SLIPS_DIR} WORKDIR ${SLIPS_DIR} RUN pip3 install --no-cache-dir --upgrade pip \ - && egrep -v "tensorflow|recommonmark|Keras|scipy|pytest|flask|tqdm|coverage|yappi|pre-commit|ruff|black|scikit-learn|scikit_learn|slackclient|matplotlib|stix2|cabby|pandas|numpy" install/requirements.txt | xargs -n 1 pip install \ + && egrep -v "tensorflow|recommonmark|Keras|scipy|pytest|flask|tqdm|coverage|yappi|pre-commit|ruff|protobuf|black|wheel|scikit-learn|scikit_learn|slackclient|matplotlib|stix2|cabby|pandas|setuptools|numpy|certifi" install/requirements.txt | xargs -n 1 pip install \ && pip freeze \ && chmod 774 slips.py \ && git init \ From 4ba97a179e922248fe1cf0fa69c7613234b13476 Mon Sep 17 00:00:00 2001 From: alya Date: Mon, 4 Nov 2024 14:02:00 +0200 Subject: [PATCH 044/128] add a light dockerfile at docker/light --- docker/light/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/light/Dockerfile b/docker/light/Dockerfile index e10469d05..5d49c661c 100644 --- a/docker/light/Dockerfile +++ b/docker/light/Dockerfile @@ -49,7 +49,7 @@ COPY . ${SLIPS_DIR} WORKDIR ${SLIPS_DIR} RUN pip3 install --no-cache-dir --upgrade pip \ - && egrep -v "tensorflow|recommonmark|Keras|scipy|pytest|flask|tqdm|coverage|yappi|pre-commit|ruff|protobuf|black|wheel|scikit-learn|scikit_learn|slackclient|matplotlib|stix2|cabby|pandas|setuptools|numpy|certifi" install/requirements.txt | xargs -n 1 pip install \ + && grep -v -f docker/light/excluded_libs.txt install/requirements.txt | xargs -n 1 pip install \ && pip freeze \ && chmod 774 slips.py \ && git init \ From 14015362dc8d4775cfb132432e9a41c828b50480 Mon Sep 17 00:00:00 2001 From: alya Date: Mon, 4 Nov 2024 14:03:46 +0200 Subject: [PATCH 045/128] add a list of excluded python libraries from the light docker image --- docker/light/excluded_libs.txt | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) create mode 100644 docker/light/excluded_libs.txt diff --git a/docker/light/excluded_libs.txt b/docker/light/excluded_libs.txt new file mode 100644 index 000000000..67503ddb8 --- /dev/null +++ b/docker/light/excluded_libs.txt @@ -0,0 +1,26 @@ +tensorflow +recommonmark +Keras +scipy +pytest +flask +tqdm +coverage +yappi +pre-commit +ruff +protobuf +black +wheel +scikit-learn +scikit_learn +slackclient +matplotlib +stix2 +cabby +pandas +setuptools +numpy +certifi +viztracer +memray From bf183f7f0bce16156da2ef36d706034f4384cab6 Mon Sep 17 00:00:00 2001 From: alya Date: Mon, 4 Nov 2024 23:58:21 +0200 Subject: [PATCH 046/128] docker/light: remove the optional modules from the Dockerfile --- docker/light/Dockerfile | 25 ++++++++++++++++++++++++- 1 file changed, 24 insertions(+), 1 deletion(-) diff --git a/docker/light/Dockerfile b/docker/light/Dockerfile index 5d49c661c..61b8a56f3 100644 --- a/docker/light/Dockerfile +++ b/docker/light/Dockerfile @@ -48,9 +48,32 @@ COPY . ${SLIPS_DIR} WORKDIR ${SLIPS_DIR} +RUN cd modules \ +&& rm -rf \ + rnn_cc_detection/ \ + timeline/ \ + kalipso/ \ + p2ptrust/ \ + flowmldetection/ \ + cyst/ \ + cesnet/ \ + exporting_alerts/ \ + riskiq/ \ + template/ \ + blocking/ \ + virustotal/ \ +&& cd .. \ +&& rm -rf dataset/ docs/ tests/ \ +&& rm kalipso.sh \ + package.json \ + pytest.ini \ + webinterface.sh \ + CITATION.cff \ + CHANGELOG.md \ + conftest.py + RUN pip3 install --no-cache-dir --upgrade pip \ && grep -v -f docker/light/excluded_libs.txt install/requirements.txt | xargs -n 1 pip install \ - && pip freeze \ && chmod 774 slips.py \ && git init \ && git remote add origin https://github.com/stratosphereips/StratosphereLinuxIPS.git From a2a19a90c7fbe6cf331d06fbb680d3bd530f2ed5 Mon Sep 17 00:00:00 2001 From: alya Date: Mon, 4 Nov 2024 23:58:48 +0200 Subject: [PATCH 047/128] CI: publish the light docker image on push to master --- ...-slips-image.yml => publish-slips-images.yml} | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) rename .github/workflows/{publish-slips-image.yml => publish-slips-images.yml} (75%) diff --git a/.github/workflows/publish-slips-image.yml b/.github/workflows/publish-slips-images.yml similarity index 75% rename from .github/workflows/publish-slips-image.yml rename to .github/workflows/publish-slips-images.yml index 75cb30587..575a7b2ce 100644 --- a/.github/workflows/publish-slips-image.yml +++ b/.github/workflows/publish-slips-images.yml @@ -1,4 +1,4 @@ -name: CI-production-publishing-slips-image +name: CI-production-publishing-slips-images on: push: @@ -40,7 +40,7 @@ jobs: - name: Set up Docker Buildx uses: docker/setup-buildx-action@v3 - - name: Build and push Slips image using dockerfile + - name: Build and push the main Slips image id: docker_build_slips uses: docker/build-push-action@v6 with: @@ -51,3 +51,15 @@ jobs: stratosphereips/slips:latest stratosphereips/slips:${{ env.SLIPS_VERSION }} push: true + + - name: Build and push the light Slips image + id: docker_build_light_slips + uses: docker/build-push-action@v6 + with: + allow: network.host + context: ./ + file: ./docker/light/Dockerfile + tags: | + stratosphereips/slips_light:latest + stratosphereips/slips_light:${{ env.SLIPS_VERSION }} + push: true From a236a86c9fe0fe0fbb89a2f1b77af23798889829 Mon Sep 17 00:00:00 2001 From: alya Date: Tue, 5 Nov 2024 00:12:22 +0200 Subject: [PATCH 048/128] installation: add the minimal slips docker image to the docs --- docs/installation.md | 27 ++++++++++++++++++++++++++- 1 file changed, 26 insertions(+), 1 deletion(-) diff --git a/docs/installation.md b/docs/installation.md index 90e446157..33e6e8354 100644 --- a/docs/installation.md +++ b/docs/installation.md @@ -10,6 +10,7 @@ There are two ways to install and run Slips: inside a Docker or in your own comp * Dockerhub (recommended) * [Linux and windows hosts](https://stratospherelinuxips.readthedocs.io/en/develop/installation.html#linux-and-windows-hosts) * [MacOS hosts](https://stratospherelinuxips.readthedocs.io/en/develop/installation.html#macos-hosts) + * [Light Slips Image](#minimal-slips-docker-image) * [Docker-compose](https://stratospherelinuxips.readthedocs.io/en/develop/installation.html#running-slips-using-docker-compose) * [Dockerfile](https://stratospherelinuxips.readthedocs.io/en/develop/installation.html#building-slips-from-the-dockerfile) * Native @@ -31,7 +32,7 @@ Slips can be run inside a Docker. Either using our docker image with from Docker or building Slips image from the Dockerfile for more advanced users. In both cases, you need to have the Docker platform installed in your computer. -Instructions how to install Docker is https://docs.docker.com/get-docker/. +For instructions how to install Docker check https://docs.docker.com/get-docker/. The recommended way of using slips would be to * Run Slips from Dockerhub @@ -74,6 +75,30 @@ To analyze your own files using slips, you can mount it to your docker using -v ./slips.py -f dataset/myfile.pcap +#### Minimal Slips Docker Image +In addition to the full stratosphereips/slips:latest image, there is now a minimal Docker image available: using `docker pull stratosphereips/slips_light:latest`. This image excludes the following modules to reduce size and resource usage: + +* rnn_cc_detection/ +* timeline/ +* kalipso/ +* p2ptrust/ +* flowmldetection/ +* cyst/ +* cesnet/ +* exporting_alerts/ +* riskiq/ +* template/ +* blocking/ +* virustotal/ + +Additionally, several directories and files have been removed from this minimal image, including: +dataset/, docs/, tests/ + +Slips' two GUIs, kalipso and the web interface, aren't available in this image. The only way to check the slips output there is by going through the generated logs. + +The stratosphereips/slips_light:latest image is recommended for users who do not require these specific modules and want a more lightweight deployment. + + ### Update slips image docker pull stratosphereips/slips:latest From 97a89f932b814f6bf83dea50f149084c39311b67 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 5 Nov 2024 07:16:39 +0000 Subject: [PATCH 049/128] build(deps): bump actions/cache from 3 to 4 Bumps [actions/cache](https://github.com/actions/cache) from 3 to 4. - [Release notes](https://github.com/actions/cache/releases) - [Changelog](https://github.com/actions/cache/blob/main/RELEASES.md) - [Commits](https://github.com/actions/cache/compare/v3...v4) --- updated-dependencies: - dependency-name: actions/cache dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] --- .github/workflows/install-slips-dependencies.yml | 2 +- .github/workflows/integration-tests.yml | 4 ++-- .github/workflows/unit-tests.yml | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/install-slips-dependencies.yml b/.github/workflows/install-slips-dependencies.yml index eda20f8a6..e9a305897 100644 --- a/.github/workflows/install-slips-dependencies.yml +++ b/.github/workflows/install-slips-dependencies.yml @@ -38,7 +38,7 @@ jobs: run: sudo apt-get update --fix-missing && sudo apt-get -y --no-install-recommends install $(cat install/apt_dependencies.txt) - name: Save APT Cache - uses: actions/cache@v3 + uses: actions/cache@v4 with: path: | /var/cache/apt/archives diff --git a/.github/workflows/integration-tests.yml b/.github/workflows/integration-tests.yml index f9e79ed26..a6909abe6 100644 --- a/.github/workflows/integration-tests.yml +++ b/.github/workflows/integration-tests.yml @@ -33,14 +33,14 @@ jobs: - name: Restore Zeek Build from Cache id: zeek-cache - uses: actions/cache@v3 + uses: actions/cache@v4 with: path: /opt/zeek key: zeek-cache - name: Restore APT cache id: apt-cache - uses: actions/cache@v3 + uses: actions/cache@v4 with: path: | /var/cache/apt/archives diff --git a/.github/workflows/unit-tests.yml b/.github/workflows/unit-tests.yml index d42089ca1..a9abe22dd 100644 --- a/.github/workflows/unit-tests.yml +++ b/.github/workflows/unit-tests.yml @@ -81,7 +81,7 @@ jobs: - name: Restore APT cache id: apt-cache - uses: actions/cache@v3 + uses: actions/cache@v4 with: path: | /var/cache/apt/archives From 2619f80bac56749689ab8675c2907ae0c4665c16 Mon Sep 17 00:00:00 2001 From: alya Date: Fri, 8 Nov 2024 13:26:30 +0200 Subject: [PATCH 050/128] update submodules --- feel_project | 2 +- fides | 2 +- iris | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/feel_project b/feel_project index e00735d9d..450e3a655 160000 --- a/feel_project +++ b/feel_project @@ -1 +1 @@ -Subproject commit e00735d9d7565e07806ac232ad67580b36ebc084 +Subproject commit 450e3a655fb9c880e0f9337cdd1b182af2ec9d37 diff --git a/fides b/fides index bfac47728..8492d6cf2 160000 --- a/fides +++ b/fides @@ -1 +1 @@ -Subproject commit bfac47728172d3a4bbb27a5bb53ceef424e45e4f +Subproject commit 8492d6cf216e0182b9f96d6ed6baffd3a4c41c24 diff --git a/iris b/iris index c492085fe..4d66f9c7c 160000 --- a/iris +++ b/iris @@ -1 +1 @@ -Subproject commit c492085fea78508dfb203a88fdeef56cc20c9421 +Subproject commit 4d66f9c7cf5f9eeef7cb566c1ac66d6c2a76eebb From 5790d2f494565af3912af56c70de839a41e78b19 Mon Sep 17 00:00:00 2001 From: alya Date: Fri, 8 Nov 2024 13:29:10 +0200 Subject: [PATCH 051/128] Add reference to feel project in the README.md --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index c80604b1a..372faab87 100644 --- a/README.md +++ b/README.md @@ -182,6 +182,7 @@ Slips key features are: * **Peer-to-Peer (P2P) Module**: Slips includes a complex automatic system to find other peers in the network and share IoC data automatically in a balanced, trusted manner. The P2P module can be enabled as needed. * **Docker Implementation**: Running Slips through Docker on Linux systems is simplified, allowing real-time traffic analysis. * **Detailed Documentation**: Slips provides detailed documentation guiding users through usage instructions for efficient utilization of its features. +* **Federated learning** Using the feel_project submodule. for more information [check the docs](https://github.com/stratosphereips/feel_project/blob/main/docs/Federated_Learning.md) # Contributing From 4ba386e9aeeb3f7fb6e19db755fecc9bdff6c979 Mon Sep 17 00:00:00 2001 From: alya Date: Wed, 13 Nov 2024 19:27:21 +0200 Subject: [PATCH 052/128] improve the docs --- docs/usage.md | 37 +++++++++---------- modules/flowalerts/set_evidence.py | 2 +- .../threat_intelligence.py | 4 +- 3 files changed, 21 insertions(+), 22 deletions(-) diff --git a/docs/usage.md b/docs/usage.md index 19abd6852..bcf3f4501 100644 --- a/docs/usage.md +++ b/docs/usage.md @@ -14,14 +14,17 @@ It's recommended to use PCAPs. All the input flows are converted to an internal format. So once read, Slips works the same with all of them. -After Slips was run on the traffic, the Slips output can be analyzed with Kalipso GUI interface. In this section, we will explain how to execute each type of file in Slips, and the output can be analyzed with Kalipso. +After Slips runs on the given traffic, the output can be analyzed with Kalipso GUI interface. -Either you are [running Slips in docker](https://stratospherelinuxips.readthedocs.io/en/develop/installation.html#installing-and-running-slips-inside-a-docker) or [locally](https://stratospherelinuxips.readthedocs.io/en/develop/installation.html#installing-slips-in-your-own-computer), you can run Slips using the same below commands and configurations. +In this section, we will explain how to execute each type of file in Slips. + +Either you are [running Slips in docker](https://stratospherelinuxips.readthedocs.io/en/develop/installation.html#slips-in-docker) or [locally](https://stratospherelinuxips.readthedocs.io/en/develop/installation.html#installing-slips-natively), you can run Slips using the same below commands and configurations. ## Reading the input -The table below shows the commands Slips uses for different inputs. The first part of the command **./slips.py -c config/slips.yaml** is same, the second part changes depending on the input type. Also, the user can execute **./slips.py --help** to find correct argument to run Slips on each type of the file. +The table below shows the different parameter Slips uses for different inputs. +