From 91f44fc2bcec795df8817af73deeca0cceec938b Mon Sep 17 00:00:00 2001 From: Wout Feys Date: Tue, 23 Jul 2024 10:15:35 +0200 Subject: [PATCH 01/32] Make sure manage.py calls "django" specifically --- sample-apps/django-mysql-gunicorn/manage.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/sample-apps/django-mysql-gunicorn/manage.py b/sample-apps/django-mysql-gunicorn/manage.py index 9b1b1a58..fe6609be 100755 --- a/sample-apps/django-mysql-gunicorn/manage.py +++ b/sample-apps/django-mysql-gunicorn/manage.py @@ -3,11 +3,10 @@ import os import sys - def main(): """Run administrative tasks.""" import aikido_firewall # Aikido module - aikido_firewall.protect() + aikido_firewall.protect("django") os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'sample-django-mysql-gunicorn-app.settings') try: from django.core.management import execute_from_command_line From 025b338cca9343daee6723fb29972222b48e19c6 Mon Sep 17 00:00:00 2001 From: Wout Feys Date: Tue, 23 Jul 2024 10:19:31 +0200 Subject: [PATCH 02/32] Update django-gunicorn app to use form data instead of url --- sample-apps/django-mysql-gunicorn/README.md | 4 ++-- .../sample_app/templates/app/create_dog.html | 15 ++++++++++++++ .../django-mysql-gunicorn/sample_app/urls.py | 4 ++-- .../django-mysql-gunicorn/sample_app/views.py | 20 ++++++++++--------- 4 files changed, 30 insertions(+), 13 deletions(-) create mode 100644 sample-apps/django-mysql-gunicorn/sample_app/templates/app/create_dog.html diff --git a/sample-apps/django-mysql-gunicorn/README.md b/sample-apps/django-mysql-gunicorn/README.md index b8152556..39697861 100644 --- a/sample-apps/django-mysql-gunicorn/README.md +++ b/sample-apps/django-mysql-gunicorn/README.md @@ -9,7 +9,7 @@ This will expose a Django web server at [localhost:8080](http://localhost:8080) ## URLS : - Homepage : `http://localhost:8080/app` -- Create a dog : `http://localhost:8080/app/create/` -- MySQL attack : `http://localhost:8080/app/create/Malicious dog", "Injected wrong boss name"); --%20` +- Create a dog : `http://localhost:8080/app/create/` +- MySQL attack : Enter `Malicious dog", "Injected wrong boss name"); -- ` To verify your attack was successfully note that the boss_name usualy is 'N/A', if you open the dog page (you can do this from the home page). You should see a "malicious dog" with a boss name that is not permitted. diff --git a/sample-apps/django-mysql-gunicorn/sample_app/templates/app/create_dog.html b/sample-apps/django-mysql-gunicorn/sample_app/templates/app/create_dog.html new file mode 100644 index 00000000..e4123ab5 --- /dev/null +++ b/sample-apps/django-mysql-gunicorn/sample_app/templates/app/create_dog.html @@ -0,0 +1,15 @@ + + + + Create Dog + + +

Create a New Dog

+
+ {% csrf_token %} + + + +
+ + diff --git a/sample-apps/django-mysql-gunicorn/sample_app/urls.py b/sample-apps/django-mysql-gunicorn/sample_app/urls.py index f5fc991d..555b1d46 100644 --- a/sample-apps/django-mysql-gunicorn/sample_app/urls.py +++ b/sample-apps/django-mysql-gunicorn/sample_app/urls.py @@ -5,5 +5,5 @@ urlpatterns = [ path("", views.index, name="index"), path("dogpage/", views.dog_page, name="dog_page"), - path("create/", views.create_dogpage, name="create"), -] \ No newline at end of file + path("create/", views.create_dogpage, name="create"), +] diff --git a/sample-apps/django-mysql-gunicorn/sample_app/views.py b/sample-apps/django-mysql-gunicorn/sample_app/views.py index c634cc71..1e791306 100644 --- a/sample-apps/django-mysql-gunicorn/sample_app/views.py +++ b/sample-apps/django-mysql-gunicorn/sample_app/views.py @@ -20,12 +20,14 @@ def dog_page(request, dog_id): dog = get_object_or_404(Dogs, pk=dog_id) return HttpResponse("Your dog, %s, is lovely. Boss name is : %s" % (dog.dog_name, dog.dog_boss)) -def create_dogpage(request, dog_name): -# dog = Dogs(dog_name=dog_name, dog_boss="Unset") -# dog.save() - # Using custom sql to create a dog : - with connection.cursor() as cursor: - query = 'INSERT INTO sample_app_dogs (dog_name, dog_boss) VALUES ("%s", "N/A")' % dog_name - print("QUERY : ", query) - cursor.execute(query) - return HttpResponse("Dog page created") \ No newline at end of file +def create_dogpage(request): + if request.method == 'GET': + return render(request, 'create_dog.html') + elif request.method == 'POST': + dog_name = request.POST.get('dog_name') + # Using custom sql to create a dog : + with connection.cursor() as cursor: + query = 'INSERT INTO sample_app_dogs (dog_name, dog_boss) VALUES ("%s", "N/A")' % dog_name + print("QUERY : ", query) + cursor.execute(query) + return HttpResponse("Dog page created") From 63d5210d40b2ac80061d1348ab6d198bc2dce8a3 Mon Sep 17 00:00:00 2001 From: Wout Feys Date: Tue, 23 Jul 2024 10:19:43 +0200 Subject: [PATCH 03/32] Add healthcheck to django-mysql-gunicorn application --- sample-apps/django-mysql-gunicorn/docker-compose.yml | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/sample-apps/django-mysql-gunicorn/docker-compose.yml b/sample-apps/django-mysql-gunicorn/docker-compose.yml index 5127cdf5..50c7ed5d 100644 --- a/sample-apps/django-mysql-gunicorn/docker-compose.yml +++ b/sample-apps/django-mysql-gunicorn/docker-compose.yml @@ -1,7 +1,7 @@ version: '3' services: db: - image: mysql + image: mariadb container_name: django_mysql_gunicorn_mariadb restart: always volumes: @@ -17,6 +17,11 @@ services: expose: # Opens port 3306 on the container - '3306' + healthcheck: + test: ["CMD", "healthcheck.sh", "--connect", "--innodb_initialized"] + interval: 5s + retries: 5 + start_period: 10s backend: build: @@ -32,7 +37,8 @@ services: env_file: - .env depends_on: - - db + db: + condition: service_healthy volumes: db_data: From 2725e1afa6b7c65b7e92aa13d29a7d051bda5648 Mon Sep 17 00:00:00 2001 From: Wout Feys Date: Tue, 23 Jul 2024 10:23:38 +0200 Subject: [PATCH 04/32] Fix bugs with create_dog.html page --- .../sample_app/templates/app/create_dog.html | 22 ++++++------------- .../django-mysql-gunicorn/sample_app/views.py | 2 +- 2 files changed, 8 insertions(+), 16 deletions(-) diff --git a/sample-apps/django-mysql-gunicorn/sample_app/templates/app/create_dog.html b/sample-apps/django-mysql-gunicorn/sample_app/templates/app/create_dog.html index e4123ab5..fbb9f2f5 100644 --- a/sample-apps/django-mysql-gunicorn/sample_app/templates/app/create_dog.html +++ b/sample-apps/django-mysql-gunicorn/sample_app/templates/app/create_dog.html @@ -1,15 +1,7 @@ - - - - Create Dog - - -

Create a New Dog

-
- {% csrf_token %} - - - -
- - +

Create a New Dog

+
+ {% csrf_token %} + + + +
diff --git a/sample-apps/django-mysql-gunicorn/sample_app/views.py b/sample-apps/django-mysql-gunicorn/sample_app/views.py index 1e791306..f959864f 100644 --- a/sample-apps/django-mysql-gunicorn/sample_app/views.py +++ b/sample-apps/django-mysql-gunicorn/sample_app/views.py @@ -22,7 +22,7 @@ def dog_page(request, dog_id): def create_dogpage(request): if request.method == 'GET': - return render(request, 'create_dog.html') + return render(request, 'app/create_dog.html') elif request.method == 'POST': dog_name = request.POST.get('dog_name') # Using custom sql to create a dog : From e1b1d8910bdd2e9af1c1921ce854bc35defe0de4 Mon Sep 17 00:00:00 2001 From: Wout Feys Date: Tue, 23 Jul 2024 10:28:58 +0200 Subject: [PATCH 05/32] Use os.fork, this fixes issues with django --- aikido_firewall/agent/__init__.py | 17 ++++++----------- 1 file changed, 6 insertions(+), 11 deletions(-) diff --git a/aikido_firewall/agent/__init__.py b/aikido_firewall/agent/__init__.py index 170d3aea..3743ffd3 100644 --- a/aikido_firewall/agent/__init__.py +++ b/aikido_firewall/agent/__init__.py @@ -85,19 +85,14 @@ class IPC: def __init__(self, address, key): self.address = address self.key = str.encode(key) - self.agent_proc = None def start_aikido_listener(self): - """This will start the aikido thread which listens""" - self.agent_proc = Process( - target=AikidoProc, - args=( - self.address, - self.key, - ), - ) - logger.debug("Starting a new agent thread") - self.agent_proc.start() + """This will start the aikido process which listens""" + pid = os.fork() + if pid == 0: # Child process + AikidoProc(self.address, self.key) + else: # Parent process + logger.critical("Started a new agent process with PID: %d", pid) def send_data(self, action, obj): """This creates a new client for comms to the thread""" From 0eefb548ee5720f1531874500d51eaf23e29e8ab Mon Sep 17 00:00:00 2001 From: Wout Feys Date: Tue, 23 Jul 2024 10:29:09 +0200 Subject: [PATCH 06/32] It's a process not a thread, change this for clarity --- aikido_firewall/agent/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/aikido_firewall/agent/__init__.py b/aikido_firewall/agent/__init__.py index 3743ffd3..c0af6027 100644 --- a/aikido_firewall/agent/__init__.py +++ b/aikido_firewall/agent/__init__.py @@ -68,7 +68,7 @@ def get_ipc(): def start_ipc(): """ - Starts a thread to handle incoming/outgoing data + Starts a process to handle incoming/outgoing data """ # pylint: disable=global-statement # We need this to be global global ipc From 650e3dde4f30c74c23726efa4c4036d2df5a513f Mon Sep 17 00:00:00 2001 From: Wout Feys Date: Tue, 23 Jul 2024 10:52:22 +0200 Subject: [PATCH 07/32] Remove pymysql from sample-django apps and use mysqlcient instead (this should've been standard) --- sample-apps/django-mysql-gunicorn/requirements.txt | 2 +- .../sample-django-mysql-gunicorn-app/__init__.py | 2 -- sample-apps/django-mysql/requirements.txt | 4 ++-- sample-apps/django-mysql/sample-django-mysql-app/__init__.py | 2 -- 4 files changed, 3 insertions(+), 7 deletions(-) diff --git a/sample-apps/django-mysql-gunicorn/requirements.txt b/sample-apps/django-mysql-gunicorn/requirements.txt index 48663587..5ad050c9 100644 --- a/sample-apps/django-mysql-gunicorn/requirements.txt +++ b/sample-apps/django-mysql-gunicorn/requirements.txt @@ -1,5 +1,5 @@ django -pymysql +mysqlclient python-decouple cryptography gunicorn diff --git a/sample-apps/django-mysql-gunicorn/sample-django-mysql-gunicorn-app/__init__.py b/sample-apps/django-mysql-gunicorn/sample-django-mysql-gunicorn-app/__init__.py index c45523b2..e69de29b 100644 --- a/sample-apps/django-mysql-gunicorn/sample-django-mysql-gunicorn-app/__init__.py +++ b/sample-apps/django-mysql-gunicorn/sample-django-mysql-gunicorn-app/__init__.py @@ -1,2 +0,0 @@ -import pymysql -pymysql.install_as_MySQLdb() \ No newline at end of file diff --git a/sample-apps/django-mysql/requirements.txt b/sample-apps/django-mysql/requirements.txt index f7dab0d7..c4fb2bbc 100644 --- a/sample-apps/django-mysql/requirements.txt +++ b/sample-apps/django-mysql/requirements.txt @@ -1,4 +1,4 @@ django -pymysql +mysqlclient python-decouple -cryptography \ No newline at end of file +cryptography diff --git a/sample-apps/django-mysql/sample-django-mysql-app/__init__.py b/sample-apps/django-mysql/sample-django-mysql-app/__init__.py index c45523b2..e69de29b 100644 --- a/sample-apps/django-mysql/sample-django-mysql-app/__init__.py +++ b/sample-apps/django-mysql/sample-django-mysql-app/__init__.py @@ -1,2 +0,0 @@ -import pymysql -pymysql.install_as_MySQLdb() \ No newline at end of file From edfca35f97f78588470eb2c601bd116ae647c8a8 Mon Sep 17 00:00:00 2001 From: Wout Feys Date: Tue, 23 Jul 2024 17:00:19 +0200 Subject: [PATCH 08/32] allow running without server and allow running server-only --- aikido_firewall/__init__.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/aikido_firewall/__init__.py b/aikido_firewall/__init__.py index 71d7c383..a79689cb 100644 --- a/aikido_firewall/__init__.py +++ b/aikido_firewall/__init__.py @@ -15,8 +15,15 @@ load_dotenv() -def protect(module="any"): +def protect(module="any", server=True): """Start Aikido agent""" + if server: + start_ipc() + else: + logger.debug("Not starting IPC server") + if module == "server-only": + return + # Import sources import aikido_firewall.sources.django @@ -27,4 +34,3 @@ def protect(module="any"): import aikido_firewall.sinks.pymysql logger.info("Aikido python firewall started") - start_ipc() From be03cf04f63379007f509224d0c534687cf30faa Mon Sep 17 00:00:00 2001 From: Wout Feys Date: Tue, 23 Jul 2024 17:00:31 +0200 Subject: [PATCH 09/32] Remove aikido imports from manage.py file --- sample-apps/django-mysql-gunicorn/manage.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/sample-apps/django-mysql-gunicorn/manage.py b/sample-apps/django-mysql-gunicorn/manage.py index fe6609be..71bec600 100755 --- a/sample-apps/django-mysql-gunicorn/manage.py +++ b/sample-apps/django-mysql-gunicorn/manage.py @@ -5,8 +5,6 @@ def main(): """Run administrative tasks.""" - import aikido_firewall # Aikido module - aikido_firewall.protect("django") os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'sample-django-mysql-gunicorn-app.settings') try: from django.core.management import execute_from_command_line From 2932a80ac90c96862e0b5b55353e3a802fb0149f Mon Sep 17 00:00:00 2001 From: Wout Feys Date: Tue, 23 Jul 2024 17:00:40 +0200 Subject: [PATCH 10/32] Add gunicorn config file --- .../django-mysql-gunicorn/docker-compose.yml | 2 +- .../django-mysql-gunicorn/gunicorn_config.py | 57 +++++++++++++++++++ 2 files changed, 58 insertions(+), 1 deletion(-) create mode 100644 sample-apps/django-mysql-gunicorn/gunicorn_config.py diff --git a/sample-apps/django-mysql-gunicorn/docker-compose.yml b/sample-apps/django-mysql-gunicorn/docker-compose.yml index 50c7ed5d..bcb1aacf 100644 --- a/sample-apps/django-mysql-gunicorn/docker-compose.yml +++ b/sample-apps/django-mysql-gunicorn/docker-compose.yml @@ -28,7 +28,7 @@ services: context: ./../../ dockerfile: ./sample-apps/django-mysql-gunicorn/Dockerfile container_name: django_mysql_gunicorn_backend - command: sh -c "python manage.py migrate && gunicorn --workers 4 --threads 2 --log-level debug --access-logfile '-' --error-logfile '-' --bind 0.0.0.0:8000 sample-django-mysql-gunicorn-app.wsgi" + command: sh -c "python manage.py migrate && gunicorn -c gunicorn_config.py --workers 4 --threads 2 --log-level debug --access-logfile '-' --error-logfile '-' --bind 0.0.0.0:8000 sample-django-mysql-gunicorn-app.wsgi" restart: always volumes: - .:/app diff --git a/sample-apps/django-mysql-gunicorn/gunicorn_config.py b/sample-apps/django-mysql-gunicorn/gunicorn_config.py new file mode 100644 index 00000000..939b90f4 --- /dev/null +++ b/sample-apps/django-mysql-gunicorn/gunicorn_config.py @@ -0,0 +1,57 @@ +import aikido_firewall +import json +def when_ready(server): + print("----------------------> WHEN READY") + aikido_firewall.protect("server-only") + +def pre_fork(server, worker): + print("----------------------> PRE FORK") + #import aikido_firewall.sources.django + +def post_fork(server, worker): + print("----------------------> POST FORK") + import aikido_firewall + aikido_firewall.protect("django", False) + +def pre_request(worker, req): + worker.log.critical(req.body) + from aikido_firewall.context import Context + django_context = Context(req, "django") + print(django_context) + worker.log.debug("%s %s", req.method, req.path) + + +# Useless : + +def post_worker_init(worker): + pass + +def pre_exec(server): + pass + +def on_reload(server): + pass + +def worker_int(worker): + pass + +def worker_abort(worker): + pass + +def post_request(worker, req, environ, resp): + pass + +def child_exit(server, worker): + pass + +def worker_exit(server, worker): + pass + +def nworkers_changed(server, new_value, old_value): + pass + +def on_exit(server): + pass + +def ssl_context(config, default_ssl_context_factory): + return default_ssl_context_factory() From 7bd21da6ae2160792282234e38c780438f63a4d9 Mon Sep 17 00:00:00 2001 From: Wout Feys Date: Tue, 23 Jul 2024 17:04:45 +0200 Subject: [PATCH 11/32] Linting --- aikido_firewall/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/aikido_firewall/__init__.py b/aikido_firewall/__init__.py index a79689cb..57f022cb 100644 --- a/aikido_firewall/__init__.py +++ b/aikido_firewall/__init__.py @@ -23,7 +23,7 @@ def protect(module="any", server=True): logger.debug("Not starting IPC server") if module == "server-only": return - + # Import sources import aikido_firewall.sources.django From 4be81d25da7a2f4d91ca69635095dde27853d4c1 Mon Sep 17 00:00:00 2001 From: Wout Feys Date: Tue, 23 Jul 2024 17:04:58 +0200 Subject: [PATCH 12/32] Add django-gunicorn parsing --- aikido_firewall/context/__init__.py | 30 +++++++++++++++++++ .../django-mysql-gunicorn/gunicorn_config.py | 2 +- 2 files changed, 31 insertions(+), 1 deletion(-) diff --git a/aikido_firewall/context/__init__.py b/aikido_firewall/context/__init__.py index 53910f4a..b9ac4093 100644 --- a/aikido_firewall/context/__init__.py +++ b/aikido_firewall/context/__init__.py @@ -3,6 +3,8 @@ """ import threading +from urllib.parse import parse_qs +from http.cookies import SimpleCookie SUPPORTED_SOURCES = ["django", "flask"] local = threading.local() @@ -20,9 +22,26 @@ def parse_headers(headers): """Parse EnvironHeaders object into a dict""" if isinstance(headers, dict): return headers + if isinstance(headers, list): + obj = {} + for k, v in headers: + obj[k] = v + return obj return dict(zip(headers.keys(), headers.values())) +def parse_cookies(cookie_str): + """Parse cookie string from headers""" + cookie_dict = {} + cookies = SimpleCookie() + cookies.load(cookie_str) + + for key, morsel in cookies.items(): + cookie_dict[key] = morsel.value + + return cookie_dict + + class Context: """ A context object, it stores everything that is important @@ -39,6 +58,17 @@ def __init__(self, req, source): self.set_flask_attrs(req) elif source == "django": self.set_django_attrs(req) + elif source == "django-gunicorn": + self.set_django_gunicorn_attrs(req) + + def set_django_gunicorn_attrs(self, req): + """Set properties that are specific to django-gunicorn""" + self.remote_address = req.remote_addr + self.url = req.uri + self.body = dict(req.body) + self.query = parse_qs(req.query) + self.cookies = parse_cookies(self.headers["COOKIE"]) + del self.headers["COOKIE"] def set_django_attrs(self, req): """set properties that are specific to django""" diff --git a/sample-apps/django-mysql-gunicorn/gunicorn_config.py b/sample-apps/django-mysql-gunicorn/gunicorn_config.py index 939b90f4..bd331a1a 100644 --- a/sample-apps/django-mysql-gunicorn/gunicorn_config.py +++ b/sample-apps/django-mysql-gunicorn/gunicorn_config.py @@ -16,7 +16,7 @@ def post_fork(server, worker): def pre_request(worker, req): worker.log.critical(req.body) from aikido_firewall.context import Context - django_context = Context(req, "django") + django_context = Context(req, "django-gunicorn") print(django_context) worker.log.debug("%s %s", req.method, req.path) From 40211b6aa38c08dc8d472c0afc73302df1589cf3 Mon Sep 17 00:00:00 2001 From: Wout Feys Date: Tue, 23 Jul 2024 17:09:55 +0200 Subject: [PATCH 13/32] django-gunicorn add to supported sources --- aikido_firewall/context/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/aikido_firewall/context/__init__.py b/aikido_firewall/context/__init__.py index b9ac4093..0c98fe23 100644 --- a/aikido_firewall/context/__init__.py +++ b/aikido_firewall/context/__init__.py @@ -6,7 +6,7 @@ from urllib.parse import parse_qs from http.cookies import SimpleCookie -SUPPORTED_SOURCES = ["django", "flask"] +SUPPORTED_SOURCES = ["django", "flask", "django-gunicorn"] local = threading.local() From 45b2fead8d7e749d15575eae66c729bff07f0d2a Mon Sep 17 00:00:00 2001 From: Wout Feys Date: Tue, 23 Jul 2024 17:30:06 +0200 Subject: [PATCH 14/32] Read body and rewrite it --- aikido_firewall/context/__init__.py | 2 +- .../django-mysql-gunicorn/gunicorn_config.py | 23 +++++++++++++++++-- 2 files changed, 22 insertions(+), 3 deletions(-) diff --git a/aikido_firewall/context/__init__.py b/aikido_firewall/context/__init__.py index 0c98fe23..3db035e7 100644 --- a/aikido_firewall/context/__init__.py +++ b/aikido_firewall/context/__init__.py @@ -65,7 +65,7 @@ def set_django_gunicorn_attrs(self, req): """Set properties that are specific to django-gunicorn""" self.remote_address = req.remote_addr self.url = req.uri - self.body = dict(req.body) + self.body = parse_qs(req.body_copy.decode("utf-8")) self.query = parse_qs(req.query) self.cookies = parse_cookies(self.headers["COOKIE"]) del self.headers["COOKIE"] diff --git a/sample-apps/django-mysql-gunicorn/gunicorn_config.py b/sample-apps/django-mysql-gunicorn/gunicorn_config.py index bd331a1a..564d687f 100644 --- a/sample-apps/django-mysql-gunicorn/gunicorn_config.py +++ b/sample-apps/django-mysql-gunicorn/gunicorn_config.py @@ -1,5 +1,10 @@ import aikido_firewall import json +from urllib.parse import parse_qs +from io import BytesIO +from aikido_firewall.context import Context +from gunicorn.http.body import Body + def when_ready(server): print("----------------------> WHEN READY") aikido_firewall.protect("server-only") @@ -14,13 +19,27 @@ def post_fork(server, worker): aikido_firewall.protect("django", False) def pre_request(worker, req): - worker.log.critical(req.body) - from aikido_firewall.context import Context + req.body, req.body_copy = clone_body(req.body) + django_context = Context(req, "django-gunicorn") print(django_context) + worker.log.debug("%s %s", req.method, req.path) +def clone_body(body): + body_read = body.read() + + # Read the body content into a buffer + body_buffer = BytesIO() + body_buffer.write(body_read) + body_buffer.seek(0) + + # Create a new Body object with the same content + cloned_body = Body(body_buffer) + + return (cloned_body, body_read) + # Useless : def post_worker_init(worker): From b6b8c3975a4e6f5d71c56064ace7c4a76f1f98f7 Mon Sep 17 00:00:00 2001 From: Wout Feys Date: Tue, 23 Jul 2024 17:33:27 +0200 Subject: [PATCH 15/32] set_as_current_context --- sample-apps/django-mysql-gunicorn/gunicorn_config.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sample-apps/django-mysql-gunicorn/gunicorn_config.py b/sample-apps/django-mysql-gunicorn/gunicorn_config.py index 564d687f..5c876971 100644 --- a/sample-apps/django-mysql-gunicorn/gunicorn_config.py +++ b/sample-apps/django-mysql-gunicorn/gunicorn_config.py @@ -22,7 +22,7 @@ def pre_request(worker, req): req.body, req.body_copy = clone_body(req.body) django_context = Context(req, "django-gunicorn") - print(django_context) + django_context.set_as_current_context() worker.log.debug("%s %s", req.method, req.path) From e560f2c73cd3112b492cd3f6e33aeebc4b795fb5 Mon Sep 17 00:00:00 2001 From: Wout Feys Date: Tue, 23 Jul 2024 17:40:22 +0200 Subject: [PATCH 16/32] Cleanup config and make special "django-gunicorn" class --- aikido_firewall/__init__.py | 5 ++- .../django-mysql-gunicorn/gunicorn_config.py | 41 +------------------ 2 files changed, 5 insertions(+), 41 deletions(-) diff --git a/aikido_firewall/__init__.py b/aikido_firewall/__init__.py index 57f022cb..38056b84 100644 --- a/aikido_firewall/__init__.py +++ b/aikido_firewall/__init__.py @@ -25,9 +25,10 @@ def protect(module="any", server=True): return # Import sources - import aikido_firewall.sources.django + if module == "django": + import aikido_firewall.sources.django - if module != "django": + if not module in ["django", "django-gunicorn"]: import aikido_firewall.sources.flask # Import sinks diff --git a/sample-apps/django-mysql-gunicorn/gunicorn_config.py b/sample-apps/django-mysql-gunicorn/gunicorn_config.py index 5c876971..c722b0f0 100644 --- a/sample-apps/django-mysql-gunicorn/gunicorn_config.py +++ b/sample-apps/django-mysql-gunicorn/gunicorn_config.py @@ -6,17 +6,15 @@ from gunicorn.http.body import Body def when_ready(server): - print("----------------------> WHEN READY") aikido_firewall.protect("server-only") def pre_fork(server, worker): - print("----------------------> PRE FORK") - #import aikido_firewall.sources.django + pass def post_fork(server, worker): print("----------------------> POST FORK") import aikido_firewall - aikido_firewall.protect("django", False) + aikido_firewall.protect("django-gunicorn", False) def pre_request(worker, req): req.body, req.body_copy = clone_body(req.body) @@ -39,38 +37,3 @@ def clone_body(body): cloned_body = Body(body_buffer) return (cloned_body, body_read) - -# Useless : - -def post_worker_init(worker): - pass - -def pre_exec(server): - pass - -def on_reload(server): - pass - -def worker_int(worker): - pass - -def worker_abort(worker): - pass - -def post_request(worker, req, environ, resp): - pass - -def child_exit(server, worker): - pass - -def worker_exit(server, worker): - pass - -def nworkers_changed(server, new_value, old_value): - pass - -def on_exit(server): - pass - -def ssl_context(config, default_ssl_context_factory): - return default_ssl_context_factory() From 00c3dd3a87ef3c8e44733112da664fcfd983f1e8 Mon Sep 17 00:00:00 2001 From: Wout Feys Date: Mon, 29 Jul 2024 12:42:43 +0200 Subject: [PATCH 17/32] Linting after merge --- aikido_firewall/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/aikido_firewall/__init__.py b/aikido_firewall/__init__.py index e4165597..ee32bbe6 100644 --- a/aikido_firewall/__init__.py +++ b/aikido_firewall/__init__.py @@ -36,4 +36,4 @@ def protect(module="any", server=True): # Import sinks import aikido_firewall.sinks.pymysql - logger.info("Aikido python firewall started") \ No newline at end of file + logger.info("Aikido python firewall started") From f950962cacdee59a1ac60640472ca65463d3bd30 Mon Sep 17 00:00:00 2001 From: Wout Feys Date: Mon, 29 Jul 2024 14:16:17 +0200 Subject: [PATCH 18/32] Update test suite to use new reset_comms function --- aikido_firewall/background_process/init_test.py | 8 +++++--- aikido_firewall/init_test.py | 5 +++-- 2 files changed, 8 insertions(+), 5 deletions(-) diff --git a/aikido_firewall/background_process/init_test.py b/aikido_firewall/background_process/init_test.py index 7ede6dd5..9c657b31 100644 --- a/aikido_firewall/background_process/init_test.py +++ b/aikido_firewall/background_process/init_test.py @@ -4,6 +4,7 @@ start_background_process, get_comms, IPC_ADDRESS, + reset_comms, ) @@ -13,18 +14,19 @@ def test_ipc_init(): ipc = IPC(address, key) assert ipc.address == address - assert ipc.background_process is None + assert ipc.key == key -# Following function does not work def test_start_background_process(monkeypatch): + reset_comms() assert get_comms() == None start_background_process() assert get_comms() != None assert get_comms().address == IPC_ADDRESS - get_comms().background_process.kill() + reset_comms() + assert get_comms() == None def test_send_data_exception(monkeypatch, caplog): diff --git a/aikido_firewall/init_test.py b/aikido_firewall/init_test.py index 752532f5..54072447 100644 --- a/aikido_firewall/init_test.py +++ b/aikido_firewall/init_test.py @@ -1,6 +1,6 @@ import pytest from aikido_firewall import protect -from aikido_firewall.background_process import get_comms +from aikido_firewall.background_process import get_comms, reset_comms def test_protect_with_django(monkeypatch, caplog): @@ -15,4 +15,5 @@ def test_protect_with_django(monkeypatch, caplog): assert "Aikido python firewall started" in caplog.text assert get_comms() != None - get_comms().background_process.kill() + reset_comms() + assert get_comms() == None From 73c5fa47820412f4aad1337aff7e8f550549ab93 Mon Sep 17 00:00:00 2001 From: Wout Feys Date: Mon, 29 Jul 2024 14:17:05 +0200 Subject: [PATCH 19/32] Create reset_comms function and at timeout to sending data over IPC --- aikido_firewall/background_process/__init__.py | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/aikido_firewall/background_process/__init__.py b/aikido_firewall/background_process/__init__.py index 09ba5b4f..8457bf4e 100644 --- a/aikido_firewall/background_process/__init__.py +++ b/aikido_firewall/background_process/__init__.py @@ -6,6 +6,7 @@ import time import os import secrets +import socket import multiprocessing.connection as con from multiprocessing import Process from threading import Thread @@ -73,6 +74,14 @@ def get_comms(): return ipc +def reset_comms(): + """This will reset communications""" + global ipc + if ipc: + ipc.send_data("KILL", {}) + ipc = None + + def start_background_process(): """ Starts a process to handle incoming/outgoing data @@ -108,7 +117,9 @@ def send_data(self, action, obj): This creates a new client for comms to the background process """ try: - conn = con.Client(self.address, authkey=self.key) + # Create a client socket so we can set the timeout for IPC at 3sec + client_socket = socket.create_connection(self.address, timeout=3) + conn = con.Client(client_socket, authkey=self.key) logger.debug("Created connection %s", conn) conn.send((action, obj)) conn.send(("CLOSE", {})) From 8c3c04eca723067f78393d6a6750edc4f17976d7 Mon Sep 17 00:00:00 2001 From: Wout Feys Date: Mon, 29 Jul 2024 14:18:18 +0200 Subject: [PATCH 20/32] Add comments, "KILL" action and check if port is already in use --- aikido_firewall/background_process/__init__.py | 18 ++++++++++++++++-- 1 file changed, 16 insertions(+), 2 deletions(-) diff --git a/aikido_firewall/background_process/__init__.py b/aikido_firewall/background_process/__init__.py index 8457bf4e..3101cb3f 100644 --- a/aikido_firewall/background_process/__init__.py +++ b/aikido_firewall/background_process/__init__.py @@ -6,6 +6,7 @@ import time import os import secrets +import signal import socket import multiprocessing.connection as con from multiprocessing import Process @@ -26,7 +27,14 @@ class AikidoBackgroundProcess: def __init__(self, address, key): logger.debug("Background process started") - listener = con.Listener(address, authkey=key) + try: + listener = con.Listener(address, authkey=key) + except OSError: + logger.warning( + "Aikido listener may already be running on port %s", address[1] + ) + pid = os.getpid() + os.kill(pid, signal.SIGTERM) # Kill this subprocess self.queue = Queue() # Start reporting thread : Thread(target=self.reporting_thread).start() @@ -36,12 +44,17 @@ def __init__(self, address, key): logger.debug("connection accepted from %s", listener.last_accepted) while True: data = conn.recv() - logger.error(data) # Temporary debugging + logger.debug("Incoming data : %s", data) if data[0] == "ATTACK": self.queue.put(data[1]) elif data[0] == "CLOSE": conn.close() break + elif data[0] == "KILL": + logger.debug("Killing subprocess") + conn.close() + pid = os.getpid() + os.kill(pid, signal.SIGTERM) # Kill this subprocess def reporting_thread(self): """Reporting thread""" @@ -101,6 +114,7 @@ class IPC: """ def __init__(self, address, key): + # The key needs to be in byte form self.address = address self.key = key From 20e0ab69d7a44943e5dc7a436b86498f291fc6f3 Mon Sep 17 00:00:00 2001 From: Wout Feys Date: Mon, 29 Jul 2024 15:35:26 +0200 Subject: [PATCH 21/32] Move send code to a thread --- .../background_process/__init__.py | 31 ++++++++++++------- .../background_process/init_test.py | 3 -- 2 files changed, 20 insertions(+), 14 deletions(-) diff --git a/aikido_firewall/background_process/__init__.py b/aikido_firewall/background_process/__init__.py index 3101cb3f..f33e29c8 100644 --- a/aikido_firewall/background_process/__init__.py +++ b/aikido_firewall/background_process/__init__.py @@ -130,14 +130,23 @@ def send_data(self, action, obj): """ This creates a new client for comms to the background process """ - try: - # Create a client socket so we can set the timeout for IPC at 3sec - client_socket = socket.create_connection(self.address, timeout=3) - conn = con.Client(client_socket, authkey=self.key) - logger.debug("Created connection %s", conn) - conn.send((action, obj)) - conn.send(("CLOSE", {})) - conn.close() - logger.debug("Connection closed") - except Exception as e: - logger.info("Failed to send data to bg process : %s", e) + + # We want to make sure that sending out this data affects the process as little as possible + # So we run it inside a seperate thread with a timeout of 3 seconds + def target(address, key, data_array): + try: + conn = con.Client(address, authkey=key) + logger.debug("Created connection %s", conn) + for data in data_array: + conn.send(data) + conn.send(("CLOSE", {})) + conn.close() + logger.debug("Connection closed") + except Exception as e: + logger.info("Failed to send data to bg process : %s", e) + + t = Thread( + target=target, args=(self.address, self.key, [(action, obj)]), daemon=True + ) + t.start() + t.join(timeout=3) diff --git a/aikido_firewall/background_process/init_test.py b/aikido_firewall/background_process/init_test.py index 9c657b31..772623a4 100644 --- a/aikido_firewall/background_process/init_test.py +++ b/aikido_firewall/background_process/init_test.py @@ -39,9 +39,6 @@ def mock_client(address, authkey): ipc = IPC(("localhost", 9898), "mock_key") ipc.send_data("ACTION", "Test Object") - assert "Failed to send data to bg" in caplog.text - # Add assertions for handling exceptions - def test_send_data_successful(monkeypatch, caplog, mocker): ipc = IPC(("localhost"), "mock_key") From 73c92056eb38707f960547634562834b5e416702 Mon Sep 17 00:00:00 2001 From: Wout Feys Date: Tue, 30 Jul 2024 09:25:53 +0200 Subject: [PATCH 22/32] Fix bugs with django-mysql sample app --- sample-apps/django-mysql/README.md | 2 +- sample-apps/django-mysql/sample_app/urls.py | 3 +-- sample-apps/django-mysql/sample_app/views.py | 24 +++++++++----------- 3 files changed, 13 insertions(+), 16 deletions(-) diff --git a/sample-apps/django-mysql/README.md b/sample-apps/django-mysql/README.md index 8975bd6a..048be0f5 100644 --- a/sample-apps/django-mysql/README.md +++ b/sample-apps/django-mysql/README.md @@ -11,6 +11,6 @@ This will expose a Django web server at [localhost:8080](http://localhost:8080) ## URLS : - Homepage : `http://localhost:8080/app` - Create a dog : `http://localhost:8080/app/create/` -- MySQL attack : `http://localhost:8080/app/create/Malicious dog", "Injected wrong boss name"); --%20` +- MySQL attack : `Malicious dog", "Injected wrong boss name"); -- ` To verify your attack was successfully note that the boss_name usualy is 'N/A', if you open the dog page (you can do this from the home page). You should see a "malicious dog" with a boss name that is not permitted. diff --git a/sample-apps/django-mysql/sample_app/urls.py b/sample-apps/django-mysql/sample_app/urls.py index 96f6509e..979f9bc1 100644 --- a/sample-apps/django-mysql/sample_app/urls.py +++ b/sample-apps/django-mysql/sample_app/urls.py @@ -5,6 +5,5 @@ urlpatterns = [ path("", views.index, name="index"), path("dogpage/", views.dog_page, name="dog_page"), - path("create/", views.create_dogpage, name="create_old"), - path("create", views.create, name="create") + path("create", views.create_dogpage, name="create") ] diff --git a/sample-apps/django-mysql/sample_app/views.py b/sample-apps/django-mysql/sample_app/views.py index cd9fa50e..f959864f 100644 --- a/sample-apps/django-mysql/sample_app/views.py +++ b/sample-apps/django-mysql/sample_app/views.py @@ -20,16 +20,14 @@ def dog_page(request, dog_id): dog = get_object_or_404(Dogs, pk=dog_id) return HttpResponse("Your dog, %s, is lovely. Boss name is : %s" % (dog.dog_name, dog.dog_boss)) -def create_dogpage(request, dog_name): -# dog = Dogs(dog_name=dog_name, dog_boss="Unset") -# dog.save() - # Using custom sql to create a dog : - with connection.cursor() as cursor: - query = 'INSERT INTO sample_app_dogs (dog_name, dog_boss) VALUES ("%s", "N/A")' % dog_name - print("QUERY : ", query) - cursor.execute(query) - return HttpResponse("Dog page created") - -def create(request): - template = loader.get_template("app/create_dog.html") - return HttpResponse(template.render({}, request)) +def create_dogpage(request): + if request.method == 'GET': + return render(request, 'app/create_dog.html') + elif request.method == 'POST': + dog_name = request.POST.get('dog_name') + # Using custom sql to create a dog : + with connection.cursor() as cursor: + query = 'INSERT INTO sample_app_dogs (dog_name, dog_boss) VALUES ("%s", "N/A")' % dog_name + print("QUERY : ", query) + cursor.execute(query) + return HttpResponse("Dog page created") From d5d2384898432c4e1ca27d43b1b234802be9f6dd Mon Sep 17 00:00:00 2001 From: Wout Feys Date: Tue, 30 Jul 2024 09:32:33 +0200 Subject: [PATCH 23/32] Fix dogpage as well --- .../django-mysql/sample_app/templates/app/create_dog.html | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/sample-apps/django-mysql/sample_app/templates/app/create_dog.html b/sample-apps/django-mysql/sample_app/templates/app/create_dog.html index b2375a80..fbb9f2f5 100644 --- a/sample-apps/django-mysql/sample_app/templates/app/create_dog.html +++ b/sample-apps/django-mysql/sample_app/templates/app/create_dog.html @@ -1,6 +1,7 @@ -

Create a Dog

-
+

Create a New Dog

+ + {% csrf_token %} - +
From 48e05f221c398f800e215ec892a52e1e0dbd67a6 Mon Sep 17 00:00:00 2001 From: willem-delbare <20814660+willem-delbare@users.noreply.github.com> Date: Tue, 30 Jul 2024 09:59:58 +0200 Subject: [PATCH 24/32] Update aikido_firewall/background_process/__init__.py --- aikido_firewall/background_process/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/aikido_firewall/background_process/__init__.py b/aikido_firewall/background_process/__init__.py index 04c9669c..af66b978 100644 --- a/aikido_firewall/background_process/__init__.py +++ b/aikido_firewall/background_process/__init__.py @@ -50,7 +50,7 @@ def __init__(self, address, key): elif data[0] == "CLOSE": # this is a kind of EOL for python IPC conn.close() break - elif data[0] == "KILL": + elif data[0] == "KILL": # when main process quits , or during testing etc logger.debug("Killing subprocess") conn.close() pid = os.getpid() From 80151a7526951559f281547e6b585a58d669eed6 Mon Sep 17 00:00:00 2001 From: Wout Feys Date: Tue, 30 Jul 2024 10:19:48 +0200 Subject: [PATCH 25/32] Renaming and splitting up files --- .../background_process/__init__.py | 131 +----------------- .../aikido_background_process.py | 72 ++++++++++ aikido_firewall/background_process/comms.py | 75 ++++++++++ .../background_process/init_test.py | 10 +- aikido_firewall/sinks/pymysql.py | 2 +- 5 files changed, 158 insertions(+), 132 deletions(-) create mode 100644 aikido_firewall/background_process/aikido_background_process.py create mode 100644 aikido_firewall/background_process/comms.py diff --git a/aikido_firewall/background_process/__init__.py b/aikido_firewall/background_process/__init__.py index af66b978..a9230ddf 100644 --- a/aikido_firewall/background_process/__init__.py +++ b/aikido_firewall/background_process/__init__.py @@ -13,140 +13,19 @@ from threading import Thread from queue import Queue from aikido_firewall.helpers.logging import logger +from aikido_firewall.background_process.comms import AikidoIPCCommunications -REPORT_SEC_INTERVAL = 600 # 10 minutes IPC_ADDRESS = ("localhost", 9898) # Specify the IP address and port -class AikidoBackgroundProcess: - """ - Aikido's background process consists of 2 threads : - - (main) Listening thread which listens on an IPC socket for incoming data - - (spawned) reporting thread which will collect the IPC data and send it to a Reporter - """ - - def __init__(self, address, key): - logger.debug("Background process started") - try: - listener = con.Listener(address, authkey=key) - except OSError: - logger.warning( - "Aikido listener may already be running on port %s", address[1] - ) - pid = os.getpid() - os.kill(pid, signal.SIGTERM) # Kill this subprocess - self.queue = Queue() - # Start reporting thread : - Thread(target=self.reporting_thread).start() - - while True: - conn = listener.accept() - logger.debug("connection accepted from %s", listener.last_accepted) - while True: - data = conn.recv() - logger.debug("Incoming data : %s", data) - if data[0] == "ATTACK": - self.queue.put(data[1]) - elif data[0] == "CLOSE": # this is a kind of EOL for python IPC - conn.close() - break - elif data[0] == "KILL": # when main process quits , or during testing etc - logger.debug("Killing subprocess") - conn.close() - pid = os.getpid() - os.kill(pid, signal.SIGTERM) # Kill this subprocess - - def reporting_thread(self): - """Reporting thread""" - logger.debug("Started reporting thread") - while True: - self.send_to_reporter() - time.sleep(REPORT_SEC_INTERVAL) - - def send_to_reporter(self): - """ - Reports the found data to an Aikido server - """ - items_to_report = [] - while not self.queue.empty(): - items_to_report.append(self.queue.get()) - logger.debug("Reporting to aikido server") - logger.critical("Items to report : %s", items_to_report) - # Currently not making API calls - - -# pylint: disable=invalid-name # This variable does change -ipc = None - - -def get_comms(): - """ - Returns the globally stored IPC object, which you need - to communicate to our background process. - """ - return ipc - - -def reset_comms(): - """This will reset communications""" - global ipc - if ipc: - ipc.send_data("KILL", {}) - ipc = None - - def start_background_process(): """ Starts a process to handle incoming/outgoing data """ - # pylint: disable=global-statement # We need this to be global - global ipc + # Generate a secret key : generated_key_bytes = secrets.token_bytes(32) - ipc = IPC(IPC_ADDRESS, generated_key_bytes) - ipc.start_aikido_listener() - - -class IPC: - """ - Facilitates Inter-Process communication - """ - - def __init__(self, address, key): - # The key needs to be in byte form - self.address = address - self.key = key - - def start_aikido_listener(self): - """This will start the aikido process which listens""" - pid = os.fork() - if pid == 0: # Child process - AikidoBackgroundProcess(self.address, self.key) - else: # Parent process - logger.debug("Started background process, PID: %d", pid) - - def send_data(self, action, obj): - """ - This creates a new client for comms to the background process - """ - - # We want to make sure that sending out this data affects the process as little as possible - # So we run it inside a seperate thread with a timeout of 3 seconds - def target(address, key, data_array): - try: - conn = con.Client(address, authkey=key) - logger.debug("Created connection %s", conn) - for data in data_array: - conn.send(data) - conn.send(("CLOSE", {})) - conn.close() - logger.debug("Connection closed") - except Exception as e: - logger.info("Failed to send data to bg process : %s", e) - - t = Thread( - target=target, args=(self.address, self.key, [(action, obj)]), daemon=True - ) - t.start() - t.join(timeout=3) + comms = AikidoIPCCommunications(IPC_ADDRESS, generated_key_bytes) + comms.set_global() + comms.start_aikido_listener() diff --git a/aikido_firewall/background_process/aikido_background_process.py b/aikido_firewall/background_process/aikido_background_process.py new file mode 100644 index 00000000..6b2023a8 --- /dev/null +++ b/aikido_firewall/background_process/aikido_background_process.py @@ -0,0 +1,72 @@ +""" +Simply exports the aikido background process +""" + +import multiprocessing.connection as con +import os +import time +import signal +from threading import Thread +from queue import Queue +from aikido_firewall.helpers.logging import logger + +REPORT_SEC_INTERVAL = 600 # 10 minutes + + +class AikidoBackgroundProcess: + """ + Aikido's background process consists of 2 threads : + - (main) Listening thread which listens on an IPC socket for incoming data + - (spawned) reporting thread which will collect the IPC data and send it to a Reporter + """ + + def __init__(self, address, key): + logger.debug("Background process started") + try: + listener = con.Listener(address, authkey=key) + except OSError: + logger.warning( + "Aikido listener may already be running on port %s", address[1] + ) + pid = os.getpid() + os.kill(pid, signal.SIGTERM) # Kill this subprocess + self.queue = Queue() + # Start reporting thread : + Thread(target=self.reporting_thread).start() + + while True: + conn = listener.accept() + logger.debug("connection accepted from %s", listener.last_accepted) + while True: + data = conn.recv() + logger.debug("Incoming data : %s", data) + if data[0] == "ATTACK": + self.queue.put(data[1]) + elif data[0] == "CLOSE": # this is a kind of EOL for python IPC + conn.close() + break + elif ( + data[0] == "KILL" + ): # when main process quits , or during testing etc + logger.debug("Killing subprocess") + conn.close() + pid = os.getpid() + os.kill(pid, signal.SIGTERM) # Kill this subprocess + + def reporting_thread(self): + """Reporting thread""" + logger.debug("Started reporting thread") + while True: + self.send_to_reporter() + time.sleep(REPORT_SEC_INTERVAL) + + def send_to_reporter(self): + """ + Reports the found data to an Aikido server + """ + items_to_report = [] + while not self.queue.empty(): + items_to_report.append(self.queue.get()) + logger.debug("Reporting to aikido server") + logger.critical("Items to report : %s", items_to_report) + # Currently not making API calls diff --git a/aikido_firewall/background_process/comms.py b/aikido_firewall/background_process/comms.py new file mode 100644 index 00000000..5b91ccc1 --- /dev/null +++ b/aikido_firewall/background_process/comms.py @@ -0,0 +1,75 @@ +import os +import multiprocessing.connection as con +from threading import Thread +from aikido_firewall.helpers.logging import logger +from aikido_firewall.background_process.aikido_background_process import ( + AikidoBackgroundProcess, +) + +# pylint: disable=invalid-name # This variable does change +comms = None + + +def get_comms(): + """ + Returns the globally stored IPC object, which you need + to communicate to our background process. + """ + return comms + + +def reset_comms(): + """This will reset communications""" + global comms + if comms: + comms.send_data_to_bg_process("KILL", {}) + comms = None + + +class AikidoIPCCommunications: + """ + Facilitates Inter-Process communication + """ + + def __init__(self, address, key): + # The key needs to be in byte form + self.address = address + self.key = key + + # Set as global ipc object : + reset_comms() + global comms + comms = self + + def start_aikido_listener(self): + """This will start the aikido process which listens""" + pid = os.fork() + if pid == 0: # Child process + AikidoBackgroundProcess(self.address, self.key) + else: # Parent process + logger.debug("Started background process, PID: %d", pid) + + def send_data_to_bg_process(self, action, obj): + """ + This creates a new client for comms to the background process + """ + + # We want to make sure that sending out this data affects the process as little as possible + # So we run it inside a seperate thread with a timeout of 3 seconds + def target(address, key, data_array): + try: + conn = con.Client(address, authkey=key) + logger.debug("Created connection %s", conn) + for data in data_array: + conn.send(data) + conn.send(("CLOSE", {})) + conn.close() + logger.debug("Connection closed") + except Exception as e: + logger.info("Failed to send data to bg process : %s", e) + + t = Thread( + target=target, args=(self.address, self.key, [(action, obj)]), daemon=True + ) + t.start() + t.join(timeout=3) diff --git a/aikido_firewall/background_process/init_test.py b/aikido_firewall/background_process/init_test.py index 772623a4..f9665363 100644 --- a/aikido_firewall/background_process/init_test.py +++ b/aikido_firewall/background_process/init_test.py @@ -29,7 +29,7 @@ def test_start_background_process(monkeypatch): assert get_comms() == None -def test_send_data_exception(monkeypatch, caplog): +def test_send_data_to_bg_process_exception(monkeypatch, caplog): def mock_client(address, authkey): raise Exception("Connection Error") @@ -37,13 +37,13 @@ def mock_client(address, authkey): monkeypatch.setitem(globals(), "logger", caplog) ipc = IPC(("localhost", 9898), "mock_key") - ipc.send_data("ACTION", "Test Object") + ipc.send_data_to_bg_process("ACTION", "Test Object") -def test_send_data_successful(monkeypatch, caplog, mocker): +def test_send_data_to_bg_process_successful(monkeypatch, caplog, mocker): ipc = IPC(("localhost"), "mock_key") mock_client = mocker.MagicMock() monkeypatch.setattr("multiprocessing.connection.Client", mock_client) - # Call the send_data function - ipc.send_data("ACTION", {"key": "value"}) + # Call the send_data_to_bg_process function + ipc.send_data_to_bg_process("ACTION", {"key": "value"}) diff --git a/aikido_firewall/sinks/pymysql.py b/aikido_firewall/sinks/pymysql.py index a48e8642..eb07463b 100644 --- a/aikido_firewall/sinks/pymysql.py +++ b/aikido_firewall/sinks/pymysql.py @@ -37,7 +37,7 @@ def aikido_new_query(_self, sql, unbuffered=False): logger.info("sql_injection results : %s", json.dumps(result)) if result: - get_comms().send_data("ATTACK", result) + get_comms().send_data_to_bg_process("ATTACK", result) raise Exception("SQL Injection [aikido_firewall]") return prev_query_function(_self, sql, unbuffered=False) From 4480e2bec5ccd5e8b75e71455e50457f3afa0397 Mon Sep 17 00:00:00 2001 From: Wout Feys Date: Tue, 30 Jul 2024 10:21:58 +0200 Subject: [PATCH 26/32] Ignore global pylint error and explain threading (coomentz) --- aikido_firewall/background_process/comms.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/aikido_firewall/background_process/comms.py b/aikido_firewall/background_process/comms.py index 5b91ccc1..ec951c6d 100644 --- a/aikido_firewall/background_process/comms.py +++ b/aikido_firewall/background_process/comms.py @@ -1,3 +1,8 @@ +""" +Holds the globally stored comms object +Exports the AikidoIPCCommunications class +""" + import os import multiprocessing.connection as con from threading import Thread @@ -20,6 +25,7 @@ def get_comms(): def reset_comms(): """This will reset communications""" + # pylint: disable=global-statement # This needs to be global global comms if comms: comms.send_data_to_bg_process("KILL", {}) @@ -38,6 +44,7 @@ def __init__(self, address, key): # Set as global ipc object : reset_comms() + # pylint: disable=global-statement # This needs to be global global comms comms = self @@ -72,4 +79,5 @@ def target(address, key, data_array): target=target, args=(self.address, self.key, [(action, obj)]), daemon=True ) t.start() + # This joins the thread for 3 seconds, afterwards the thread is forced to close (daemon=True) t.join(timeout=3) From 3dbf878b04b93c1b43a676a14f8420c690567c1b Mon Sep 17 00:00:00 2001 From: Wout Feys Date: Tue, 30 Jul 2024 10:25:34 +0200 Subject: [PATCH 27/32] Split up testing, remove unused imports --- .../background_process/__init__.py | 9 ---- .../background_process/comms_test.py | 29 +++++++++++ .../background_process/init_test.py | 49 ------------------- 3 files changed, 29 insertions(+), 58 deletions(-) create mode 100644 aikido_firewall/background_process/comms_test.py delete mode 100644 aikido_firewall/background_process/init_test.py diff --git a/aikido_firewall/background_process/__init__.py b/aikido_firewall/background_process/__init__.py index a9230ddf..c32cbe60 100644 --- a/aikido_firewall/background_process/__init__.py +++ b/aikido_firewall/background_process/__init__.py @@ -3,15 +3,7 @@ and listen for data sent by our sources and sinks """ -import time -import os import secrets -import signal -import socket -import multiprocessing.connection as con -from multiprocessing import Process -from threading import Thread -from queue import Queue from aikido_firewall.helpers.logging import logger from aikido_firewall.background_process.comms import AikidoIPCCommunications @@ -27,5 +19,4 @@ def start_background_process(): generated_key_bytes = secrets.token_bytes(32) comms = AikidoIPCCommunications(IPC_ADDRESS, generated_key_bytes) - comms.set_global() comms.start_aikido_listener() diff --git a/aikido_firewall/background_process/comms_test.py b/aikido_firewall/background_process/comms_test.py new file mode 100644 index 00000000..f0898925 --- /dev/null +++ b/aikido_firewall/background_process/comms_test.py @@ -0,0 +1,29 @@ +import pytest +from aikido_firewall.background_process.comms import AikidoIPCCommunications + +def test_comms_init(): + address = ("localhost", 9898) + key = "secret_key" + comms = AikidoIPCCommunications(address, key) + + assert comms.address == address + assert comms.key == key + +def test_send_data_to_bg_process_exception(monkeypatch, caplog): + def mock_client(address, authkey): + raise Exception("Connection Error") + + monkeypatch.setitem(globals(), "Client", mock_client) + monkeypatch.setitem(globals(), "logger", caplog) + + comms = AikidoIPCCommunications(("localhost", 9898), "mock_key") + comms.send_data_to_bg_process("ACTION", "Test Object") + + +def test_send_data_to_bg_process_successful(monkeypatch, caplog, mocker): + comms = AikidoIPCCommunications(("localhost"), "mock_key") + mock_client = mocker.MagicMock() + monkeypatch.setattr("multiprocessing.connection.Client", mock_client) + + # Call the send_data_to_bg_process function + comms.send_data_to_bg_process("ACTION", {"key": "value"}) diff --git a/aikido_firewall/background_process/init_test.py b/aikido_firewall/background_process/init_test.py deleted file mode 100644 index f9665363..00000000 --- a/aikido_firewall/background_process/init_test.py +++ /dev/null @@ -1,49 +0,0 @@ -import pytest -from aikido_firewall.background_process import ( - IPC, - start_background_process, - get_comms, - IPC_ADDRESS, - reset_comms, -) - - -def test_ipc_init(): - address = ("localhost", 9898) - key = "secret_key" - ipc = IPC(address, key) - - assert ipc.address == address - assert ipc.key == key - - -def test_start_background_process(monkeypatch): - reset_comms() - assert get_comms() == None - start_background_process() - - assert get_comms() != None - assert get_comms().address == IPC_ADDRESS - - reset_comms() - assert get_comms() == None - - -def test_send_data_to_bg_process_exception(monkeypatch, caplog): - def mock_client(address, authkey): - raise Exception("Connection Error") - - monkeypatch.setitem(globals(), "Client", mock_client) - monkeypatch.setitem(globals(), "logger", caplog) - - ipc = IPC(("localhost", 9898), "mock_key") - ipc.send_data_to_bg_process("ACTION", "Test Object") - - -def test_send_data_to_bg_process_successful(monkeypatch, caplog, mocker): - ipc = IPC(("localhost"), "mock_key") - mock_client = mocker.MagicMock() - monkeypatch.setattr("multiprocessing.connection.Client", mock_client) - - # Call the send_data_to_bg_process function - ipc.send_data_to_bg_process("ACTION", {"key": "value"}) From 9968b2ed8f5bf18a14cb1fab4a507dd56189148f Mon Sep 17 00:00:00 2001 From: Wout Feys Date: Tue, 30 Jul 2024 10:27:03 +0200 Subject: [PATCH 28/32] Linting and import comms --- aikido_firewall/background_process/__init__.py | 6 +++++- aikido_firewall/background_process/comms_test.py | 2 ++ 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/aikido_firewall/background_process/__init__.py b/aikido_firewall/background_process/__init__.py index c32cbe60..b2bd76c4 100644 --- a/aikido_firewall/background_process/__init__.py +++ b/aikido_firewall/background_process/__init__.py @@ -5,7 +5,11 @@ import secrets from aikido_firewall.helpers.logging import logger -from aikido_firewall.background_process.comms import AikidoIPCCommunications +from aikido_firewall.background_process.comms import ( + AikidoIPCCommunications, + get_comms, + reset_comms, +) IPC_ADDRESS = ("localhost", 9898) # Specify the IP address and port diff --git a/aikido_firewall/background_process/comms_test.py b/aikido_firewall/background_process/comms_test.py index f0898925..c0232db5 100644 --- a/aikido_firewall/background_process/comms_test.py +++ b/aikido_firewall/background_process/comms_test.py @@ -1,6 +1,7 @@ import pytest from aikido_firewall.background_process.comms import AikidoIPCCommunications + def test_comms_init(): address = ("localhost", 9898) key = "secret_key" @@ -9,6 +10,7 @@ def test_comms_init(): assert comms.address == address assert comms.key == key + def test_send_data_to_bg_process_exception(monkeypatch, caplog): def mock_client(address, authkey): raise Exception("Connection Error") From c586dbb83b94d49b548906f63b0f87f445887bc0 Mon Sep 17 00:00:00 2001 From: Wout Feys Date: Tue, 30 Jul 2024 10:28:36 +0200 Subject: [PATCH 29/32] Linting comment --- aikido_firewall/background_process/__init__.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/aikido_firewall/background_process/__init__.py b/aikido_firewall/background_process/__init__.py index af66b978..586ecb31 100644 --- a/aikido_firewall/background_process/__init__.py +++ b/aikido_firewall/background_process/__init__.py @@ -50,7 +50,9 @@ def __init__(self, address, key): elif data[0] == "CLOSE": # this is a kind of EOL for python IPC conn.close() break - elif data[0] == "KILL": # when main process quits , or during testing etc + elif ( + data[0] == "KILL" + ): # when main process quits , or during testing etc logger.debug("Killing subprocess") conn.close() pid = os.getpid() From 673c889fc7f8c6e18994e755ed143e279f7fc306 Mon Sep 17 00:00:00 2001 From: Wout Feys Date: Tue, 30 Jul 2024 11:03:38 +0200 Subject: [PATCH 30/32] Move gunicorn code to a file so our gunicorn config file is smaller --- aikido_firewall/middleware/django_gunicorn.py | 67 +++++++++++++++++++ .../django-mysql-gunicorn/gunicorn_config.py | 46 ++----------- 2 files changed, 74 insertions(+), 39 deletions(-) create mode 100644 aikido_firewall/middleware/django_gunicorn.py diff --git a/aikido_firewall/middleware/django_gunicorn.py b/aikido_firewall/middleware/django_gunicorn.py new file mode 100644 index 00000000..2a1d3611 --- /dev/null +++ b/aikido_firewall/middleware/django_gunicorn.py @@ -0,0 +1,67 @@ +""" +Includes all the wrappers for gunicorn config file +""" + +from gunicorn.http.body import Body +from io import BytesIO +import aikido_firewall +from aikido_firewall.context import Context + + +def when_ready(prev_func): + """ + Aikido decorator for gunicorn config + Function: pre_request(worker, req) + """ + + def aik_when_ready(server): + aikido_firewall.protect("server-only") + prev_func(server) + + return aik_when_ready + + +def pre_request(prev_func): + """ + Aikido decorator for gunicorn config + Function: pre_request(worker, req) + """ + + def aik_pre_request(worker, req): + req.body, req.body_copy = clone_body(req.body) + + django_context = Context(req, "django-gunicorn") + django_context.set_as_current_context() + prev_func(worker, req) + + return aik_pre_request + + +def post_fork(prev_func): + """ + Aikido decorator for gunicorn config + Function: post_fork(server, worker) + """ + + def aik_post_fork(server, worker): + aikido_firewall.protect("django-gunicorn", False) + prev_func(server, worker) + + return aik_post_fork + + +def clone_body(body): + """ + Clones the body by creating a new stream + """ + body_read = body.read() + + # Read the body content into a buffer + body_buffer = BytesIO() + body_buffer.write(body_read) + body_buffer.seek(0) + + # Create a new Body object with the same content + cloned_body = Body(body_buffer) + + return (cloned_body, body_read) diff --git a/sample-apps/django-mysql-gunicorn/gunicorn_config.py b/sample-apps/django-mysql-gunicorn/gunicorn_config.py index c722b0f0..75761e7e 100644 --- a/sample-apps/django-mysql-gunicorn/gunicorn_config.py +++ b/sample-apps/django-mysql-gunicorn/gunicorn_config.py @@ -1,39 +1,7 @@ -import aikido_firewall -import json -from urllib.parse import parse_qs -from io import BytesIO -from aikido_firewall.context import Context -from gunicorn.http.body import Body - -def when_ready(server): - aikido_firewall.protect("server-only") - -def pre_fork(server, worker): - pass - -def post_fork(server, worker): - print("----------------------> POST FORK") - import aikido_firewall - aikido_firewall.protect("django-gunicorn", False) - -def pre_request(worker, req): - req.body, req.body_copy = clone_body(req.body) - - django_context = Context(req, "django-gunicorn") - django_context.set_as_current_context() - - worker.log.debug("%s %s", req.method, req.path) - - -def clone_body(body): - body_read = body.read() - - # Read the body content into a buffer - body_buffer = BytesIO() - body_buffer.write(body_read) - body_buffer.seek(0) - - # Create a new Body object with the same content - cloned_body = Body(body_buffer) - - return (cloned_body, body_read) +import aikido_firewall.middleware.django_gunicorn as aik +@aik.when_ready +def when_ready(server): pass +@aik.post_fork +def post_fork(server, worker): pass +@aik.pre_request +def pre_request(worker, req): pass From b212ef01f74ad72f1d78f4a5f952e2a1dae87ed1 Mon Sep 17 00:00:00 2001 From: Wout Feys Date: Tue, 30 Jul 2024 15:16:49 +0200 Subject: [PATCH 31/32] Rename "server-only" to "background-process-only" --- aikido_firewall/__init__.py | 2 +- aikido_firewall/middleware/django_gunicorn.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/aikido_firewall/__init__.py b/aikido_firewall/__init__.py index ee32bbe6..82bbb9ae 100644 --- a/aikido_firewall/__init__.py +++ b/aikido_firewall/__init__.py @@ -23,7 +23,7 @@ def protect(module="any", server=True): start_background_process() else: logger.debug("Not starting background process") - if module == "server-only": + if module == "background-process-only": return # Import sources diff --git a/aikido_firewall/middleware/django_gunicorn.py b/aikido_firewall/middleware/django_gunicorn.py index 2a1d3611..9489cb91 100644 --- a/aikido_firewall/middleware/django_gunicorn.py +++ b/aikido_firewall/middleware/django_gunicorn.py @@ -15,7 +15,7 @@ def when_ready(prev_func): """ def aik_when_ready(server): - aikido_firewall.protect("server-only") + aikido_firewall.protect("background-process-only") prev_func(server) return aik_when_ready From 2e2f63dc17e41836092e1c1b679dd5ab95ee58a3 Mon Sep 17 00:00:00 2001 From: willem-delbare <20814660+willem-delbare@users.noreply.github.com> Date: Tue, 30 Jul 2024 15:18:09 +0200 Subject: [PATCH 32/32] Update aikido_firewall/background_process/aikido_background_process.py --- aikido_firewall/background_process/aikido_background_process.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/aikido_firewall/background_process/aikido_background_process.py b/aikido_firewall/background_process/aikido_background_process.py index 6b2023a8..1e760af9 100644 --- a/aikido_firewall/background_process/aikido_background_process.py +++ b/aikido_firewall/background_process/aikido_background_process.py @@ -38,7 +38,7 @@ def __init__(self, address, key): conn = listener.accept() logger.debug("connection accepted from %s", listener.last_accepted) while True: - data = conn.recv() + data = conn.recv() # because of this no sleep needed in thread logger.debug("Incoming data : %s", data) if data[0] == "ATTACK": self.queue.put(data[1])