Squashed changes from develop into main

Makefile

@@ -19,19 +19,19 @@ local: npm mkcert docs
 	sudo docker compose down --remove-orphans
 	sudo URL='https:\/\/localhost:8443' docker compose up --build -d
 	sleep 60
-	cd ./elk && ./seed.sh
+	cd ./elk && ./seed.sh add_mappings add_configs add_tokens add_users
 
 demo: npm mkcert docs
 	sudo docker compose down --remove-orphans
 	sudo URL='https:\/\/demo.paradrop.io' docker compose up --build -d
 	sleep 60
-	cd ./elk && ./seed.sh
+	cd ./elk && ./seed.sh demo
 
 develop: npm mkcert docs
 	sudo docker compose down --remove-orphans
 	sudo URL='https:\/\/develop.paradrop.io' docker compose up --build -d
 	sleep 60
-	cd ./elk && ./seed.sh
+	cd ./elk && ./seed.sh demo
 
 ui: npm
 	./ui/http_server.py
@@ -47,7 +47,7 @@ up:
 	sudo URL='https:\/\/127.0.0.1' docker compose up --build -d
 
 seed:
-	cd ./elk && ./seed.sh
+	cd ./elk && ./seed.sh demo
 
 elk:
 	sudo docker rm -f opensearch
@@ -65,7 +65,7 @@ elk:
 		-v "${PWD}"/elk/opensearch_dashboards.yml:/usr/share/opensearch-dashboards/config/opensearch_dashboards.yml \
 		opensearchproject/opensearch-dashboards:2.11.1
 
-	cd ./elk && ./seed.sh
+	cd ./elk && ./seed.sh demo
 
 cbuilds: npm mkcert
 	cd ./api && sudo docker build -t $(API_NAME) .

README.md

@@ -18,6 +18,8 @@ paradrop is an open source, cybersecurity asset management tool that collects op
 
 ```bash
 make local
+# Optional: Load test / example demo data
+make seed
 ```
 
 > 🖥️  **Vagrant** Method

Vagrantfile

@@ -3,9 +3,9 @@
 
 Vagrant.configure(2) do |config|
     config.vm.box = "ubuntu/jammy64"
-    config.vm.hostname = "paradrop-ubuntu22-1"
+    config.vm.hostname = "paradrop-ubuntu22-01"
     config.vm.provider "virtualbox" do |v|
-        v.name = "paradrop-ubuntu22-1"
+        v.name = "paradrop-ubuntu22-01"
         v.memory = 8192
         v.cpus = 4
         v.customize ["modifyvm", :id, "--natdnsproxy1", "on"]
@@ -43,7 +43,7 @@ ACCEPT_EULA=Y DEBIAN_FRONTEND=noninteractive apt-get update -y
 ACCEPT_EULA=Y DEBIAN_FRONTEND=noninteractive apt-get remove -y whoopsie apport apport-gtk ubuntu-report unattended-upgrades kerneloops plymouth thunderbird transmission-common cheese aisleriot gnome-mahjongg gnome-mines gnome-sudoku remmina mlocate
 ACCEPT_EULA=Y DEBIAN_FRONTEND=noninteractive apt-get autoremove -y
 ACCEPT_EULA=Y DEBIAN_FRONTEND=noninteractive apt-get upgrade -y
-ACCEPT_EULA=Y DEBIAN_FRONTEND=noninteractive apt-get install -y curl jq vim net-tools dnsutils screen nodejs python3-pip python3-dev make unzip htop clamav libopenscap8 dmidecode
+ACCEPT_EULA=Y DEBIAN_FRONTEND=noninteractive apt-get install -y curl jq vim net-tools dnsutils screen nodejs python3-pip python3-dev make unzip htop clamav libopenscap8 dmidecode shellcheck wget apt-transport-https gnupg lsb-release
 
 # Setup Docker
 curl -fsSL https://get.docker.com -o ./get-docker.sh
@@ -54,21 +54,43 @@ systemctl enable docker
 systemctl restart docker
 
 # Install Trivy Scanner
-apt-get install wget apt-transport-https gnupg lsb-release
 wget -qO - https://aquasecurity.github.io/trivy-repo/deb/public.key | sudo apt-key add -
 echo deb https://aquasecurity.github.io/trivy-repo/deb $(lsb_release -sc) main | sudo tee -a /etc/apt/sources.list.d/trivy.list
 apt-get update
 apt-get install trivy
 
 # Install OpenScap Content Guides
-wget -c https://github.com/ComplianceAsCode/content/releases/download/v0.1.72/scap-security-guide-0.1.72.zip
+curl -OLs https://github.com/ComplianceAsCode/content/releases/download/v0.1.72/scap-security-guide-0.1.72.zip
 unzip scap-security-guide-0.1.72.zip
 mkdir -p /usr/share/scap-security-guide
 cp -rf scap-security-guide-0.1.72/* /usr/share/scap-security-guide/
 rm -rf scap-security-guide-0.1.72*
 
 # Install Python Deps
-pip3 install -r /paradrop/api/requirements.txt
+cd /paradrop
+make pip
+pip3 install flake8
+
+# Setup Golang
+GOVER="1.22.3"
+curl -OLs "https://golang.org/dl/go$GOVER.linux-amd64.tar.gz"
+tar -zxf ./"go$GOVER.linux-amd64.tar.gz"
+mv -f ./go /usr/local/
+rm -f ./"go$GOVER.linux-amd64.tar.gz"
+ln -s /usr/local/go/bin/go /usr/bin/go
+
+# Setup Golang Env & Build Agent
+mkdir -p /home/vagrant/go/{src/github.com/Metrostar,bin,pkg}
+ln -s /paradrop/agent /home/vagrant/go/src/github.com/Metrostar/paradrop
+chown -Rf vagrant:vagrant /home/vagrant
+export GOPATH=/home/vagrant/go
+cd /home/vagrant/go/src/github.com/Metrostar/paradrop
+make
+chmod -f 0755 ./paradrop-agent
+
+# Run paradrop Stack
+cd /paradrop
+make local
 
 # Setup paradrop-agent
 mkdir -p /etc/paradrop
@@ -82,9 +104,9 @@ tags: ["app=paradrop-agent-vagrant","health=https://localhost:8443/v1/health"]
 oscap_xccdf_xml: "/usr/share/scap-security-guide/ssg-ubuntu2204-ds.xml"
 EOF
 
-cp -f /paradrop/paradrop-agent /usr/bin/
+cp -f /paradrop/agent/paradrop-agent /usr/bin/
 
-# timeout 15 paradrop-agent -d
+timeout 200 paradrop-agent -d
 
 # Restart
 systemctl reboot

api/api_setup.py

@@ -3,7 +3,7 @@
 from view_resources.search_view import GetModalData, HostsView, ContainersView, SoftwareView, VulnerabilitiesView, HostAssessmentView, EventsView, AuditView, ChangesView
 from health_resources.health_check import HealthCheck
 from host_resources.add_host import AddHost
-from auth_resources.auth_tokens import AddUserToken, GetUserToken, UpdateUserToken, GetAgentToken, UpdateAgentToken#, GetCsrfToken
+from auth_resources.auth_tokens import AddUserToken, GetUserToken, UpdateUserToken, GetAgentToken, UpdateAgentToken
 from configs_resources.configs import ListConfigurations, UpdateConfigurations
 from event_resources.event_triggers import ListEventTriggers, AddEventTrigger, UpdateEventTrigger, DeleteEventTrigger
 from report_resources.reports import ListHostFields, ListReports, AddReport, UpdateReport, DeleteReport, DownloadReports
@@ -142,8 +142,6 @@ async def add_resources() -> Optional[bool]:
 
         api.add_resource(AuthorizationCheck, "/v1/authorization-check")
 
-        # api.add_resource(GetCsrfToken, "/v1/get-csrf-token")
-
         api.add_resource(CreateAccount, "/v1/create-user")
 
         api.add_resource(UpdateUser, "/v1/update-user")

api/auth_resources/auth_tokens.py

@@ -2,7 +2,6 @@
 from flask import session, Response, request
 from flask_restful import Resource
 from asyncio import run
-# from flask_wtf.csrf import generate_csrf
 import uuid
 import json
 from flask_setup import logger
@@ -154,16 +153,3 @@ def post(self) -> json:
             logger.error(e)
             return Response(response=f"Something went wrong..:{e}",
                             status=500)
-
-
-class GetCsrfToken(Resource):
-    @swag_from("endpoints_spec/get_csrf_token.yml")
-    def get(self) -> json:
-        try:
-            # token: str = generate_csrf()
-            return json.dumps({"csrf_token": "token"}), 200
-
-        except BaseException as e:
-            logger.error(e)
-            return Response(response=f"Something went wrong..:{e}",
-                            status=500)

api/auth_resources/endpoints_spec/user_login.yml

@@ -13,12 +13,6 @@ tags:
   - Login/Logout Endpoints
 
 parameters:
-  - name: X-CSRFToken
-    in: header
-    description: CSRFT Token is required because of security reasons. You can generate it in "/v1/get-csrf-token" endpoint.
-    x-example: Ijk1OTQ5YTk1NTBmNzExZTZmNzBkOWEwYzFiMWZlOTNlN2UxOGNiM2Mi.Ys_wKw.6v-BR0UAcQng56FiCPwxsC_c0_Y
-    type: string
-
   - name: body
     in: body
     required: true

api/auth_resources/login.py

@@ -2,7 +2,6 @@
 from flask import session, request, Response
 from flask_restful import Resource
 from asyncio import run
-# from utils.csrf_protection import csrf_protection_enabled
 from db.users.read_users import check_pwd
 from db.users.update_users import db_update_user_attribute
 from utils.timestamps import gen_timestamp
@@ -11,7 +10,6 @@
 
 
 class UserLogin(Resource):
-    # @csrf_protection_enabled
     @swag_from("endpoints_spec/user_login.yml")
     def post(self) -> Response:
         try:

api/db/changes.py

@@ -7,6 +7,7 @@
 from utils.timestamps import gen_timestamp
 import json
 
+
 async def db_add_changes(original_host_data: dict, new_host_data: dict) -> dict:
     """
     Function that takes original host data and new host data, creates a new dictionary
@@ -18,11 +19,32 @@ async def db_add_changes(original_host_data: dict, new_host_data: dict) -> dict:
 
         # List of all keys that we don't want to include in the comparison.
         keys_to_skip: list = [
-        "dmesg_errors", "users_loggedin",
-        "last_run", "failed_logins", "id", "processes",
-        "network_interfaces"
+            "diskfree_gb",
+            "diskused_gb",
+            "diskused_pct",
+            "docker_containers",
+            "docker_images_count",
+            "docker_images",
+            "docker_labels",
+            "id",
+            "network_interfaces",
+            "journalctl_logs",
+            "last_run",
+            "load1",
+            "load5",
+            "load15",
+            "memoryfree_gb",
+            "memoryused_gb",
+            "memoryused_pct",
+            "processes",
+            "sysctl",
+            "systemd_timers",
+            "trivy",
+            "uptime_days",
+            "users_loggedin",
+            "dmesg_errors",
         ]
-        
+
         changes: dict = {}
 
         for key in original_host_data.keys():
@@ -35,10 +57,15 @@ async def db_add_changes(original_host_data: dict, new_host_data: dict) -> dict:
 
                         # Add details to changes
                         changes_summary += f" - {key}"
-                        changes[key] = json.dumps({"+++" : new_host_data[key], "---" : original_host_data[key]}, indent=1)
+                        changes[key] = json.dumps(
+                            {"+++": new_host_data[key], "---": original_host_data[key]},
+                            indent=1,
+                        )
                 else:
                     changes_summary += f" - {key}"
-                    changes[key] = json.dumps({"---" : original_host_data[key]}, indent=1)
+                    changes[key] = json.dumps(
+                        {"---": original_host_data[key]}, indent=1
+                    )
 
         for key in new_host_data.keys():
             if key not in keys_to_skip:
@@ -48,8 +75,8 @@ async def db_add_changes(original_host_data: dict, new_host_data: dict) -> dict:
 
                     # Add details to changes if there is a new key
                     changes_summary += f" - {key}"
-                    changes[key] = json.dumps({"+++" : new_host_data[key]}, indent=1)
-        
+                    changes[key] = json.dumps({"+++": new_host_data[key]}, indent=1)
+
         # If there were any changes made, add them to the Changes index
         if changes != {}:
             changes["id"] = gen_id()
@@ -61,10 +88,8 @@ async def db_add_changes(original_host_data: dict, new_host_data: dict) -> dict:
 
     except BaseException as e:
         logger.error(e)
-        return {
-            "valid": False,
-            "code": 500,
-            "message": f"Something went wrong: {e}.."}
+        return {"valid": False, "code": 500, "message": f"Something went wrong: {e}.."}
+
 
 async def db_get_changes(query: dict = {"query": {"match_all": {}}}) -> dict:
     """
@@ -93,18 +118,17 @@ async def db_get_changes(query: dict = {"query": {"match_all": {}}}) -> dict:
                 "code": 404,
                 "message": "No changes found in the database..",
                 "data": changes,
-                "number_of_results": number_of_results}
+                "number_of_results": number_of_results,
+            }
         else:
             return {
                 "valid": True,
                 "code": 200,
                 "message": "Changes found, returning data..",
                 "data": changes,
-                "number_of_results": number_of_results}
+                "number_of_results": number_of_results,
+            }
 
     except BaseException as e:
         logger.error(e)
-        return {
-            "valid": False,
-            "code": 500,
-            "message": f"Something went wrong: {e}.."}
+        return {"valid": False, "code": 500, "message": f"Something went wrong: {e}.."}

api/db/db_requests.py

@@ -6,8 +6,10 @@
 from urllib3.exceptions import InsecureRequestWarning
 from utils.id_generator import gen_id
 
+
 requests.packages.urllib3.disable_warnings(category=InsecureRequestWarning)
 
+
 def post_request(request_url: str, request_data: dict) -> Response:
     """
     Function to make POST request to the database.