debug

.github/workflows/_build-package.yml

@@ -10,7 +10,7 @@ jobs:
       - name: Set up Python
         uses: actions/setup-python@v4
         with: 
-          python-version: 3.11
+          python-version: 3.12
       - name: Install build dependencies
         run: pip install --no-cache-dir -U pip .['build']
       - name: Build package

.github/workflows/_integration-tests.yml

@@ -1,22 +1,46 @@
 name: integration-tests
 on: 
   workflow_call:
-    secrets:
-      DOCKER_TOKEN:
-        required: true
 jobs:
-  integration-tests:
-    name: Run integration tests
+  test-alert-endpoints:
+    name: Test alert endpoints
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v4
       - name: Set up Python
         uses: actions/setup-python@v4
         with:
-          python-version: 3.11
+          python-version: 3.12
       - name: Install dependencies
         run: pip install --no-cache-dir -U pip .['test']
-      - name: Docker login
-        run: docker login -u kamforka -p ${{ secrets.DOCKER_TOKEN }}
       - name: Run integration tests
-        run: scripts/ci.py --test
+        run: pytest -v tests/test_alert_endpoint.py
+
+  test-case-endpoints:
+    name: Test case endpoints
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - name: Set up Python
+        uses: actions/setup-python@v4
+        with:
+          python-version: 3.12
+      - name: Install dependencies
+        run: pip install --no-cache-dir -U pip .['test']
+      - name: Run integration tests
+        run: pytest -v tests/test_case_endpoint.py
+
+  test-other-endpoints:
+    name: Test other endpoints
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - name: Set up Python
+        uses: actions/setup-python@v4
+        with:
+          python-version: 3.12
+      - name: Install dependencies
+        run: pip install --no-cache-dir -U pip .['test']
+      - name: Run integration tests
+        run: |
+          pytest -v --ignore=tests/test_alert_endpoint.py --ignore=tests/test_case_endpoint.py

.github/workflows/_static-checks.yml

@@ -7,7 +7,7 @@ jobs:
     runs-on: ubuntu-latest
     strategy:
       matrix:
-        python-version: ["3.8", "3.9", "3.10", "3.11"]
+        python-version: ["3.8", "3.9", "3.10", "3.11", "3.12"]
     steps:
       - uses: actions/checkout@v4
       - name: Set up Python ${{ matrix.python-version }}

.github/workflows/_upload-package.yml

@@ -26,7 +26,7 @@ jobs:
       - name: Set up Python
         uses: actions/setup-python@v4
         with: 
-          python-version: 3.11
+          python-version: 3.12
       - name: Install build dependencies
         run: pip install --no-cache-dir -U pip .['build']
       - name: Upload to PyPI

.github/workflows/integrator.yml

@@ -0,0 +1,61 @@
+name: integrator-image
+on: 
+  push:
+    branches:
+      - main
+  pull_request:
+jobs:
+  changes:
+    name: Change detection
+    runs-on: ubuntu-latest
+    outputs:
+      integrator: ${{ steps.filter.outputs.integrator }}
+    steps:
+      - uses: actions/checkout@v4
+      - uses: dorny/paths-filter@v3
+        id: filter
+        with:
+          filters: |
+            integrator:
+              - 'docker/thehive4py-integrator/**'
+  build:
+    name: Build and push
+    needs: changes
+    if: ${{ needs.changes.outputs.integrator == 'true' }}
+    runs-on: ubuntu-latest
+    env:
+      INTEGRATOR_BUILD_CTX: docker/thehive4py-integrator
+      INTEGRATOR_IMAGE_NAME: kamforka/thehive4py-integrator
+      THEHIVE_VERSION: 5.3.0
+
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+
+      - name: Set variables
+        id: variables
+        run: |
+          echo "integrator_image_fullname=$INTEGRATOR_IMAGE_NAME:thehive-$THEHIVE_VERSION" >> "$GITHUB_OUTPUT"
+          echo "integrator_image_fullname_with_hash=$INTEGRATOR_IMAGE_NAME:thehive-$THEHIVE_VERSION-$(git rev-parse --short HEAD)" >> "$GITHUB_OUTPUT"
+
+      - name: Set up QEMU
+        uses: docker/setup-qemu-action@v3
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
+
+      - name: Login to Docker Hub
+        uses: docker/login-action@v3
+        with:
+          username: kamforka
+          password: ${{ secrets.DOCKER_TOKEN }}
+
+      - name: Build and push
+        uses: docker/build-push-action@v5
+        with:
+          context: ${{ env.INTEGRATOR_BUILD_CTX }}
+          platforms: linux/amd64,linux/arm64
+          push: ${{ github.ref == 'refs/heads/main' }}
+          tags: ${{ steps.variables.outputs.integrator_image_fullname }},${{ steps.variables.outputs.integrator_image_fullname_with_hash}}
+          build-args: |
+            THEHIVE_VERSION=${{ env.THEHIVE_VERSION }}

.github/workflows/main-cicd.yml

@@ -10,8 +10,6 @@ jobs:
     uses: ./.github/workflows/_static-checks.yml
   integration-tests:
     uses: ./.github/workflows/_integration-tests.yml
-    secrets:
-      DOCKER_TOKEN: ${{ secrets.DOCKER_TOKEN }}
   build-package:
     uses: ./.github/workflows/_build-package.yml
   upload-package:

README.md

@@ -304,14 +304,37 @@ With pre-commit hooks in place, your changes will be automatically validated for
 
 ## Testing
 
+> IMPORTANT NOTE: Since TheHive 5.3 the licensing constraints has been partially lifted therefore a public integrator image is available for running tests both locally and in github.
+
 `thehive4py` primarily relies on integration tests, which are designed to execute against a live TheHive 5.x instance. These tests ensure that the library functions correctly in an environment closely resembling real-world usage.
 
-However, due to licensing constraints with TheHive 5.x, the integration tests are currently not available for public or local use.
+### Test requirements
+
+Since the test suite relies on the existence of a live TheHive docker container a local docker engine installation is a must.
+If you are unfamiliar with docker please check out the [official documentation][get-docker].
+
+### Test setup
+
+The test suite relies on a self-contained TheHive image which we call: [thehive4py-integrator]. 
+This image contains everything that is needed to run a barebone TheHive instance. 
+
+The test suite uses this image to create a container locally with the predefined name `thehive4py-integration-tester` which will act as a unique id. 
+The container will expose TheHive on a random port to make sure it causes no conflicts for any other containers which expose ports.
+The suite can identify this random port by querying the container info based on the predefined name. 
+During the very first test run the container will take a longer time to initialize due to the ElasticSearch and TheHive startups.
+Once TheHive is responsive the suite will initialize the instance with a setup required by the tests (e.g.: test users, organisations, etc.).
+After a successful initialization the 
+
+### Testing locally
+To execute the whole test suite locally one can use the `scripts/ci.py` utility script like:
 
-To ensure code quality and prevent broken code from being merged, a private image is available for the integration-test workflow. This means that any issues should be detected and addressed during the PR phase.
+    ./scripts/ci.py --test
 
-The project is actively working on a solution to enable developers to run integration tests locally, providing a more accessible and comprehensive testing experience.
+Note however that the above will execute the entire test suite which can take several minutes to complete.
+In case one wants to execute only a portion of the test suite then the easiest workaround is to use `pytest` and pass the path to the specific test module. For example to only execute tests for the alert endpoints one can do:
 
-While local testing is in development, relying on the automated PR checks ensures the reliability and quality of the `thehive4py` library.
+    pytest -v tests/test_alert_endpoint.py
 
-[query-api-docs]: https://docs.strangebee.com/thehive/api-docs/#operation/Query%20API
+[get-docker]: https://docs.docker.com/get-docker/
+[query-api-docs]: https://docs.strangebee.com/thehive/api-docs/#operation/Query%20API
+[thehive4py-integrator]: https://hub.docker.com/repository/docker/kamforka/thehive4py-integrator/general

docker/thehive4py-integrator/Dockerfile

@@ -0,0 +1,50 @@
+FROM alpine:3.17 as base
+
+# BUILDER STAGE
+FROM base as builder
+
+ARG ES_VERSION=7.17.19
+ARG THEHIVE_VERSION=5.3.0
+
+RUN apk update && apk upgrade && apk add curl
+
+## ES DOWNLOAD
+ARG ES_DOWNLOAD_URL=https://artifacts.elastic.co/downloads/elasticsearch/elasticsearch-${ES_VERSION}-linux-x86_64.tar.gz
+
+RUN curl -Lo /tmp/elasticsearch.tgz ${ES_DOWNLOAD_URL} \
+    && tar -xzf /tmp/elasticsearch.tgz -C /tmp \
+    && mv /tmp/elasticsearch-${ES_VERSION} /tmp/elasticsearch
+
+## THEHIVE DOWNLOAD
+ARG THEHIVE_DOWNLOAD_URL=https://archives.strangebee.com/zip/thehive-${THEHIVE_VERSION}-1.zip 
+
+RUN curl -Lo /tmp/thehive.zip ${THEHIVE_DOWNLOAD_URL}
+RUN unzip -qo /tmp/thehive.zip -d /tmp \
+    && mv /tmp/thehive-${THEHIVE_VERSION}-1 /tmp/thehive
+
+# FINAL STAGE
+FROM base
+RUN apk update && apk upgrade && apk add --no-cache openjdk11-jre-headless bash su-exec curl
+
+## ES SETUP
+COPY --from=builder /tmp/elasticsearch /usr/share/elasticsearch
+COPY configs/elasticsearch.yml /usr/share/elasticsearch/config/elasticsearch.yml
+
+RUN adduser -u 1000 -g 1000 -Dh /usr/share/elasticsearch elasticsearch \
+    && mkdir -p /usr/share/elasticsearch/data \
+    && chown -R elasticsearch:elasticsearch /usr/share/elasticsearch \
+    && rm -rf /usr/share/elasticsearch/modules/x-pack-ml/platform/linux-x86_64
+
+## THEHIVE SETUP
+COPY --from=builder /tmp/thehive /opt/thehive/
+COPY configs/thehive.conf /opt/thehive/conf/application.conf
+
+RUN adduser -u 1001 -g 1001 -Dh /opt/thehive thehive \
+    && mkdir /var/log/thehive \
+    && chown -R thehive:thehive /opt/thehive /var/log/thehive
+
+## ENTRYPOINT
+COPY entrypoint.sh /
+RUN chmod +x entrypoint.sh
+EXPOSE 9000
+ENTRYPOINT /entrypoint.sh

docker/thehive4py-integrator/configs/elasticsearch.yml

@@ -0,0 +1,7 @@
+http.host: 0.0.0.0
+transport.host: 0.0.0.0
+discovery.type: single-node
+cluster.name: thehive4py
+xpack.security.enabled: false
+xpack.ml.enabled: false
+script.allowed_types: "inline,stored"

docker/thehive4py-integrator/configs/thehive.conf

@@ -0,0 +1,22 @@
+play.http.secret.key="supersecret"
+play.http.parser.maxDiskBuffer: 20MB
+
+db {
+  provider: janusgraph
+  janusgraph {
+    storage {
+      backend: berkeleyje
+      directory: /opt/thehive/db
+    }
+
+    index.search {
+      backend: elasticsearch
+      hostname: ["127.0.0.1"]
+    }
+  }
+}
+
+storage {
+  provider: localfs
+  localfs.location: /opt/thehive/data
+}

docker/thehive4py-integrator/entrypoint.sh

@@ -0,0 +1,35 @@
+#!/bin/bash
+
+wait_for_elastic() {
+    local health_url="http://localhost:9200/_cat/health"
+    local timeout=30
+
+    local start_time=$(date +%s)
+    while true; do
+        local current_time=$(date +%s)
+        local elapsed_time=$((current_time - start_time))
+
+        if [ "$elapsed_time" -ge "$timeout" ]; then
+            echo "error: elastic couldn't start in $timeout seconds"
+            exit 1
+        fi
+
+        local status_code=$(curl -so /dev/null -w %{http_code} ${health_url})
+        if [ "$status_code" -eq 200 ]; then
+            return
+        fi
+
+        sleep 0.25
+    done
+}
+
+
+echo "starting elasticsearch in the background"
+export ES_JAVA_HOME=$(dirname $(dirname $(readlink -f $(which java))))
+su-exec elasticsearch /usr/share/elasticsearch/bin/elasticsearch > /dev/null 2>&1 &
+
+echo "waiting for elastic to start up..."
+wait_for_elastic
+
+echo "starting thehive in the foreground"
+su-exec thehive /opt/thehive/bin/thehive -Dconfig.file=/opt/thehive/conf/application.conf

pyproject.toml

@@ -20,14 +20,15 @@ classifiers = [
     "Programming Language :: Python :: 3.9",
     "Programming Language :: Python :: 3.10",
     "Programming Language :: Python :: 3.11",
+    "Programming Language :: Python :: 3.12",
     "License :: OSI Approved :: GNU Affero General Public License v3",
 ]
 authors = [{ name = "Szabolcs Antal", email = "[email protected]" }]
 
 [project.optional-dependencies]
 audit = ["bandit", "pip-audit"]
 build = ["build", "twine"]
-lint = ["black", "flake8", "flake8-pyproject", "mypy", "pre-commit"]
+lint = ["black",  "flake8-pyproject", "mypy", "pre-commit"]
 test = ["pytest", "pytest-cov"]
 dev = ["thehive4py[audit, lint, test, build]"]
 

tests/conftest.py

@@ -2,7 +2,7 @@
 
 import pytest
 
-from tests.utils import TestConfig, reinit_hive_container, spawn_hive_container
+from tests.utils import TestConfig, reset_hive_instance, spawn_hive_container
 from thehive4py.client import TheHiveApi
 from thehive4py.helpers import now_to_ts
 from thehive4py.types.alert import InputAlert, OutputAlert
@@ -23,8 +23,8 @@
 @pytest.fixture(scope="session")
 def test_config():
     return TestConfig(
-        image_name="kamforka/thehive4py-integrator:thehive-5.2.11",
-        container_name="thehive4py-integration-tests",
+        image_name="kamforka/thehive4py-integrator:thehive-5.3.0",
+        container_name="thehive4py-integration-tester",
         user="[email protected]",
         password="secret",
         admin_org="admin",
@@ -34,8 +34,8 @@ def test_config():
 
 
 @pytest.fixture(scope="function", autouse=True)
-def init_hive_container(test_config: TestConfig):
-    reinit_hive_container(test_config=test_config)
+def auto_reset_hive_instance(thehive: TheHiveApi, test_config: TestConfig):
+    reset_hive_instance(hive_url=thehive.session.hive_url, test_config=test_config)
 
 
 @pytest.fixture(scope="session")

tests/test_case_endpoint.py

@@ -197,6 +197,7 @@ def test_share_and_unshare(self, thehive: TheHiveApi, test_case: OutputCase):
         thehive.case.unshare(case_id=test_case["_id"], organisation_ids=[organisation])
         assert len(thehive.case.list_shares(case_id=test_case["_id"])) == 1
 
+    @pytest.mark.skip(reason="integrator container only supports a single org ")
     def test_share_and_remove_share(self, thehive: TheHiveApi, test_case: OutputCase):
         organisation = "share-org"
         share: InputShare = {"organisation": organisation}
@@ -220,6 +221,7 @@ def test_update_share(self, thehive: TheHiveApi, test_case: OutputCase):
         updated_share = thehive.case.share(case_id=test_case["_id"], shares=[share])[0]
         assert updated_share["profileName"] == update_profile
 
+    @pytest.mark.skip(reason="integrator container only supports a single org ")
     def test_share_and_set_share(self, thehive: TheHiveApi, test_case: OutputCase):
         organisation = "share-org"
         share: InputShare = {"organisation": organisation}

tests/test_user_endpoint.py

@@ -61,6 +61,7 @@ def test_delete(self, thehive: TheHiveApi, test_user: OutputUser):
         with pytest.raises(TheHiveError):
             thehive.user.get(user_id=user_id)
 
+    @pytest.mark.skip(reason="integrator container only supports a single org ")
     def test_set_organisations(
         self, test_config: TestConfig, thehive: TheHiveApi, test_user: OutputUser
     ):