Merge branch 'main' into feature/report-plugins-enhacement

.github/workflows/build-rdo-package.yml

@@ -16,6 +16,9 @@ env:
 jobs:
   build:
     runs-on: ubuntu-22.04
+    strategy:
+      matrix:
+        python_version: ["3.10", "3.11"]
 
     # Generic bits
     steps:
@@ -26,7 +29,7 @@ jobs:
 
       - uses: actions/setup-python@v5
         with:
-          python-version: "3.10"
+          python-version: ${{ matrix.python_version }}
           cache: "pip"
 
       - name: Prep Install requirements
@@ -48,18 +51,18 @@ jobs:
 
       - name: Octopoes Build whl package
         run: |
-          python3.10 -m pip install build
-          python3.10 -m build
+          python${{ matrix.python_version }} -m pip install build
+          python${{ matrix.python_version }} -m build
         working-directory: ./octopoes
 
       - name: Octopoes Create env
-        run: python3.10 -m venv /var/www/html/.venv
+        run: python${{ matrix.python_version }} -m venv /var/www/html/.venv
 
       - name: Octopoes Install requirements
         run: cd /var/www/html; source .venv/bin/activate; pip install --upgrade pip; pip install --requirement requirements.txt
 
       - name: Octopoes Create venv archive
-        run: tar -zcvf ${{ env.PKGDIR }}/octopoes_venv_${{ env.RELEASE_VERSION }}.tar.gz -C /var/www/html/ .venv
+        run: tar -zcvf ${{ env.PKGDIR }}/octopoes_venv_${{ env.RELEASE_VERSION }}_python${{ matrix.python_version }}.tar.gz -C /var/www/html/ .venv
 
       - name: Octopoes Enable production logging config
         run: mv prod.logging.yml logging.yml
@@ -84,13 +87,13 @@ jobs:
         working-directory: ./rocky
 
       - name: Rocky Create env
-        run: python3.10 -m venv /var/www/html/.venv
+        run: python${{ matrix.python_version }} -m venv /var/www/html/.venv
 
       - name: Rocky Install requirements
         run: cd /var/www/html; source .venv/bin/activate; pip install --upgrade pip; grep -v git+https:// requirements.txt | pip install -r /dev/stdin ; grep git+https:// requirements.txt | pip install -r /dev/stdin; pip install ${{ github.workspace }}/octopoes/dist/octopoes*.whl
 
       - name: Rocky Create rocky_venv tarball
-        run: tar -zcvf ${{ env.PKGDIR }}/rocky_venv_${{ env.RELEASE_VERSION }}.tar.gz -C /var/www/html/ .venv
+        run: tar -zcvf ${{ env.PKGDIR }}/rocky_venv_${{ env.RELEASE_VERSION }}_python${{ matrix.python_version }}.tar.gz -C /var/www/html/ .venv
 
       - name: Rocky Run nvm install
         run: nvm install v16
@@ -113,7 +116,7 @@ jobs:
         working-directory: ./rocky
 
       - name: Rocky Compilemessages
-        run: /var/www/html/.venv/bin/python3.10 manage.py collectstatic && /var/www/html/.venv/bin/python3.10 manage.py compress && /var/www/html/.venv/bin/python3.10 manage.py compilemessages
+        run: /var/www/html/.venv/bin/python${{ matrix.python_version }} manage.py collectstatic && /var/www/html/.venv/bin/python${{ matrix.python_version }} manage.py compress && /var/www/html/.venv/bin/python${{ matrix.python_version }} manage.py compilemessages
         working-directory: ./rocky
         env:
           BYTES_API: http://bytes:8000
@@ -138,7 +141,7 @@ jobs:
         working-directory: ./bytes
 
       - name: Bytes Create env
-        run: python3.10 -m venv /var/www/html/.venv
+        run: python${{ matrix.python_version }} -m venv /var/www/html/.venv
 
       - name: Bytes Install requirements
         run: cd /var/www/html; source .venv/bin/activate; pip install --upgrade pip; pip install --requirement requirements.txt
@@ -148,7 +151,7 @@ jobs:
         working-directory: ./bytes
 
       - name: Bytes Create bytes venv package
-        run: tar -zcvf ${{ env.PKGDIR }}/bytes_venv_${{ env.RELEASE_VERSION }}.tar.gz -C /var/www/html/ .venv
+        run: tar -zcvf ${{ env.PKGDIR }}/bytes_venv_${{ env.RELEASE_VERSION }}_python${{ matrix.python_version }}.tar.gz -C /var/www/html/ .venv
 
       # Mula / Scheduler
       - name: Mula Create /var/lib/html
@@ -159,21 +162,21 @@ jobs:
         working-directory: ./mula
 
       - name: Mula Create env
-        run: python3.10 -m venv /var/www/html/.venv
+        run: python${{ matrix.python_version }} -m venv /var/www/html/.venv
 
       - name: Create scheduler release archive
         run: tar -cvzf ${{ env.PKGDIR }}/scheduler_${{ env.RELEASE_VERSION }}.tar.gz --exclude=./.git* --exclude=Makefile --exclude=Dockerfile --exclude=requirements* --exclude=tests .
         working-directory: ./mula
 
       - name: Create virtual env
-        run: python3.10 -m venv /var/www/html/.venv
+        run: python${{ matrix.python_version }} -m venv /var/www/html/.venv
 
       - name: Install requirements
         run: source .venv/bin/activate; pip install --upgrade pip; pip install --requirement requirements.txt
         working-directory: /var/www/html
 
       - name: Create venv archive
-        run: tar -zcvf ${{ env.PKGDIR }}/scheduler_venv_${{ env.RELEASE_VERSION }}.tar.gz -C /var/www/html/ .venv
+        run: tar -zcvf ${{ env.PKGDIR }}/scheduler_venv_${{ env.RELEASE_VERSION }}_python${{ matrix.python_version }}.tar.gz -C /var/www/html/ .venv
 
       # Boefjes
       - name: Boefjes Create /var/lib/html
@@ -184,14 +187,14 @@ jobs:
         working-directory: ./boefjes
 
       - name: Boefjes Create env
-        run: python3.10 -m venv /var/www/html/.venv
+        run: python${{ matrix.python_version }} -m venv /var/www/html/.venv
 
       - name: Install requirements
         run: source .venv/bin/activate; pip install --upgrade pip; grep -v git+https:// requirements.txt | pip install -r /dev/stdin ; grep git+https:// requirements.txt | pip install -r /dev/stdin; pip install ${{ github.workspace }}/octopoes/dist/octopoes*.whl
         working-directory: /var/www/html
 
       - name: Create archive
-        run: tar -zcvf ${{ env.PKGDIR }}/boefjes_venv_${{ env.RELEASE_VERSION }}.tar.gz -C /var/www/html/ .venv
+        run: tar -zcvf ${{ env.PKGDIR }}/boefjes_venv_${{ env.RELEASE_VERSION }}_python${{ matrix.python_version }}.tar.gz -C /var/www/html/ .venv
 
       - name: Create boefjes release
         run: tar -cvzf ${{ env.PKGDIR }}/boefjes_${{ env.RELEASE_VERSION }}.tar.gz .
@@ -206,14 +209,14 @@ jobs:
         working-directory: ./keiko
 
       - name: Keiko Create env
-        run: python3.10 -m venv /var/www/html/.venv
+        run: python${{ matrix.python_version }} -m venv /var/www/html/.venv
 
       - name: Keiko Install requirements
         run: source .venv/bin/activate; pip install --upgrade pip; find . -name requirements.txt | xargs -L 1 pip install -r
         working-directory: /var/www/html
 
       - name: Keiko Create archive
-        run: tar -zcvf ${{ env.PKGDIR }}/katkeiko_venv_${{ env.RELEASE_VERSION }}.tar.gz -C /var/www/html/ .venv
+        run: tar -zcvf ${{ env.PKGDIR }}/katkeiko_venv_${{ env.RELEASE_VERSION }}_python${{ matrix.python_version }}.tar.gz -C /var/www/html/ .venv
 
       - name: Keiko Create katkeiko release
         run: tar -cvzf ${{ env.PKGDIR }}/katkeiko_${{ env.RELEASE_VERSION }}.tar.gz .
@@ -229,8 +232,8 @@ jobs:
       - name: Octopoes Upload venv tar
         uses: actions/upload-artifact@v4
         with:
-          name: octopoes_venv_${{ env.RELEASE_VERSION }}
-          path: ${{ env.PKGDIR }}/octopoes_venv_${{ env.RELEASE_VERSION }}.tar.gz
+          name: octopoes_venv_${{ env.RELEASE_VERSION }}_python${{ matrix.python_version }}
+          path: ${{ env.PKGDIR }}/octopoes_venv_${{ env.RELEASE_VERSION }}_python${{ matrix.python_version }}.tar.gz
 
       - name: Octopoes Upload octopoes tar
         uses: actions/upload-artifact@v4
@@ -247,8 +250,8 @@ jobs:
       - name: Rocky Upload venv tar
         uses: actions/upload-artifact@v4
         with:
-          name: rocky_venv_${{ env.RELEASE_VERSION }}
-          path: ${{ env.PKGDIR }}/rocky_venv_${{ env.RELEASE_VERSION }}.tar.gz
+          name: rocky_venv_${{ env.RELEASE_VERSION }}_python${{ matrix.python_version }}
+          path: ${{ env.PKGDIR }}/rocky_venv_${{ env.RELEASE_VERSION }}_python${{ matrix.python_version }}.tar.gz
 
       - name: Bytes Upload bytes tar
         uses: actions/upload-artifact@v4
@@ -259,8 +262,8 @@ jobs:
       - name: Bytes Upload bytes venv tar
         uses: actions/upload-artifact@v4
         with:
-          name: bytes_venv_${{ env.RELEASE_VERSION }}
-          path: ${{ env.PKGDIR }}/bytes_venv_${{ env.RELEASE_VERSION }}.tar.gz
+          name: bytes_venv_${{ env.RELEASE_VERSION }}_python${{ matrix.python_version }}
+          path: ${{ env.PKGDIR }}/bytes_venv_${{ env.RELEASE_VERSION }}_python${{ matrix.python_version }}.tar.gz
 
       - name: Upload scheduler release
         uses: actions/upload-artifact@v4
@@ -271,8 +274,8 @@ jobs:
       - name: Upload venv archive
         uses: actions/upload-artifact@v4
         with:
-          name: scheduler_venv_${{ env.RELEASE_VERSION }}
-          path: ${{ env.PKGDIR }}/scheduler_venv_${{ env.RELEASE_VERSION }}.tar.gz
+          name: scheduler_venv_${{ env.RELEASE_VERSION }}_python${{ matrix.python_version }}
+          path: ${{ env.PKGDIR }}/scheduler_venv_${{ env.RELEASE_VERSION }}_python${{ matrix.python_version }}.tar.gz
 
       - name: Upload boefjes tar
         uses: actions/upload-artifact@v4
@@ -283,8 +286,8 @@ jobs:
       - name: Upload venv tar
         uses: actions/upload-artifact@v4
         with:
-          name: boefjes_venv_${{ env.RELEASE_VERSION }}
-          path: ${{ env.PKGDIR }}/boefjes_venv_${{ env.RELEASE_VERSION }}.tar.gz
+          name: boefjes_venv_${{ env.RELEASE_VERSION }}_python${{ matrix.python_version }}
+          path: ${{ env.PKGDIR }}/boefjes_venv_${{ env.RELEASE_VERSION }}_python${{ matrix.python_version }}.tar.gz
 
       - name: Keiko Upload tar
         uses: actions/upload-artifact@v4
@@ -295,5 +298,5 @@ jobs:
       - name: Keiko Upload venv tar
         uses: actions/upload-artifact@v4
         with:
-          name: katkeiko_venv_${{ env.RELEASE_VERSION }}
-          path: ${{ env.PKGDIR }}/katkeiko_venv_${{ env.RELEASE_VERSION }}.tar.gz
+          name: katkeiko_venv_${{ env.RELEASE_VERSION }}_python${{ matrix.python_version }}
+          path: ${{ env.PKGDIR }}/katkeiko_venv_${{ env.RELEASE_VERSION }}_python${{ matrix.python_version }}.tar.gz

.github/workflows/build_docs_on_pr.yml

@@ -3,7 +3,14 @@ name: Build docs artifact for PR
 on:
   pull_request:
     paths:
+      # We generate documentation for boefje, bytes, keiko, mula and octopoes
+      # from code so the workflow should also depend on it.
+      - "boefje/**"
+      - "bytes/**"
       - "docs/**"
+      - "keiko/**"
+      - "mula/**"
+      - "octopoes/**"
       - "requirements.txt"
       - ".github/workflows/build_docs_on_pr.yml"
 

.gitignore

@@ -437,7 +437,14 @@ nl-kat-*
 /_version.py
 
 # Automatically generated markdown files for the environment variables
-/docs/source/technical_design/environment_settings/*.md
+docs/source/installation-and-deployment/environment-settings/boefjes.md
+docs/source/installation-and-deployment/environment-settings/bytes.md
+docs/source/installation-and-deployment/environment-settings/keiko.md
+docs/source/installation-and-deployment/environment-settings/mula.md
+docs/source/installation-and-deployment/environment-settings/octopoes.md
+
+docs/source/_static/d3.min.js
+docs/source/_static/mermaid.min.js
 
 # rpki cache
 /boefjes/boefjes/plugins/kat_rpki/rpki.json

.pre-commit-config.yaml

@@ -188,5 +188,6 @@ repos:
           \.json$ |
           \.min\.js$ |
           ^rocky/assets/css/themes/soft/fonts |
-          ^rocky/assets/vendors
+          ^rocky/assets/vendors |
+          ^docs/source/_static
           )

Makefile

@@ -12,12 +12,14 @@ UNAME := $(shell uname)
 export DOCKER_BUILDKIT=1
 export COMPOSE_DOCKER_CLI_BUILD=1
 
+# We can't really return an error here, so if settings-doc fails we delete the
+# file which will result in sphinx-build returning an error later on
 define build-settings-doc
-	echo "# $(4)" > docs/source/installation_and_deployment/environment_settings/$(3).md
+	echo "# $(4)" > docs/source/installation-and-deployment/environment-settings/$(3).md
 	DOCS=True PYTHONPATH=./$(1) settings-doc generate \
 	-f markdown -m $(2) \
 	--templates docs/settings-doc-templates \
-	>> docs/source/installation_and_deployment/environment_settings/$(3).md
+	>> docs/source/installation-and-deployment/environment-settings/$(3).md || exit 1
 endef
 
 
@@ -114,7 +116,12 @@ docs:
 	$(call build-settings-doc,bytes,bytes.config,bytes,Bytes)
 	$(call build-settings-doc,mula/scheduler,config.settings,mula,Mula)
 
-	PYTHONPATH=$(PYTHONPATH):boefjes/:bytes/:keiko/:mula/:octopoes/ sphinx-build -b html docs/source docs/_build
+	curl -sL -o - https://registry.npmjs.org/d3/-/d3-7.9.0.tgz | tar -Oxzf - package/dist/d3.min.js > docs/source/_static/d3.min.js
+	curl -sL -o - https://registry.npmjs.org/mermaid/-/mermaid-11.3.0.tgz | tar -Oxzf - package/dist/mermaid.min.js > docs/source/_static/mermaid.min.js
+	echo "f2094bbf6141b359722c4fe454eb6c4b0f0e42cc10cc7af921fc158fceb86539  docs/source/_static/d3.min.js" | sha256sum --quiet --check || exit 1
+	echo "0d2b6f2361e7e0ce466a6ed458e03daa5584b42ef6926c3beb62eb64670ca261  docs/source/_static/mermaid.min.js" | sha256sum --quiet --check || exit 1
+
+	PYTHONPATH=$(PYTHONPATH):boefjes/:bytes/:keiko/:mula/:octopoes/ sphinx-build -b html --fail-on-warning docs/source docs/_build
 
 
 poetry-dependencies:

docs/source/conf.py

@@ -59,5 +59,13 @@
 html_static_path = ["_static"]
 html_css_files = ["openkat.css"]
 
-mermaid_version = ""  # Do not fetch from the CDN
-html_js_files = ["mermaid-9.4.3.min.js"]
+mermaid_use_local = "mermaid.min.js"
+mermaid_include_elk = ""
+d3_use_local = "d3.min.js"
+
+autosectionlabel_prefix_document = True
+
+suppress_warnings = [
+    f"autosectionlabel.installation-and-deployment/environment-settings/{document}"
+    for document in ("boefjes", "bytes", "keiko", "mula", "octopoes")
+]

docs/source/developer_documentation/boefjes-runner.md → docs/source/developer-documentation/boefjes-runner.md

@@ -265,7 +265,7 @@ having been implemented in these PRs:
 - https://github.com/minvws/nl-kat-coordination/pull/2709
 - https://github.com/minvws/nl-kat-coordination/pull/2832
 
-#### Summary of decisions
+### Summary of decisions
 
 We decided not to focus on the following:
 

docs/source/developer_documentation/boefjes.md → docs/source/developer-documentation/boefjes.md

@@ -3,7 +3,7 @@
 This module has several entry points discussed below, but let us first consider the prerequisites and scope.
 If you already have a running setup and want to learn where each bit of functionality goes, read the following page:
 
-[Developing Openkat Plugins](../introduction/makeyourown.rst)
+[Developing Openkat Plugins](../introduction/make-your-own.rst)
 
 ## Prerequisites
 

docs/source/developer_documentation/development_tutorial/creating_a_report.md → docs/source/developer-documentation/development-tutorial/creating-report.md

@@ -77,7 +77,7 @@ After having done all that. The user should be able to create their own Greeting
 
 Let's go to the reports tab, and change our filters again so we can see our Greetings OOI. Check one of their boxes and press the _Continue with selection_ button. Now in the grid of available report types you can make, you should see 2 options. The Findings Report and the Greetings Report. Let's check them both and press once more on the _Continue with selection_ button. Now you will see a report that includes both the Findings Report and the Greetings report inside a single web page. In the top right, you can press the _Export_ button which will make a pdf of your report. Including information about every finding of your project.
 
-If you want to get more information inside your report, make sure to check our [technical report documentation](https://docs.openkat.nl/developer_documentation/rocky.html#testing) about reports.
+If you want to get more information inside your report, make sure to check our documentation about {ref}`developer-documentation/reports:Creating reports`.
 
 ## Conclusion
 

docs/source/developer_documentation/development_tutorial/index.rst → docs/source/developer-documentation/development-tutorial/index.rst

@@ -24,9 +24,9 @@ Glossary
    :caption: Contents
    :maxdepth: 1
 
-   creating_a_boefje
-   testing_the_boefje
-   creating_a_new_model
-   creating_a_normalizer
-   creating_a_bit
-   creating_a_report
+   creating-boefje
+   testing-boefje
+   creating-model
+   creating-normalizer
+   creating-bit
+   creating-report

docs/source/developer_documentation/index.rst → docs/source/developer-documentation/index.rst

@@ -14,8 +14,8 @@ Contains documentation for developers and contributors.
    normalisers-runner
    bytes
    octopoes
-   octopoes_models
-   octopoes_research
+   octopoes-models
+   octopoes-research
    keiko
    reports
-   development_tutorial/index
+   development-tutorial/index

docs/source/developer_documentation/keiko.md → docs/source/developer-documentation/keiko.md

@@ -49,7 +49,7 @@ template is started. After the report is generated, the PDF is available at `/re
 ## Logging
 
 Keiko logging can be configured through by supplying a `logging.json`, relative to the current working directory of the
-process. See [logging.json](logging.json) for an example.
+process. See `keiko/logging.json` for an example.
 
 ## Building a new template
 

docs/source/developer_documentation/normalisers-runner.md → docs/source/developer-documentation/normalisers-runner.md

@@ -6,7 +6,7 @@ _Status: Draft_
 
 The current workflow is as follows:
 
-```mermaid
+```{mermaid}
 graph LR;
     Mula--Tasks-->Boefjes;
     Boefjes--Information-->Bytes;

docs/source/developer_documentation/octopoes.md → docs/source/developer-documentation/octopoes.md

@@ -413,7 +413,7 @@ python3 octopoes/tools/xtdb-cli.py -n "aa" history --with-docs "BIT_METRIC" |jq
 
 If you want to query the XTDB database directly, you can use the XTDB-tool. This is explained on the following page: https://docs.openkat.nl/developer_documentation/octopoes.html#xtdb-cli-tool.
 
-## OOI
+## OOI Objects
 
 OOI objects are instances of relatively simple classes, which inherit from `OOI`.
 

docs/source/developer_documentation/reports.md → docs/source/developer-documentation/reports.md

@@ -31,7 +31,7 @@ class YourNameReport(Report):
 ```
 
 5. Open `reports/report_types/helpers.py` and add your new class to the `REPORTS` constant list.
-6. Implement a method within `report.py` to gather the required data for report generation. See the [Collecting data](#collecting-data) section for more information.
+6. Implement a method within `report.py` to gather the required data for report generation. See the {ref}`developer-documentation/reports:Collecting data` section for more information.
 7. Design the HTML structure for your report within `report.html`. The generated data from `report.py` can be used with Django Template in this file. For example by referring to the returned value like `{{ data }}`.
 8. Save your changes and refresh the page to see the changes you made immediately.
 
@@ -43,7 +43,7 @@ The `generate_data` method only works on single OOIs and should not be used in n
 Use all existing reports as examples to gather data for your report.
 
 - In the file `rocky/reports/report_types/definitions.py` you can find some methods that may be useful.
-- For querying data from Octopoes, consult `octopoes/octopoes/connector/octopoes.py` which contains various useful methods. Additional information on how to write queries can be found [here](https://docs.openkat.nl/developer_documentation/octopoes.html#querying).
+- For querying data from Octopoes, consult `octopoes/octopoes/connector/octopoes.py` which contains various useful methods. Additional information on how to write queries can be found {ref}`here <developer-documentation/octopoes:Querying>`.
 
 ## Writing report unit tests
 
@@ -91,4 +91,4 @@ def test_my_new_report_multiple_results(mock_octopoes_api_connector, valid_time,
 
 ### Executing unit tests
 
-Information about how to execute unit tests can be found [here](https://docs.openkat.nl/developer_documentation/rocky.html#testing).
+Information about how to execute unit tests can be found {ref}`here <rocky-testing>`.