More accurate orthanc-anon system test (#272)

* Set STUDY_TIME_OFFSET in .env.test

Required to run tests that hit pixl_dcmd.

* Add PIXL_DB_* environment variables to orthanc_anon container

pixl_dcmd needs these to persist anonmyised identifierst to postgres.

* Change `_hash_values` to return a string instead of bytes.

This fixes an error when trying to store the hash in the database and aligns
with what the hasher endpoints return.

* Improve orthanc-anon check

The existing test script checked the orthanc-anon log for receipt of a DICOM but
this was not catching that the received DICOM was not being stored due problems
during anonymisation.  The new check script queries the orthanc-anon API to make
sure it has received and stored the test DICOMs.

* Add license header

* Account for responses from hasher being text

* Correct types when logging the response

Co-authored-by: Stef Piatek <[email protected]>

* Clearer separation between header and documentation

Co-authored-by: Stef Piatek <[email protected]>

* Attempt to satisfy the linter

* Don't actually need interactive or TTY for the test

---------

Co-authored-by: Stef Piatek <[email protected]>

docker-compose.yml

@@ -119,6 +119,11 @@ services:
       ORTHANC_RAW_AE_TITLE: ${ORTHANC_RAW_AE_TITLE}
       ORTHANC_RAW_DICOM_PORT: "4242"
       ORTHANC_RAW_HOSTNAME: "orthanc-raw"
+      PIXL_DB_HOST: ${PIXL_DB_HOST}
+      PIXL_DB_PORT: ${PIXL_DB_PORT}
+      PIXL_DB_NAME: ${PIXL_DB_NAME}
+      PIXL_DB_USER: ${PIXL_DB_USER}
+      PIXL_DB_PASSWORD: ${PIXL_DB_PASSWORD}
       DICOM_WEB_PLUGIN_ENABLED: ${ENABLE_DICOM_WEB}
       HASHER_API_AZ_NAME: "hasher-api"
       HASHER_API_PORT: 8000

pixl_dcmd/src/pixl_dcmd/main.py

@@ -353,10 +353,12 @@ def hash_endpoint_path_for_tag(group: bytes, element: bytes) -> str:
     return "/hash"
 
 
-def _hash_values(grp: bytes, el: bytes, pat_value: str, hasher_host_url: str) -> bytes:
+def _hash_values(grp: bytes, el: bytes, pat_value: str, hasher_host_url: str) -> str:
     ep_path = hash_endpoint_path_for_tag(group=grp, element=el)
     payload = ep_path + "?message=" + pat_value
     request_url = hasher_host_url + payload
     response = requests.get(request_url)
-    logging.info(b"RESPONSE = %a}" % response.content)
-    return response.content
+    # All three hashing endpoints return application/text so should be fine to
+    # use response.text here
+    logging.info("RESPONSE = %s}" % response.text)
+    return response.text

pixl_dcmd/tests/conftest.py

@@ -110,6 +110,7 @@ class MockResponse(object):
     def __init__(self, content: str):
         self.status_code = 200
         self.content = "-".join(list(content)).encode("utf-8")
+        self.text = self.content.decode("utf-8")
 
 
 # monkeypatched requests.get moved to a fixture

pixl_dcmd/tests/test_main.py

@@ -76,7 +76,7 @@ def test_pseudo_identifier_processing(rows_in_session, tag_scheme):
 
     accession_number = "AA12345605"
     mrn = "987654321"
-    fake_hash = "-".join(list(f"{mrn}{accession_number}")).encode("utf-8")
+    fake_hash = "-".join(list(f"{mrn}{accession_number}"))
     print("fake_hash = ", fake_hash)
     output_dataset = apply_tag_scheme(input_dataset, tag_scheme)
     image = (

test/.env.test

@@ -45,6 +45,7 @@ ORTHANC_ANON_HTTP_TIMEOUT=60
 ENABLE_DICOM_WEB=true
 ORTHANC_AUTOROUTE_ANON_TO_AZURE=false
 PIXL_DICOM_TRANSFER_TIMEOUT=240
+STUDY_TIME_OFFSET=0
 
 # UCVNAQR DICOM node information
 VNAQR_AE_TITLE=VNAQR

test/run-system-test.sh

@@ -32,7 +32,7 @@ pixl populate "${PACKAGE_DIR}/test/resources/omop"
 pixl start
 sleep 65  # need to wait until the DICOM image is "stable" = 60s
 ./scripts/check_entry_in_pixl_anon.sh
-./scripts/check_entry_in_orthanc_anon.sh
+./scripts/check_entry_in_orthanc_anon.py
 ./scripts/check_max_storage_in_orthanc_raw.sh
 
 pixl extract-radiology-reports "${PACKAGE_DIR}/test/resources/omop"

test/scripts/check_entry_in_orthanc_anon.sh → test/scripts/check_entry_in_orthanc_anon.py

@@ -1,4 +1,5 @@
-#!/bin/bash
+#!/usr/bin/env python3
+
 #  Copyright (c) University College London Hospitals NHS Foundation Trust
 #
 #  Licensed under the Apache License, Version 2.0 (the "License");
@@ -12,8 +13,19 @@
 #  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 #  See the License for the specific language governing permissions and
 #  limitations under the License.
-set -euxo pipefail
 
-# This could be much improved by having more realistic test data some of
-# which actually was persisted
-docker logs system-test-orthanc-anon-1 2>&1 | grep "DICOM instance received"
+"""
+After pixl has run this script will query the orthanc-anon REST API to check
+that the correct number of instances have been received.
+"""
+
+import json
+import shlex
+import subprocess
+
+
+instances_cmd = shlex.split('docker exec system-test-orthanc-anon-1 curl -u "orthanc_anon_username:orthanc_anon_password" http://orthanc-anon:8042/instances')
+instances_output = subprocess.run(instances_cmd, capture_output=True, check=True, text=True)
+instances = json.loads(instances_output.stdout)
+print("orthanc-anon instances:", instances)
+assert len(instances) == 2