Add FTPS server to system test and test successful upload

.env.sample

@@ -43,16 +43,16 @@ ORTHANC_RAW_MAXIMUM_STORAGE_SIZE= // MB
 
 # PIXL Orthanc anon instance
 ORTHANC_ANON_USERNAME=
-ORTHANC_ANON_PASSWORD=     
+ORTHANC_ANON_PASSWORD=
 ORTHANC_ANON_AE_TITLE=
 ORTHANC_ANON_HTTP_TIMEOUT=60
-ORTHANC_AUTOROUTE_ANON_TO_AZURE=false
+ORTHANC_AUTOROUTE_ANON_TO_FTPS=true
 ENABLE_DICOM_WEB=true
 STUDY_TIME_OFFSET=
 SALT_VALUE=PIXL
 
 
-# UCVNAQR DICOM node information  
+# UCVNAQR DICOM node information
 VNAQR_AE_TITLE=
 VNAQR_DICOM_PORT=
 VNAQR_IP_ADDR=

docker-compose.yml

@@ -115,7 +115,7 @@ services:
       ORTHANC_USERNAME: ${ORTHANC_ANON_USERNAME}
       ORTHANC_PASSWORD: ${ORTHANC_ANON_PASSWORD}
       ORTHANC_ANON_AE_TITLE: ${ORTHANC_ANON_AE_TITLE}
-      ORTHANC_AUTOROUTE_ANON_TO_AZURE: ${ORTHANC_AUTOROUTE_ANON_TO_AZURE}
+      ORTHANC_AUTOROUTE_ANON_TO_FTPS: ${ORTHANC_AUTOROUTE_ANON_TO_FTPS}
       ORTHANC_RAW_AE_TITLE: ${ORTHANC_RAW_AE_TITLE}
       ORTHANC_RAW_DICOM_PORT: "4242"
       ORTHANC_RAW_HOSTNAME: "orthanc-raw"

orthanc/orthanc-anon/plugin/pixl.py

@@ -28,6 +28,7 @@
 from io import BytesIO
 from pathlib import Path
 from time import sleep
+from typing import TYPE_CHECKING
 
 import requests
 import yaml
@@ -38,6 +39,9 @@
 import orthanc
 import pixl_dcmd
 
+if TYPE_CHECKING:
+    from typing import Any
+
 ORTHANC_USERNAME = config("ORTHANC_USERNAME")
 ORTHANC_PASSWORD = config("ORTHANC_PASSWORD")
 ORTHANC_URL = "http://localhost:8042"
@@ -180,17 +184,17 @@ def SendViaFTPS(resourceId: str) -> None:
 
 def ShouldAutoRoute():
     """
-    Checks whether ORTHANC_AUTOROUTE_ANON_TO_AZURE environment variable is
+    Checks whether ORTHANC_AUTOROUTE_ANON_TO_FTPS environment variable is
     set to true or false
     """
-    return os.environ.get("ORTHANC_AUTOROUTE_ANON_TO_AZURE", "false").lower() == "true"
+    return os.environ.get("ORTHANC_AUTOROUTE_ANON_TO_FTPS", "false").lower() == "true"
 
 
-def OnChange(changeType, level, resource) -> None:  # noqa: ARG001
+def OnChange(changeType: str, _level: None, resource: str) -> Any:
     """
     Three ChangeTypes included in this function:
     - If a study if stable and if ShouldAutoRoute returns true
-    then SendViaStow is called
+    then SendViaFTPS is called
     - If orthanc has started then message added to Orthanc LogWarning
     and AzureDICOMTokenRefresh called
     - If orthanc has stopped and TIMER is not none then message added
@@ -202,7 +206,7 @@ def OnChange(changeType, level, resource) -> None:  # noqa: ARG001
 
     if changeType == orthanc.ChangeType.STABLE_STUDY and ShouldAutoRoute():
         print("Stable study: %s" % resource)  # noqa: T201
-        SendViaStow(resource)
+        SendViaFTPS(resource)
 
     if changeType == orthanc.ChangeType.ORTHANC_STARTED:
         orthanc.LogWarning("Starting the scheduler")
@@ -213,13 +217,13 @@ def OnChange(changeType, level, resource) -> None:  # noqa: ARG001
             TIMER.cancel()
 
 
-def OnHeartBeat(output, uri, **request):  # noqa: ARG001
+def OnHeartBeat(output, uri, **request) -> Any:  # noqa: ARG001
     """Extends the REST API by registering a new route in the REST API"""
     orthanc.LogWarning("OK")
     output.AnswerBuffer("OK\n", "text/plain")
 
 
-def ReceivedInstanceCallback(receivedDicom, origin):
+def ReceivedInstanceCallback(receivedDicom: BytesIO, origin: str) -> Any:
     """Modifies a DICOM instance received by Orthanc and applies anonymisation."""
     if origin == orthanc.InstanceOrigin.REST_API:
         orthanc.LogWarning("DICOM instance received from the REST API")

pixl_core/tests/conftest.py

@@ -35,6 +35,7 @@
 os.environ["FTP_PORT"] = "20021"
 
 TEST_DIR = Path(__file__).parent
+DUMMY_SERVICES_DIR = Path(__file__).parents[2] / "test" / "dummy-services"
 STUDY_DATE = datetime.date.fromisoformat("2023-01-01")
 
 
@@ -71,9 +72,11 @@ def mounted_data() -> Path:
     The mounted data directory for the ftp server.
     This will contain the data after successful upload.
     """
-    yield TEST_DIR / "ftp-server" / "mounts" / "data"
+    yield DUMMY_SERVICES_DIR / "ftp-server" / "mounts" / "data"
     sub_dirs = [
-        f.path for f in os.scandir(TEST_DIR / "ftp-server" / "mounts" / "data") if f.is_dir()
+        f.path
+        for f in os.scandir(DUMMY_SERVICES_DIR / "ftp-server" / "mounts" / "data")
+        if f.is_dir()
     ]
     # Tear down the directory after tests
     for sub_dir in sub_dirs:

pixl_core/tests/docker-compose.yml

@@ -32,16 +32,16 @@ services:
       retries: 5
   ftp-server:
     container_name: test-ftp-server
-    build: 
-      context: ftp-server
+    build:
+      context: ../../test/dummy-services/ftp-server
     ports:
       - "20021:21"
       - "21000-21010:21000-21010"
     volumes:
       # Mount for uploaded data
-      - "./ftp-server/mounts/data/:/home/pixl/"
+      - "../../test/dummy-services/ftp-server/mounts/data/:/home/pixl/"
       # Mount SSL keys for TLS
-      - "./ftp-server/mounts/ssl/:/etc/ssl/private/"
+      - "../../test/dummy-services/ftp-server/mounts/ssl/:/etc/ssl/private/"
     environment:
       ADDRESS: "localhost"
       USERS: pixl|longpassword|/home/pixl

pixl_ehr/README.md

@@ -43,7 +43,8 @@ and the processing tests with
 ```
 
 To test the availability of a CogStack instance, we mock up a *FastAPI* server which simply takes in
-some input text and returns the same text. The configuration of this mock instance is defined in [`test/dummy-services`](/test/dummy-services/).
+some input text and returns the same text. The mocking is handled by a *pytest* fixture in
+`test_processing.py()` (`_mock_requests`).
 
 ## Usage
 

test/.env.test

@@ -43,7 +43,7 @@ ORTHANC_ANON_PASSWORD=orthanc_anon_password
 ORTHANC_ANON_AE_TITLE=ORTHANCANON
 ORTHANC_ANON_HTTP_TIMEOUT=60
 ENABLE_DICOM_WEB=true
-ORTHANC_AUTOROUTE_ANON_TO_AZURE=false
+ORTHANC_AUTOROUTE_ANON_TO_FTPS=true
 PIXL_DICOM_TRANSFER_TIMEOUT=240
 
 # UCVNAQR DICOM node information

test/docker-compose.yml

@@ -67,7 +67,7 @@ services:
   cogstack-api:
     container_name: system-test-cogstack
     build:
-      context: ./dummy-services/
+      context: ./dummy-services/cogstack/
     healthcheck:
       test: [ "CMD", "curl", "-f", "http://localhost:8000/heart-beat" ]
       interval: 10s
@@ -76,4 +76,25 @@ services:
     networks:
       pixl-net:
 
-
+  ftp-server:
+    container_name: test-ftp-server
+    build:
+      context: ./dummy-services/ftp-server
+    ports:
+      - "20021:21"
+      - "21000-21010:21000-21010"
+    volumes:
+      # Mount for uploaded data
+      - "./dummy-services/ftp-server/mounts/data/:/home/pixl/"
+      # Mount SSL keys for TLS
+      - "./dummy-services/ftp-server/mounts/ssl/:/etc/ssl/private/"
+    environment:
+      ADDRESS: "localhost"
+      USERS: pixl|longpassword|/home/pixl
+      TLS_KEY: /etc/ssl/private/localhost.key
+      TLS_CERT: /etc/ssl/private/localhost.crt
+    healthcheck:
+      test: ping localhost:21 -w 2
+      interval: 10s
+      timeout: 5s
+      retries: 5

test/run-system-test.sh

@@ -34,6 +34,7 @@ sleep 65  # need to wait until the DICOM image is "stable" = 60s
 ./scripts/check_entry_in_pixl_anon.sh
 ./scripts/check_entry_in_orthanc_anon.sh
 ./scripts/check_max_storage_in_orthanc_raw.sh
+./scripts/check_ftps_upload.py
 
 pixl extract-radiology-reports "${PACKAGE_DIR}/test/resources/omop"
 
@@ -45,4 +46,3 @@ docker exec system-test-ehr-api-1 rm -r /run/exports/test-extract-uclh-omop-cdm/
 
 cd "${PACKAGE_DIR}"
 docker compose -f docker-compose.yml -f test/docker-compose.yml -p system-test down --volumes
-

test/scripts/check_ftps_upload.py

@@ -0,0 +1,33 @@
+#!/usr/bin/env python
+#  Copyright (c) University College London Hospitals NHS Foundation Trust
+#
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+import logging
+from pathlib import Path
+from pixl_cli._io import config_from_log_file
+
+logger = logging.getLogger(__name__)
+
+PARQUET_PATH = Path(__file__).parents[1] / "resources" / "omop"
+MOUNTED_DATA_DIR = Path(__file__).parents[1] / "dummy-services" / "ftp-server" / "mounts" / "data"
+
+project_name, omop_es_timestamp = config_from_log_file(PARQUET_PATH)
+expected_output_dir = MOUNTED_DATA_DIR / project_name
+
+# Test whether DICOM images have been uploaded
+glob_list = [expected_output_dir.glob("*.zip")]
+logger.info(f"glob_list: {glob_list}")
+
+assert len(glob_list) == 1
+
+# TODO: check parquet files upload