Merge branch 'main' into milanmlft/232-multi-project-config

cli/README.md

@@ -37,7 +37,7 @@ pixl --help
 ```
 ### Configuration
 
-The `rabbitmq` and `postgress` services are configured by setting the following environment variables
+The `rabbitmq` and `postgres` services are configured by setting the following environment variables
 (default values shown):
 
 ```sh
@@ -87,19 +87,11 @@ parquet_dir
     └── PROCEDURE_OCCURRENCE.parquet
 ```
 
-Start the imaging extraction
+Extraction will start automatically after populating the queues.  If granular
+customisation of the rate per queue is required or a queue should not be started
+then supply the argument `--no-start` and use `pixl start...` to launch
+processing.
 
-```bash
-pixl start --queues imaging
-```
-
-and equivalently the EHR extraction
-
-```bash
-pixl start --queues ehr
-```
-
-Use `pixl start --help` for information.
 
 Stop Imaging and EHR database extraction
 

cli/pyproject.toml

@@ -17,7 +17,8 @@ dependencies = [
 [project.optional-dependencies]
 test = [
     "pytest==7.4.*",
-    "pytest-pixl",
+    "pytest-mock==3.12.*",
+    "pytest-pixl"
 ]
 dev = [
     "mypy",

cli/src/pixl_cli/main.py

@@ -92,10 +92,25 @@ def check_env(*, error: bool, sample_env_file: click.Path) -> None:
     default=True,
     help="Restart from a saved state. Otherwise will use the given input file(s)",
 )
+@click.option(
+    "--start/--no-start",
+    "start_processing",
+    show_default=True,
+    default=True,
+    help="Start processing from the queues after population is complete",
+)
+@click.option(
+    "--rate",
+    type=float,
+    default=None,
+    help="Rate at which to process items from a queue (in items per second).",
+)
 @click.argument(
     "parquet-path", required=True, type=click.Path(path_type=Path, exists=True, file_okay=True)
 )
-def populate(parquet_path: Path, *, restart: bool, queues: str) -> None:
+def populate(
+    parquet_path: Path, *, queues: str, restart: bool, start_processing: bool, rate: Optional[float]
+) -> None:
     """
     Populate a (set of) queue(s) from a parquet file directory
     PARQUET_DIR: Directory containing the public and private parquet input files and an
@@ -135,6 +150,9 @@ def populate(parquet_path: Path, *, restart: bool, queues: str) -> None:
         with PixlProducer(queue_name=queue, **SERVICE_SETTINGS["rabbitmq"]) as producer:
             producer.publish(sorted_messages)
 
+    if start_processing:
+        _start_or_update_extract(queues=queues.split(","), rate=rate)
+
 
 @cli.command()
 @click.argument(

cli/tests/test_populate.py

@@ -27,25 +27,47 @@
     from core.patient_queue.message import Message
 
 
+class MockProducer(PixlProducer):
+    """Mock producer of PIXL task messages."""
+
+    def __enter__(self) -> PixlProducer:
+        """Context entrypoint."""
+        return self
+
+    def __exit__(self, *args: object, **kwargs) -> None:
+        """Context exit point."""
+        return
+
+    def publish(self, messages: list[Message]) -> None:  # noqa: ARG002 don't access messages
+        """Dummy method for publish."""
+        return
+
+
 def test_populate_queue_parquet(
     monkeypatch, omop_resources: Path, queue_name: str = "test_populate"
 ) -> None:
     """Checks that patient queue can be populated without error."""
     omop_parquet_dir = str(omop_resources / "omop")
     runner = CliRunner()
 
-    # mock producer
-    class MockProducer(PixlProducer):
-        def __enter__(self) -> PixlProducer:
-            return self
+    monkeypatch.setattr(pixl_cli.main, "PixlProducer", MockProducer)
+
+    result = runner.invoke(populate, args=[omop_parquet_dir, "--queues", queue_name, "--no-start"])
+    assert result.exit_code == 0
 
-        def __exit__(self, *args: object, **kwargs) -> None:
-            return
 
-        def publish(self, messages: list[Message]) -> None:  # noqa: ARG002 don't access messages
-            return
+def test_populate_queue_and_start(
+    mocker, monkeypatch, omop_resources: Path, queue_name: str = "test_populate"
+) -> None:
+    """Checks that patient queue can be populated without error."""
+    omop_parquet_dir = str(omop_resources / "omop")
+    runner = CliRunner()
 
+    mocked_start = mocker.patch("pixl_cli.main._start_or_update_extract")
     monkeypatch.setattr(pixl_cli.main, "PixlProducer", MockProducer)
 
-    result = runner.invoke(populate, args=[omop_parquet_dir, "--queues", queue_name])
-    assert result.exit_code == 0
+    runner.invoke(populate, args=[omop_parquet_dir, "--queues", queue_name, "--no-start"])
+    mocked_start.assert_not_called()
+
+    runner.invoke(populate, args=[omop_parquet_dir, "--queues", queue_name])
+    mocked_start.assert_called_with(queues=queue_name.split(","), rate=None)

docker/ehr-api/Dockerfile

@@ -18,8 +18,14 @@ ARG TEST="false"
 
 RUN export DEBIAN_FRONTEND=noninteractive && \
     apt-get update && \
-    apt-get install --yes --no-install-recommends procps ca-certificates \
-    iproute2 git curl libpq-dev curl gnupg g++ locales
+    apt-get install --yes --no-install-recommends \
+        procps \
+        ca-certificates \
+        iproute2 \
+        libpq-dev \
+        curl \
+        gnupg \
+        locales
 RUN sed -i '/en_GB.UTF-8/s/^# //g' /etc/locale.gen && locale-gen
 RUN  apt-get autoremove && apt-get clean && rm -rf /var/lib/apt/lists/*
 

docker/imaging-api/Dockerfile

@@ -18,8 +18,14 @@ ARG TEST="false"
 
 RUN export DEBIAN_FRONTEND=noninteractive && \
     apt-get update && \
-    apt-get install --yes --no-install-recommends procps ca-certificates \
-    iproute2 git curl libpq-dev curl gnupg g++ locales
+    apt-get install --yes --no-install-recommends \
+        procps \
+        ca-certificates \
+        iproute2 \
+        libpq-dev \
+        curl \
+        gnupg \
+        locales
 RUN sed -i '/en_GB.UTF-8/s/^# //g' /etc/locale.gen && locale-gen
 RUN  apt-get autoremove && apt-get clean && rm -rf /var/lib/apt/lists/*
 

pixl_ehr/pyproject.toml

@@ -10,7 +10,7 @@ dependencies = [
     "core",
     "uvicorn==0.23.2",
     "python-decouple==3.6",
-    "psycopg2==2.9.5",
+    "psycopg2-binary==2.9.9",
     "azure-identity==1.12.0",
     "azure-storage-blob==12.14.1",
     "pyarrow==14.0.1",

test/conftest.py

@@ -38,7 +38,6 @@ def _setup_pixl_cli(ftps_server) -> None:
     """Run pixl populate/start. Cleanup intermediate export dir on exit."""
     # CLI calls need to have CWD = test dir so they can find the pixl_config.yml file
     run_subprocess(["pixl", "populate", str(RESOURCES_OMOP_DIR.absolute())], TEST_DIR)
-    run_subprocess(["pixl", "start"], TEST_DIR)
     # poll here for two minutes to check for imaging to be processed, printing progress
     wait_for_stable_orthanc_anon(121, 5)
     yield

test/dummy-services/cogstack/Dockerfile

@@ -15,8 +15,13 @@ FROM python:3.11.7-slim-bullseye
 
 RUN export DEBIAN_FRONTEND=noninteractive && \
     apt-get update && \
-    apt-get install --yes --no-install-recommends procps ca-certificates \
-    iproute2 git curl libpq-dev curl gnupg g++ locales
+    apt-get install --yes --no-install-recommends \
+        procps \
+        ca-certificates \
+        iproute2 \
+        curl \
+        gnupg \
+        locales
 
 WORKDIR /app