resolving merge conflict from master

.github/workflows/code-checks.yaml

@@ -4,7 +4,7 @@ on: push
 
 jobs:
   code-checks:
-    runs-on: ubuntu-20.04
+    runs-on: ubuntu-latest
     steps:
       - name: checkout
         uses: actions/checkout@v4

.github/workflows/integration-tests.yaml

@@ -23,9 +23,9 @@ jobs:
         to_test:
           - "mnist-keras numpyhelper"
           - "mnist-pytorch numpyhelper"
-        python_version: ["3.8", "3.9", "3.10", "3.11", "3.12"]
+        python_version: ["3.9", "3.10", "3.11", "3.12"]
         os:
-          - ubuntu-22.04
+          - ubuntu-24.04
     runs-on: ${{ matrix.os }}
     steps:
       - name: checkout

Dockerfile

@@ -1,54 +1,61 @@
-# Base image
-ARG BASE_IMG=python:3.10-slim
-FROM $BASE_IMG
+# Stage 1: Builder
+ARG BASE_IMG=python:3.12-slim
+FROM $BASE_IMG as builder
 
 ARG GRPC_HEALTH_PROBE_VERSION=""
-
-# Requirements (use MNIST Keras as default)
 ARG REQUIREMENTS=""
 
+WORKDIR /build
+
+# Install build dependencies
+RUN apt-get update && apt-get upgrade -y && apt-get install -y --no-install-recommends python3-dev gcc wget \
+  && rm -rf /var/lib/apt/lists/*
+
 # Add FEDn and default configs
-COPY . /app
-COPY config/settings-client.yaml.template /app/config/settings-client.yaml
-COPY config/settings-combiner.yaml.template /app/config/settings-combiner.yaml
-COPY config/settings-hooks.yaml.template /app/config/settings-hooks.yaml
-COPY config/settings-reducer.yaml.template /app/config/settings-reducer.yaml
-COPY $REQUIREMENTS /app/config/requirements.txt
+COPY . /build
+COPY $REQUIREMENTS /build/requirements.txt
 
-# Install developer tools (needed for psutil)
-RUN apt-get update && apt-get install -y python3-dev gcc
+# Install dependencies
+RUN python -m venv /venv \
+  && /venv/bin/pip install --upgrade pip \
+  && /venv/bin/pip install --no-cache-dir 'setuptools>=65' \
+  && /venv/bin/pip install --no-cache-dir . \
+  && if [[ ! -z "$REQUIREMENTS" ]]; then \
+  /venv/bin/pip install --no-cache-dir -r /build/requirements.txt; \
+  fi \
+  && rm -rf /build/requirements.txt
 
-# Install grpc health probe checker
+
+# Install grpc health probe
 RUN if [ ! -z "$GRPC_HEALTH_PROBE_VERSION" ]; then \
-  apt-get install -y wget && \
-  wget -qO/bin/grpc_health_probe https://github.com/grpc-ecosystem/grpc-health-probe/releases/download/${GRPC_HEALTH_PROBE_VERSION}/grpc_health_probe-linux-amd64 && \
-  chmod +x /bin/grpc_health_probe && \
-  apt-get remove -y wget && apt autoremove -y; \
-  else \
-  echo "No grpc_health_probe version specified, skipping installation"; \
+  wget -qO /build/grpc_health_probe https://github.com/grpc-ecosystem/grpc-health-probe/releases/download/${GRPC_HEALTH_PROBE_VERSION}/grpc_health_probe-linux-amd64 && \
+  chmod +x /build/grpc_health_probe; \
   fi
 
-# Setup working directory
+# Stage 2: Runtime
+FROM $BASE_IMG
+
 WORKDIR /app
 
-# Create FEDn app directory
-SHELL ["/bin/bash", "-c"]
-RUN mkdir -p /app \
-  && mkdir -p /app/client \
-  && mkdir -p /app/certs \
-  && mkdir -p /app/client/package \
-  && mkdir -p /app/certs \
-  #
-  # Install FEDn and requirements
-  && python -m venv /venv \
-  && /venv/bin/pip install --upgrade pip \
-  && /venv/bin/pip install --no-cache-dir 'setuptools>=65' \
-  && /venv/bin/pip install --no-cache-dir -e . \
-  && if [[ ! -z "$REQUIREMENTS" ]]; then \
-  /venv/bin/pip install --no-cache-dir -r /app/config/requirements.txt; \
-  fi \
-  #
-  # Clean up
-  && rm -r /app/config/requirements.txt
+# Copy application and venv from the builder stage
+COPY --from=builder /venv /venv
+COPY --from=builder /build /app
+
+# Use a non-root user
+RUN set -ex \
+  # Create a non-root user
+  && addgroup --system --gid 1001 appgroup \
+  && adduser --system --uid 1001 --gid 1001 --no-create-home appuser \
+  # Creare application specific tmp directory, set ENV TMPDIR to /app/tmp
+  && mkdir -p /app/tmp \
+  && chown -R appuser:appgroup /venv /app \
+  # Upgrade the package index and install security upgrades
+  && apt-get update \
+  && apt-get upgrade -y \
+  && apt-get autoremove -y \
+  && apt-get clean -y \
+  && rm -rf /var/lib/apt/lists/*
+USER appuser
+
+ENTRYPOINT [ "/venv/bin/fedn" ]
 
-ENTRYPOINT [ "/venv/bin/fedn" ]

docker-compose.yaml

@@ -58,21 +58,23 @@ services:
       - USER=test
       - PROJECT=project
       - FLASK_DEBUG=1
-      - STATESTORE_CONFIG=/app/config/settings-reducer.yaml
-      - MODELSTORAGE_CONFIG=/app/config/settings-reducer.yaml
+      - STATESTORE_CONFIG=/app/config/settings-reducer.yaml.template
+      - MODELSTORAGE_CONFIG=/app/config/settings-reducer.yaml.template
+      - FEDN_COMPUTE_PACKAGE_DIR=/app
+      - TMPDIR=/app/tmp
     build:
       context: .
       args:
-        BASE_IMG: ${BASE_IMG:-python:3.10-slim}
+        BASE_IMG: ${BASE_IMG:-python:3.12-slim}
     working_dir: /app
     volumes:
       - ${HOST_REPO_DIR:-.}/fedn:/app/fedn
     depends_on:
       - minio
       - mongo
-    entrypoint: [ "sh", "-c" ]
     command:
-      - "/venv/bin/pip install --no-cache-dir -e . && /venv/bin/fedn controller start"
+      - controller
+      - start
     ports:
       - 8092:8092
 
@@ -81,24 +83,27 @@ services:
     environment:
       - PYTHONUNBUFFERED=0
       - GET_HOSTS_FROM=dns
-      - STATESTORE_CONFIG=/app/config/settings-combiner.yaml
-      - MODELSTORAGE_CONFIG=/app/config/settings-combiner.yaml
+      - STATESTORE_CONFIG=/app/config/settings-combiner.yaml.template
+      - MODELSTORAGE_CONFIG=/app/config/settings-combiner.yaml.template
       - HOOK_SERVICE_HOST=hook:12081
+      - TMPDIR=/app/tmp
     build:
       context: .
       args:
-        BASE_IMG: ${BASE_IMG:-python:3.10-slim}
-        GRPC_HEALTH_PROBE_VERSION: v0.4.24
+        BASE_IMG: ${BASE_IMG:-python:3.12-slim}
+        GRPC_HEALTH_PROBE_VERSION: v0.4.35
     working_dir: /app
     volumes:
       - ${HOST_REPO_DIR:-.}/fedn:/app/fedn
-    entrypoint: [ "sh", "-c" ]
     command:
-      - "/venv/bin/pip install --no-cache-dir -e . && /venv/bin/fedn combiner start --init config/settings-combiner.yaml"
+      - combiner
+      - start
+      - --init
+      - config/settings-combiner.yaml.template
     ports:
       - 12080:12080
     healthcheck:
-      test: [ "CMD", "/bin/grpc_health_probe", "-addr=localhost:12080" ]
+      test: [ "CMD", "/app/grpc_health_probe", "-addr=localhost:12080" ]
       interval: 20s
       timeout: 10s
       retries: 5
@@ -110,11 +115,12 @@ services:
     container_name: hook
     environment:
       - GET_HOSTS_FROM=dns
+      - TMPDIR=/app/tmp
     build:
       context: .
       args:
-        BASE_IMG: ${BASE_IMG:-python:3.10-slim}
-        GRPC_HEALTH_PROBE_VERSION: v0.4.24
+        BASE_IMG: ${BASE_IMG:-python:3.12-slim}
+        GRPC_HEALTH_PROBE_VERSION: v0.4.35
     working_dir: /app
     volumes:
       - ${HOST_REPO_DIR:-.}/fedn:/app/fedn
@@ -141,9 +147,11 @@ services:
     working_dir: /app
     volumes:
       - ${HOST_REPO_DIR:-.}/fedn:/app/fedn
-    entrypoint: [ "sh", "-c" ]
     command:
-      - "/venv/bin/pip install --no-cache-dir -e . && /venv/bin/fedn client start --api-url http://api-server:8092"
+      - client
+      - start
+      - --api-url
+      - http://api-server:8092
     deploy:
       replicas: 0
     depends_on:

docs/conf.py

@@ -11,7 +11,7 @@
 author = "Scaleout Systems AB"
 
 # The full version, including alpha/beta/rc tags
-release = "0.19.0"
+release = "0.20.0"
 
 # Add any Sphinx extension module names here, as strings
 extensions = [

docs/quickstart.rst

@@ -9,7 +9,7 @@ Getting started with FEDn
 
 **Prerequisites**
 
--  `Python >=3.8, <=3.12 <https://www.python.org/downloads>`__
+-  `Python >=3.9, <=3.12 <https://www.python.org/downloads>`__
 -  `A FEDn Studio account <https://fedn.scaleoutsystems.com/signup>`__ 
 
 

examples/FedSimSiam/README.rst

@@ -16,7 +16,7 @@ To run the example, follow the steps below. For a more detailed explanation, fol
 Prerequisites
 -------------
 
--  `Python >=3.8, <=3.12 <https://www.python.org/downloads>`__
+-  `Python >=3.9, <=3.12 <https://www.python.org/downloads>`__
 -  `A project in FEDn Studio  <https://fedn.scaleoutsystems.com/signup>`__   
 
 Creating the compute package and seed model

examples/FedSimSiam/client/python_env.yaml

@@ -9,7 +9,6 @@ dependencies:
   - torch==2.2.2; sys_platform == "darwin" and platform_machine == "x86_64"
   - torchvision==0.19.1; (sys_platform == "darwin" and platform_machine == "arm64") or (sys_platform == "win32" or sys_platform == "win64" or sys_platform == "linux")
   - torchvision==0.17.2; sys_platform == "darwin" and platform_machine == "x86_64"
-  - numpy==2.0.2; (sys_platform == "darwin" and platform_machine == "arm64" and python_version >= "3.9") or (sys_platform == "win32" or sys_platform == "win64" or sys_platform == "linux" and python_version >= "3.9")
-  - numpy==1.26.4; (sys_platform == "darwin" and platform_machine == "x86_64" and python_version >= "3.9")
-  - numpy==1.24.4; python_version == "3.8"
+  - numpy==2.0.2; (sys_platform == "darwin" and platform_machine == "arm64") or (sys_platform == "win32" or sys_platform == "win64" or sys_platform == "linux")
+  - numpy==1.26.4; (sys_platform == "darwin" and platform_machine == "x86_64")
   - fedn

examples/flower-client/client/python_env.yaml

@@ -10,8 +10,7 @@ dependencies:
   - torch==2.2.2; sys_platform == "darwin" and platform_machine == "x86_64"
   - torchvision==0.19.1; (sys_platform == "darwin" and platform_machine == "arm64") or (sys_platform == "win32" or sys_platform == "win64" or sys_platform == "linux")
   - torchvision==0.17.2; sys_platform == "darwin" and platform_machine == "x86_64"
-  - numpy==2.0.2; (sys_platform == "darwin" and platform_machine == "arm64" and python_version >= "3.9") or (sys_platform == "win32" or sys_platform == "win64" or sys_platform == "linux" and python_version >= "3.9")
-  - numpy==1.26.4; (sys_platform == "darwin" and platform_machine == "x86_64" and python_version >= "3.9")
-  - numpy==1.24.4; python_version == "3.8"
+  - numpy==2.0.2; (sys_platform == "darwin" and platform_machine == "arm64") or (sys_platform == "win32" or sys_platform == "win64" or sys_platform == "linux")
+  - numpy==1.26.4; (sys_platform == "darwin" and platform_machine == "x86_64")
   - fire==0.3.1
   - flwr-datasets[vision]==0.1.0

examples/huggingface/README.rst

@@ -27,7 +27,7 @@ To run the example, follow the steps below. For a more detailed explanation, fol
 Prerequisites
 -------------
 
--  `Python >=3.8, <=3.12 <https://www.python.org/downloads>`__
+-  `Python >=3.9, <=3.12 <https://www.python.org/downloads>`__
 -  `A project in FEDn Studio  <https://fedn.scaleoutsystems.com/signup>`__   
 
 Creating the compute package and seed model

examples/huggingface/client/python_env.yaml

@@ -9,9 +9,8 @@ dependencies:
   - torch==2.2.2; sys_platform == "darwin" and platform_machine == "x86_64"
   - torchvision==0.19.1; (sys_platform == "darwin" and platform_machine == "arm64") or (sys_platform == "win32" or sys_platform == "win64" or sys_platform == "linux")
   - torchvision==0.17.2; sys_platform == "darwin" and platform_machine == "x86_64"
-  - numpy==2.0.2; (sys_platform == "darwin" and platform_machine == "arm64" and python_version >= "3.9") or (sys_platform == "win32" or sys_platform == "win64" or sys_platform == "linux" and python_version >= "3.9")
-  - numpy==1.26.4; (sys_platform == "darwin" and platform_machine == "x86_64" and python_version >= "3.9")
-  - numpy==1.24.4; python_version == "3.8"
+  - numpy==2.0.2; (sys_platform == "darwin" and platform_machine == "arm64") or (sys_platform == "win32" or sys_platform == "win64" or sys_platform == "linux")
+  - numpy==1.26.4; (sys_platform == "darwin" and platform_machine == "x86_64")
   - transformers
   - datasets
   - fedn

examples/mnist-keras/README.rst

@@ -8,7 +8,7 @@ This is a TF/Keras version of the PyTorch Quickstart Tutorial. For a step-by-ste
 Prerequisites
 -------------------------------------------
 
--  `Python >=3.8, <=3.12 <https://www.python.org/downloads>`__
+-  `Python >=3.9, <=3.12 <https://www.python.org/downloads>`__
 
 Creating the compute package and seed model
 -------------------------------------------

examples/mnist-pytorch-DPSGD/README.rst

@@ -2,21 +2,30 @@ FEDn Project: Federated Differential Privacy MNIST (Opacus + PyTorch)
 ----------------------------------------------------------------------
 
 This example FEDn Project demonstrates how Differential Privacy can be integrated to enhance the confidentiality of client data.
-We have expanded our baseline MNIST-PyTorch example by incorporating the Opacus framework, which is specifically designed for PyTorch models.
+We have expanded our baseline MNIST-PyTorch example by incorporating the Opacus framework, which is specifically designed for PyTorch models. To learn more about differential privacy, read our `blogpost <https://www.scaleoutsystems.com/post/guaranteeing-data-privacy-for-clients-in-federated-machine-learning>`__  about it.
 
 
 
 Prerequisites
 -------------
 
--  `Python >=3.8, <=3.12 <https://www.python.org/downloads>`__
+-  `Python >=3.9, <=3.12 <https://www.python.org/downloads>`__
 -  `A project in FEDn Studio  <https://fedn.scaleoutsystems.com/signup>`__   
 
-Edit Differential Privacy budget
+
+Edit Client-Specific Differential Privacy Parameters 
 --------------------------
-- The **Differential Privacy budget** (`FINAL_EPSILON`, `DELTA`) is configured in the `compute` package at `client/train.py` (lines 35 and 39).
-- If `HARDLIMIT` (line 40) is set to `True`, the `FINAL_EPSILON` will not exceed its specified limit.
-- If `HARDLIMIT` is set to `False`, the expected `FINAL_EPSILON` will be around its specified value given the server runs `GLOBAL_ROUNDS` variable (line 36).
+The **Differential Privacy budget** (``epsilon``, ``delta``), along with other settings, is configurable in the ``client_settings.yaml`` file:
+
+- **epochs**: Number of local epochs per round.
+- **epsilon**: Total epsilon budget to spend, determined by the ``global_rounds`` set on the server side.
+- **delta**: Total delta budget to spend.
+- **max_grad_norm**: Clipping threshold for gradients.
+- **global_rounds**: Number of rounds the server will run.
+- **hardlimit**:
+
+   - If ``hardlimit`` is set to ``True``, the ``epsilon`` budget will not exceed its specified limit, even if it means skipping updates for some rounds.
+   - If ``hardlimit`` is set to ``False``, the expected ``epsilon`` will be approximately equal to its specified value, assuming the server completes the specified ``global_rounds`` of updates.
 
 Creating the compute package and seed model
 -------------------------------------------

examples/mnist-pytorch-DPSGD/client/fedn.yaml

@@ -1,4 +1,6 @@
-python_env: python_env.yaml
+# Remove the python_env tag below to handle the environment manually
+python_env: python_env.yaml 
+
 entry_points:
   build:
     command: python model.py

examples/mnist-pytorch-DPSGD/client/python_env.yaml

@@ -10,7 +10,6 @@ dependencies:
   - torch==2.2.2; sys_platform == "darwin" and platform_machine == "x86_64"
   - torchvision==0.19.1; (sys_platform == "darwin" and platform_machine == "arm64") or (sys_platform == "win32" or sys_platform == "win64" or sys_platform == "linux")
   - torchvision==0.17.2; sys_platform == "darwin" and platform_machine == "x86_64"
-  - numpy==2.0.2; (sys_platform == "darwin" and platform_machine == "arm64" and python_version >= "3.9") or (sys_platform == "win32" or sys_platform == "win64" or sys_platform == "linux" and python_version >= "3.9")
-  - numpy==1.26.4; (sys_platform == "darwin" and platform_machine == "x86_64" and python_version >= "3.9")
-  - numpy==1.24.4; python_version == "3.8"
+  - numpy==2.0.2; (sys_platform == "darwin" and platform_machine == "arm64") or (sys_platform == "win32" or sys_platform == "win64" or sys_platform == "linux")
+  - numpy==1.26.4; (sys_platform == "darwin" and platform_machine == "x86_64")
   - opacus