Feature/SK-736 | Ruff linter (#552)

* revert --fix etc

* move setup.py configs

* reset repo change

* move init for pyproject

* include setup.py changes

* remove generated

* fix minor ruff errors

* ruff fix

* remove setup.py

* update test build path

* cli path update

* update cli path

* update cli path

* fix cli path

* change dockerfile path

* change compose paths

* update compose paths

* change path

* change pash

* add editable

* add virtualenv to pyproject

* fix pyproject.toml

* change project structure

* trigger CI

* trigger CI

* fix paths compose

* fix compose paths

* setuptools for -e . in dockers

* revert setuptools enforce

* enforce setuptools>65 in dockerfile

.ci/tests/examples/run.sh

@@ -11,7 +11,7 @@ helper="$2"
 
 python -m venv ".$example"
 source ".$example/bin/activate"
-".$example/bin/pip" install ./fedn/ fire
+".$example/bin/pip" install . fire
 
 >&2 echo "Start FEDn"
 pushd "examples/$example"

.devcontainer/bin/init_venv.sh

@@ -10,8 +10,5 @@ python -m venv .venv
     sphinx==4.4.0 \
     sphinx_press_theme==0.8.0 \
     sphinx-autobuild==2021.3.14 \
-    autopep8==1.5.7 \
-    isort==5.10.1 \
-    flake8==4.0.1 \
     sphinx_rtd_theme==0.5.2
-.venv/bin/pip install -e fedn
+.venv/bin/pip install -e .

.devcontainer/devcontainer.json.tpl

@@ -5,23 +5,26 @@
   "remoteUser": "default",
   // "workspaceFolder": "/fedn",
   // "workspaceMount": "source=/path/to/fedn,target=/fedn,type=bind,consistency=default",
-  "extensions": [
-    "ms-azuretools.vscode-docker",
-    "ms-python.python",
-    "exiasr.hadolint",
-    "yzhang.markdown-all-in-one",
-    "ms-python.isort"
-  ],
+  "customizations": {
+    "vscode": {
+      "extensions": [
+        "ms-azuretools.vscode-docker",
+        "ms-python.python",
+        "exiasr.hadolint",
+        "yzhang.markdown-all-in-one",
+        "charliermarsh.ruff"
+      ]
+    }
+  },
   "mounts": [
-    "source=/var/run/docker.sock,target=/var/run/docker.sock,type=bind,consistency=default",
+    "source=/var/run/docker.sock,target=/var/run/docker.sock,type=bind,consistency=default"
   ],
   "runArgs": [
     "--net=host"
   ],
   "build": {
     "args": {
-      "BASE_IMG": "python:3.9"
+      "BASE_IMG": "python:3.11"
     }
   }
-}
-
+}

.dockerignore

@@ -9,4 +9,5 @@ docs/*
 **/.mnist-pytorch
 **/*.npz
 **/data
-**/*.tgz
+**/*.tgz
+dist

.github/workflows/code-checks.yaml

@@ -11,30 +11,9 @@ jobs:
 
       - name: init venv
         run: .devcontainer/bin/init_venv.sh
-
-      - name: check Python imports
-        run: >
-          .venv/bin/isort . --check --diff 
-          --skip .venv
-          --skip .mnist-keras
-          --skip .mnist-pytorch
-          --skip fedn_pb2.py
-          --skip fedn_pb2_grpc.py
-
-      - name: check Python formatting
-        run: >
-          .venv/bin/autopep8 --recursive --diff
-          --exclude .venv
-          --exclude .mnist-keras
-          --exclude .mnist-pytorch
-          --exclude fedn_pb2.py
-          --exclude fedn_pb2_grpc.py
-          .
 
-      - name: run Python linter
-        run: >
-          .venv/bin/flake8 . 
-          --exclude ".venv,.mnist-keras,.mnist-pytorch,fedn_pb2.py,fedn_pb2_grpc.py"
+      - name: Ruff Linting
+        uses: chartboost/ruff-action@v1
 
       - name: check for floating imports
         run: >

.github/workflows/push-to-pypi.yaml

@@ -16,15 +16,15 @@ jobs:
 
       - name: Install pypa/build
         run: python -m pip install build
-        working-directory: ./fedn
+        working-directory: ./
 
       - name: Build package
         run: python -m build
-        working-directory: ./fedn
+        working-directory: ./
 
       - name: Publish to Test PyPI
         uses: pypa/[email protected]
         with:
           user: __token__
           password: ${{ secrets.PYPI_TOKEN }}
-          packages_dir: fedn/dist
+          packages_dir: ./dist

.gitignore

@@ -22,7 +22,6 @@ __pycache__/
 .Python
 build/
 develop-eggs/
-#dist/
 downloads/
 eggs/
 .eggs/
@@ -38,6 +37,7 @@ share/python-wheels/
 .installed.cfg
 *.egg
 MANIFEST
+dist
 
 # PyInstaller
 #  Usually these files are written by a python script from a template

Dockerfile

@@ -8,7 +8,7 @@ ARG GRPC_HEALTH_PROBE_VERSION=""
 ARG REQUIREMENTS=""
 
 # Add FEDn and default configs
-COPY fedn /app/fedn
+COPY . /app
 COPY config/settings-client.yaml.template /app/config/settings-client.yaml
 COPY config/settings-combiner.yaml.template /app/config/settings-combiner.yaml
 COPY config/settings-reducer.yaml.template /app/config/settings-reducer.yaml
@@ -27,6 +27,8 @@ RUN if [ ! -z "$GRPC_HEALTH_PROBE_VERSION" ]; then \
   echo "No grpc_health_probe version specified, skipping installation"; \
   fi
 
+# Setup working directory
+WORKDIR /app
 
 # Create FEDn app directory
 SHELL ["/bin/bash", "-c"]
@@ -39,14 +41,13 @@ RUN mkdir -p /app \
   # Install FEDn and requirements
   && python -m venv /venv \
   && /venv/bin/pip install --upgrade pip \
-  && /venv/bin/pip install --no-cache-dir -e /app/fedn \
+  && /venv/bin/pip install --no-cache-dir setuptools>=65 \
+  && /venv/bin/pip install --no-cache-dir -e . \
   && if [[ ! -z "$REQUIREMENTS" ]]; then \
   /venv/bin/pip install --no-cache-dir -r /app/config/requirements.txt; \
   fi \
   #
   # Clean up
   && rm -r /app/config/requirements.txt
 
-# Setup working directory
-WORKDIR /app
 ENTRYPOINT [ "/venv/bin/fedn" ]

config/combiner-settings.override.yaml

@@ -5,5 +5,5 @@ version: '3.3'
 services:
   combiner:
     volumes:
-      - ${HOST_REPO_DIR:-.}/fedn:/app/fedn
+      - ${HOST_REPO_DIR:-.}:/app
       - ${HOST_REPO_DIR:-.}/config/settings-combiner.yaml:/app/config/settings-combiner.yaml

config/reducer-settings.override.yaml

@@ -5,5 +5,5 @@ version: '3.3'
 services:
   reducer:
     volumes:
-      - ${HOST_REPO_DIR:-.}/fedn:/app/fedn
+      - ${HOST_REPO_DIR:-.}:/app
       - ${HOST_REPO_DIR:-.}/config/settings-reducer.yaml:/app/config/settings-reducer.yaml

docker-compose.yaml

@@ -78,7 +78,7 @@ services:
       - mongo
     entrypoint: [ "sh", "-c" ]
     command:
-      - "/venv/bin/pip install --no-cache-dir -e /app/fedn && /venv/bin/python fedn/fedn/network/api/server.py"
+      - "/venv/bin/pip install --no-cache-dir -e . && /venv/bin/python fedn/network/api/server.py"
     ports:
       - 8092:8092
 
@@ -97,7 +97,7 @@ services:
       - ${HOST_REPO_DIR:-.}/fedn:/app/fedn
     entrypoint: [ "sh", "-c" ]
     command:
-      - "/venv/bin/pip install --no-cache-dir -e /app/fedn && /venv/bin/fedn run combiner --init config/settings-combiner.yaml"
+      - "/venv/bin/pip install --no-cache-dir -e . && /venv/bin/fedn run combiner --init config/settings-combiner.yaml"
     ports:
       - 12080:12080
     healthcheck:
@@ -127,7 +127,7 @@ services:
       - ${HOST_REPO_DIR:-.}/fedn:/app/fedn
     entrypoint: [ "sh", "-c" ]
     command:
-      - "/venv/bin/pip install --no-cache-dir -e /app/fedn && /venv/bin/fedn run client --init config/settings-client.yaml"
+      - "/venv/bin/pip install --no-cache-dir -e . && /venv/bin/fedn run client --init config/settings-client.yaml"
     deploy:
       replicas: 0
     depends_on:

docs/quickstart.rst

@@ -34,8 +34,8 @@ Clone the FEDn repository and install the package:
 .. code-block:: bash
 
    git clone https://github.com/scaleoutsystems/fedn.git
-   cd fedn/fedn
-   pip install -e .
+   cd fedn
+   pip install .
 
 It is recommended to use a virtual environment when installing FEDn.
 

examples/flower-client/client/flwr_task.py

@@ -5,9 +5,9 @@
 from collections import OrderedDict
 
 import torch
-import torch.nn as nn
-import torch.nn.functional as F
+import torch.nn.functional as F  # noqa: N812
 from flwr_datasets import FederatedDataset
+from torch import nn
 from torch.utils.data import DataLoader
 from torchvision.transforms import Compose, Normalize, ToTensor
 from tqdm import tqdm
@@ -42,9 +42,7 @@ def load_data(partition_id, num_clients):
     partition = fds.load_partition(partition_id)
     # Divide data on each node: 80% train, 20% test
     partition_train_test = partition.train_test_split(test_size=0.2)
-    pytorch_transforms = Compose(
-        [ToTensor(), Normalize((0.5, 0.5, 0.5), (0.5, 0.5, 0.5))]
-    )
+    pytorch_transforms = Compose([ToTensor(), Normalize((0.5, 0.5, 0.5), (0.5, 0.5, 0.5))])
 
     def apply_transforms(batch):
         """Apply transforms to the partition from FederatedDataset."""

examples/flower-client/client/python_env.yaml

@@ -8,4 +8,4 @@ dependencies:
   - torchvision==0.17.1
   - fire==0.3.1
   - fedn[flower]==0.9.0
-  - flwr-datasets[vision]==0.1.0
+  - flwr-datasets[vision]==0.1.0

examples/mnist-pytorch/docker-compose.override.yaml

@@ -3,8 +3,7 @@ version: '3.4'
 
 # Overriding requirements
 
-x-env:
-  &defaults
+x-env: &defaults
   GET_HOSTS_FROM: dns
   FEDN_PACKAGE_EXTRACT_DIR: package
   FEDN_NUM_DATA_SPLITS: 2

fedn/__init__.py

@@ -1,3 +1,23 @@
-"""The fedn package."""
+import glob
+import os
+from os.path import basename, dirname, isfile
+
+from fedn.network.api.client import APIClient
 
 # flake8: noqa
+
+
+modules = glob.glob(dirname(__file__) + "/fedn" + "/*.py")
+__all__ = [basename(f)[:-3] for f in modules if isfile(f) and not f.endswith("__init__.py")]
+
+
+_ROOT = os.path.abspath(os.path.dirname(__file__))
+
+
+def get_data(path):
+    """
+
+    :param path:
+    :return:
+    """
+    return os.path.join(_ROOT, "data", path)