Merge branch 'develop' into feature/android

.ci/tests/examples/api_test.py

@@ -0,0 +1,83 @@
+import fire
+import yaml
+
+from fedn import APIClient
+
+
+def _download_config(output):
+    """ Download the client configuration file from the controller.
+
+    :param output: The output file path.
+    :type output: str
+    """
+    client = APIClient(host="localhost", port=8092)
+    config = client.get_client_config(checksum=True)
+    with open(output, 'w') as f:
+        f.write(yaml.dump(config))
+
+
+def test_api_get_methods():
+    client = APIClient(host="localhost", port=8092)
+    status = client.get_controller_status()
+    assert status
+    print("Controller status: ", status, flush=True)
+
+    events = client.get_events()
+    assert events
+    print("Events: ", events, flush=True)
+
+    validations = client.list_validations()
+    assert validations
+    print("Validations: ", validations, flush=True)
+
+    models = client.get_model_trail()
+    assert models
+    print("Models: ", models, flush=True)
+
+    clients = client.list_clients()
+    assert clients
+    print("Clients: ", clients, flush=True)
+
+    combiners = client.list_combiners()
+    assert combiners
+    print("Combiners: ", combiners, flush=True)
+
+    combiner = client.get_combiner("combiner")
+    assert combiner
+    print("Combiner: ", combiner, flush=True)
+
+    first_model = client.get_initial_model()
+    assert first_model
+    print("First model: ", first_model, flush=True)
+
+    package = client.get_package()
+    assert package
+    print("Package: ", package, flush=True)
+
+    checksum = client.get_package_checksum()
+    assert checksum
+    print("Checksum: ", checksum, flush=True)
+
+    rounds = client.list_rounds()
+    assert rounds
+    print("Rounds: ", rounds, flush=True)
+
+    round = client.get_round(1)
+    assert round
+    print("Round: ", round, flush=True)
+
+    sessions = client.list_sessions()
+    assert sessions
+    print("Sessions: ", sessions, flush=True)
+
+
+if __name__ == '__main__':
+
+    client = APIClient(host="localhost", port=8092)
+    fire.Fire({
+        'set_seed': client.set_initial_model,
+        'set_package': client.set_package,
+        'start_session': client.start_session,
+        'get_client_config': _download_config,
+        'test_api_get_methods': test_api_get_methods,
+    })

.ci/tests/examples/print_logs.sh

@@ -5,8 +5,11 @@ docker logs "$(basename $PWD)_minio_1"
 echo "Mongo logs"
 docker logs "$(basename $PWD)_mongo_1"
 
-echo "Reducer logs"
-docker logs "$(basename $PWD)_reducer_1"
+echo "Dashboard logs"
+docker logs "$(basename $PWD)_dashboard_1"
+
+echo "API-Server logs"
+docker logs "$(basename $PWD)_api-server_1"
 
 echo "Combiner logs"
 docker logs "$(basename $PWD)_combiner_1"

.ci/tests/examples/run.sh

@@ -23,34 +23,23 @@ docker-compose \
 ".$example/bin/python" ../../.ci/tests/examples/wait_for.py combiners
 
 >&2 echo "Upload compute package"
-curl -k -X POST \
-    -F [email protected] \
-    -F helper="$helper" \
-    http://localhost:8090/context
-printf '\n'
+".$example/bin/python" ../../.ci/tests/examples/api_test.py set_package --path package.tgz --helper "$helper"
 
 >&2 echo "Upload seed"
-curl -k -X POST \
-    -F [email protected] \
-    http://localhost:8090/models
-printf '\n'
+".$example/bin/python" ../../.ci/tests/examples/api_test.py set_seed --path seed.npz
 
 >&2 echo "Wait for clients to connect"
 ".$example/bin/python" ../../.ci/tests/examples/wait_for.py clients
 
->&2 echo "Start round"
-curl -k -X POST \
-    -F rounds=3 \
-    -F validate=True \
-    http://localhost:8090/control
-printf '\n'
+>&2 echo "Start session"
+".$example/bin/python" ../../.ci/tests/examples/api_test.py start_session --rounds 3 --helper "$helper"
 
 >&2 echo "Checking rounds success"
 ".$example/bin/python" ../../.ci/tests/examples/wait_for.py rounds
 
 >&2 echo "Test client connection with dowloaded settings"
 # Get config
-curl -k http://localhost:8090/config/download > ../../client.yaml
+".$example/bin/python" ../../.ci/tests/examples/api_test.py get_client_config --output ../../client.yaml
 
 # Redeploy clients with config
 docker-compose \
@@ -62,5 +51,8 @@ docker-compose \
 >&2 echo "Wait for clients to reconnect"
 ".$example/bin/python" ../../.ci/tests/examples/wait_for.py clients
 
+>&2 echo "Test API GET requests"
+".$example/bin/python" ../../.ci/tests/examples/api_test.py test_api_get_methods
+
 popd
 >&2 echo "Test completed successfully"

.ci/tests/examples/wait_for.py

@@ -37,21 +37,31 @@ def _test_rounds(n_rounds):
     return n == n_rounds
 
 
-def _test_nodes(n_nodes, node_type, reducer_host='localhost', reducer_port='8090'):
+def _test_nodes(n_nodes, node_type, reducer_host='localhost', reducer_port='8092'):
     try:
-        resp = requests.get(
-            f'http://{reducer_host}:{reducer_port}/netgraph', verify=False)
+
+        endpoint = "list_clients" if node_type == "client" else "list_combiners"
+
+        response = requests.get(
+            f'http://{reducer_host}:{reducer_port}/{endpoint}', verify=False)
+
+        if response.status_code == 200:
+
+            data = json.loads(response.content)
+
+            count = 0
+            if node_type == "client":
+                arr = data.get('result')
+                count = sum(element.get('status') == "online" for element in arr)
+            else:
+                count = data.get('count')
+
+            _eprint(f'Active {node_type}s: {count}.')
+            return count == n_nodes
+
     except Exception as e:
         _eprint(f'Reques exception econuntered: {e}.')
         return False
-    if resp.status_code == 200:
-        gr = json.loads(resp.content)
-        n = sum(values.get('type') == node_type and values.get(
-            'status') == 'active' for values in gr['nodes'])
-        _eprint(f'Active {node_type}s: {n}.')
-        return n == n_nodes
-    _eprint(f'Reducer returned {resp.status_code}.')
-    return False
 
 
 def rounds(n_rounds=3):

.github/workflows/code-checks.yaml

@@ -18,27 +18,32 @@ jobs:
           --skip .venv
           --skip .mnist-keras
           --skip .mnist-pytorch
+          --skip fedn_pb2.py
+          --skip fedn_pb2_grpc.py
 
       - name: check Python formatting
         run: >
           .venv/bin/autopep8 --recursive --diff
           --exclude .venv
           --exclude .mnist-keras
           --exclude .mnist-pytorch
+          --exclude fedn_pb2.py
+          --exclude fedn_pb2_grpc.py
           .
 
       - name: run Python linter
         run: >
           .venv/bin/flake8 . 
-          --exclude ".venv,.mnist-keras,.mnist-pytorch,fedn_pb2.py"
+          --exclude ".venv,.mnist-keras,.mnist-pytorch,fedn_pb2.py,fedn_pb2_grpc.py"
 
       - name: check for floating imports
         run: >
           ! grep -E -R 
           --exclude-dir='.venv' 
           --exclude-dir='.mnist-pytorch' 
           --exclude-dir='.mnist-keras' 
-          --exclude-dir='docs' 
+          --exclude-dir='docs'
+          --exclude='tests.py'
           '^[ \t]+(import|from) ' -I . 
 
       # TODO: add linting/formatting for all file types

.readthedocs.yaml

@@ -6,4 +6,9 @@ build:
     python: "3.9"
 
 sphinx:
-   configuration: docs/conf.py
+   configuration: docs/conf.py
+
+python:
+  install:
+    - method: pip
+      path: ./fedn

README.rst

@@ -101,7 +101,7 @@ To connect a client that uses the data partition 'data/clients/1/mnist.pt':
   -v $PWD/data/clients/1:/var/data \
   -e ENTRYPOINT_OPTS=--data_path=/var/data/mnist.pt \
   --network=fedn_default \
-  ghcr.io/scaleoutsystems/fedn/fedn:develop-mnist-pytorch run client -in client.yaml --name client1 
+  ghcr.io/scaleoutsystems/fedn/fedn:master-mnist-pytorch run client -in client.yaml --name client1 
 
 You are now ready to start training the model at http://localhost:8090/control.
 

config/settings-client.yaml.template

@@ -1,3 +1,3 @@
 network_id: fedn-network
-discover_host: reducer
-discover_port: 8090
+discover_host: api-server
+discover_port: 8092

config/settings-combiner.yaml.template

@@ -1,6 +1,6 @@
 network_id: fedn-network
-discover_host: reducer
-discover_port: 8090
+discover_host: api-server
+discover_port: 8092
 
 name: combiner
 host: combiner

config/settings-reducer.yaml.template

@@ -1,4 +1,8 @@
 network_id: fedn-network
+controller:
+  host: api-server
+  port: 8092
+  debug: True
 
 statestore:
   type: MongoDB

docker-compose.yaml

@@ -58,14 +58,13 @@ services:
     ports:
       - 8081:8081
 
-  # Reducer
-  reducer:
+  dashboard:
     environment:
       - GET_HOSTS_FROM=dns
       - USER=test
       - PROJECT=project
       - FLASK_DEBUG=1
-      - FLASK_ENV=development
+      - STATESTORE_CONFIG=/app/config/settings-reducer.yaml
     build:
       context: .
       args:
@@ -75,10 +74,34 @@ services:
       - ${HOST_REPO_DIR:-.}/fedn:/app/fedn
     entrypoint: [ "sh", "-c" ]
     command:
-      - "/venv/bin/pip install --no-cache-dir -e /app/fedn && /venv/bin/fedn run reducer -n reducer --init=config/settings-reducer.yaml"
+      - "/venv/bin/pip install --no-cache-dir -e /app/fedn && /venv/bin/fedn run dashboard -n reducer --init=config/settings-reducer.yaml"
     ports:
       - 8090:8090
 
+  api-server:
+    environment:
+      - GET_HOSTS_FROM=dns
+      - USER=test
+      - PROJECT=project
+      - FLASK_DEBUG=1
+      - STATESTORE_CONFIG=/app/config/settings-reducer.yaml
+      - MODELSTORAGE_CONFIG=/app/config/settings-reducer.yaml
+    build:
+      context: .
+      args:
+        BASE_IMG: ${BASE_IMG:-python:3.9-slim}
+    working_dir: /app
+    volumes:
+      - ${HOST_REPO_DIR:-.}/fedn:/app/fedn
+    depends_on:
+      - minio
+      - mongo
+    entrypoint: [ "sh", "-c" ]
+    command:
+      - "/venv/bin/pip install --no-cache-dir -e /app/fedn && /venv/bin/python fedn/fedn/network/api/server.py"
+    ports:
+      - 8092:8092
+
   # Combiner
   combiner:
     environment:

docs/README.md

@@ -0,0 +1,4 @@
+FEDn is using sphinx with reStructuredText.
+
+sphinx-apidoc --ext-autodoc --module-first -o _source  ../fedn/fedn ../*tests* ../*exceptions* ../*common* ../ ../fedn/fedn/network/api/server.py ../fedn/fedn/network/controller/controlbase.py
+sphinx-build . _build