Merge remote-tracking branch 'origin/master' into feature/SK-946

.github/workflows/branch-name-check.yaml

@@ -7,7 +7,7 @@ on:
       - master
 
 env:
-  BRANCH_REGEX: '^((feature|github|hotfix|bugfix|fix|bug|docs|refactor)\/.+)|(release\/v((([0-9]+)\.([0-9]+)\.([0-9]+)(?:-([0-9a-zA-Z-]+(?:\.[0-9a-zA-Z-]+)*))?)(?:\+([0-9a-zA-Z-]+(?:\.[0-9a-zA-Z-]+)*))?))$'
+  BRANCH_REGEX: '^((feature|github|dependabot|hotfix|bugfix|fix|bug|docs|refactor)\/.+)|(release\/v((([0-9]+)\.([0-9]+)\.([0-9]+)(?:-([0-9a-zA-Z-]+(?:\.[0-9a-zA-Z-]+)*))?)(?:\+([0-9a-zA-Z-]+(?:\.[0-9a-zA-Z-]+)*))?))$'
 
 jobs:
   branch-name-check:

.github/workflows/build-containers.yaml

@@ -1,6 +1,7 @@
 name: "build containers"
 
 on:
+  workflow_dispatch:
   push:
     branches:
       - master

.github/workflows/code-checks.yaml

@@ -25,6 +25,7 @@ jobs:
           --exclude-dir='docs'
           --exclude-dir='flower-client'
           --exclude='tests.py'
+          --exclude='controller_cmd.py'
           --exclude='README.rst'
           '^[ \t]+(import|from) ' -I . 
 

.github/workflows/pr-title-check.yaml

@@ -0,0 +1,35 @@
+name: PR Title Check
+
+on:
+  pull_request:
+    types: [opened, edited, reopened, synchronize]
+
+jobs:
+  title-check:
+    runs-on: ubuntu-latest
+
+    steps:
+    - name: Check out repository
+      uses: actions/checkout@v3
+
+    - name: Check if PR is internal
+      id: check_internal
+      run: |
+        if [[ "${{ github.event.pull_request.head.repo.full_name }}" == "${{ github.repository }}" ]]; then
+          echo "internal=true" >> $GITHUB_ENV
+        else
+          echo "internal=false" >> $GITHUB_ENV
+        fi
+
+    - name: Run title check script
+      if: env.internal == 'true'
+      run: |
+        pr_title="${{ github.event.pull_request.title }}"
+        pattern="^(Feature|Fix|Bug|Bugfix|Docs|Refactor|Chore|Github)\/SK-[0-9]+ \| .+"
+        if [[ ! "$pr_title" =~ $pattern ]]; then
+          echo "Error: PR title does not follow the required pattern."
+          echo "Please ensure the title follows the pattern: 'Feature|Fix|Bug|Bugfix|Docs|Refactor|Chore|Github/SK-<numbers> | <Summary>'"
+          exit 1
+        else
+          echo "PR title is valid."
+        fi

.github/workflows/push-to-pypi.yaml

@@ -1,8 +1,9 @@
 name: Publish Python distribution to PyPI
 
 on:
+  workflow_dispatch:
   release:
-    types: [created]
+    types: published
 
 jobs:
   build-and-publish:

README.rst

@@ -9,50 +9,50 @@
 .. |pic3| image:: https://readthedocs.org/projects/fedn/badge/?version=latest&style=flat
    :target: https://fedn.readthedocs.io
 
-FEDn
---------
+FEDn: An enterprise-ready federated learning framework 
+-------------------------------------------------------
 
-FEDn empowers its users to create federated learning applications that seamlessly transition from local proofs-of-concept to secure distributed deployments. 
+Our goal is to provide a federated learning framework that is both secure, scalable and easy-to-use. We believe that that minimal code change should be needed to progress from early proof-of-concepts to production. This is reflected in our core design: 
 
-Leverage a flexible pseudo-local sandbox to rapidly transition your existing ML project to a federated setting. Test and scale in real-world scenarios using FEDn Studio - a fully managed, secure deployment of all server-side components (SaaS). 
+-  **Minimal server-side complexity for the end-user**. Running a proper distributed FL deployment is hard. With FEDn Studio we seek to handle all server-side complexity and provide a UI, REST API and a Python interface to help users manage FL experiments and track metrics in real time.
 
-We develop the FEDn framework following these core design principles:
+-  **Secure by design.** FL clients do not need to open any ingress ports. Industry-standard communication protocols (gRPC) and token-based authentication and RBAC (Jason Web Tokens) provides flexible integration in a range of production environments.  
 
--  **Seamless transition from proof-of-concepts to real-world FL**. FEDn has been designed to make the journey from R&D to real-world deployments as smooth as possibe. Develop your federated learning use case in a pseudo-local environment, then deploy it to FEDn Studio (cloud or on-premise) for real-world scenarios. No code change is required to go from development and testing to production. 
+-  **ML-framework agnostic**. A black-box client-side architecture lets data scientists interface with their framework of choice. 
 
--  **Designed for scalability and resilience.** FEDn enables model aggregation through multiple aggregation servers sharing the workload. A hierarchical architecture makes the framework well suited borh for cross-silo and cross-device use-cases. FEDn seamlessly recover from failures in all critical components, and manages intermittent client-connections, ensuring robust deployment in production environments.
+-  **Cloud native.** By following cloud native design principles, we ensure a wide range of deployment options including private cloud and on-premise infrastructure. 
 
--  **Secure by design.** FL clients do not need to open any ingress ports, facilitating distributed deployments across a wide variety of settings. Additionally, FEDn utilizes secure, industry-standard communication protocols and supports token-based authentication and RBAC for FL clients (JWT), providing flexible integration in production environments.   
+-  **Scalability and resilience.** Multiple aggregation servers (combiners) can share the workload. FEDn seamlessly recover from failures in all critical components and manages intermittent client-connections. 
 
--  **Developer and data scientist friendly.** Extensive event logging and distributed tracing enables developers to monitor experiments in real-time, simplifying troubleshooting and auditing. Machine learning metrics can be accessed via both a Python API and visualized in an intuitive UI that helps the data scientists analyze and communicate ML-model training progress.
+-  **Developer and DevOps friendly.** Extensive event logging and distributed tracing enables developers to monitor the sytem in real-time, simplifying troubleshooting and auditing. Extensions and integrations are facilitated by a flexible plug-in architecture.  
 
+We provide a fully managed deployment for testing, academic, and personal use. Sign up for a `FEDn Studio account <https://fedn.scaleoutsystems.com/signup>`__  and take the `Quickstart tutorial <https://fedn.readthedocs.io/en/stable/quickstart.html>`__ to get started with FEDn. 
 
 Features
 =========
 
-Core FL framework (this repository): 
+Federated learning: 
 
 - Tiered federated learning architecture enabling massive scalability and resilience. 
 - Support for any ML framework (examples for PyTorch, Tensforflow/Keras and Scikit-learn)
 - Extendable via a plug-in architecture (aggregators, load balancers, object storage backends, databases  etc.)
 - Built-in federated algorithms (FedAvg, FedAdam, FedYogi, FedAdaGrad, etc.)
-- CLI and Python API.
+- UI, CLI and Python API.
 - Implement clients in any language (Python, C++, Kotlin etc.)
 - No open ports needed client-side.
-- Flexible deployment of server-side components using Docker / docker compose.   
 
 
-FEDn Studio - From development to FL in production: 
+From development to FL in production: 
 
 -  Secure deployment of server-side / control-plane on Kubernetes.
--  UI with dashboards for orchestrating experiments and visualizing results
+-  UI with dashboards for orchestrating FL experiments and for visualizing results
 -  Team features - collaborate with other users in shared project workspaces. 
 -  Features for the trusted-third party: Manage access to the FL network, FL clients and training progress.
 -  REST API for handling experiments/jobs. 
 -  View and export logging and tracing information. 
 -  Public cloud, dedicated cloud and on-premise deployment options.
 
-Available clients:
+Available client APIs:
 
 - Python client (this repository)
 - C++ client (`FEDn C++ client <https://github.com/scaleoutsystems/fedn-cpp-client>`__)
@@ -64,11 +64,11 @@ Getting started
 
 Get started with FEDn in two steps:  
 
-1. Sign up for a `Free FEDn Studio account <https://fedn.scaleoutsystems.com/signup>`__
+1. Register for a `FEDn Studio account <https://fedn.scaleoutsystems.com/signup>`__
 2. Take the `Quickstart tutorial <https://fedn.readthedocs.io/en/stable/quickstart.html>`__
 
-FEDn Studio (SaaS) is free for academic use and personal development / small-scale testing and exploration. For users and teams requiring
-additional project resources, dedicated support or other hosting options, `explore our plans <https://www.scaleoutsystems.com/start#pricing>`__.  
+Use of our multi-tenant, managed deployment of FEDn Studio (SaaS) is free forever for academic research and personal development/testing purposes.
+For users and teams requiring additional resources, more storage and cpu, dedicated support, and other hosting options (private cloud, on-premise), `explore our plans <https://www.scaleoutsystems.com/start#pricing>`__.  
 
 Documentation
 =============

docker-compose.yaml

@@ -78,7 +78,7 @@ services:
       - mongo
     entrypoint: [ "sh", "-c" ]
     command:
-      - "/venv/bin/pip install --no-cache-dir -e . && /venv/bin/python fedn/network/api/server.py"
+      - "/venv/bin/pip install --no-cache-dir -e . && /venv/bin/fedn controller start"
     ports:
       - 8092:8092
 

docs/apiclient.rst

@@ -1,11 +1,9 @@
+.. _apiclient-label:
+
 APIClient
 =========
 
-.. note:: 
-
-   For access to FEDn Studio API, please see :ref:`studio-api`.
-
-FEDn comes with an *APIClient* for interacting with the FEDn network. The APIClient is a Python3 library that can be used to interact with the FEDn network programmatically. 
+FEDn comes with an *APIClient* - a Python3 library that can be used to interact with FEDn programmatically. 
 
 **Installation**
 
@@ -17,12 +15,15 @@ The APIClient is available as a Python package on PyPI, and can be installed usi
 
 **Initialize the APIClient**
 
-To initialize the APIClient, you need to provide the hostname and port of the FEDn API server. The default port is 8092. The following code snippet shows how to initialize the APIClient:
+The FEDn REST API is available at <controller-host>/api/v1/. To access this API you need the url to the controller-host, as well as an admin API token. The controller host can be found in the project dashboard (top right corner).
+To obtain an admin API token, navigate to the "Settings" tab in your Studio project and click on the "Generate token" button. Copy the 'access' token and use it to access the API using the instructions below. 
+
 
 .. code-block:: python
-   
-   from fedn import APIClient
-   client = APIClient("localhost", 8092)
+
+   >>> from fedn import APIClient
+   >>> client = APIClient(host="<controller-host>", token="<access-token>", secure=True, verify=True)
+
 
 **Set active package and seed model**
 
@@ -38,9 +39,9 @@ To set the initial seed model, you can use the following code snippet:
    
    client.set_active_model(path="path/to/seed.npz")
 
-**Start training session**
+**Start a training session**
 
-Once the active package and seed model are set, you can connect clients to the network and start training models. The following code snippet initializes a session (training rounds):
+Once the active package and seed model are set, you can connect clients to the network and start training models. The following code snippet starts a traing session:
 
 .. code-block:: python
    

docs/architecture.rst

@@ -1,3 +1,5 @@
+.. _architecture-label:
+
 Architecture overview
 =====================
 

docs/conf.py

@@ -12,7 +12,7 @@
 author = "Scaleout Systems AB"
 
 # The full version, including alpha/beta/rc tags
-release = "0.9.6"
+release = "0.11.1"
 
 # Add any Sphinx extension module names here, as strings
 extensions = [
@@ -24,7 +24,7 @@
     "sphinx.ext.ifconfig",
     "sphinx.ext.viewcode",
     "sphinx_rtd_theme",
-    "sphinx_code_tabs"
+    "sphinx_code_tabs",
 ]
 
 # The master toctree document.
@@ -71,15 +71,12 @@
     # The paper size ('letterpaper' or 'a4paper').
     #
     # 'papersize': 'letterpaper',
-
     # The font size ('10pt', '11pt' or '12pt').
     #
     # 'pointsize': '10pt',
-
     # Additional stuff for the LaTeX preamble.
     #
     # 'preamble': '',
-
     # Latex figure (float) alignment
     #
     # 'figure_align': 'htbp',
@@ -89,24 +86,18 @@
 # (source start file, target name, title,
 #  author, documentclass [howto, manual, or own class]).
 latex_documents = [
-    (master_doc, "fedn.tex", "FEDn Documentation",
-     "Scaleout Systems AB", "manual"),
+    (master_doc, "fedn.tex", "FEDn Documentation", "Scaleout Systems AB", "manual"),
 ]
 
 # One entry per manual page. List of tuples
 # (source start file, name, description, authors, manual section).
-man_pages = [
-    (master_doc, "fedn", "FEDn Documentation",
-     [author], 1)
-]
+man_pages = [(master_doc, "fedn", "FEDn Documentation", [author], 1)]
 
 # Grouping the document tree into Texinfo files. List of tuples
 # (source start file, target name, title, author,
 #  dir menu entry, description, category)
 texinfo_documents = [
-    (master_doc, "fedn", "FEDn Documentation",
-     author, "fedn", "One line description of project.",
-     "Miscellaneous"),
+    (master_doc, "fedn", "FEDn Documentation", author, "fedn", "One line description of project.", "Miscellaneous"),
 ]
 
 # Bibliographic Dublin Core info.