diff --git a/.github/workflows/build-containers.yaml b/.github/workflows/build-containers.yaml index 8dd2f561a..89648764e 100644 --- a/.github/workflows/build-containers.yaml +++ b/.github/workflows/build-containers.yaml @@ -1,6 +1,7 @@ name: "build containers" on: + workflow_dispatch: push: branches: - master diff --git a/.github/workflows/code-checks.yaml b/.github/workflows/code-checks.yaml index dbe378c63..8c48a3015 100644 --- a/.github/workflows/code-checks.yaml +++ b/.github/workflows/code-checks.yaml @@ -25,6 +25,7 @@ jobs: --exclude-dir='docs' --exclude-dir='flower-client' --exclude='tests.py' + --exclude='controller_cmd.py' --exclude='README.rst' '^[ \t]+(import|from) ' -I . diff --git a/.github/workflows/push-to-pypi.yaml b/.github/workflows/push-to-pypi.yaml index 1b59835ad..1e184c17f 100644 --- a/.github/workflows/push-to-pypi.yaml +++ b/.github/workflows/push-to-pypi.yaml @@ -1,8 +1,9 @@ name: Publish Python distribution to PyPI on: + workflow_dispatch: release: - types: [created] + types: published jobs: build-and-publish: diff --git a/docker-compose.yaml b/docker-compose.yaml index 85386c6da..c3620e79d 100644 --- a/docker-compose.yaml +++ b/docker-compose.yaml @@ -78,7 +78,7 @@ services: - mongo entrypoint: [ "sh", "-c" ] command: - - "/venv/bin/pip install --no-cache-dir -e . && /venv/bin/python fedn/network/api/server.py" + - "/venv/bin/pip install --no-cache-dir -e . && /venv/bin/fedn controller start" ports: - 8092:8092 diff --git a/docs/apiclient.rst b/docs/apiclient.rst index b4dfd789f..2806ebe86 100644 --- a/docs/apiclient.rst +++ b/docs/apiclient.rst @@ -1,11 +1,9 @@ +.. _apiclient-label: + APIClient ========= -.. note:: - - For access to FEDn Studio API, please see :ref:`studio-api`. - -FEDn comes with an *APIClient* for interacting with the FEDn network. The APIClient is a Python3 library that can be used to interact with the FEDn network programmatically. +FEDn comes with an *APIClient* - a Python3 library that can be used to interact with FEDn programmatically. **Installation** @@ -17,12 +15,15 @@ The APIClient is available as a Python package on PyPI, and can be installed usi **Initialize the APIClient** -To initialize the APIClient, you need to provide the hostname and port of the FEDn API server. The default port is 8092. The following code snippet shows how to initialize the APIClient: +The FEDn REST API is available at /api/v1/. To access this API you need the url to the controller-host, as well as an admin API token. The controller host can be found in the project dashboard (top right corner). +To obtain an admin API token, navigate to the "Settings" tab in your Studio project and click on the "Generate token" button. Copy the 'access' token and use it to access the API using the instructions below. + .. code-block:: python - - from fedn import APIClient - client = APIClient("localhost", 8092) + + >>> from fedn import APIClient + >>> client = APIClient(host="", token="", secure=True, verify=True) + **Set active package and seed model** @@ -38,9 +39,9 @@ To set the initial seed model, you can use the following code snippet: client.set_active_model(path="path/to/seed.npz") -**Start training session** +**Start a training session** -Once the active package and seed model are set, you can connect clients to the network and start training models. The following code snippet initializes a session (training rounds): +Once the active package and seed model are set, you can connect clients to the network and start training models. The following code snippet starts a traing session: .. code-block:: python diff --git a/docs/architecture.rst b/docs/architecture.rst index a820e7e20..85e2430da 100644 --- a/docs/architecture.rst +++ b/docs/architecture.rst @@ -1,3 +1,5 @@ +.. _architecture-label: + Architecture overview ===================== diff --git a/docs/conf.py b/docs/conf.py index bebc3a80e..c45e90846 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -12,7 +12,7 @@ author = "Scaleout Systems AB" # The full version, including alpha/beta/rc tags -release = "0.11.0" +release = "0.11.1" # Add any Sphinx extension module names here, as strings extensions = [ diff --git a/docs/developer.rst b/docs/developer.rst new file mode 100644 index 000000000..3e05357b4 --- /dev/null +++ b/docs/developer.rst @@ -0,0 +1,168 @@ +.. _developer-label: + +Local development +================= + +.. note:: + These instructions are for users wanting to set up a local development deployment of FEDn (i.e. without FEDn Studio). + This requires practical knowledge of Docker and docker-compose. + +Running the FEDn development sandbox (docker-compose) +===================================================== + +During development on FEDn, and when working on own aggregators/helpers, it is +useful to have a local development setup of the core FEDn services (controller, combiner, database, object store). +For this, we provide Dockerfiles and docker-compose template. + +To start a development sandbox for FEDn using docker-compose: + +.. code-block:: + + docker compose \ + -f ../../docker-compose.yaml \ + -f docker-compose.override.yaml \ + up + +This starts up local services for MongoDB, Minio, the API Server, one Combiner and two clients. +You can verify the deployment using these urls: + +- API Server: http://localhost:8092/get_controller_status +- Minio: http://localhost:9000 +- Mongo Express: http://localhost:8081 + +This setup does not include the security features of Studio, and thus will not require authentication of clients. +To use the APIClient to test a compute package and seed model against a local FEDn deployment: + +.. code-block:: + + from fedn import APIClient + client = APIClient(host="localhost", port=8092) + client.set_active_package("package.tgz", helper="numpyhelper") + client.set_active_model("seed.npz") + + +To connect a native FEDn client, you need to make sure that the combiner service can be resolved using the name "combiner". +One way to achieve this is to edit your '/etc/hosts' and add a line '127.0.0.1 combiner'. + +Access message logs and validation data from MongoDB +==================================================== + +You can access and download event logs and validation data via the API, and you can also as a developer obtain +the MongoDB backend data using pymongo or via the MongoExpress interface: + +- http://localhost:8081/db/fedn-network/ + +Username and password are found in 'docker-compose.yaml'. + +Access global models +==================== + +You can obtain global model updates from the 'fedn-models' bucket in Minio: + +- http://localhost:9000 + +Username and password are found in 'docker-compose.yaml'. + +Reset the FEDn deployment +========================= + +To purge all data from a deployment incuding all session and round data, access the MongoExpress UI interface and +delete the entire ``fedn-network`` collection. Then restart all services. + +Clean up +======== +You can clean up by running + +.. code-block:: + + docker-compose -f ../../docker-compose.yaml -f docker-compose.override.yaml down -v + + +Connecting clients using Docker: +================================ + +For convenience, we distribute a Docker image hosted on ghrc.io with FEDn preinstalled. For example, to start a client for the MNIST PyTorch example using Docker +and FEDN 0.10.0, run this from the example folder: + +.. code-block:: + + docker run \ + -v $PWD/client.yaml:/app/client.yaml \ + -e FEDN_PACKAGE_EXTRACT_DIR=package \ + -e FEDN_NUM_DATA_SPLITS=2 \ + -e FEDN_DATA_PATH=/app/package/data/clients/1/mnist.pt \ + ghcr.io/scaleoutsystems/fedn/fedn:0.10.0 run client -in client.yaml --force-ssl --secure=True + + +Self-managed distributed deployment +=================================== + +You can use different hosts for the various FEDn services. These instructions shows how to set up FEDn on a **local network** using a single workstation or laptop as +the host for the servier-side components, and other hosts or devices as clients. + +.. note:: + For a secure and production-grade deployment solution over **public networks**, explore the FEDn Studio service at + **fedn.scaleoutsystems.com**. + + Alternatively follow this tutorial substituting the hosts local IP with your public IP, open the neccesary + ports (see which ports are used in docker-compose.yaml), and ensure you have taken additional neccesary security + precautions. + +**Prerequisites** +- `One host workstation and atleast one client device` +- `Python 3.8, 3.9, 3.10 or 3.11 `__ +- `Docker `__ +- `Docker Compose `__ + +Launch a distributed FEDn Network +--------------------------------- + + +Start by noting your host's local IP address, used within your network. Discover it by running ifconfig on UNIX or +ipconfig on Windows, typically listed under inet for Unix and IPv4 for Windows. + +Continue by following the standard procedure to initiate a FEDn network, for example using the provided docker-compose template. +Once the network is active, upload your compute package and seed (for comprehensive details, see the quickstart tutorials). + +.. note:: + This guide covers general local networks where server and client may be on different hosts but able to communicate on their private IPs. + A common scenario is also to run fedn and the clients on **localhost** on a single machine. In that case, you can replace + by "127.0.0.1" below. + +Configuring and Attaching Clients +--------------------------------- + +On your client device, continue with initializing your client. To connect to the host machine we need to ensure we are +routing the correct DNS to our hosts local IP address. We can do this using the standard FEDn `client.yaml`: + +.. code-block:: + + network_id: fedn-network + discover_host: api-server + discover_port: 8092 + + +We can then run a client using docker by adding the hostname:ip mapping in the docker run command: + +.. code-block:: + + docker run \ + -v $PWD/client.yaml: \ + + —add-host=api-server: \ + —add-host=combiner: \ + run client -in client.yaml --name client1 + + +Alternatively updating the `/etc/hosts` file, appending the following lines for running naitively: + +.. code-block:: + + api-server + combiner + + +Start a training session +------------------------ + +After connecting with your clients, you are ready to start training sessions from the host machine. \ No newline at end of file diff --git a/docs/distributed.rst b/docs/distributed.rst deleted file mode 100644 index 13803dd3f..000000000 --- a/docs/distributed.rst +++ /dev/null @@ -1,73 +0,0 @@ -Self-managed distributed deployment -=================================== - -This tutorial outlines the steps for deploying the FEDn framework over a **local network**, using a single workstation or laptop as -the host for the servier-side components, and other hosts or devices as clients. For general steps on how to run FEDn, see the quickstart tutorials. - - -.. note:: - For a secure and production-grade deployment solution over **public networks**, explore the FEDn Studio service at - **fedn.scaleoutsystems.com**. - - Alternatively follow this tutorial substituting the hosts local IP with your public IP, open the neccesary - ports (see which ports are used in docker-compose.yaml), and ensure you have taken additional neccesary security - precautions. - -**Prerequisites** -- `One host workstation and atleast one client device` -- `Python 3.8, 3.9, 3.10 or 3.11 `__ -- `Docker `__ -- `Docker Compose `__ - -Launch a distributed FEDn Network ---------------------------------- - - -Start by noting your host's local IP address, used within your network. Discover it by running ifconfig on UNIX or -ipconfig on Windows, typically listed under inet for Unix and IPv4 for Windows. - -Continue by following the standard procedure to initiate a FEDn network, for example using the provided docker-compose template. -Once the network is active, upload your compute package and seed (for comprehensive details, see the quickstart tutorials). - -.. note:: - This guide covers general local networks where server and client may be on different hosts but able to communicate on their private IPs. - A common scenario is also to run fedn and the clients on **localhost** on a single machine. In that case, you can replace - by "127.0.0.1" below. - -Configuring and Attaching Clients ---------------------------------- - -On your client device, continue with initializing your client. To connect to the host machine we need to ensure we are -routing the correct DNS to our hosts local IP address. We can do this using the standard FEDn `client.yaml`: - -.. code-block:: - - network_id: fedn-network - discover_host: api-server - discover_port: 8092 - - -We can then run a client using docker by adding the hostname:ip mapping in the docker run command: - -.. code-block:: - - docker run \ - -v $PWD/client.yaml: \ - - —add-host=api-server: \ - —add-host=combiner: \ - run client -in client.yaml --name client1 - - -Alternatively updating the `/etc/hosts` file, appending the following lines for running naitively: - -.. code-block:: - - api-server - combiner - - -Start a training session ------------------------- - -After connecting with your clients, you are ready to start training sessions from the host machine. \ No newline at end of file diff --git a/docs/index.rst b/docs/index.rst index f73a9aa5c..79b862c0e 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -4,19 +4,18 @@ introduction quickstart + projects .. toctree:: :maxdepth: 1 :caption: Documentation - studio - distributed apiclient - projects architecture aggregators helpers auth + developer .. toctree:: :maxdepth: 1 diff --git a/docs/projects.rst b/docs/projects.rst index 6397a3a56..2b86faa62 100644 --- a/docs/projects.rst +++ b/docs/projects.rst @@ -1,8 +1,11 @@ .. _projects-label: -FEDn Projects +Develop your own project ================================================ +This guide explains how a FEDn project is structured, and details how to develop your own +projects for your own use-cases. + A FEDn project is a convention for packaging/wrapping machine learning code to be used for federated learning with FEDn. At the core, a project is a directory of files (often a Git repository), containing your machine learning code, FEDn entry points, and a specification of the runtime environment (python environment or a Docker image). The FEDn API and command-line tools provides functionality @@ -28,9 +31,9 @@ We recommend that projects have roughly the following folder and file structure: | └ Dockerfile / docker-compose.yaml | -The "client" folder is referred to as the *compute package*. The file fedn.yaml is the FEDn Project File. It informs the FEDn Client of the code entry points to execute when computing model updates (local training) and validating models (optionally) . -When deploying the project to FEDn, the client folder will be compressed as a .tgz bundle and uploaded to the FEDn controller. FEDn can then manage the distribution of the compute package to each client/data provider when they connect. -Upon recipt of the bundle, the client will unpack it and stage it locally. +The ``client`` folder is commonly referred to as the *compute package*. The file ``fedn.yaml`` is the FEDn Project File. It contains information about the ``entry points``. The entry points are used by the client to compute model updates (local training) and local validations (optional) . +To run a project in FEDn, the client folder is compressed as a .tgz bundle and pushed to the FEDn controller. FEDn then manages the distribution of the compute package to each client. +Upon recipt of the package, a client will unpack it and stage it locally. .. image:: img/ComputePackageOverview.png :alt: Compute package overview @@ -41,7 +44,7 @@ The above figure provides a logical view of how FEDn uses the compute package (c recieves a model update request, it calls upon a Dispatcher that looks up entry point definitions in the compute package from the FEDn Project File. -FEDn Project File (fedn.yaml) +The Project File (fedn.yaml) ------------------------------ FEDn uses on a project file named 'fedn.yaml' to specify which entrypoints to execute when the client recieves a training or validation request, and @@ -60,22 +63,28 @@ what environment to execute those entrypoints in. command: python validate.py -Environment -^^^^^^^^^^^ - -The software environment to be used to exectute the entry points. This should specify all client side dependencies of the project. -FEDn currently supports Virtualenv environments, with packages on PyPI. When a project specifies a **python_env**, the FEDn -client will create an isolated virtual environment and install the project dependencies into it before starting up the client. +**Environment** + +It is assumed that all entry points are executable within the client runtime environment. As a user, you have two main options +to specify the environment: + 1. Provide a ``python_env`` in the ``fedn.yaml`` file. In this case, FEDn will create an isolated virtual environment and install the project dependencies into it before starting up the client. FEDn currently supports Virtualenv environments, with packages on PyPI. + 2. Manage the environment manually. Here you have several options, such as managing your own virtualenv, running in a Docker container, etc. Remove the ``python_env`` tag from ``fedn.yaml`` to handle the environment manually. -Entry Points -^^^^^^^^^^^^ +**Entry Points** There are up to four Entry Points to be specified. **Build Entrypoint (build, optional):** -This entrypoint is usually called **once** for building artifacts such as initial seed models. However, it not limited to artifacts, and can be used for any kind of setup that needs to be done before the client starts up. +This entrypoint is intended to be called **once** for building artifacts such as initial seed models. However, it not limited to artifacts, and can be used for any kind of setup that needs to be done before the client starts up. + +To invoke the build entrypoint using the CLI: + +.. code-block:: bash + fedn build -- + + **Startup Entrypoint (startup, optional):** @@ -107,8 +116,7 @@ model on local test/validation data. It should read a model update from file, va The validate entrypoint is optional. -Example train entry point -^^^^^^^^^^^^^^^^^^^^^^^^^^^ +**Example train entry point** Below is an example training entry point taken from the PyTorch getting stated project. @@ -202,7 +210,7 @@ using the pytorch helper module. The fifth function (_init_seed) is used to init The seventh function (_validate) is used to validate the model, again observe the two first arguments which will be set by the FEDn client. -Packaging for distribution +Build a compute package -------------------------- To deploy a project to FEDn (Studio or pseudo-local) we simply compress the *client* folder as .tgz file. using fedn command line tool or manually: @@ -223,15 +231,15 @@ by looking at the code above. Here we assume that the dataset is present in a fi the exection of entrypoint.py. Then, independent on the preferred way to run the client (native, Docker, K8s etc) this structure needs to be maintained for this particular compute package. Note however, that there are many ways to accompish this on a local operational level. -Testing the entry points before deploying the package to FEDn --------------------------------------------------------------- +Testing the entry points locally +--------------------------------- -We recommend you to test your code before deploying it to FEDn for distibution to clients. You can conveniently test *train* and *validate* by: +We recommend you to test your entrypoints locally before uploading the compute package to Studio. You can test *train* and *validate* by (example for the mnist-keras +project): .. code-block:: bash python train.py ../seed.npz ../model_update.npz --data_path ../data/mnist.npz python validate.py ../model_update.npz ../validation.json --data_path ../data/mnist.npz -Once everything works as expected you can start the federated network, upload the .tgz compute package and the initial model (use :py:meth:`fedn.network.api.client.APIClient.set_initial_model` for uploading an initial model). - +Note that we here assume execution in the correct Python environment. diff --git a/docs/quickstart.rst b/docs/quickstart.rst index d231aca9e..7723e7f28 100644 --- a/docs/quickstart.rst +++ b/docs/quickstart.rst @@ -2,26 +2,34 @@ Getting started with FEDn ========================= .. note:: - This tutorial is a quickstart guide to FEDn based on a pre-made FEDn Project. It is designed to serve as a minimalistic starting point for developers. - To learn about FEDn Projects in order to develop your own federated machine learning projects, see :ref:`projects-label`. + This tutorial is a quickstart guide to FEDn based on a pre-made FEDn Project. It is designed to serve as a starting point for new developers. + To learn how to develop your own project from scratch, see :ref:`projects-label`. **Prerequisites** -- `Python >=3.8, <=3.11 `__ +- `Python >=3.8, <=3.12 `__ - `A FEDn Studio account `__ -Set up a FEDn Studio Project ----------------------------- +1. Start a FEDn Studio Project +------------------------------ -Start by creating an account in FEDn Studio and set up a project by following the instruction here: :ref:`studio`. +Start by creating an account in Studio. Head over to `fedn.scaleoutsystems.com/signup `_ and sign up. -Install FEDn ------------- +Logged into Studio, create a new project by clicking on the "New Project" button in the top right corner of the screen. +You will see a Studio project similar to the image below. The Studio project provides all the necessary server side components of FEDn. +We will use this project in a later stage to run the federated experiments. But first, we will set up the local client. + + +.. image:: img/studio_project_overview.png + + +2. Install FEDn on your client +------------------------------- **Using pip** -Install the FEDn package using pip: +On you local machine/client, install the FEDn package using pip: .. code-block:: bash @@ -41,44 +49,64 @@ It is recommended to use a virtual environment when installing FEDn. .. _package-creation: -Initialize FEDn with the client code bundle and seed model ----------------------------------------------------------- +Next, we will prepare the client. We will use one of the pre-defined projects in the FEDn repository, ``mnist-pytorch``. -Next, we will prepare the client. The key part of a FEDn Project is the client definition - -code that contains entrypoints for training and (optionally) validating a model update on the client. +3. Create the compute package and seed model +-------------------------------------------- + +In order to train a federated model using FEDn, your Studio project needs to be initialized with a ``compute package`` and a ``seed model``. The compute package is a code bundle containing the +code used by the client to execute local training and local validation. The seed model is a first version of the global model. +For a detailed explaination of the compute package and seed model, see this guide: :ref:`projects-label` + +To work through this quick start you need a local copy of the ``mnist-pytorch`` example project contained in the main FEDn Git repository. +The following command clones the entire repository but you will only use the example: + +.. code-block:: bash -Locate into ``examples/mnist-pytorch`` and familiarize yourself with the project structure. The dependencies needed in the client environment are specified -in ``client/python_env.yaml``. + git clone https://github.com/scaleoutsystems/fedn.git -In order to train a federated model using FEDn, your Studio project needs to be initialized with a compute package and a seed model. The compute package is a bundle -of the client specification, and the seed model is a first version of the global model. +Locate into the ``fedn/examples/mnist-pytorch`` folder. The compute package is located in the folder ``client``. -Create a package of the fedn project (assumes your current working directory is in the root of the project /examples/mnist-pytorch): +Create a compute package: .. code-block:: fedn package create --path client -This will create a package called 'package.tgz' in the root of the project. +This will create a file called ``package.tgz`` in the root of the project. -Next, run the build entrypoint defined in ``client/fedn.yaml`` to build the model artifact. +Next, create the seed model: .. code-block:: fedn run build --path client -This will create a seed model called 'seed.npz' in the root of the project. We will now upload these to your Studio project using the FEDn APIClient. +This will create a file called ``seed.npz`` in the root of the project. + +.. note:: + This example automatically creates the runtime environment for the compute package using Virtualenv. + When you first exectue the above commands, FEDn will build a venv, and this takes + a bit of time. For more information on the various options to manage the environement, see :ref:`projects-label`. + +Next will now upload these files to your Studio project: + +4. Initialize your FEDn Studio Project +-------------------------------------- + +In the Studio UI, navigate to the project you created above and click on the "Sessions" tab. Click on the "New Session" button. Under the "Compute package" tab, select a name and upload the generated package file. Under the "Seed model" tab, upload the generated seed file: + +.. image:: img/upload_package.png + +**Upload the package and seed model using the Python APIClient** -**Upload the package and seed model** +It is also possible to upload a package and seed model using the Python API Client. .. note:: You need to create an API admin token and use the token to authenticate the APIClient. Do this by going to the 'Settings' tab in FEDn Studio and click 'Generate token'. Copy the access token and use it in the APIClient below. - The controller host can be found on the main Dashboard in FEDn Studio. + The controller host can be found on the main Dashboard in FEDn Studio. More information on the use of the APIClient can be found here: :ref:`apiclient-label. - You can also upload the file via the FEDn Studio UI. Please see :ref:`studio-upload-files` for more details. - -Upload the package and seed model using the APIClient: +To upload the package and seed model using the APIClient: .. code:: python @@ -88,23 +116,30 @@ Upload the package and seed model using the APIClient: >>> client.set_active_model("seed.npz") -Configure and attach clients ----------------------------- +5. Configure and attach clients +------------------------------- + +**Generate an access token for the client (in Studio)** + +Each local client needs an access token in order to connect securely to the FEDn server. These tokens are issued from your Studio Project. +Go to the Clients' tab and click 'Connect client'. Download a client configuration file and save it to the root of the ``examples/mnist-pytorch folder``. +Rename the file to 'client.yaml'. + +**Start the client (on your local machine)** -Each local client needs an access token in order to connect. These tokens are issued from your Studio Project. Go to the 'Clients' tab and click 'Connect client'. -Download a client configuration file and save it to the root of the examples/mnist-pytorch folder. Rename the file to 'client.yaml'. -Then start the client by running the following command in the root of the project: +Now we can start the client by running the following command: .. code-block:: fedn run client -in client.yaml --secure=True --force-ssl -Repeat the above for the number of clients you want to use. A normal laptop should be able to handle several clients for this example. +Repeat these two steps (generate an access token and start a local client) for the number of clients you want to use. +A normal laptop should be able to handle several clients for this example. -**Modifing the data split:** +**Modifying the data split (multiple-clients, optional):** -The default traning and test data for this example (MNIST) is for convenience downloaded and split by the client when it starts up (see 'startup' entrypoint). -The number of splits and which split used by a client can be controlled via the environment variables ``FEDN_NUM_DATA_SPLITS`` and ``FEDN_DATA_PATH``. +The default traning and test data for this particular example (mnist-pytorch) is for convenience downloaded and split automatically by the client when it starts up (see the 'startup' entrypoint). +The number of splits and which split to use by a client can be controlled via the environment variables ``FEDN_NUM_DATA_SPLITS`` and ``FEDN_DATA_PATH``. For example, to split the data in 10 parts and start a client using the 8th partiton: .. tabs:: @@ -126,10 +161,27 @@ For example, to split the data in 10 parts and start a client using the 8th part fedn client start -in client.yaml --secure=True --force-ssl -Start a training session ------------------------- +6. Start a training session +--------------------------- + +In Studio click on the "Sessions" link, then the "New session" button in the upper right corner. Click the "Start session" tab and enter your desirable settings (the default settings are good for this example) and hit the "Start run" button. +In the terminal where your are running your client you should now see some activity. When a round is completed, you can see the results on the "Models" page. + +**Watch the training progress** + +Once a training session is started, you can monitor the progress of the training by navigating to "Sessions" and click on the "Open" button of the active session. The session page will list the models as soon as they are generated. +To get more information about a particular model, navigate to the model page by clicking the model name. From the model page you can download the model weights and get validation metrics. + +.. image:: img/studio_model_overview.png -You are now ready to start training the model using the APIClient: +.. _studio-api: + +Congratulations, you have now completed your first federated training session with FEDn! Below you find additional information that can +be useful as you progress in your federated learning journey. + +**Control training sessions using the Python APIClient** + +You can also issue training sessions using the APIClient: .. code:: python @@ -144,125 +196,39 @@ You are now ready to start training the model using the APIClient: >>> validations = client.get_validations(model_id=model_id) -Please see :py:mod:`fedn.network.api` for more details on the APIClient. - -.. note:: - - In FEDn Studio, you can start a training session by going to the 'Sessions' tab and click 'Start session'. See :ref:`studio` for a - step-by-step guide for how to control experiments using the UI. +Please see :py:mod:`fedn.network.api` for more details on how to use the APIClient. -Access model updates --------------------- +**Downloading global model updates** .. note:: In FEDn Studio, you can access global model updates by going to the 'Models' or 'Sessions' tab. Here you can download model updates, metrics (as csv) and view the model trail. -You can access global model updates via the APIClient: +You can also access global model updates via the APIClient: .. code:: python >>> ... >>> client.download_model("", path="model.npz") - -**Connecting clients using Docker** - -You can also use Docker to containerize the client. -For convenience, there is a Docker image hosted on ghrc.io with fedn preinstalled. -To start a client using Docker: - -.. code-block:: - - docker run \ - -v $PWD/client.yaml:/app/client.yaml \ - -e FEDN_PACKAGE_EXTRACT_DIR=package \ - -e FEDN_NUM_DATA_SPLITS=2 \ - -e FEDN_DATA_PATH=/app/package/data/clients/1/mnist.pt \ - ghcr.io/scaleoutsystems/fedn/fedn:0.9.0 run client -in client.yaml --force-ssl --secure=True - - **Where to go from here?** +-------------------------- -With you first FEDn federation set up, we suggest that you take a close look at how a FEDn project is structured +With you first FEDn federated project set up, we suggest that you take a close look at how a FEDn project is structured and how you develop your own FEDn projects: - :ref:`projects-label` +You can also dive into the architecture overview to learn more about how FEDn is designed and works under the hood: -Local development deployment (using docker compose) ----------------------------------------------------------- - -.. note:: - These instructions are for users wanting to set up a local development deployment of FEDn (wihout Studio). - This requires basic knowledge of Docker and docker-compose. - The main use-case for this is rapid iteration while developing the FEDn Project, - development of aggregator plugins, etc. - -Follow the steps above to install FEDn, generate 'package.tgz' and 'seed.tgz'. Then, instead of -using a Studio project for a managed FEDn server-side, start a local FEDn network -using docker-compose: - -.. code-block:: - - docker compose \ - -f ../../docker-compose.yaml \ - -f docker-compose.override.yaml \ - up - -This starts up local services for MongoDB, Minio, the API Server, one Combiner and two clients. -You can verify the deployment using these urls: +- :ref:`architecture-label` -- API Server: http://localhost:8092/get_controller_status -- Minio: http://localhost:9000 -- Mongo Express: http://localhost:8081 +For developers looking to cutomize FEDn and develop own aggregators, check out the local development guide. +This page also has instructions for using Docker to run clients: -Upload the package and seed model to FEDn controller using the APIClient. In Python: +- :ref:`developer-label` -.. code-block:: - - from fedn import APIClient - client = APIClient(host="localhost", port=8092) - client.set_active_package("package.tgz", helper="numpyhelper") - client.set_active_model("seed.npz") -You can now start a training session with 5 rounds (default): - -.. code-block:: - client.start_session() -**Automate experimentation with several clients** - -If you want to scale the number of clients, you can do so by modifying ``docker-compose.override.yaml``. For example, -in order to run with 3 clients, change the environment variable ``FEDN_NUM_DATA_SPLITS`` to 3, and add one more client -by copying ``client1`` and setting ``FEDN_DATA_PATH`` to ``/app/package/data/clients/3/mnist.pt`` - - -**Access message logs and validation data from MongoDB** - -You can access and download event logs and validation data via the API, and you can also as a developer obtain -the MongoDB backend data using pymongo or via the MongoExpress interface: - -- http://localhost:8081/db/fedn-network/ - -The credentials are as set in docker-compose.yaml in the root of the repository. - -**Access global models** - -You can obtain global model updates from the 'fedn-models' bucket in Minio: - -- http://localhost:9000 - -**Reset the FEDn deployment** - -To purge all data from a deployment incuding all session and round data, access the MongoExpress UI interface and -delete the entire ``fedn-network`` collection. Then restart all services. - -**Clean up** - -You can clean up by running - -.. code-block:: - docker-compose -f ../../docker-compose.yaml -f docker-compose.override.yaml down -v diff --git a/examples/mnist-keras/README.rst b/examples/mnist-keras/README.rst index 0f2ec9cf1..aaf13c21d 100644 --- a/examples/mnist-keras/README.rst +++ b/examples/mnist-keras/README.rst @@ -1,14 +1,14 @@ FEDn Project: Keras/Tensorflow (MNIST) ------------------------------------------- -This is a TF/Keras version of the Quickstart Tutorial (PyTorch) FEDn Project. For a step-by-step guide, refer to that tutorial. +This is a TF/Keras version of the PyTorch Quickstart Tutorial. For a step-by-step guide, refer to that tutorial. **Note: We recommend all new users to start by following the Quickstart Tutorial: https://fedn.readthedocs.io/en/latest/quickstart.html** Prerequisites ------------------------------------------- -- `Python 3.8, 3.9, 3.10 or 3.11 `__ +- `Python >=3.8, <=3.12 `__ Creating the compute package and seed model ------------------------------------------- @@ -24,7 +24,7 @@ Clone this repository, then locate into this directory: .. code-block:: git clone https://github.com/scaleoutsystems/fedn.git - cd fedn/examples/mnist-pytorch + cd fedn/examples/mnist-keras Create the compute package: @@ -42,13 +42,8 @@ Next, generate a seed model (the first model in a global model trail): This step will take a few minutes, depending on hardware and internet connection (builds a virtualenv). -Using FEDn Studio ------------------- +Running the project on FEDn +---------------------------- -To set up your FEDn Studio project and learn how to connect clients (using JWT token authentication), follow this guide: https://fedn.readthedocs.io/en/latest/studio.html. On the -step "Upload Files", upload 'package.tgz' and 'seed.npz' created above. +To learn how to set up your FEDn Studio project and connect clients, take the quickstart tutorial: https://fedn.readthedocs.io/en/stable/quickstart.html. -Using pseudo-distributed mode (local development) -------------------------------------------------- - -See the PyTorch version of this example for detailed instructions on how to deploy FEDn in `local development mode <../mnist-pytorch/README.rst>`_. diff --git a/examples/mnist-pytorch/README.rst b/examples/mnist-pytorch/README.rst index 71e5de2d1..990b902b2 100644 --- a/examples/mnist-pytorch/README.rst +++ b/examples/mnist-pytorch/README.rst @@ -2,25 +2,15 @@ FEDn Project: MNIST (PyTorch) ----------------------------- This is an example FEDn Project based on the classic hand-written text recognition dataset MNIST. -The example is intented as a minimalistic quickstart and automates the handling of training data -by letting the client download and create its partition of the dataset as it starts up. +The example is intented as a minimalistic quickstart to learn how to use FEDn. - **Note: These instructions are geared towards users seeking to learn how to work - with FEDn in local development mode using Docker/docker-compose. We recommend all new users - to start by following the Quickstart Tutorial: https://fedn.readthedocs.io/en/stable/quickstart.html** + **Note: We recommend that all new users start by taking the Quickstart Tutorial: https://fedn.readthedocs.io/en/stable/quickstart.html** Prerequisites ------------- -Using FEDn Studio: - -- `Python 3.8, 3.9, 3.10 or 3.11 `__ -- `A FEDn Studio account `__ - -If using pseudo-distributed mode with docker-compose: - -- `Docker `__ -- `Docker Compose `__ +- `Python >=3.8, <=3.12 `__ +- `A project in FEDn Studio `__ Creating the compute package and seed model ------------------------------------------- @@ -44,126 +34,17 @@ Create the compute package: fedn package create --path client -This should create a file 'package.tgz' in the project folder. +This creates a file 'package.tgz' in the project folder. -Next, generate a seed model (the first model in a global model trail): +Next, generate the seed model: .. code-block:: fedn run build --path client -This will create a seed model called 'seed.npz' in the root of the project. This step will take a few minutes, depending on hardware and internet connection (builds a virtualenv). - -Using FEDn Studio ------------------ - -Follow the guide here to set up your FEDn Studio project and learn how to connect clients (using token authentication): `Studio guide `__. -On the step "Upload Files", upload 'package.tgz' and 'seed.npz' created above. - - -Modifing the data split: -======================== - -The default traning and test data for this example is downloaded and split direcly by the client when it starts up (see 'startup' entrypoint). -The number of splits and which split used by a client can be controlled via the environment variables ``FEDN_NUM_DATA_SPLITS`` and ``FEDN_DATA_PATH``. -For example, to split the data in 10 parts and start a client using the 8th partiton: - -.. code-block:: - - export FEDN_PACKAGE_EXTRACT_DIR=package - export FEDN_NUM_DATA_SPLITS=10 - export FEDN_DATA_PATH=./data/clients/8/mnist.pt - fedn client start -in client.yaml --secure=True --force-ssl - -The default is to split the data into 2 partitions and use the first partition. - - -Connecting clients using Docker: -================================ - -For convenience, there is a Docker image hosted on ghrc.io with fedn preinstalled. To start a client using Docker: - -.. code-block:: - - docker run \ - -v $PWD/client.yaml:/app/client.yaml \ - -e FEDN_PACKAGE_EXTRACT_DIR=package \ - -e FEDN_NUM_DATA_SPLITS=2 \ - -e FEDN_DATA_PATH=/app/package/data/clients/1/mnist.pt \ - ghcr.io/scaleoutsystems/fedn/fedn:0.9.0 run client -in client.yaml --force-ssl --secure=True - - -Local development mode using Docker/docker compose --------------------------------------------------- - -Follow the steps above to install FEDn, generate 'package.tgz' and 'seed.tgz'. - -Start a pseudo-distributed FEDn network using docker-compose: - -.. code-block:: - - docker compose \ - -f ../../docker-compose.yaml \ - -f docker-compose.override.yaml \ - up +This will create a model file 'seed.npz' in the root of the project. This step will take a few minutes, depending on hardware and internet connection (builds a virtualenv). -This starts up local services for MongoDB, Minio, the API Server, one Combiner and two clients. -You can verify the deployment using these urls: - -- API Server: http://localhost:8092/get_controller_status -- Minio: http://localhost:9000 -- Mongo Express: http://localhost:8081 - -Upload the package and seed model to FEDn controller using the APIClient. In Python: - -.. code-block:: - - from fedn import APIClient - client = APIClient(host="localhost", port=8092) - client.set_active_package("package.tgz", helper="numpyhelper") - client.set_active_model("seed.npz") - -You can now start a training session with 5 rounds (default): - -.. code-block:: - - client.start_session() - -Automate experimentation with several clients -============================================= - -If you want to scale the number of clients, you can do so by modifying ``docker-compose.override.yaml``. For example, -in order to run with 3 clients, change the environment variable ``FEDN_NUM_DATA_SPLITS`` to 3, and add one more client -by copying ``client1`` and setting ``FEDN_DATA_PATH`` to ``/app/package/data/clients/3/mnist.pt`` - - -Access message logs and validation data from MongoDB -==================================================== - -You can access and download event logs and validation data via the API, and you can also as a developer obtain -the MongoDB backend data using pymongo or via the MongoExpress interface: - -- http://localhost:8081/db/fedn-network/ - -The credentials are as set in docker-compose.yaml in the root of the repository. - -Access global models -==================== - -You can obtain global model updates from the 'fedn-models' bucket in Minio: - -- http://localhost:9000 - -Reset the FEDn deployment -========================= - -To purge all data from a deployment incuding all session and round data, access the MongoExpress UI interface and -delete the entire ``fedn-network`` collection. Then restart all services. - -Clean up -======== -You can clean up by running - -.. code-block:: +Running the project on FEDn +---------------------------- - docker-compose -f ../../docker-compose.yaml -f docker-compose.override.yaml down -v +To learn how to set up your FEDn Studio project and connect clients, take the quickstart tutorial: https://fedn.readthedocs.io/en/stable/quickstart.html. diff --git a/fedn/cli/__init__.py b/fedn/cli/__init__.py index 137fc9b9c..bcd27dc53 100644 --- a/fedn/cli/__init__.py +++ b/fedn/cli/__init__.py @@ -9,3 +9,4 @@ from .session_cmd import session_cmd # noqa: F401 from .status_cmd import status_cmd # noqa: F401 from .validation_cmd import validation_cmd # noqa: F401 +from .controller_cmd import controller_cmd # noqa: F401 diff --git a/fedn/cli/controller_cmd.py b/fedn/cli/controller_cmd.py new file mode 100644 index 000000000..ab8727b27 --- /dev/null +++ b/fedn/cli/controller_cmd.py @@ -0,0 +1,18 @@ +import click + +from .main import main + + +@main.group("controller") +@click.pass_context +def controller_cmd(ctx): + """:param ctx:""" + pass + + +@controller_cmd.command("start") +@click.pass_context +def controller_cmd(ctx): + from fedn.network.api.server import start_server_api + + start_server_api() diff --git a/fedn/cli/main.py b/fedn/cli/main.py index d6f912e62..ab1dd448e 100644 --- a/fedn/cli/main.py +++ b/fedn/cli/main.py @@ -1,3 +1,4 @@ +from fedn.utils.dist import get_version import click CONTEXT_SETTINGS = dict( @@ -5,10 +6,12 @@ help_option_names=["-h", "--help"], ) +version=get_version("fedn") + @click.group(context_settings=CONTEXT_SETTINGS) +@click.version_option(version) @click.pass_context def main(ctx): - """:param ctx: - """ + """:param ctx:""" ctx.obj = dict() diff --git a/fedn/cli/run_cmd.py b/fedn/cli/run_cmd.py index 123f17320..20836cf96 100644 --- a/fedn/cli/run_cmd.py +++ b/fedn/cli/run_cmd.py @@ -45,6 +45,36 @@ def run_cmd(ctx): """ pass +@run_cmd.command("startup") +@click.option("-p", "--path", required=True, help="Path to package directory containing fedn.yaml") +@click.pass_context +def startup_cmd(ctx, path): + """Execute 'startup' entrypoint in fedn.yaml. + + :param ctx: + :param path: Path to folder containing fedn.yaml + :type path: str + """ + path = os.path.abspath(path) + yaml_file = os.path.join(path, "fedn.yaml") + if not os.path.exists(yaml_file): + logger.error(f"Could not find fedn.yaml in {path}") + exit(-1) + + config = _read_yaml_file(yaml_file) + # Check that startup is defined in fedn.yaml under entry_points + if "startup" not in config["entry_points"]: + logger.error("No startup command defined in fedn.yaml") + exit(-1) + + dispatcher = Dispatcher(config, path) + _ = dispatcher._get_or_create_python_env() + dispatcher.run_cmd("startup") + + # delete the virtualenv + if dispatcher.python_env_path: + logger.info(f"Removing virtualenv {dispatcher.python_env_path}") + shutil.rmtree(dispatcher.python_env_path) @run_cmd.command("build") @click.option("-p", "--path", required=True, help="Path to package directory containing fedn.yaml") diff --git a/fedn/common/config.py b/fedn/common/config.py index 4864ce1ef..23d873ff7 100644 --- a/fedn/common/config.py +++ b/fedn/common/config.py @@ -2,8 +2,7 @@ import yaml -global STATESTORE_CONFIG -global MODELSTORAGE_CONFIG +from fedn.utils.dist import get_package_path SECRET_KEY = os.environ.get("FEDN_JWT_SECRET_KEY", False) FEDN_JWT_CUSTOM_CLAIM_KEY = os.environ.get("FEDN_JWT_CUSTOM_CLAIM_KEY", False) @@ -23,9 +22,15 @@ def get_environment_config(): """Get the configuration from environment variables.""" global STATESTORE_CONFIG global MODELSTORAGE_CONFIG - - STATESTORE_CONFIG = os.environ.get("STATESTORE_CONFIG", "/workspaces/fedn/config/settings-reducer.yaml.template") - MODELSTORAGE_CONFIG = os.environ.get("MODELSTORAGE_CONFIG", "/workspaces/fedn/config/settings-reducer.yaml.template") + if not os.environ.get("STATESTORE_CONFIG", False): + STATESTORE_CONFIG = get_package_path() + "/common/settings-controller.yaml.template" + else: + STATESTORE_CONFIG = os.environ.get("STATESTORE_CONFIG") + + if not os.environ.get("MODELSTORAGE_CONFIG", False): + MODELSTORAGE_CONFIG = get_package_path() + "/common/settings-controller.yaml.template" + else: + MODELSTORAGE_CONFIG = os.environ.get("MODELSTORAGE_CONFIG") def get_statestore_config(file=None): diff --git a/fedn/common/settings-controller.yaml.template b/fedn/common/settings-controller.yaml.template new file mode 100644 index 000000000..a5266a38b --- /dev/null +++ b/fedn/common/settings-controller.yaml.template @@ -0,0 +1,24 @@ +network_id: fedn-network +controller: + host: localhost + port: 8092 + debug: True + +statestore: + type: MongoDB + mongo_config: + username: fedn_admin + password: password + host: localhost + port: 6534 + +storage: + storage_type: S3 + storage_config: + storage_hostname: localhost + storage_port: 9000 + storage_access_key: fedn_admin + storage_secret_key: password + storage_bucket: fedn-models + context_bucket: fedn-context + storage_secure_mode: False diff --git a/fedn/network/api/server.py b/fedn/network/api/server.py index 820ac93ac..d56c3ab0b 100644 --- a/fedn/network/api/server.py +++ b/fedn/network/api/server.py @@ -9,6 +9,7 @@ from fedn.network.api.v1 import _routes custom_url_prefix = os.environ.get("FEDN_CUSTOM_URL_PREFIX", False) +# statestore_config,modelstorage_config,network_id,control=set_statestore_config() api = API(statestore, control) app = Flask(__name__) for bp in _routes: @@ -626,12 +627,13 @@ def list_combiners_data(): app.add_url_rule(f"{custom_url_prefix}/list_combiners_data", view_func=list_combiners_data, methods=["POST"]) -def start(): +def start_server_api(): config = get_controller_config() port = config["port"] debug = config["debug"] - app.run(debug=debug, port=port, host="0.0.0.0") + host = "0.0.0.0" + app.run(debug=debug, port=port, host=host) if __name__ == "__main__": - start() + start_server_api() diff --git a/fedn/network/api/shared.py b/fedn/network/api/shared.py index fc8d4ae57..9e0e5acbd 100644 --- a/fedn/network/api/shared.py +++ b/fedn/network/api/shared.py @@ -5,7 +5,6 @@ statestore_config = get_statestore_config() modelstorage_config = get_modelstorage_config() network_id = get_network_config() - statestore = MongoStateStore(network_id, statestore_config["mongo_config"]) statestore.set_storage_backend(modelstorage_config) control = Control(statestore=statestore) diff --git a/fedn/network/api/v1/shared.py b/fedn/network/api/v1/shared.py index b7ae170af..a27a6f637 100644 --- a/fedn/network/api/v1/shared.py +++ b/fedn/network/api/v1/shared.py @@ -3,10 +3,9 @@ import pymongo from pymongo.database import Database -from fedn.network.api.shared import statestore_config, network_id +from fedn.network.api.shared import statestore_config,network_id api_version = "v1" - mc = pymongo.MongoClient(**statestore_config["mongo_config"]) mc.server_info() mdb: Database = mc[network_id] diff --git a/fedn/network/combiner/combiner.py b/fedn/network/combiner/combiner.py index 37c4c3264..d336932c5 100644 --- a/fedn/network/combiner/combiner.py +++ b/fedn/network/combiner/combiner.py @@ -115,7 +115,10 @@ def __init__(self, config): # Set the status to offline for previous clients. previous_clients = self.statestore.clients.find({"combiner": config["name"]}) for client in previous_clients: - self.statestore.set_client({"name": client["name"], "status": "offline", "client_id": client["client_id"]}) + try: + self.statestore.set_client({"name": client["name"], "status": "offline", "client_id": client["client_id"]}) + except KeyError: + self.statestore.set_client({"name": client["name"], "status": "offline"}) # Set up gRPC server configuration if config["secure"]: diff --git a/fedn/network/combiner/roundhandler.py b/fedn/network/combiner/roundhandler.py index 8cad774b0..6853fe01c 100644 --- a/fedn/network/combiner/roundhandler.py +++ b/fedn/network/combiner/roundhandler.py @@ -310,7 +310,6 @@ def _assign_round_clients(self, n, type="trainers"): clients = self.server.get_active_trainers() else: logger.error("(ERROR): {} is not a supported type of client".format(type)) - raise # If the number of requested trainers exceeds the number of available, use all available. if n > len(clients): diff --git a/fedn/network/storage/statestore/mongostatestore.py b/fedn/network/storage/statestore/mongostatestore.py index 1a7111fe3..3ef204b5c 100644 --- a/fedn/network/storage/statestore/mongostatestore.py +++ b/fedn/network/storage/statestore/mongostatestore.py @@ -737,7 +737,13 @@ def set_client(self, client_data): :return: """ client_data["updated_at"] = str(datetime.now()) - self.clients.update_one({"client_id": client_data["client_id"]}, {"$set": client_data}, True) + try: + self.clients.update_one({"client_id": client_data["client_id"]}, {"$set": client_data}, True) + except KeyError: + # If client_id is not present, use name as identifier, for backwards compatibility + id = str(uuid.uuid4()) + client_data["client_id"] = id + self.clients.update_one({"name": client_data["name"]}, {"$set": client_data}, True) def get_client(self, client_id): """Get client by client_id. diff --git a/fedn/utils/dist.py b/fedn/utils/dist.py new file mode 100644 index 000000000..e5fa7192b --- /dev/null +++ b/fedn/utils/dist.py @@ -0,0 +1,17 @@ +import importlib.metadata + +import fedn + + +def get_version(pacakge): + # Dynamically get the version of the package + try: + version = importlib.metadata.version("fedn") + except importlib.metadata.PackageNotFoundError: + version = "unknown" + return version + + +def get_package_path(): + # Get the path of the package + return fedn.__path__[0] diff --git a/pyproject.toml b/pyproject.toml index 5773a38a5..3806fee56 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -6,7 +6,7 @@ build-backend = "setuptools.build_meta" [project] name = "fedn" -version = "0.11.0" +version = "0.11.1" description = "Scaleout Federated Learning" authors = [{ name = "Scaleout Systems AB", email = "contact@scaleoutsystems.com" }] readme = "README.rst"