Merge branch 'master' into feature/SK-974

.github/workflows/code-checks.yaml

@@ -26,6 +26,8 @@ jobs:
           --exclude-dir='flower-client'
           --exclude='tests.py'
           --exclude='controller_cmd.py'
+          --exclude='combiner_cmd.py'
+          --exclude='run_cmd.py'
           --exclude='README.rst'
           '^[ \t]+(import|from) ' -I . 
 

LICENSE

@@ -1,3 +1,5 @@
+Copyright 2021 Scaleout Systems AB.  All rights reserved.
+
                                  Apache License
                            Version 2.0, January 2004
                         http://www.apache.org/licenses/

MANIFEST.in

@@ -0,0 +1,2 @@
+include README.rst
+include fedn/common/settings-controller.yaml.template

config/settings-client.yaml.local.template

@@ -0,0 +1,3 @@
+network_id: fedn-network
+discover_host: localhost
+discover_port: 8092

config/settings-combiner.yaml.local.template

@@ -0,0 +1,31 @@
+network_id: fedn-network
+
+name: combiner
+host: localhost
+address: localhost
+port: 12080
+max_clients: 30
+
+cert_path: tmp/server.crt
+key_path: tmp/server.key
+
+statestore:
+  type: MongoDB
+  mongo_config:
+    username: fedn_admin
+    password: password
+    host: localhost
+    port: 6534
+
+storage:
+  storage_type: S3
+  storage_config:
+    storage_hostname: localhost
+    storage_port: 9000
+    storage_access_key: fedn_admin
+    storage_secret_key: password
+    storage_bucket: fedn-models
+    context_bucket: fedn-context
+    storage_secure_mode: False 
+
+

config/settings-combiner.yaml.template

@@ -7,4 +7,23 @@ host: combiner
 port: 12080
 max_clients: 30
 
+statestore:
+  type: MongoDB
+  mongo_config:
+    username: fedn_admin
+    password: password
+    host: mongo
+    port: 6534
+
+storage:
+  storage_type: S3
+  storage_config:
+    storage_hostname: minio
+    storage_port: 9000
+    storage_access_key: fedn_admin
+    storage_secret_key: password
+    storage_bucket: fedn-models
+    context_bucket: fedn-context
+    storage_secure_mode: False
+
 

config/settings-controller.yaml.local.template

@@ -0,0 +1,24 @@
+network_id: fedn-network
+controller:
+  host: localhost
+  port: 8092
+  debug: True
+
+statestore:
+  type: MongoDB
+  mongo_config:
+    username: fedn_admin
+    password: password
+    host: localhost
+    port: 6534
+
+storage:
+  storage_type: S3
+  storage_config:
+    storage_hostname: localhost
+    storage_port: 9000
+    storage_access_key: fedn_admin
+    storage_secret_key: password
+    storage_bucket: fedn-models
+    context_bucket: fedn-context
+    storage_secure_mode: False 

docker-compose.yaml

@@ -87,6 +87,8 @@ services:
     environment:
       - PYTHONUNBUFFERED=0
       - GET_HOSTS_FROM=dns
+      - STATESTORE_CONFIG=/app/config/settings-combiner.yaml
+      - MODELSTORAGE_CONFIG=/app/config/settings-combiner.yaml
     build:
       context: .
       args:

docs/_static/css/elements.css

@@ -19,7 +19,6 @@ article ul {
 
 .rst-content .section ul li,
 .rst-content .toctree-wrapper ul li,
-.rst-content section ul li,
 .wy-plain-list-disc li,
 article ul li {
     list-style: none;

docs/conf.py

@@ -8,11 +8,10 @@
 
 # Project info
 project = "FEDn"
-copyright = "2021, Scaleout Systems AB"
 author = "Scaleout Systems AB"
 
 # The full version, including alpha/beta/rc tags
-release = "0.13.0"
+release = "0.14.0"
 
 # Add any Sphinx extension module names here, as strings
 extensions = [

docs/projects.rst

@@ -385,6 +385,15 @@ Then, standing inside the 'client folder', you can test *train* and *validate* b
     python train.py ../seed.npz ../model_update.npz --data_path data/clients/1/mnist.pt
     python validate.py ../model_update.npz ../validation.json --data_path data/clients/1/mnist.pt
 
+You can also test *train* and *validate* entrypoint using CLI command:
+
+.. note:: Before running the fedn run train or fedn run validate commands, make sure to download the training and test data. The downloads are usually handled by the "fedn run startup" command in the examples provided by FEDn.
+
+.. code-block:: bash
+
+    fedn run train --path client --input <path to input model parameters> --output <path to write the updated model parameters>
+    fedn run validate --path client --input <path to input model parameters> --output <path to write the output JSON containing validation metrics>
+
 Packaging for training on FEDn
 ===============================
 

examples/mnist-pytorch/client/python_env.yaml

@@ -4,6 +4,6 @@ build_dependencies:
   - setuptools
   - wheel
 dependencies:
-  - torch==2.3.1
-  - torchvision==0.18.1
+  - torch
+  - torchvision
   - fedn

fedn/cli/combiner_cmd.py

@@ -3,17 +3,14 @@
 import click
 import requests
 
-from fedn.network.combiner.combiner import Combiner
-
 from .main import main
 from .shared import CONTROLLER_DEFAULTS, apply_config, get_api_url, get_token, print_response
 
 
 @main.group("combiner")
 @click.pass_context
 def combiner_cmd(ctx):
-    """:param ctx:
-    """
+    """:param ctx:"""
     pass
 
 
@@ -60,6 +57,8 @@ def start_cmd(ctx, discoverhost, discoverport, token, name, host, port, fqdn, se
         click.echo(f"\nCombiner configuration loaded from file: {init}")
         click.echo("Values set in file override defaults and command line arguments...\n")
 
+    from fedn.network.combiner.combiner import Combiner
+
     combiner = Combiner(config)
     combiner.run()
 

fedn/cli/run_cmd.py

@@ -4,15 +4,15 @@
 
 import click
 import yaml
+
+from fedn.cli.client_cmd import validate_client_config
+from fedn.cli.main import main
+from fedn.cli.shared import apply_config
 from fedn.common.exceptions import InvalidClientConfig
 from fedn.common.log_config import logger
 from fedn.network.clients.client import Client
-from fedn.network.combiner.combiner import Combiner
 from fedn.utils.dispatcher import Dispatcher, _read_yaml_file
 
-from fedn.cli.client_cmd import validate_client_config
-from fedn.cli.main import main
-from fedn.cli.shared import apply_config
 
 def get_statestore_config_from_file(init):
     """:param init:
@@ -35,6 +35,7 @@ def check_helper_config_file(config):
         exit(-1)
     return helper
 
+
 def check_yaml_exists(path):
     """Check if fedn.yaml exists in the given path."""
     yaml_file = os.path.join(path, "fedn.yaml")
@@ -43,21 +44,26 @@ def check_yaml_exists(path):
         click.echo(f"Could not find fedn.yaml in {path}")
         exit(-1)
     return yaml_file
+
+
 def delete_virtual_environment(dispatcher):
     if dispatcher.python_env_path:
         logger.info(f"Removing virtualenv {dispatcher.python_env_path}")
         shutil.rmtree(dispatcher.python_env_path)
     else:
         logger.warning("No virtualenv found to remove.")
+
+
 @main.group("run")
 @click.pass_context
 def run_cmd(ctx):
-    """:param ctx:
-    """
+    """:param ctx:"""
     pass
+
+
 @run_cmd.command("validate")
 @click.option("-p", "--path", required=True, help="Path to package directory containing fedn.yaml")
-@click.option("-i", "--input", required=True, help="Path to input model" )
+@click.option("-i", "--input", required=True, help="Path to input model")
 @click.option("-o", "--output", required=True, help="Path to write the output JSON containing validation metrics")
 @click.option("-v", "--keep-venv", is_flag=True, required=False, help="Use flag to keep the python virtual environment (python_env in fedn.yaml)")
 @click.pass_context
@@ -82,9 +88,11 @@ def validate_cmd(ctx, path, input, output, keep_venv):
     dispatcher.run_cmd("validate {} {}".format(input, output))
     if not keep_venv:
         delete_virtual_environment(dispatcher)
+
+
 @run_cmd.command("train")
 @click.option("-p", "--path", required=True, help="Path to package directory containing fedn.yaml")
-@click.option("-i", "--input", required=True, help="Path to input model parameters" )
+@click.option("-i", "--input", required=True, help="Path to input model parameters")
 @click.option("-o", "--output", required=True, help="Path to write the updated model parameters ")
 @click.option("-v", "--keep-venv", is_flag=True, required=False, help="Use flag to keep the python virtual environment (python_env in fedn.yaml)")
 @click.pass_context
@@ -109,6 +117,8 @@ def train_cmd(ctx, path, input, output, keep_venv):
     dispatcher.run_cmd("train {} {}".format(input, output))
     if not keep_venv:
         delete_virtual_environment(dispatcher)
+
+
 @run_cmd.command("startup")
 @click.option("-p", "--path", required=True, help="Path to package directory containing fedn.yaml")
 @click.option("-v", "--keep-venv", is_flag=True, required=False, help="Use flag to keep the python virtual environment (python_env in fedn.yaml)")
@@ -134,6 +144,7 @@ def startup_cmd(ctx, path, keep_venv):
     if not keep_venv:
         delete_virtual_environment(dispatcher)
 
+
 @run_cmd.command("build")
 @click.option("-p", "--path", required=True, help="Path to package directory containing fedn.yaml")
 @click.option("-v", "--keep-venv", is_flag=True, required=False, help="Use flag to keep the python virtual environment (python_env in fedn.yaml)")
@@ -173,7 +184,7 @@ def build_cmd(ctx, path, keep_venv):
 @click.option("-s", "--secure", required=False, default=False)
 @click.option("-pc", "--preshared-cert", required=False, default=False)
 @click.option("-v", "--verify", is_flag=True, help="Verify SSL/TLS for REST service")
-@click.option("-c", "--preferred-combiner", required=False,type=str, default="",help="url to the combiner or name of the preferred combiner")
+@click.option("-c", "--preferred-combiner", required=False, type=str, default="", help="url to the combiner or name of the preferred combiner")
 @click.option("-va", "--validator", required=False, default=True)
 @click.option("-tr", "--trainer", required=False, default=True)
 @click.option("-in", "--init", required=False, default=None, help="Set to a filename to (re)init client from file state.")
@@ -310,5 +321,7 @@ def combiner_cmd(ctx, discoverhost, discoverport, token, name, host, port, fqdn,
         click.echo(f"\nCombiner configuration loaded from file: {init}")
         click.echo("Values set in file override defaults and command line arguments...\n")
 
+    from fedn.network.combiner.combiner import Combiner
+
     combiner = Combiner(config)
     combiner.run()

fedn/common/certificate/certificate.py

@@ -9,24 +9,27 @@
 
 
 class Certificate:
-    """Utility to generate unsigned certificates.
-
-    """
+    """Utility to generate unsigned certificates."""
 
     CERT_NAME = "cert.pem"
     KEY_NAME = "key.pem"
     BITS = 2048
 
-    def __init__(self, cwd, name=None, key_name="key.pem", cert_name="cert.pem", create_dirs=True):
-        try:
-            os.makedirs(cwd)
-        except OSError:
-            logger.info("Directory exists, will store all cert and keys here.")
+    def __init__(self, name=None, key_path="", cert_path="", create_dirs=False):
+        if create_dirs:
+            try:
+                cwd = os.getcwd()
+                os.makedirs(cwd)
+            except OSError:
+                logger.info("Directory exists, will store all cert and keys here.")
+            else:
+                logger.info("Successfully created the directory to store cert and keys in {}".format(cwd))
+
+            self.key_path = os.path.join(cwd, "key.pem")
+            self.cert_path = os.path.join(cwd, "cert.pem")
         else:
-            logger.info("Successfully created the directory to store cert and keys in {}".format(cwd))
-
-        self.key_path = os.path.join(cwd, key_name)
-        self.cert_path = os.path.join(cwd, cert_name)
+            self.key_path = key_path
+            self.cert_path = cert_path
 
         if name:
             self.name = name
@@ -36,9 +39,7 @@ def __init__(self, cwd, name=None, key_name="key.pem", cert_name="cert.pem", cre
     def gen_keypair(
         self,
     ):
-        """Generate keypair.
-
-        """
+        """Generate keypair."""
         key = crypto.PKey()
         key.generate_key(crypto.TYPE_RSA, 2048)
         cert = crypto.X509()
@@ -73,25 +74,22 @@ def set_keypair_raw(self, certificate, privatekey):
             certfile.write(crypto.dump_certificate(crypto.FILETYPE_PEM, certificate))
 
     def get_keypair_raw(self):
-        """:return:
-        """
+        """:return:"""
         with open(self.key_path, "rb") as keyfile:
             key_buf = keyfile.read()
         with open(self.cert_path, "rb") as certfile:
             cert_buf = certfile.read()
         return copy.deepcopy(cert_buf), copy.deepcopy(key_buf)
 
     def get_key(self):
-        """:return:
-        """
+        """:return:"""
         with open(self.key_path, "rb") as keyfile:
             key_buf = keyfile.read()
         key = crypto.load_privatekey(crypto.FILETYPE_PEM, key_buf)
         return key
 
     def get_cert(self):
-        """:return:
-        """
+        """:return:"""
         with open(self.cert_path, "rb") as certfile:
             cert_buf = certfile.read()
         cert = crypto.load_certificate(crypto.FILETYPE_PEM, cert_buf)

fedn/common/config.py

@@ -15,6 +15,8 @@
 FEDN_AUTH_REFRESH_TOKEN = os.environ.get("FEDN_AUTH_REFRESH_TOKEN", False)
 FEDN_CUSTOM_URL_PREFIX = os.environ.get("FEDN_CUSTOM_URL_PREFIX", "")
 
+
+FEDN_ALLOW_LOCAL_PACKAGE = os.environ.get("FEDN_ALLOW_LOCAL_PACKAGE", False)
 FEDN_PACKAGE_EXTRACT_DIR = os.environ.get("FEDN_PACKAGE_EXTRACT_DIR", "package")