From 4bbee52f16a61019b92d376ab817c5cc2afa12ef Mon Sep 17 00:00:00 2001 From: Niklas Date: Thu, 18 Apr 2024 14:55:26 +0200 Subject: [PATCH 01/12] added package_cmd --- fedn/cli/__init__.py | 1 + fedn/cli/package_cmd.py | 41 +++++++++++++++++++++++++++++++++++++++++ fedn/cli/run_cmd.py | 34 ---------------------------------- 3 files changed, 42 insertions(+), 34 deletions(-) create mode 100644 fedn/cli/package_cmd.py diff --git a/fedn/cli/__init__.py b/fedn/cli/__init__.py index 840d4252b..0ff210966 100644 --- a/fedn/cli/__init__.py +++ b/fedn/cli/__init__.py @@ -1,2 +1,3 @@ from .main import main # noqa: F401 from .run_cmd import run_cmd # noqa: F401 +from .package_cmd import package_cmd # noqa: F401 \ No newline at end of file diff --git a/fedn/cli/package_cmd.py b/fedn/cli/package_cmd.py new file mode 100644 index 000000000..c13ed0470 --- /dev/null +++ b/fedn/cli/package_cmd.py @@ -0,0 +1,41 @@ +import os +import tarfile + +import click + +from fedn.common.log_config import logger + +from .main import main + + +@main.group('package') +@click.pass_context +def package_cmd(ctx): + """ + + :param ctx: + """ + pass + + +@package_cmd.command('create') +@click.option('-p', '--path', required=True, help='Path to package directory containing fedn.yaml') +@click.option('-n', '--name', required=False, default='package.tgz', help='Name of package tarball') +@click.pass_context +def create_cmd(ctx, path, name): + """ Create compute package. + + Make a tar.gz archive of folder given by --path + + :param ctx: + :param path: + """ + path = os.path.abspath(path) + yaml_file = os.path.join(path, 'fedn.yaml') + if not os.path.exists(yaml_file): + logger.error(f"Could not find fedn.yaml in {path}") + exit(-1) + + with tarfile.open(name, "w:gz") as tar: + tar.add(path, arcname=os.path.basename(path)) + logger.info(f"Created package {name}") \ No newline at end of file diff --git a/fedn/cli/run_cmd.py b/fedn/cli/run_cmd.py index 4c007131f..44e87563c 100644 --- a/fedn/cli/run_cmd.py +++ b/fedn/cli/run_cmd.py @@ -1,6 +1,5 @@ import os import shutil -import tarfile import uuid import click @@ -215,36 +214,3 @@ def build_cmd(ctx, path): if dispatcher.python_env_path: logger.info(f"Removing virtualenv {dispatcher.python_env_path}") shutil.rmtree(dispatcher.python_env_path) - - -@main.group('package') -@click.pass_context -def package_cmd(ctx): - """ - - :param ctx: - """ - pass - - -@package_cmd.command('create') -@click.option('-p', '--path', required=True, help='Path to package directory containing fedn.yaml') -@click.option('-n', '--name', required=False, default='package.tgz', help='Name of package tarball') -@click.pass_context -def create_cmd(ctx, path, name): - """ Create compute package. - - Make a tar.gz archive of folder given by --path - - :param ctx: - :param path: - """ - path = os.path.abspath(path) - yaml_file = os.path.join(path, 'fedn.yaml') - if not os.path.exists(yaml_file): - logger.error(f"Could not find fedn.yaml in {path}") - exit(-1) - - with tarfile.open(name, "w:gz") as tar: - tar.add(path, arcname=os.path.basename(path)) - logger.info(f"Created package {name}") From 6aa59b7594747cf63a74ec0061f268d3e5cc9a09 Mon Sep 17 00:00:00 2001 From: Niklas Date: Wed, 24 Apr 2024 15:31:38 +0200 Subject: [PATCH 02/12] added fedn combiner start command --- fedn/cli/combiner_cmd.py | 56 ++++++++++++++++++++++++++++++++++++++++ fedn/cli/shared.py | 22 ++++++++++++++++ 2 files changed, 78 insertions(+) create mode 100644 fedn/cli/combiner_cmd.py create mode 100644 fedn/cli/shared.py diff --git a/fedn/cli/combiner_cmd.py b/fedn/cli/combiner_cmd.py new file mode 100644 index 000000000..54a3028c4 --- /dev/null +++ b/fedn/cli/combiner_cmd.py @@ -0,0 +1,56 @@ +import uuid + +import click + +from .shared import apply_config +from fedn.network.combiner.combiner import Combiner + +from .main import main + +@main.group('combiner') +@click.pass_context +def combiner_cmd(ctx): + """ + + :param ctx: + """ + pass + + +@combiner_cmd.command('start') +@click.option('-d', '--discoverhost', required=False, help='Hostname for discovery services (reducer).') +@click.option('-p', '--discoverport', required=False, help='Port for discovery services (reducer).') +@click.option('-t', '--token', required=False, help='Set token provided by reducer if enabled') +@click.option('-n', '--name', required=False, default="combiner" + str(uuid.uuid4())[:8], help='Set name for combiner.') +@click.option('-h', '--host', required=False, default="combiner", help='Set hostname.') +@click.option('-i', '--port', required=False, default=12080, help='Set port.') +@click.option('-f', '--fqdn', required=False, default=None, help='Set fully qualified domain name') +@click.option('-s', '--secure', is_flag=True, help='Enable SSL/TLS encrypted gRPC channels.') +@click.option('-v', '--verify', is_flag=True, help='Verify SSL/TLS for REST discovery service (reducer)') +@click.option('-c', '--max_clients', required=False, default=30, help='The maximal number of client connections allowed.') +@click.option('-in', '--init', required=False, default=None, + help='Path to configuration file to (re)init combiner.') +@click.pass_context +def start_cmd(ctx, discoverhost, discoverport, token, name, host, port, fqdn, secure, verify, max_clients, init): + """ + + :param ctx: + :param discoverhost: + :param discoverport: + :param token: + :param name: + :param hostname: + :param port: + :param secure: + :param max_clients: + :param init: + """ + config = {'discover_host': discoverhost, 'discover_port': discoverport, 'token': token, 'host': host, + 'port': port, 'fqdn': fqdn, 'name': name, 'secure': secure, 'verify': verify, 'max_clients': max_clients, + 'init': init} + + if config['init']: + apply_config(config) + + combiner = Combiner(config) + combiner.run() \ No newline at end of file diff --git a/fedn/cli/shared.py b/fedn/cli/shared.py new file mode 100644 index 000000000..754f8df6b --- /dev/null +++ b/fedn/cli/shared.py @@ -0,0 +1,22 @@ + +import yaml + +from fedn.common.log_config import logger + + +def apply_config(config): + """Parse client config from file. + + Override configs from the CLI with settings in config file. + + :param config: Client config (dict). + """ + with open(config['init'], 'r') as file: + try: + settings = dict(yaml.safe_load(file)) + except Exception: + logger.error('Failed to read config from settings file, exiting.') + return + + for key, val in settings.items(): + config[key] = val \ No newline at end of file From 3657d4e2bf121e40a338faa846d389ff5d56eafa Mon Sep 17 00:00:00 2001 From: Niklas Date: Wed, 24 Apr 2024 16:19:54 +0200 Subject: [PATCH 03/12] fedn run client => fedn client start --- fedn/cli/__init__.py | 3 +- fedn/cli/client_cmd.py | 98 +++++++++++++++++++++++++++ fedn/cli/combiner_cmd.py | 5 +- fedn/cli/run_cmd.py | 140 --------------------------------------- 4 files changed, 103 insertions(+), 143 deletions(-) create mode 100644 fedn/cli/client_cmd.py diff --git a/fedn/cli/__init__.py b/fedn/cli/__init__.py index 0ff210966..3235c51b2 100644 --- a/fedn/cli/__init__.py +++ b/fedn/cli/__init__.py @@ -1,3 +1,4 @@ +from .client_cmd import client_cmd # noqa: F401 from .main import main # noqa: F401 +from .package_cmd import package_cmd # noqa: F401 from .run_cmd import run_cmd # noqa: F401 -from .package_cmd import package_cmd # noqa: F401 \ No newline at end of file diff --git a/fedn/cli/client_cmd.py b/fedn/cli/client_cmd.py new file mode 100644 index 000000000..f63ccd2c3 --- /dev/null +++ b/fedn/cli/client_cmd.py @@ -0,0 +1,98 @@ +import uuid + +import click + +from fedn.common.exceptions import InvalidClientConfig +from fedn.network.clients.client import Client + +from .main import main +from .shared import apply_config + + +def validate_client_config(config): + """Validate client configuration. + + :param config: Client config (dict). + """ + + try: + if config['discover_host'] is None or \ + config['discover_host'] == '': + raise InvalidClientConfig("Missing required configuration: discover_host") + if 'discover_port' not in config.keys(): + config['discover_port'] = None + except Exception: + raise InvalidClientConfig("Could not load config from file. Check config") + + +@main.group('client') +@click.pass_context +def client_cmd(ctx): + """ + + :param ctx: + """ + pass + + +@client_cmd.command('start') +@click.option('-d', '--discoverhost', required=False, help='Hostname for discovery services(reducer).') +@click.option('-p', '--discoverport', required=False, help='Port for discovery services (reducer).') +@click.option('--token', required=False, help='Set token provided by reducer if enabled') +@click.option('-n', '--name', required=False, default="client" + str(uuid.uuid4())[:8]) +@click.option('-i', '--client_id', required=False) +@click.option('--local-package', is_flag=True, help='Enable local compute package') +@click.option('--force-ssl', is_flag=True, help='Force SSL/TLS for REST service') +@click.option('-u', '--dry-run', required=False, default=False) +@click.option('-s', '--secure', required=False, default=False) +@click.option('-pc', '--preshared-cert', required=False, default=False) +@click.option('-v', '--verify', is_flag=True, help='Verify SSL/TLS for REST service') +@click.option('-c', '--preferred-combiner', required=False, default=False) +@click.option('-va', '--validator', required=False, default=True) +@click.option('-tr', '--trainer', required=False, default=True) +@click.option('-in', '--init', required=False, default=None, + help='Set to a filename to (re)init client from file state.') +@click.option('-l', '--logfile', required=False, default=None, + help='Set logfile for client log to file.') +@click.option('--heartbeat-interval', required=False, default=2) +@click.option('--reconnect-after-missed-heartbeat', required=False, default=30) +@click.option('--verbosity', required=False, default='INFO', type=click.Choice(['CRITICAL', 'ERROR', 'WARNING', 'INFO', 'DEBUG'], case_sensitive=False)) +@click.pass_context +def client_cmd(ctx, discoverhost, discoverport, token, name, client_id, local_package, force_ssl, dry_run, secure, preshared_cert, + verify, preferred_combiner, validator, trainer, init, logfile, heartbeat_interval, reconnect_after_missed_heartbeat, + verbosity): + """ + + :param ctx: + :param discoverhost: + :param discoverport: + :param token: + :param name: + :param client_id: + :param remote: + :param dry_run: + :param secure: + :param preshared_cert: + :param verify_cert: + :param preferred_combiner: + :param init: + :param logfile: + :param hearbeat_interval + :param reconnect_after_missed_heartbeat + :param verbosity + :return: + """ + remote = False if local_package else True + config = {'discover_host': discoverhost, 'discover_port': discoverport, 'token': token, 'name': name, + 'client_id': client_id, 'remote_compute_context': remote, 'force_ssl': force_ssl, 'dry_run': dry_run, 'secure': secure, + 'preshared_cert': preshared_cert, 'verify': verify, 'preferred_combiner': preferred_combiner, + 'validator': validator, 'trainer': trainer, 'init': init, 'logfile': logfile, 'heartbeat_interval': heartbeat_interval, + 'reconnect_after_missed_heartbeat': reconnect_after_missed_heartbeat, 'verbosity': verbosity} + + if init: + apply_config(config) + + validate_client_config(config) + + client = Client(config) + client.run() diff --git a/fedn/cli/combiner_cmd.py b/fedn/cli/combiner_cmd.py index 54a3028c4..9ff451661 100644 --- a/fedn/cli/combiner_cmd.py +++ b/fedn/cli/combiner_cmd.py @@ -2,10 +2,11 @@ import click -from .shared import apply_config from fedn.network.combiner.combiner import Combiner from .main import main +from .shared import apply_config + @main.group('combiner') @click.pass_context @@ -53,4 +54,4 @@ def start_cmd(ctx, discoverhost, discoverport, token, name, host, port, fqdn, se apply_config(config) combiner = Combiner(config) - combiner.run() \ No newline at end of file + combiner.run() diff --git a/fedn/cli/run_cmd.py b/fedn/cli/run_cmd.py index 44e87563c..ce57403b4 100644 --- a/fedn/cli/run_cmd.py +++ b/fedn/cli/run_cmd.py @@ -1,14 +1,10 @@ import os import shutil -import uuid import click import yaml -from fedn.common.exceptions import InvalidClientConfig from fedn.common.log_config import logger -from fedn.network.clients.client import Client -from fedn.network.combiner.combiner import Combiner from fedn.utils.dispatcher import Dispatcher, _read_yaml_file from .main import main @@ -38,40 +34,6 @@ def check_helper_config_file(config): return helper -def apply_config(config): - """Parse client config from file. - - Override configs from the CLI with settings in config file. - - :param config: Client config (dict). - """ - with open(config['init'], 'r') as file: - try: - settings = dict(yaml.safe_load(file)) - except Exception: - logger.error('Failed to read config from settings file, exiting.') - return - - for key, val in settings.items(): - config[key] = val - - -def validate_client_config(config): - """Validate client configuration. - - :param config: Client config (dict). - """ - - try: - if config['discover_host'] is None or \ - config['discover_host'] == '': - raise InvalidClientConfig("Missing required configuration: discover_host") - if 'discover_port' not in config.keys(): - config['discover_port'] = None - except Exception: - raise InvalidClientConfig("Could not load config from file. Check config") - - @main.group('run') @click.pass_context def run_cmd(ctx): @@ -82,108 +44,6 @@ def run_cmd(ctx): pass -@run_cmd.command('client') -@click.option('-d', '--discoverhost', required=False, help='Hostname for discovery services(reducer).') -@click.option('-p', '--discoverport', required=False, help='Port for discovery services (reducer).') -@click.option('--token', required=False, help='Set token provided by reducer if enabled') -@click.option('-n', '--name', required=False, default="client" + str(uuid.uuid4())[:8]) -@click.option('-i', '--client_id', required=False) -@click.option('--local-package', is_flag=True, help='Enable local compute package') -@click.option('--force-ssl', is_flag=True, help='Force SSL/TLS for REST service') -@click.option('-u', '--dry-run', required=False, default=False) -@click.option('-s', '--secure', required=False, default=False) -@click.option('-pc', '--preshared-cert', required=False, default=False) -@click.option('-v', '--verify', is_flag=True, help='Verify SSL/TLS for REST service') -@click.option('-c', '--preferred-combiner', required=False, default=False) -@click.option('-va', '--validator', required=False, default=True) -@click.option('-tr', '--trainer', required=False, default=True) -@click.option('-in', '--init', required=False, default=None, - help='Set to a filename to (re)init client from file state.') -@click.option('-l', '--logfile', required=False, default=None, - help='Set logfile for client log to file.') -@click.option('--heartbeat-interval', required=False, default=2) -@click.option('--reconnect-after-missed-heartbeat', required=False, default=30) -@click.option('--verbosity', required=False, default='INFO', type=click.Choice(['CRITICAL', 'ERROR', 'WARNING', 'INFO', 'DEBUG'], case_sensitive=False)) -@click.pass_context -def client_cmd(ctx, discoverhost, discoverport, token, name, client_id, local_package, force_ssl, dry_run, secure, preshared_cert, - verify, preferred_combiner, validator, trainer, init, logfile, heartbeat_interval, reconnect_after_missed_heartbeat, - verbosity): - """ - - :param ctx: - :param discoverhost: - :param discoverport: - :param token: - :param name: - :param client_id: - :param remote: - :param dry_run: - :param secure: - :param preshared_cert: - :param verify_cert: - :param preferred_combiner: - :param init: - :param logfile: - :param hearbeat_interval - :param reconnect_after_missed_heartbeat - :param verbosity - :return: - """ - remote = False if local_package else True - config = {'discover_host': discoverhost, 'discover_port': discoverport, 'token': token, 'name': name, - 'client_id': client_id, 'remote_compute_context': remote, 'force_ssl': force_ssl, 'dry_run': dry_run, 'secure': secure, - 'preshared_cert': preshared_cert, 'verify': verify, 'preferred_combiner': preferred_combiner, - 'validator': validator, 'trainer': trainer, 'init': init, 'logfile': logfile, 'heartbeat_interval': heartbeat_interval, - 'reconnect_after_missed_heartbeat': reconnect_after_missed_heartbeat, 'verbosity': verbosity} - - if init: - apply_config(config) - - validate_client_config(config) - - client = Client(config) - client.run() - - -@run_cmd.command('combiner') -@click.option('-d', '--discoverhost', required=False, help='Hostname for discovery services (reducer).') -@click.option('-p', '--discoverport', required=False, help='Port for discovery services (reducer).') -@click.option('-t', '--token', required=False, help='Set token provided by reducer if enabled') -@click.option('-n', '--name', required=False, default="combiner" + str(uuid.uuid4())[:8], help='Set name for combiner.') -@click.option('-h', '--host', required=False, default="combiner", help='Set hostname.') -@click.option('-i', '--port', required=False, default=12080, help='Set port.') -@click.option('-f', '--fqdn', required=False, default=None, help='Set fully qualified domain name') -@click.option('-s', '--secure', is_flag=True, help='Enable SSL/TLS encrypted gRPC channels.') -@click.option('-v', '--verify', is_flag=True, help='Verify SSL/TLS for REST discovery service (reducer)') -@click.option('-c', '--max_clients', required=False, default=30, help='The maximal number of client connections allowed.') -@click.option('-in', '--init', required=False, default=None, - help='Path to configuration file to (re)init combiner.') -@click.pass_context -def combiner_cmd(ctx, discoverhost, discoverport, token, name, host, port, fqdn, secure, verify, max_clients, init): - """ - - :param ctx: - :param discoverhost: - :param discoverport: - :param token: - :param name: - :param hostname: - :param port: - :param secure: - :param max_clients: - :param init: - """ - config = {'discover_host': discoverhost, 'discover_port': discoverport, 'token': token, 'host': host, - 'port': port, 'fqdn': fqdn, 'name': name, 'secure': secure, 'verify': verify, 'max_clients': max_clients, - 'init': init} - - if config['init']: - apply_config(config) - - combiner = Combiner(config) - combiner.run() - - @run_cmd.command('build') @click.option('-p', '--path', required=True, help='Path to package directory containing fedn.yaml') @click.pass_context From 9eb5c9fa99c2b4104a662a7effc78f1830aba3c9 Mon Sep 17 00:00:00 2001 From: Niklas Date: Thu, 25 Apr 2024 09:58:29 +0200 Subject: [PATCH 04/12] fedn run client => fedn client start (in docs & docker) --- docker-compose.yaml | 2 +- docs/faq.rst | 4 ++-- docs/quickstart.rst | 2 +- examples/flower-client/README.rst | 2 +- examples/mnist-pytorch/README.rst | 2 +- 5 files changed, 6 insertions(+), 6 deletions(-) diff --git a/docker-compose.yaml b/docker-compose.yaml index ca6045301..1182f9a66 100644 --- a/docker-compose.yaml +++ b/docker-compose.yaml @@ -127,7 +127,7 @@ services: - ${HOST_REPO_DIR:-.}/fedn:/app/fedn entrypoint: [ "sh", "-c" ] command: - - "/venv/bin/pip install --no-cache-dir -e /app/fedn && /venv/bin/fedn run client --init config/settings-client.yaml" + - "/venv/bin/pip install --no-cache-dir -e /app/fedn && /venv/bin/fedn client start --init config/settings-client.yaml" deploy: replicas: 0 depends_on: diff --git a/docs/faq.rst b/docs/faq.rst index e7a0ed28b..8db6e375b 100644 --- a/docs/faq.rst +++ b/docs/faq.rst @@ -26,7 +26,7 @@ Yes, to facilitate interactive development of the compute package you can start .. code-block:: bash - fedn run client --remote=False -in client.yaml + fedn client start --remote=False -in client.yaml Note that in production federations this options should in most cases be disallowed. @@ -56,7 +56,7 @@ Yes! You can toggle which message streams a client subscibes to when starting th .. code-block:: bash - fedn run client --trainer=False -in client.yaml + fedn client start --trainer=False -in client.yaml Q: How do you approach the question of output privacy? diff --git a/docs/quickstart.rst b/docs/quickstart.rst index f6e17c8fd..da2186c8f 100644 --- a/docs/quickstart.rst +++ b/docs/quickstart.rst @@ -141,7 +141,7 @@ This will build a container image for the client, start two clients and connect .. code-block:: export FEDN_AUTH_SCHEME=Bearer - fedn run client -in client.yaml --secure=True --force-ssl + fedn client start -in client.yaml --secure=True --force-ssl Start a training session ------------------------ diff --git a/examples/flower-client/README.rst b/examples/flower-client/README.rst index 2093caf3f..79de30b62 100644 --- a/examples/flower-client/README.rst +++ b/examples/flower-client/README.rst @@ -64,7 +64,7 @@ On your local machine / client, start the FEDn client: export FEDN_AUTH_SCHEME=Bearer export FEDN_PACKAGE_EXTRACT_DIR=package - CLIENT_NUMBER=0 fedn run client -in client.yaml --secure=True --force-ssl + CLIENT_NUMBER=0 fedn client start -in client.yaml --secure=True --force-ssl Or, if you prefer to use Docker (this might take a long time): diff --git a/examples/mnist-pytorch/README.rst b/examples/mnist-pytorch/README.rst index eaa09c435..212b12e8b 100644 --- a/examples/mnist-pytorch/README.rst +++ b/examples/mnist-pytorch/README.rst @@ -58,7 +58,7 @@ Then, start the client using the client.yaml file: export FEDN_AUTH_SCHEME=Bearer export FEDN_PACKAGE_EXTRACT_DIR=package - fedn run client -in client.yaml --secure=True --force-ssl + fedn client start -in client.yaml --secure=True --force-ssl The default traning and test data is for this example downloaded and split direcly by the client when it starts up. The data will be found in package/data/clients/1/mnist.pt and can be changed to other partitions by exporting the environment variable FEDN_DATA_PATH. From 149c88878dbaef38af436381e6a240d46ac56f74 Mon Sep 17 00:00:00 2001 From: Niklas Date: Thu, 25 Apr 2024 13:34:36 +0200 Subject: [PATCH 05/12] fedn package list - added --- docker-compose.yaml | 2 +- fedn/cli/__init__.py | 1 + fedn/cli/package_cmd.py | 38 ++++++++++++++++++- fedn/cli/shared.py | 84 ++++++++++++++++++++++++++++++++++++++++- 4 files changed, 122 insertions(+), 3 deletions(-) diff --git a/docker-compose.yaml b/docker-compose.yaml index 1182f9a66..70e8f17ab 100644 --- a/docker-compose.yaml +++ b/docker-compose.yaml @@ -97,7 +97,7 @@ services: - ${HOST_REPO_DIR:-.}/fedn:/app/fedn entrypoint: [ "sh", "-c" ] command: - - "/venv/bin/pip install --no-cache-dir -e /app/fedn && /venv/bin/fedn run combiner --init config/settings-combiner.yaml" + - "/venv/bin/pip install --no-cache-dir -e /app/fedn && /venv/bin/fedn combiner start --init config/settings-combiner.yaml" ports: - 12080:12080 healthcheck: diff --git a/fedn/cli/__init__.py b/fedn/cli/__init__.py index 3235c51b2..0c583336e 100644 --- a/fedn/cli/__init__.py +++ b/fedn/cli/__init__.py @@ -1,4 +1,5 @@ from .client_cmd import client_cmd # noqa: F401 +from .combiner_cmd import combiner_cmd # noqa: F401 from .main import main # noqa: F401 from .package_cmd import package_cmd # noqa: F401 from .run_cmd import run_cmd # noqa: F401 diff --git a/fedn/cli/package_cmd.py b/fedn/cli/package_cmd.py index c13ed0470..4026cb2ef 100644 --- a/fedn/cli/package_cmd.py +++ b/fedn/cli/package_cmd.py @@ -2,10 +2,12 @@ import tarfile import click +import requests from fedn.common.log_config import logger from .main import main +from .shared import CONTROLLER_DEFAULTS, get_api_url, get_token, print_response @main.group('package') @@ -38,4 +40,38 @@ def create_cmd(ctx, path, name): with tarfile.open(name, "w:gz") as tar: tar.add(path, arcname=os.path.basename(path)) - logger.info(f"Created package {name}") \ No newline at end of file + logger.info(f"Created package {name}") + + +@click.option('-p', '--protocol', required=False, default=CONTROLLER_DEFAULTS['protocol'], help='Communication protocol of controller (api)') +@click.option('-H', '--host', required=False, default=CONTROLLER_DEFAULTS['host'], help='Hostname of controller (api)') +@click.option('-P', '--port', required=False, default=CONTROLLER_DEFAULTS['port'], help='Port of controller (api)') +@click.option('-t', '--token', required=False, help='Authentication token') +@click.option('--n_max', required=False, help='Number of items to list') +@package_cmd.command('list') +@click.pass_context +def list_packages(ctx, protocol: str, host: str, port: str, token: str = None, n_max: int = None): + """ + return: + - count: number of packages + - result: list of packages + """ + url = get_api_url(protocol=protocol, host=host, port=port, endpoint='packages') + headers = {} + + if n_max: + headers['X-Limit'] = n_max + + _token = get_token(token) + + if _token: + headers['Authorization'] = _token + + click.echo(f'\nListing packages: {url}\n') + click.echo(f'Headers: {headers}') + + try: + response = requests.get(url, headers=headers) + print_response(response, 'packages') + except requests.exceptions.ConnectionError: + click.echo(f'Error: Could not connect to {url}') diff --git a/fedn/cli/shared.py b/fedn/cli/shared.py index 754f8df6b..109768a15 100644 --- a/fedn/cli/shared.py +++ b/fedn/cli/shared.py @@ -1,8 +1,33 @@ +import os +import click import yaml from fedn.common.log_config import logger +CONTROLLER_DEFAULTS = { + 'protocol': 'http', + 'host': 'localhost', + 'port': 8092, + 'debug': False +} + +COMBINER_DEFAULTS = { + 'discover_host': 'localhost', + 'discover_port': 8092, + 'host': 'localhost', + 'port': 12080, + "name": "combiner", + "max_clients": 30 +} + +CLIENT_DEFAULTS = { + 'discover_host': 'localhost', + 'discover_port': 8092, +} + +API_VERSION = 'v1' + def apply_config(config): """Parse client config from file. @@ -19,4 +44,61 @@ def apply_config(config): return for key, val in settings.items(): - config[key] = val \ No newline at end of file + config[key] = val + + +def get_api_url(protocol: str, host: str, port: str, endpoint: str) -> str: + _url = os.environ.get('FEDN_CONTROLLER_URL') + + if _url: + return f'{_url}/api/{API_VERSION}/{endpoint}' + + _protocol = protocol or os.environ.get('FEDN_PROTOCOL') or CONTROLLER_DEFAULTS['protocol'] + _host = host or os.environ.get('FEDN_HOST') or CONTROLLER_DEFAULTS['host'] + _port = port or os.environ.get('FEDN_PORT') or CONTROLLER_DEFAULTS['port'] + + return f'{_protocol}://{_host}:{_port}/api/{API_VERSION}/{endpoint}' + + +def get_token(token: str) -> str: + _token = token or os.environ.get("FEDN_TOKEN", None) + + if _token is None: + return None + + scheme = os.environ.get("FEDN_AUTH_SCHEME", "Bearer") + + return f"{scheme} {_token}" + + +def get_client_package_dir(path: str) -> str: + return path or os.environ.get('FEDN_PACKAGE_DIR', None) + + +# Print response from api (list of entities) +def print_response(response, entity_name: str): + """ + Prints the api response to the cli. + :param response: + type: array + description: list of entities + :param entity_name: + type: string + description: name of entity + return: None + """ + if response.status_code == 200: + json_data = response.json() + count, result = json_data.values() + click.echo(f'Found {count} {entity_name}') + click.echo('\n---------------------------------\n') + for obj in result: + click.echo('{') + for k, v in obj.items(): + click.echo(f'\t{k}: {v}') + click.echo('}') + elif response.status_code == 500: + json_data = response.json() + click.echo(f'Error: {json_data["message"]}') + else: + click.echo(f'Error: {response.status_code}') From 8c65d72a889f844c7191dd60ca6f99dc94e32623 Mon Sep 17 00:00:00 2001 From: Niklas Date: Thu, 25 Apr 2024 13:53:00 +0200 Subject: [PATCH 06/12] fedn model list - added --- fedn/cli/__init__.py | 1 + fedn/cli/model_cmd.py | 50 +++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 51 insertions(+) create mode 100644 fedn/cli/model_cmd.py diff --git a/fedn/cli/__init__.py b/fedn/cli/__init__.py index 0c583336e..a41538941 100644 --- a/fedn/cli/__init__.py +++ b/fedn/cli/__init__.py @@ -1,5 +1,6 @@ from .client_cmd import client_cmd # noqa: F401 from .combiner_cmd import combiner_cmd # noqa: F401 from .main import main # noqa: F401 +from .model_cmd import model_cmd # noqa: F401 from .package_cmd import package_cmd # noqa: F401 from .run_cmd import run_cmd # noqa: F401 diff --git a/fedn/cli/model_cmd.py b/fedn/cli/model_cmd.py new file mode 100644 index 000000000..2af526305 --- /dev/null +++ b/fedn/cli/model_cmd.py @@ -0,0 +1,50 @@ + +import click +import requests + +from .main import main +from .shared import CONTROLLER_DEFAULTS, get_api_url, get_token, print_response + + +@main.group('model') +@click.pass_context +def model_cmd(ctx): + """ + + :param ctx: + """ + pass + + +@click.option('-p', '--protocol', required=False, default=CONTROLLER_DEFAULTS['protocol'], help='Communication protocol of controller (api)') +@click.option('-H', '--host', required=False, default=CONTROLLER_DEFAULTS['host'], help='Hostname of controller (api)') +@click.option('-P', '--port', required=False, default=CONTROLLER_DEFAULTS['port'], help='Port of controller (api)') +@click.option('-t', '--token', required=False, help='Authentication token') +@click.option('--n_max', required=False, help='Number of items to list') +@model_cmd.command('list') +@click.pass_context +def list_models(ctx, protocol: str, host: str, port: str, token: str = None, n_max: int = None): + """ + return: + - count: number of models + - result: list of models + """ + url = get_api_url(protocol=protocol, host=host, port=port, endpoint='models') + headers = {} + + if n_max: + headers['X-Limit'] = n_max + + _token = get_token(token) + + if _token: + headers['Authorization'] = _token + + click.echo(f'\nListing models: {url}\n') + click.echo(f'Headers: {headers}') + + try: + response = requests.get(url, headers=headers) + print_response(response, 'models') + except requests.exceptions.ConnectionError: + click.echo(f'Error: Could not connect to {url}') From 1d6f73059f69dccee3878b2ef3322b8c7c02d012 Mon Sep 17 00:00:00 2001 From: Niklas Date: Thu, 25 Apr 2024 15:08:22 +0200 Subject: [PATCH 07/12] fedn client/combiner/round/session/status/validation list -added --- fedn/cli/__init__.py | 4 +++ fedn/cli/client_cmd.py | 38 ++++++++++++++++++++++++++++- fedn/cli/combiner_cmd.py | 38 ++++++++++++++++++++++++++++- fedn/cli/round_cmd.py | 50 ++++++++++++++++++++++++++++++++++++++ fedn/cli/session_cmd.py | 50 ++++++++++++++++++++++++++++++++++++++ fedn/cli/status_cmd.py | 49 +++++++++++++++++++++++++++++++++++++ fedn/cli/validation_cmd.py | 49 +++++++++++++++++++++++++++++++++++++ 7 files changed, 276 insertions(+), 2 deletions(-) create mode 100644 fedn/cli/round_cmd.py create mode 100644 fedn/cli/session_cmd.py create mode 100644 fedn/cli/status_cmd.py create mode 100644 fedn/cli/validation_cmd.py diff --git a/fedn/cli/__init__.py b/fedn/cli/__init__.py index a41538941..88b38ccf0 100644 --- a/fedn/cli/__init__.py +++ b/fedn/cli/__init__.py @@ -3,4 +3,8 @@ from .main import main # noqa: F401 from .model_cmd import model_cmd # noqa: F401 from .package_cmd import package_cmd # noqa: F401 +from .round_cmd import round_cmd # noqa: F401 from .run_cmd import run_cmd # noqa: F401 +from .session_cmd import session_cmd # noqa: F401 +from .status_cmd import status_cmd # noqa: F401 +from .validation_cmd import validation_cmd # noqa: F401 diff --git a/fedn/cli/client_cmd.py b/fedn/cli/client_cmd.py index f63ccd2c3..47dbc40e1 100644 --- a/fedn/cli/client_cmd.py +++ b/fedn/cli/client_cmd.py @@ -1,12 +1,14 @@ import uuid import click +import requests from fedn.common.exceptions import InvalidClientConfig from fedn.network.clients.client import Client from .main import main -from .shared import apply_config +from .shared import (CONTROLLER_DEFAULTS, apply_config, get_api_url, get_token, + print_response) def validate_client_config(config): @@ -35,6 +37,40 @@ def client_cmd(ctx): pass +@click.option('-p', '--protocol', required=False, default=CONTROLLER_DEFAULTS['protocol'], help='Communication protocol of controller (api)') +@click.option('-H', '--host', required=False, default=CONTROLLER_DEFAULTS['host'], help='Hostname of controller (api)') +@click.option('-P', '--port', required=False, default=CONTROLLER_DEFAULTS['port'], help='Port of controller (api)') +@click.option('-t', '--token', required=False, help='Authentication token') +@click.option('--n_max', required=False, help='Number of items to list') +@client_cmd.command('list') +@click.pass_context +def list_clients(ctx, protocol: str, host: str, port: str, token: str = None, n_max: int = None): + """ + return: + - count: number of clients + - result: list of clients + """ + url = get_api_url(protocol=protocol, host=host, port=port, endpoint='clients') + headers = {} + + if n_max: + headers['X-Limit'] = n_max + + _token = get_token(token) + + if _token: + headers['Authorization'] = _token + + click.echo(f'\nListing clients: {url}\n') + click.echo(f'Headers: {headers}') + + try: + response = requests.get(url, headers=headers) + print_response(response, 'clients') + except requests.exceptions.ConnectionError: + click.echo(f'Error: Could not connect to {url}') + + @client_cmd.command('start') @click.option('-d', '--discoverhost', required=False, help='Hostname for discovery services(reducer).') @click.option('-p', '--discoverport', required=False, help='Port for discovery services (reducer).') diff --git a/fedn/cli/combiner_cmd.py b/fedn/cli/combiner_cmd.py index 9ff451661..cca37c693 100644 --- a/fedn/cli/combiner_cmd.py +++ b/fedn/cli/combiner_cmd.py @@ -1,11 +1,13 @@ import uuid import click +import requests from fedn.network.combiner.combiner import Combiner from .main import main -from .shared import apply_config +from .shared import (CONTROLLER_DEFAULTS, apply_config, get_api_url, get_token, + print_response) @main.group('combiner') @@ -55,3 +57,37 @@ def start_cmd(ctx, discoverhost, discoverport, token, name, host, port, fqdn, se combiner = Combiner(config) combiner.run() + + +@click.option('-p', '--protocol', required=False, default=CONTROLLER_DEFAULTS['protocol'], help='Communication protocol of controller (api)') +@click.option('-H', '--host', required=False, default=CONTROLLER_DEFAULTS['host'], help='Hostname of controller (api)') +@click.option('-P', '--port', required=False, default=CONTROLLER_DEFAULTS['port'], help='Port of controller (api)') +@click.option('-t', '--token', required=False, help='Authentication token') +@click.option('--n_max', required=False, help='Number of items to list') +@combiner_cmd.command('list') +@click.pass_context +def list_combiners(ctx, protocol: str, host: str, port: str, token: str = None, n_max: int = None): + """ + return: + - count: number of combiners + - result: list of combiners + """ + url = get_api_url(protocol=protocol, host=host, port=port, endpoint='combiners') + headers = {} + + if n_max: + headers['X-Limit'] = n_max + + _token = get_token(token) + + if _token: + headers['Authorization'] = _token + + click.echo(f'\nListing combiners: {url}\n') + click.echo(f'Headers: {headers}') + + try: + response = requests.get(url, headers=headers) + print_response(response, 'combiners') + except requests.exceptions.ConnectionError: + click.echo(f'Error: Could not connect to {url}') \ No newline at end of file diff --git a/fedn/cli/round_cmd.py b/fedn/cli/round_cmd.py new file mode 100644 index 000000000..028675be7 --- /dev/null +++ b/fedn/cli/round_cmd.py @@ -0,0 +1,50 @@ + +import click +import requests + +from .main import main +from .shared import CONTROLLER_DEFAULTS, get_api_url, get_token, print_response + + +@main.group('round') +@click.pass_context +def round_cmd(ctx): + """ + + :param ctx: + """ + pass + + +@click.option('-p', '--protocol', required=False, default=CONTROLLER_DEFAULTS['protocol'], help='Communication protocol of controller (api)') +@click.option('-H', '--host', required=False, default=CONTROLLER_DEFAULTS['host'], help='Hostname of controller (api)') +@click.option('-P', '--port', required=False, default=CONTROLLER_DEFAULTS['port'], help='Port of controller (api)') +@click.option('-t', '--token', required=False, help='Authentication token') +@click.option('--n_max', required=False, help='Number of items to list') +@round_cmd.command('list') +@click.pass_context +def list_rounds(ctx, protocol: str, host: str, port: str, token: str = None, n_max: int = None): + """ + return: + - count: number of rounds + - result: list of rounds + """ + url = get_api_url(protocol=protocol, host=host, port=port, endpoint='rounds') + headers = {} + + if n_max: + headers['X-Limit'] = n_max + + _token = get_token(token) + + if _token: + headers['Authorization'] = _token + + click.echo(f'\nListing rounds: {url}\n') + click.echo(f'Headers: {headers}') + + try: + response = requests.get(url, headers=headers) + print_response(response, 'rounds') + except requests.exceptions.ConnectionError: + click.echo(f'Error: Could not connect to {url}') diff --git a/fedn/cli/session_cmd.py b/fedn/cli/session_cmd.py new file mode 100644 index 000000000..b18cb1925 --- /dev/null +++ b/fedn/cli/session_cmd.py @@ -0,0 +1,50 @@ + +import click +import requests + +from .main import main +from .shared import CONTROLLER_DEFAULTS, get_api_url, get_token, print_response + + +@main.group('session') +@click.pass_context +def session_cmd(ctx): + """ + + :param ctx: + """ + pass + + +@click.option('-p', '--protocol', required=False, default=CONTROLLER_DEFAULTS['protocol'], help='Communication protocol of controller (api)') +@click.option('-H', '--host', required=False, default=CONTROLLER_DEFAULTS['host'], help='Hostname of controller (api)') +@click.option('-P', '--port', required=False, default=CONTROLLER_DEFAULTS['port'], help='Port of controller (api)') +@click.option('-t', '--token', required=False, help='Authentication token') +@click.option('--n_max', required=False, help='Number of items to list') +@session_cmd.command('list') +@click.pass_context +def list_sessions(ctx, protocol: str, host: str, port: str, token: str = None, n_max: int = None): + """ + return: + - count: number of sessions + - result: list of sessions + """ + url = get_api_url(protocol=protocol, host=host, port=port, endpoint='sessions') + headers = {} + + if n_max: + headers['X-Limit'] = n_max + + _token = get_token(token) + + if _token: + headers['Authorization'] = _token + + click.echo(f'\nListing sessions: {url}\n') + click.echo(f'Headers: {headers}') + + try: + response = requests.get(url, headers=headers) + print_response(response, 'sessions') + except requests.exceptions.ConnectionError: + click.echo(f'Error: Could not connect to {url}') \ No newline at end of file diff --git a/fedn/cli/status_cmd.py b/fedn/cli/status_cmd.py new file mode 100644 index 000000000..9a658a9a7 --- /dev/null +++ b/fedn/cli/status_cmd.py @@ -0,0 +1,49 @@ +import click +import requests + +from .main import main +from .shared import CONTROLLER_DEFAULTS, get_api_url, get_token, print_response + + +@main.group('status') +@click.pass_context +def status_cmd(ctx): + """ + + :param ctx: + """ + pass + + +@click.option('-p', '--protocol', required=False, default=CONTROLLER_DEFAULTS['protocol'], help='Communication protocol of controller (api)') +@click.option('-H', '--host', required=False, default=CONTROLLER_DEFAULTS['host'], help='Hostname of controller (api)') +@click.option('-P', '--port', required=False, default=CONTROLLER_DEFAULTS['port'], help='Port of controller (api)') +@click.option('-t', '--token', required=False, help='Authentication token') +@click.option('--n_max', required=False, help='Number of items to list') +@status_cmd.command('list') +@click.pass_context +def list_statuses(ctx, protocol: str, host: str, port: str, token: str = None, n_max: int = None): + """ + return: + - count: number of statuses + - result: list of statuses + """ + url = get_api_url(protocol=protocol, host=host, port=port, endpoint='statuses') + headers = {} + + if n_max: + headers['X-Limit'] = n_max + + _token = get_token(token) + + if _token: + headers['Authorization'] = _token + + click.echo(f'\nListing statuses: {url}\n') + click.echo(f'Headers: {headers}') + + try: + response = requests.get(url, headers=headers) + print_response(response, 'statuses') + except requests.exceptions.ConnectionError: + click.echo(f'Error: Could not connect to {url}') \ No newline at end of file diff --git a/fedn/cli/validation_cmd.py b/fedn/cli/validation_cmd.py new file mode 100644 index 000000000..2e6cb41d7 --- /dev/null +++ b/fedn/cli/validation_cmd.py @@ -0,0 +1,49 @@ +import click +import requests + +from .main import main +from .shared import CONTROLLER_DEFAULTS, get_api_url, get_token, print_response + + +@main.group('validation') +@click.pass_context +def validation_cmd(ctx): + """ + + :param ctx: + """ + pass + + +@click.option('-p', '--protocol', required=False, default=CONTROLLER_DEFAULTS['protocol'], help='Communication protocol of controller (api)') +@click.option('-H', '--host', required=False, default=CONTROLLER_DEFAULTS['host'], help='Hostname of controller (api)') +@click.option('-P', '--port', required=False, default=CONTROLLER_DEFAULTS['port'], help='Port of controller (api)') +@click.option('-t', '--token', required=False, help='Authentication token') +@click.option('--n_max', required=False, help='Number of items to list') +@validation_cmd.command('list') +@click.pass_context +def list_validations(ctx, protocol: str, host: str, port: str, token: str = None, n_max: int = None): + """ + return: + - count: number of validations + - result: list of validations + """ + url = get_api_url(protocol=protocol, host=host, port=port, endpoint='validations') + headers = {} + + if n_max: + headers['X-Limit'] = n_max + + _token = get_token(token) + + if _token: + headers['Authorization'] = _token + + click.echo(f'\nListing validations: {url}\n') + click.echo(f'Headers: {headers}') + + try: + response = requests.get(url, headers=headers) + print_response(response, 'validations') + except requests.exceptions.ConnectionError: + click.echo(f'Error: Could not connect to {url}') \ No newline at end of file From bf2eaf1f35c15a643b59a2476a67c3856fb3846f Mon Sep 17 00:00:00 2001 From: Niklas Date: Fri, 26 Apr 2024 11:17:18 +0200 Subject: [PATCH 08/12] fedn config - to see env:s set --- fedn/cli/__init__.py | 1 + fedn/cli/config_cmd.py | 54 ++++++++++++++++++++++++++++++++++++++++++ fedn/cli/shared.py | 8 +++---- 3 files changed, 59 insertions(+), 4 deletions(-) create mode 100644 fedn/cli/config_cmd.py diff --git a/fedn/cli/__init__.py b/fedn/cli/__init__.py index 88b38ccf0..137fc9b9c 100644 --- a/fedn/cli/__init__.py +++ b/fedn/cli/__init__.py @@ -1,5 +1,6 @@ from .client_cmd import client_cmd # noqa: F401 from .combiner_cmd import combiner_cmd # noqa: F401 +from .config_cmd import config_cmd # noqa: F401 from .main import main # noqa: F401 from .model_cmd import model_cmd # noqa: F401 from .package_cmd import package_cmd # noqa: F401 diff --git a/fedn/cli/config_cmd.py b/fedn/cli/config_cmd.py new file mode 100644 index 000000000..856882e62 --- /dev/null +++ b/fedn/cli/config_cmd.py @@ -0,0 +1,54 @@ +import os + +import click + +from .main import main + +envs = [ + { + "name": "FEDN_CONTROLLER_PROTOCOL", + "description": "The protocol to use for communication with the controller." + }, + { + "name": "FEDN_CONTROLLER_HOST", + "description": "The host to use for communication with the controller." + }, + { + "name": "FEDN_CONTROLLER_PORT", + "description": "The port to use for communication with the controller." + }, + { + "name": "FEDN_AUTH_TOKEN", + "description": "The authentication token to use for communication with the controller and combiner." + }, + { + "name": "FEDN_AUTH_SCHEME", + "description": "The authentication scheme to use for communication with the controller and combiner." + }, + { + "name": "FEDN_CONTROLLER_URL", + "description": "The URL of the controller. Overrides FEDN_CONTROLLER_PROTOCOL, FEDN_CONTROLLER_HOST and FEDN_CONTROLLER_PORT." + }, + { + "name": "FEDN_PACKAGE_EXTRACT_DIR", + "description": "The directory to extract packages to." + } +] + + +@main.group('config', invoke_without_command=True) +@click.pass_context +def config_cmd(ctx): + """ + - Configuration commands for the FEDn CLI. + """ + if ctx.invoked_subcommand is None: + click.echo('\n--- FEDn Cli Configuration ---\n') + click.echo('Current configuration:\n') + + for env in envs: + name = env['name'] + value = os.environ.get(name) + click.echo(f'{name}: {value or "Not set"}') + click.echo(f'{env["description"]}\n') + click.echo('\n') diff --git a/fedn/cli/shared.py b/fedn/cli/shared.py index 109768a15..7171f8ec5 100644 --- a/fedn/cli/shared.py +++ b/fedn/cli/shared.py @@ -53,15 +53,15 @@ def get_api_url(protocol: str, host: str, port: str, endpoint: str) -> str: if _url: return f'{_url}/api/{API_VERSION}/{endpoint}' - _protocol = protocol or os.environ.get('FEDN_PROTOCOL') or CONTROLLER_DEFAULTS['protocol'] - _host = host or os.environ.get('FEDN_HOST') or CONTROLLER_DEFAULTS['host'] - _port = port or os.environ.get('FEDN_PORT') or CONTROLLER_DEFAULTS['port'] + _protocol = protocol or os.environ.get('FEDN_CONTROLLER_PROTOCOL') or CONTROLLER_DEFAULTS['protocol'] + _host = host or os.environ.get('FEDN_CONTROLLER_HOST') or CONTROLLER_DEFAULTS['host'] + _port = port or os.environ.get('FEDN_CONTROLLER_PORT') or CONTROLLER_DEFAULTS['port'] return f'{_protocol}://{_host}:{_port}/api/{API_VERSION}/{endpoint}' def get_token(token: str) -> str: - _token = token or os.environ.get("FEDN_TOKEN", None) + _token = token or os.environ.get("FEDN_AUTH_TOKEN", None) if _token is None: return None From 1507981274b2a3c71e19b343f09fee2a1189baec Mon Sep 17 00:00:00 2001 From: Niklas Date: Fri, 26 Apr 2024 15:36:25 +0200 Subject: [PATCH 09/12] inform user input file has priority over input params --- fedn/cli/client_cmd.py | 12 +++++++++--- fedn/cli/combiner_cmd.py | 9 +++++---- fedn/cli/shared.py | 8 ++++---- 3 files changed, 18 insertions(+), 11 deletions(-) diff --git a/fedn/cli/client_cmd.py b/fedn/cli/client_cmd.py index 47dbc40e1..76ccd2b3c 100644 --- a/fedn/cli/client_cmd.py +++ b/fedn/cli/client_cmd.py @@ -122,13 +122,19 @@ def client_cmd(ctx, discoverhost, discoverport, token, name, client_id, local_pa config = {'discover_host': discoverhost, 'discover_port': discoverport, 'token': token, 'name': name, 'client_id': client_id, 'remote_compute_context': remote, 'force_ssl': force_ssl, 'dry_run': dry_run, 'secure': secure, 'preshared_cert': preshared_cert, 'verify': verify, 'preferred_combiner': preferred_combiner, - 'validator': validator, 'trainer': trainer, 'init': init, 'logfile': logfile, 'heartbeat_interval': heartbeat_interval, + 'validator': validator, 'trainer': trainer, 'logfile': logfile, 'heartbeat_interval': heartbeat_interval, 'reconnect_after_missed_heartbeat': reconnect_after_missed_heartbeat, 'verbosity': verbosity} if init: - apply_config(config) + apply_config(init, config) + click.echo(f'\nClient configuration loaded from file: {init}') + click.echo('Values set in file override defaults and command line arguments...\n') - validate_client_config(config) + try: + validate_client_config(config) + except InvalidClientConfig as e: + click.echo(f'Error: {e}') + return client = Client(config) client.run() diff --git a/fedn/cli/combiner_cmd.py b/fedn/cli/combiner_cmd.py index cca37c693..f74be38ba 100644 --- a/fedn/cli/combiner_cmd.py +++ b/fedn/cli/combiner_cmd.py @@ -49,11 +49,12 @@ def start_cmd(ctx, discoverhost, discoverport, token, name, host, port, fqdn, se :param init: """ config = {'discover_host': discoverhost, 'discover_port': discoverport, 'token': token, 'host': host, - 'port': port, 'fqdn': fqdn, 'name': name, 'secure': secure, 'verify': verify, 'max_clients': max_clients, - 'init': init} + 'port': port, 'fqdn': fqdn, 'name': name, 'secure': secure, 'verify': verify, 'max_clients': max_clients} - if config['init']: - apply_config(config) + if init: + apply_config(init, config) + click.echo(f'\nCombiner configuration loaded from file: {init}') + click.echo('Values set in file override defaults and command line arguments...\n') combiner = Combiner(config) combiner.run() diff --git a/fedn/cli/shared.py b/fedn/cli/shared.py index 7171f8ec5..81e5ebd07 100644 --- a/fedn/cli/shared.py +++ b/fedn/cli/shared.py @@ -29,14 +29,14 @@ API_VERSION = 'v1' -def apply_config(config): +def apply_config(path: str, config: dict): """Parse client config from file. Override configs from the CLI with settings in config file. :param config: Client config (dict). """ - with open(config['init'], 'r') as file: + with open(path, 'r') as file: try: settings = dict(yaml.safe_load(file)) except Exception: @@ -51,13 +51,13 @@ def get_api_url(protocol: str, host: str, port: str, endpoint: str) -> str: _url = os.environ.get('FEDN_CONTROLLER_URL') if _url: - return f'{_url}/api/{API_VERSION}/{endpoint}' + return f'{_url}/api/{API_VERSION}/{endpoint}/' _protocol = protocol or os.environ.get('FEDN_CONTROLLER_PROTOCOL') or CONTROLLER_DEFAULTS['protocol'] _host = host or os.environ.get('FEDN_CONTROLLER_HOST') or CONTROLLER_DEFAULTS['host'] _port = port or os.environ.get('FEDN_CONTROLLER_PORT') or CONTROLLER_DEFAULTS['port'] - return f'{_protocol}://{_host}:{_port}/api/{API_VERSION}/{endpoint}' + return f'{_protocol}://{_host}:{_port}/api/{API_VERSION}/{endpoint}/' def get_token(token: str) -> str: From 7fe1549ff019fef37915f2dedc416d024224d5da Mon Sep 17 00:00:00 2001 From: Niklas Date: Mon, 29 Apr 2024 15:03:48 +0200 Subject: [PATCH 10/12] linter fix --- fedn/cli/combiner_cmd.py | 2 +- fedn/cli/session_cmd.py | 2 +- fedn/cli/status_cmd.py | 2 +- fedn/cli/validation_cmd.py | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/fedn/cli/combiner_cmd.py b/fedn/cli/combiner_cmd.py index f74be38ba..69d9f2ba7 100644 --- a/fedn/cli/combiner_cmd.py +++ b/fedn/cli/combiner_cmd.py @@ -91,4 +91,4 @@ def list_combiners(ctx, protocol: str, host: str, port: str, token: str = None, response = requests.get(url, headers=headers) print_response(response, 'combiners') except requests.exceptions.ConnectionError: - click.echo(f'Error: Could not connect to {url}') \ No newline at end of file + click.echo(f'Error: Could not connect to {url}') diff --git a/fedn/cli/session_cmd.py b/fedn/cli/session_cmd.py index b18cb1925..a113ad400 100644 --- a/fedn/cli/session_cmd.py +++ b/fedn/cli/session_cmd.py @@ -47,4 +47,4 @@ def list_sessions(ctx, protocol: str, host: str, port: str, token: str = None, n response = requests.get(url, headers=headers) print_response(response, 'sessions') except requests.exceptions.ConnectionError: - click.echo(f'Error: Could not connect to {url}') \ No newline at end of file + click.echo(f'Error: Could not connect to {url}') diff --git a/fedn/cli/status_cmd.py b/fedn/cli/status_cmd.py index 9a658a9a7..72612a416 100644 --- a/fedn/cli/status_cmd.py +++ b/fedn/cli/status_cmd.py @@ -46,4 +46,4 @@ def list_statuses(ctx, protocol: str, host: str, port: str, token: str = None, n response = requests.get(url, headers=headers) print_response(response, 'statuses') except requests.exceptions.ConnectionError: - click.echo(f'Error: Could not connect to {url}') \ No newline at end of file + click.echo(f'Error: Could not connect to {url}') diff --git a/fedn/cli/validation_cmd.py b/fedn/cli/validation_cmd.py index 2e6cb41d7..469c46ac6 100644 --- a/fedn/cli/validation_cmd.py +++ b/fedn/cli/validation_cmd.py @@ -46,4 +46,4 @@ def list_validations(ctx, protocol: str, host: str, port: str, token: str = None response = requests.get(url, headers=headers) print_response(response, 'validations') except requests.exceptions.ConnectionError: - click.echo(f'Error: Could not connect to {url}') \ No newline at end of file + click.echo(f'Error: Could not connect to {url}') From 1f5bc75088eff2b9ebca5c7d501f0f054ba684fa Mon Sep 17 00:00:00 2001 From: Niklas Date: Tue, 7 May 2024 09:38:03 +0200 Subject: [PATCH 11/12] fedn run combiner/client added back with deprecation warning --- fedn/cli/run_cmd.py | 133 ++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 133 insertions(+) diff --git a/fedn/cli/run_cmd.py b/fedn/cli/run_cmd.py index ce57403b4..87a54f7f1 100644 --- a/fedn/cli/run_cmd.py +++ b/fedn/cli/run_cmd.py @@ -1,13 +1,19 @@ import os import shutil +import uuid import click import yaml +from fedn.common.exceptions import InvalidClientConfig from fedn.common.log_config import logger +from fedn.network.clients.client import Client +from fedn.network.combiner.combiner import Combiner from fedn.utils.dispatcher import Dispatcher, _read_yaml_file +from .client_cmd import validate_client_config from .main import main +from .shared import apply_config def get_statestore_config_from_file(init): @@ -74,3 +80,130 @@ def build_cmd(ctx, path): if dispatcher.python_env_path: logger.info(f"Removing virtualenv {dispatcher.python_env_path}") shutil.rmtree(dispatcher.python_env_path) + + +@run_cmd.command('client') +@click.option('-d', '--discoverhost', required=False, help='Hostname for discovery services(reducer).') +@click.option('-p', '--discoverport', required=False, help='Port for discovery services (reducer).') +@click.option('--token', required=False, help='Set token provided by reducer if enabled') +@click.option('-n', '--name', required=False, default="client" + str(uuid.uuid4())[:8]) +@click.option('-i', '--client_id', required=False) +@click.option('--local-package', is_flag=True, help='Enable local compute package') +@click.option('--force-ssl', is_flag=True, help='Force SSL/TLS for REST service') +@click.option('-u', '--dry-run', required=False, default=False) +@click.option('-s', '--secure', required=False, default=False) +@click.option('-pc', '--preshared-cert', required=False, default=False) +@click.option('-v', '--verify', is_flag=True, help='Verify SSL/TLS for REST service') +@click.option('-c', '--preferred-combiner', required=False, default=False) +@click.option('-va', '--validator', required=False, default=True) +@click.option('-tr', '--trainer', required=False, default=True) +@click.option('-in', '--init', required=False, default=None, + help='Set to a filename to (re)init client from file state.') +@click.option('-l', '--logfile', required=False, default=None, + help='Set logfile for client log to file.') +@click.option('--heartbeat-interval', required=False, default=2) +@click.option('--reconnect-after-missed-heartbeat', required=False, default=30) +@click.option('--verbosity', required=False, default='INFO', type=click.Choice(['CRITICAL', 'ERROR', 'WARNING', 'INFO', 'DEBUG'], case_sensitive=False)) +@click.pass_context +def client_cmd(ctx, discoverhost, discoverport, token, name, client_id, local_package, force_ssl, dry_run, secure, preshared_cert, + verify, preferred_combiner, validator, trainer, init, logfile, heartbeat_interval, reconnect_after_missed_heartbeat, + verbosity): + """ + + :param ctx: + :param discoverhost: + :param discoverport: + :param token: + :param name: + :param client_id: + :param remote: + :param dry_run: + :param secure: + :param preshared_cert: + :param verify_cert: + :param preferred_combiner: + :param init: + :param logfile: + :param hearbeat_interval + :param reconnect_after_missed_heartbeat + :param verbosity + :return: + """ + remote = False if local_package else True + config = {'discover_host': discoverhost, 'discover_port': discoverport, 'token': token, 'name': name, + 'client_id': client_id, 'remote_compute_context': remote, 'force_ssl': force_ssl, 'dry_run': dry_run, 'secure': secure, + 'preshared_cert': preshared_cert, 'verify': verify, 'preferred_combiner': preferred_combiner, + 'validator': validator, 'trainer': trainer, 'logfile': logfile, 'heartbeat_interval': heartbeat_interval, + 'reconnect_after_missed_heartbeat': reconnect_after_missed_heartbeat, 'verbosity': verbosity} + + click.echo( + click.style( + '\n*** fedn run client is deprecated and will be removed. Please use fedn client start instead. ***\n', + blink=True, + bold=True, + fg='red' + ) + ) + + if init: + apply_config(init, config) + click.echo(f'\nClient configuration loaded from file: {init}') + click.echo('Values set in file override defaults and command line arguments...\n') + + try: + validate_client_config(config) + except InvalidClientConfig as e: + click.echo(f'Error: {e}') + return + + client = Client(config) + client.run() + + +@run_cmd.command('combiner') +@click.option('-d', '--discoverhost', required=False, help='Hostname for discovery services (reducer).') +@click.option('-p', '--discoverport', required=False, help='Port for discovery services (reducer).') +@click.option('-t', '--token', required=False, help='Set token provided by reducer if enabled') +@click.option('-n', '--name', required=False, default="combiner" + str(uuid.uuid4())[:8], help='Set name for combiner.') +@click.option('-h', '--host', required=False, default="combiner", help='Set hostname.') +@click.option('-i', '--port', required=False, default=12080, help='Set port.') +@click.option('-f', '--fqdn', required=False, default=None, help='Set fully qualified domain name') +@click.option('-s', '--secure', is_flag=True, help='Enable SSL/TLS encrypted gRPC channels.') +@click.option('-v', '--verify', is_flag=True, help='Verify SSL/TLS for REST discovery service (reducer)') +@click.option('-c', '--max_clients', required=False, default=30, help='The maximal number of client connections allowed.') +@click.option('-in', '--init', required=False, default=None, + help='Path to configuration file to (re)init combiner.') +@click.pass_context +def combiner_cmd(ctx, discoverhost, discoverport, token, name, host, port, fqdn, secure, verify, max_clients, init): + """ + + :param ctx: + :param discoverhost: + :param discoverport: + :param token: + :param name: + :param hostname: + :param port: + :param secure: + :param max_clients: + :param init: + """ + config = {'discover_host': discoverhost, 'discover_port': discoverport, 'token': token, 'host': host, + 'port': port, 'fqdn': fqdn, 'name': name, 'secure': secure, 'verify': verify, 'max_clients': max_clients} + + click.echo( + click.style( + '\n*** fedn run combiner is deprecated and will be removed. Please use fedn combiner start instead. ***\n', + blink=True, + bold=True, + fg='red' + ) + ) + + if init: + apply_config(init, config) + click.echo(f'\nCombiner configuration loaded from file: {init}') + click.echo('Values set in file override defaults and command line arguments...\n') + + combiner = Combiner(config) + combiner.run() From 138035c6435fa4ef6b3fec544e9079a2f63a3ae3 Mon Sep 17 00:00:00 2001 From: Niklas Date: Tue, 7 May 2024 10:11:37 +0200 Subject: [PATCH 12/12] ruff linter fix --- fedn/cli/client_cmd.py | 4 +++- fedn/cli/combiner_cmd.py | 4 +++- fedn/cli/model_cmd.py | 4 +++- fedn/cli/package_cmd.py | 4 +++- fedn/cli/round_cmd.py | 4 +++- fedn/cli/session_cmd.py | 4 +++- fedn/cli/status_cmd.py | 4 +++- fedn/cli/validation_cmd.py | 4 +++- 8 files changed, 24 insertions(+), 8 deletions(-) diff --git a/fedn/cli/client_cmd.py b/fedn/cli/client_cmd.py index 76ccd2b3c..f9916985c 100644 --- a/fedn/cli/client_cmd.py +++ b/fedn/cli/client_cmd.py @@ -46,9 +46,11 @@ def client_cmd(ctx): @click.pass_context def list_clients(ctx, protocol: str, host: str, port: str, token: str = None, n_max: int = None): """ - return: + Return: + ------ - count: number of clients - result: list of clients + """ url = get_api_url(protocol=protocol, host=host, port=port, endpoint='clients') headers = {} diff --git a/fedn/cli/combiner_cmd.py b/fedn/cli/combiner_cmd.py index 69d9f2ba7..758dda718 100644 --- a/fedn/cli/combiner_cmd.py +++ b/fedn/cli/combiner_cmd.py @@ -69,9 +69,11 @@ def start_cmd(ctx, discoverhost, discoverport, token, name, host, port, fqdn, se @click.pass_context def list_combiners(ctx, protocol: str, host: str, port: str, token: str = None, n_max: int = None): """ - return: + Return: + ------ - count: number of combiners - result: list of combiners + """ url = get_api_url(protocol=protocol, host=host, port=port, endpoint='combiners') headers = {} diff --git a/fedn/cli/model_cmd.py b/fedn/cli/model_cmd.py index 2af526305..ddccd9e2d 100644 --- a/fedn/cli/model_cmd.py +++ b/fedn/cli/model_cmd.py @@ -25,9 +25,11 @@ def model_cmd(ctx): @click.pass_context def list_models(ctx, protocol: str, host: str, port: str, token: str = None, n_max: int = None): """ - return: + Return: + ------ - count: number of models - result: list of models + """ url = get_api_url(protocol=protocol, host=host, port=port, endpoint='models') headers = {} diff --git a/fedn/cli/package_cmd.py b/fedn/cli/package_cmd.py index 4026cb2ef..a19ed2f9e 100644 --- a/fedn/cli/package_cmd.py +++ b/fedn/cli/package_cmd.py @@ -52,9 +52,11 @@ def create_cmd(ctx, path, name): @click.pass_context def list_packages(ctx, protocol: str, host: str, port: str, token: str = None, n_max: int = None): """ - return: + Return: + ------ - count: number of packages - result: list of packages + """ url = get_api_url(protocol=protocol, host=host, port=port, endpoint='packages') headers = {} diff --git a/fedn/cli/round_cmd.py b/fedn/cli/round_cmd.py index 028675be7..31f4accc4 100644 --- a/fedn/cli/round_cmd.py +++ b/fedn/cli/round_cmd.py @@ -25,9 +25,11 @@ def round_cmd(ctx): @click.pass_context def list_rounds(ctx, protocol: str, host: str, port: str, token: str = None, n_max: int = None): """ - return: + Return: + ------ - count: number of rounds - result: list of rounds + """ url = get_api_url(protocol=protocol, host=host, port=port, endpoint='rounds') headers = {} diff --git a/fedn/cli/session_cmd.py b/fedn/cli/session_cmd.py index a113ad400..37eb3a8a6 100644 --- a/fedn/cli/session_cmd.py +++ b/fedn/cli/session_cmd.py @@ -25,9 +25,11 @@ def session_cmd(ctx): @click.pass_context def list_sessions(ctx, protocol: str, host: str, port: str, token: str = None, n_max: int = None): """ - return: + Return: + ------ - count: number of sessions - result: list of sessions + """ url = get_api_url(protocol=protocol, host=host, port=port, endpoint='sessions') headers = {} diff --git a/fedn/cli/status_cmd.py b/fedn/cli/status_cmd.py index 72612a416..457fc9c00 100644 --- a/fedn/cli/status_cmd.py +++ b/fedn/cli/status_cmd.py @@ -24,9 +24,11 @@ def status_cmd(ctx): @click.pass_context def list_statuses(ctx, protocol: str, host: str, port: str, token: str = None, n_max: int = None): """ - return: + Return: + ------ - count: number of statuses - result: list of statuses + """ url = get_api_url(protocol=protocol, host=host, port=port, endpoint='statuses') headers = {} diff --git a/fedn/cli/validation_cmd.py b/fedn/cli/validation_cmd.py index 469c46ac6..3707f9bb8 100644 --- a/fedn/cli/validation_cmd.py +++ b/fedn/cli/validation_cmd.py @@ -24,9 +24,11 @@ def validation_cmd(ctx): @click.pass_context def list_validations(ctx, protocol: str, host: str, port: str, token: str = None, n_max: int = None): """ - return: + Return: + ------ - count: number of validations - result: list of validations + """ url = get_api_url(protocol=protocol, host=host, port=port, endpoint='validations') headers = {}