diff --git a/docker-compose.yaml b/docker-compose.yaml index 969626582..b6563ace6 100644 --- a/docker-compose.yaml +++ b/docker-compose.yaml @@ -97,7 +97,7 @@ services: - ${HOST_REPO_DIR:-.}/fedn:/app/fedn entrypoint: [ "sh", "-c" ] command: - - "/venv/bin/pip install --no-cache-dir -e . && /venv/bin/fedn run combiner --init config/settings-combiner.yaml" + - "/venv/bin/pip install --no-cache-dir -e . && /venv/bin/fedn combiner start --init config/settings-combiner.yaml" ports: - 12080:12080 healthcheck: @@ -127,7 +127,7 @@ services: - ${HOST_REPO_DIR:-.}/fedn:/app/fedn entrypoint: [ "sh", "-c" ] command: - - "/venv/bin/pip install --no-cache-dir -e . && /venv/bin/fedn run client --init config/settings-client.yaml" + - "/venv/bin/pip install --no-cache-dir -e . && /venv/bin/fedn client start --init config/settings-client.yaml" deploy: replicas: 0 depends_on: diff --git a/docs/faq.rst b/docs/faq.rst index f817a0dee..223aa2e49 100644 --- a/docs/faq.rst +++ b/docs/faq.rst @@ -19,6 +19,17 @@ However, during development of a new model it will be necessary to reinitialize. 2. Restart the clients. +Q: Can I skip fetching the remote package and instead use a local folder when developing the compute package +------------------------------------------------------------------------------------------------------------ + +Yes, to facilitate interactive development of the compute package you can start a client that uses a local folder 'client' in your current working directory by: + +.. code-block:: bash + + fedn client start --remote=False -in client.yaml + + +Note that in production federations this options should in most cases be disallowed. Q: How can other aggregation algorithms can be defined? ------------------------------------------------------- @@ -45,7 +56,7 @@ Yes! You can toggle which message streams a client subscibes to when starting th .. code-block:: bash - fedn run client --trainer=False -in client.yaml + fedn client start --trainer=False -in client.yaml Q: How do you approach the question of output privacy? diff --git a/docs/quickstart.rst b/docs/quickstart.rst index 314d40c01..fa5b83eaa 100644 --- a/docs/quickstart.rst +++ b/docs/quickstart.rst @@ -115,7 +115,7 @@ For example, to split the data in 10 parts and start a client using the 8th part export FEDN_PACKAGE_EXTRACT_DIR=package export FEDN_NUM_DATA_SPLITS=10 export FEDN_DATA_PATH=./data/clients/8/mnist.pt - fedn run client -in client.yaml --secure=True --force-ssl + fedn client start -in client.yaml --secure=True --force-ssl .. code-tab:: bash :caption: Windows (Powershell) diff --git a/examples/flower-client/README.rst b/examples/flower-client/README.rst index 9cfa617dc..fff8e20b3 100644 --- a/examples/flower-client/README.rst +++ b/examples/flower-client/README.rst @@ -62,7 +62,7 @@ On your local machine / client, start the FEDn client: .. code-block:: - fedn run client -in client.yaml --secure=True --force-ssl + fedn client start -in client.yaml --secure=True --force-ssl Or, if you prefer to use Docker (this might take a long time): diff --git a/examples/mnist-pytorch/README.rst b/examples/mnist-pytorch/README.rst index 8f0b5276c..71e5de2d1 100644 --- a/examples/mnist-pytorch/README.rst +++ b/examples/mnist-pytorch/README.rst @@ -73,7 +73,7 @@ For example, to split the data in 10 parts and start a client using the 8th part export FEDN_PACKAGE_EXTRACT_DIR=package export FEDN_NUM_DATA_SPLITS=10 export FEDN_DATA_PATH=./data/clients/8/mnist.pt - fedn run client -in client.yaml --secure=True --force-ssl + fedn client start -in client.yaml --secure=True --force-ssl The default is to split the data into 2 partitions and use the first partition. diff --git a/fedn/cli/__init__.py b/fedn/cli/__init__.py index 840d4252b..137fc9b9c 100644 --- a/fedn/cli/__init__.py +++ b/fedn/cli/__init__.py @@ -1,2 +1,11 @@ +from .client_cmd import client_cmd # noqa: F401 +from .combiner_cmd import combiner_cmd # noqa: F401 +from .config_cmd import config_cmd # noqa: F401 from .main import main # noqa: F401 +from .model_cmd import model_cmd # noqa: F401 +from .package_cmd import package_cmd # noqa: F401 +from .round_cmd import round_cmd # noqa: F401 from .run_cmd import run_cmd # noqa: F401 +from .session_cmd import session_cmd # noqa: F401 +from .status_cmd import status_cmd # noqa: F401 +from .validation_cmd import validation_cmd # noqa: F401 diff --git a/fedn/cli/client_cmd.py b/fedn/cli/client_cmd.py new file mode 100644 index 000000000..f9916985c --- /dev/null +++ b/fedn/cli/client_cmd.py @@ -0,0 +1,142 @@ +import uuid + +import click +import requests + +from fedn.common.exceptions import InvalidClientConfig +from fedn.network.clients.client import Client + +from .main import main +from .shared import (CONTROLLER_DEFAULTS, apply_config, get_api_url, get_token, + print_response) + + +def validate_client_config(config): + """Validate client configuration. + + :param config: Client config (dict). + """ + + try: + if config['discover_host'] is None or \ + config['discover_host'] == '': + raise InvalidClientConfig("Missing required configuration: discover_host") + if 'discover_port' not in config.keys(): + config['discover_port'] = None + except Exception: + raise InvalidClientConfig("Could not load config from file. Check config") + + +@main.group('client') +@click.pass_context +def client_cmd(ctx): + """ + + :param ctx: + """ + pass + + +@click.option('-p', '--protocol', required=False, default=CONTROLLER_DEFAULTS['protocol'], help='Communication protocol of controller (api)') +@click.option('-H', '--host', required=False, default=CONTROLLER_DEFAULTS['host'], help='Hostname of controller (api)') +@click.option('-P', '--port', required=False, default=CONTROLLER_DEFAULTS['port'], help='Port of controller (api)') +@click.option('-t', '--token', required=False, help='Authentication token') +@click.option('--n_max', required=False, help='Number of items to list') +@client_cmd.command('list') +@click.pass_context +def list_clients(ctx, protocol: str, host: str, port: str, token: str = None, n_max: int = None): + """ + Return: + ------ + - count: number of clients + - result: list of clients + + """ + url = get_api_url(protocol=protocol, host=host, port=port, endpoint='clients') + headers = {} + + if n_max: + headers['X-Limit'] = n_max + + _token = get_token(token) + + if _token: + headers['Authorization'] = _token + + click.echo(f'\nListing clients: {url}\n') + click.echo(f'Headers: {headers}') + + try: + response = requests.get(url, headers=headers) + print_response(response, 'clients') + except requests.exceptions.ConnectionError: + click.echo(f'Error: Could not connect to {url}') + + +@client_cmd.command('start') +@click.option('-d', '--discoverhost', required=False, help='Hostname for discovery services(reducer).') +@click.option('-p', '--discoverport', required=False, help='Port for discovery services (reducer).') +@click.option('--token', required=False, help='Set token provided by reducer if enabled') +@click.option('-n', '--name', required=False, default="client" + str(uuid.uuid4())[:8]) +@click.option('-i', '--client_id', required=False) +@click.option('--local-package', is_flag=True, help='Enable local compute package') +@click.option('--force-ssl', is_flag=True, help='Force SSL/TLS for REST service') +@click.option('-u', '--dry-run', required=False, default=False) +@click.option('-s', '--secure', required=False, default=False) +@click.option('-pc', '--preshared-cert', required=False, default=False) +@click.option('-v', '--verify', is_flag=True, help='Verify SSL/TLS for REST service') +@click.option('-c', '--preferred-combiner', required=False, default=False) +@click.option('-va', '--validator', required=False, default=True) +@click.option('-tr', '--trainer', required=False, default=True) +@click.option('-in', '--init', required=False, default=None, + help='Set to a filename to (re)init client from file state.') +@click.option('-l', '--logfile', required=False, default=None, + help='Set logfile for client log to file.') +@click.option('--heartbeat-interval', required=False, default=2) +@click.option('--reconnect-after-missed-heartbeat', required=False, default=30) +@click.option('--verbosity', required=False, default='INFO', type=click.Choice(['CRITICAL', 'ERROR', 'WARNING', 'INFO', 'DEBUG'], case_sensitive=False)) +@click.pass_context +def client_cmd(ctx, discoverhost, discoverport, token, name, client_id, local_package, force_ssl, dry_run, secure, preshared_cert, + verify, preferred_combiner, validator, trainer, init, logfile, heartbeat_interval, reconnect_after_missed_heartbeat, + verbosity): + """ + + :param ctx: + :param discoverhost: + :param discoverport: + :param token: + :param name: + :param client_id: + :param remote: + :param dry_run: + :param secure: + :param preshared_cert: + :param verify_cert: + :param preferred_combiner: + :param init: + :param logfile: + :param hearbeat_interval + :param reconnect_after_missed_heartbeat + :param verbosity + :return: + """ + remote = False if local_package else True + config = {'discover_host': discoverhost, 'discover_port': discoverport, 'token': token, 'name': name, + 'client_id': client_id, 'remote_compute_context': remote, 'force_ssl': force_ssl, 'dry_run': dry_run, 'secure': secure, + 'preshared_cert': preshared_cert, 'verify': verify, 'preferred_combiner': preferred_combiner, + 'validator': validator, 'trainer': trainer, 'logfile': logfile, 'heartbeat_interval': heartbeat_interval, + 'reconnect_after_missed_heartbeat': reconnect_after_missed_heartbeat, 'verbosity': verbosity} + + if init: + apply_config(init, config) + click.echo(f'\nClient configuration loaded from file: {init}') + click.echo('Values set in file override defaults and command line arguments...\n') + + try: + validate_client_config(config) + except InvalidClientConfig as e: + click.echo(f'Error: {e}') + return + + client = Client(config) + client.run() diff --git a/fedn/cli/combiner_cmd.py b/fedn/cli/combiner_cmd.py new file mode 100644 index 000000000..758dda718 --- /dev/null +++ b/fedn/cli/combiner_cmd.py @@ -0,0 +1,96 @@ +import uuid + +import click +import requests + +from fedn.network.combiner.combiner import Combiner + +from .main import main +from .shared import (CONTROLLER_DEFAULTS, apply_config, get_api_url, get_token, + print_response) + + +@main.group('combiner') +@click.pass_context +def combiner_cmd(ctx): + """ + + :param ctx: + """ + pass + + +@combiner_cmd.command('start') +@click.option('-d', '--discoverhost', required=False, help='Hostname for discovery services (reducer).') +@click.option('-p', '--discoverport', required=False, help='Port for discovery services (reducer).') +@click.option('-t', '--token', required=False, help='Set token provided by reducer if enabled') +@click.option('-n', '--name', required=False, default="combiner" + str(uuid.uuid4())[:8], help='Set name for combiner.') +@click.option('-h', '--host', required=False, default="combiner", help='Set hostname.') +@click.option('-i', '--port', required=False, default=12080, help='Set port.') +@click.option('-f', '--fqdn', required=False, default=None, help='Set fully qualified domain name') +@click.option('-s', '--secure', is_flag=True, help='Enable SSL/TLS encrypted gRPC channels.') +@click.option('-v', '--verify', is_flag=True, help='Verify SSL/TLS for REST discovery service (reducer)') +@click.option('-c', '--max_clients', required=False, default=30, help='The maximal number of client connections allowed.') +@click.option('-in', '--init', required=False, default=None, + help='Path to configuration file to (re)init combiner.') +@click.pass_context +def start_cmd(ctx, discoverhost, discoverport, token, name, host, port, fqdn, secure, verify, max_clients, init): + """ + + :param ctx: + :param discoverhost: + :param discoverport: + :param token: + :param name: + :param hostname: + :param port: + :param secure: + :param max_clients: + :param init: + """ + config = {'discover_host': discoverhost, 'discover_port': discoverport, 'token': token, 'host': host, + 'port': port, 'fqdn': fqdn, 'name': name, 'secure': secure, 'verify': verify, 'max_clients': max_clients} + + if init: + apply_config(init, config) + click.echo(f'\nCombiner configuration loaded from file: {init}') + click.echo('Values set in file override defaults and command line arguments...\n') + + combiner = Combiner(config) + combiner.run() + + +@click.option('-p', '--protocol', required=False, default=CONTROLLER_DEFAULTS['protocol'], help='Communication protocol of controller (api)') +@click.option('-H', '--host', required=False, default=CONTROLLER_DEFAULTS['host'], help='Hostname of controller (api)') +@click.option('-P', '--port', required=False, default=CONTROLLER_DEFAULTS['port'], help='Port of controller (api)') +@click.option('-t', '--token', required=False, help='Authentication token') +@click.option('--n_max', required=False, help='Number of items to list') +@combiner_cmd.command('list') +@click.pass_context +def list_combiners(ctx, protocol: str, host: str, port: str, token: str = None, n_max: int = None): + """ + Return: + ------ + - count: number of combiners + - result: list of combiners + + """ + url = get_api_url(protocol=protocol, host=host, port=port, endpoint='combiners') + headers = {} + + if n_max: + headers['X-Limit'] = n_max + + _token = get_token(token) + + if _token: + headers['Authorization'] = _token + + click.echo(f'\nListing combiners: {url}\n') + click.echo(f'Headers: {headers}') + + try: + response = requests.get(url, headers=headers) + print_response(response, 'combiners') + except requests.exceptions.ConnectionError: + click.echo(f'Error: Could not connect to {url}') diff --git a/fedn/cli/config_cmd.py b/fedn/cli/config_cmd.py new file mode 100644 index 000000000..856882e62 --- /dev/null +++ b/fedn/cli/config_cmd.py @@ -0,0 +1,54 @@ +import os + +import click + +from .main import main + +envs = [ + { + "name": "FEDN_CONTROLLER_PROTOCOL", + "description": "The protocol to use for communication with the controller." + }, + { + "name": "FEDN_CONTROLLER_HOST", + "description": "The host to use for communication with the controller." + }, + { + "name": "FEDN_CONTROLLER_PORT", + "description": "The port to use for communication with the controller." + }, + { + "name": "FEDN_AUTH_TOKEN", + "description": "The authentication token to use for communication with the controller and combiner." + }, + { + "name": "FEDN_AUTH_SCHEME", + "description": "The authentication scheme to use for communication with the controller and combiner." + }, + { + "name": "FEDN_CONTROLLER_URL", + "description": "The URL of the controller. Overrides FEDN_CONTROLLER_PROTOCOL, FEDN_CONTROLLER_HOST and FEDN_CONTROLLER_PORT." + }, + { + "name": "FEDN_PACKAGE_EXTRACT_DIR", + "description": "The directory to extract packages to." + } +] + + +@main.group('config', invoke_without_command=True) +@click.pass_context +def config_cmd(ctx): + """ + - Configuration commands for the FEDn CLI. + """ + if ctx.invoked_subcommand is None: + click.echo('\n--- FEDn Cli Configuration ---\n') + click.echo('Current configuration:\n') + + for env in envs: + name = env['name'] + value = os.environ.get(name) + click.echo(f'{name}: {value or "Not set"}') + click.echo(f'{env["description"]}\n') + click.echo('\n') diff --git a/fedn/cli/model_cmd.py b/fedn/cli/model_cmd.py new file mode 100644 index 000000000..ddccd9e2d --- /dev/null +++ b/fedn/cli/model_cmd.py @@ -0,0 +1,52 @@ + +import click +import requests + +from .main import main +from .shared import CONTROLLER_DEFAULTS, get_api_url, get_token, print_response + + +@main.group('model') +@click.pass_context +def model_cmd(ctx): + """ + + :param ctx: + """ + pass + + +@click.option('-p', '--protocol', required=False, default=CONTROLLER_DEFAULTS['protocol'], help='Communication protocol of controller (api)') +@click.option('-H', '--host', required=False, default=CONTROLLER_DEFAULTS['host'], help='Hostname of controller (api)') +@click.option('-P', '--port', required=False, default=CONTROLLER_DEFAULTS['port'], help='Port of controller (api)') +@click.option('-t', '--token', required=False, help='Authentication token') +@click.option('--n_max', required=False, help='Number of items to list') +@model_cmd.command('list') +@click.pass_context +def list_models(ctx, protocol: str, host: str, port: str, token: str = None, n_max: int = None): + """ + Return: + ------ + - count: number of models + - result: list of models + + """ + url = get_api_url(protocol=protocol, host=host, port=port, endpoint='models') + headers = {} + + if n_max: + headers['X-Limit'] = n_max + + _token = get_token(token) + + if _token: + headers['Authorization'] = _token + + click.echo(f'\nListing models: {url}\n') + click.echo(f'Headers: {headers}') + + try: + response = requests.get(url, headers=headers) + print_response(response, 'models') + except requests.exceptions.ConnectionError: + click.echo(f'Error: Could not connect to {url}') diff --git a/fedn/cli/package_cmd.py b/fedn/cli/package_cmd.py new file mode 100644 index 000000000..a19ed2f9e --- /dev/null +++ b/fedn/cli/package_cmd.py @@ -0,0 +1,79 @@ +import os +import tarfile + +import click +import requests + +from fedn.common.log_config import logger + +from .main import main +from .shared import CONTROLLER_DEFAULTS, get_api_url, get_token, print_response + + +@main.group('package') +@click.pass_context +def package_cmd(ctx): + """ + + :param ctx: + """ + pass + + +@package_cmd.command('create') +@click.option('-p', '--path', required=True, help='Path to package directory containing fedn.yaml') +@click.option('-n', '--name', required=False, default='package.tgz', help='Name of package tarball') +@click.pass_context +def create_cmd(ctx, path, name): + """ Create compute package. + + Make a tar.gz archive of folder given by --path + + :param ctx: + :param path: + """ + path = os.path.abspath(path) + yaml_file = os.path.join(path, 'fedn.yaml') + if not os.path.exists(yaml_file): + logger.error(f"Could not find fedn.yaml in {path}") + exit(-1) + + with tarfile.open(name, "w:gz") as tar: + tar.add(path, arcname=os.path.basename(path)) + logger.info(f"Created package {name}") + + +@click.option('-p', '--protocol', required=False, default=CONTROLLER_DEFAULTS['protocol'], help='Communication protocol of controller (api)') +@click.option('-H', '--host', required=False, default=CONTROLLER_DEFAULTS['host'], help='Hostname of controller (api)') +@click.option('-P', '--port', required=False, default=CONTROLLER_DEFAULTS['port'], help='Port of controller (api)') +@click.option('-t', '--token', required=False, help='Authentication token') +@click.option('--n_max', required=False, help='Number of items to list') +@package_cmd.command('list') +@click.pass_context +def list_packages(ctx, protocol: str, host: str, port: str, token: str = None, n_max: int = None): + """ + Return: + ------ + - count: number of packages + - result: list of packages + + """ + url = get_api_url(protocol=protocol, host=host, port=port, endpoint='packages') + headers = {} + + if n_max: + headers['X-Limit'] = n_max + + _token = get_token(token) + + if _token: + headers['Authorization'] = _token + + click.echo(f'\nListing packages: {url}\n') + click.echo(f'Headers: {headers}') + + try: + response = requests.get(url, headers=headers) + print_response(response, 'packages') + except requests.exceptions.ConnectionError: + click.echo(f'Error: Could not connect to {url}') diff --git a/fedn/cli/round_cmd.py b/fedn/cli/round_cmd.py new file mode 100644 index 000000000..31f4accc4 --- /dev/null +++ b/fedn/cli/round_cmd.py @@ -0,0 +1,52 @@ + +import click +import requests + +from .main import main +from .shared import CONTROLLER_DEFAULTS, get_api_url, get_token, print_response + + +@main.group('round') +@click.pass_context +def round_cmd(ctx): + """ + + :param ctx: + """ + pass + + +@click.option('-p', '--protocol', required=False, default=CONTROLLER_DEFAULTS['protocol'], help='Communication protocol of controller (api)') +@click.option('-H', '--host', required=False, default=CONTROLLER_DEFAULTS['host'], help='Hostname of controller (api)') +@click.option('-P', '--port', required=False, default=CONTROLLER_DEFAULTS['port'], help='Port of controller (api)') +@click.option('-t', '--token', required=False, help='Authentication token') +@click.option('--n_max', required=False, help='Number of items to list') +@round_cmd.command('list') +@click.pass_context +def list_rounds(ctx, protocol: str, host: str, port: str, token: str = None, n_max: int = None): + """ + Return: + ------ + - count: number of rounds + - result: list of rounds + + """ + url = get_api_url(protocol=protocol, host=host, port=port, endpoint='rounds') + headers = {} + + if n_max: + headers['X-Limit'] = n_max + + _token = get_token(token) + + if _token: + headers['Authorization'] = _token + + click.echo(f'\nListing rounds: {url}\n') + click.echo(f'Headers: {headers}') + + try: + response = requests.get(url, headers=headers) + print_response(response, 'rounds') + except requests.exceptions.ConnectionError: + click.echo(f'Error: Could not connect to {url}') diff --git a/fedn/cli/run_cmd.py b/fedn/cli/run_cmd.py index 4c007131f..87a54f7f1 100644 --- a/fedn/cli/run_cmd.py +++ b/fedn/cli/run_cmd.py @@ -1,6 +1,5 @@ import os import shutil -import tarfile import uuid import click @@ -12,7 +11,9 @@ from fedn.network.combiner.combiner import Combiner from fedn.utils.dispatcher import Dispatcher, _read_yaml_file +from .client_cmd import validate_client_config from .main import main +from .shared import apply_config def get_statestore_config_from_file(init): @@ -39,48 +40,46 @@ def check_helper_config_file(config): return helper -def apply_config(config): - """Parse client config from file. - - Override configs from the CLI with settings in config file. - - :param config: Client config (dict). +@main.group('run') +@click.pass_context +def run_cmd(ctx): """ - with open(config['init'], 'r') as file: - try: - settings = dict(yaml.safe_load(file)) - except Exception: - logger.error('Failed to read config from settings file, exiting.') - return - for key, val in settings.items(): - config[key] = val - - -def validate_client_config(config): - """Validate client configuration. - - :param config: Client config (dict). + :param ctx: """ - - try: - if config['discover_host'] is None or \ - config['discover_host'] == '': - raise InvalidClientConfig("Missing required configuration: discover_host") - if 'discover_port' not in config.keys(): - config['discover_port'] = None - except Exception: - raise InvalidClientConfig("Could not load config from file. Check config") + pass -@main.group('run') +@run_cmd.command('build') +@click.option('-p', '--path', required=True, help='Path to package directory containing fedn.yaml') @click.pass_context -def run_cmd(ctx): - """ +def build_cmd(ctx, path): + """ Execute 'build' entrypoint in fedn.yaml. :param ctx: + :param path: Path to folder containing fedn.yaml + :type path: str """ - pass + path = os.path.abspath(path) + yaml_file = os.path.join(path, 'fedn.yaml') + if not os.path.exists(yaml_file): + logger.error(f"Could not find fedn.yaml in {path}") + exit(-1) + + config = _read_yaml_file(yaml_file) + # Check that build is defined in fedn.yaml under entry_points + if 'build' not in config['entry_points']: + logger.error("No build command defined in fedn.yaml") + exit(-1) + + dispatcher = Dispatcher(config, path) + _ = dispatcher._get_or_create_python_env() + dispatcher.run_cmd("build") + + # delete the virtualenv + if dispatcher.python_env_path: + logger.info(f"Removing virtualenv {dispatcher.python_env_path}") + shutil.rmtree(dispatcher.python_env_path) @run_cmd.command('client') @@ -134,13 +133,28 @@ def client_cmd(ctx, discoverhost, discoverport, token, name, client_id, local_pa config = {'discover_host': discoverhost, 'discover_port': discoverport, 'token': token, 'name': name, 'client_id': client_id, 'remote_compute_context': remote, 'force_ssl': force_ssl, 'dry_run': dry_run, 'secure': secure, 'preshared_cert': preshared_cert, 'verify': verify, 'preferred_combiner': preferred_combiner, - 'validator': validator, 'trainer': trainer, 'init': init, 'logfile': logfile, 'heartbeat_interval': heartbeat_interval, + 'validator': validator, 'trainer': trainer, 'logfile': logfile, 'heartbeat_interval': heartbeat_interval, 'reconnect_after_missed_heartbeat': reconnect_after_missed_heartbeat, 'verbosity': verbosity} + click.echo( + click.style( + '\n*** fedn run client is deprecated and will be removed. Please use fedn client start instead. ***\n', + blink=True, + bold=True, + fg='red' + ) + ) + if init: - apply_config(config) + apply_config(init, config) + click.echo(f'\nClient configuration loaded from file: {init}') + click.echo('Values set in file override defaults and command line arguments...\n') - validate_client_config(config) + try: + validate_client_config(config) + except InvalidClientConfig as e: + click.echo(f'Error: {e}') + return client = Client(config) client.run() @@ -175,76 +189,21 @@ def combiner_cmd(ctx, discoverhost, discoverport, token, name, host, port, fqdn, :param init: """ config = {'discover_host': discoverhost, 'discover_port': discoverport, 'token': token, 'host': host, - 'port': port, 'fqdn': fqdn, 'name': name, 'secure': secure, 'verify': verify, 'max_clients': max_clients, - 'init': init} + 'port': port, 'fqdn': fqdn, 'name': name, 'secure': secure, 'verify': verify, 'max_clients': max_clients} + + click.echo( + click.style( + '\n*** fedn run combiner is deprecated and will be removed. Please use fedn combiner start instead. ***\n', + blink=True, + bold=True, + fg='red' + ) + ) - if config['init']: - apply_config(config) + if init: + apply_config(init, config) + click.echo(f'\nCombiner configuration loaded from file: {init}') + click.echo('Values set in file override defaults and command line arguments...\n') combiner = Combiner(config) combiner.run() - - -@run_cmd.command('build') -@click.option('-p', '--path', required=True, help='Path to package directory containing fedn.yaml') -@click.pass_context -def build_cmd(ctx, path): - """ Execute 'build' entrypoint in fedn.yaml. - - :param ctx: - :param path: Path to folder containing fedn.yaml - :type path: str - """ - path = os.path.abspath(path) - yaml_file = os.path.join(path, 'fedn.yaml') - if not os.path.exists(yaml_file): - logger.error(f"Could not find fedn.yaml in {path}") - exit(-1) - - config = _read_yaml_file(yaml_file) - # Check that build is defined in fedn.yaml under entry_points - if 'build' not in config['entry_points']: - logger.error("No build command defined in fedn.yaml") - exit(-1) - - dispatcher = Dispatcher(config, path) - _ = dispatcher._get_or_create_python_env() - dispatcher.run_cmd("build") - - # delete the virtualenv - if dispatcher.python_env_path: - logger.info(f"Removing virtualenv {dispatcher.python_env_path}") - shutil.rmtree(dispatcher.python_env_path) - - -@main.group('package') -@click.pass_context -def package_cmd(ctx): - """ - - :param ctx: - """ - pass - - -@package_cmd.command('create') -@click.option('-p', '--path', required=True, help='Path to package directory containing fedn.yaml') -@click.option('-n', '--name', required=False, default='package.tgz', help='Name of package tarball') -@click.pass_context -def create_cmd(ctx, path, name): - """ Create compute package. - - Make a tar.gz archive of folder given by --path - - :param ctx: - :param path: - """ - path = os.path.abspath(path) - yaml_file = os.path.join(path, 'fedn.yaml') - if not os.path.exists(yaml_file): - logger.error(f"Could not find fedn.yaml in {path}") - exit(-1) - - with tarfile.open(name, "w:gz") as tar: - tar.add(path, arcname=os.path.basename(path)) - logger.info(f"Created package {name}") diff --git a/fedn/cli/session_cmd.py b/fedn/cli/session_cmd.py new file mode 100644 index 000000000..37eb3a8a6 --- /dev/null +++ b/fedn/cli/session_cmd.py @@ -0,0 +1,52 @@ + +import click +import requests + +from .main import main +from .shared import CONTROLLER_DEFAULTS, get_api_url, get_token, print_response + + +@main.group('session') +@click.pass_context +def session_cmd(ctx): + """ + + :param ctx: + """ + pass + + +@click.option('-p', '--protocol', required=False, default=CONTROLLER_DEFAULTS['protocol'], help='Communication protocol of controller (api)') +@click.option('-H', '--host', required=False, default=CONTROLLER_DEFAULTS['host'], help='Hostname of controller (api)') +@click.option('-P', '--port', required=False, default=CONTROLLER_DEFAULTS['port'], help='Port of controller (api)') +@click.option('-t', '--token', required=False, help='Authentication token') +@click.option('--n_max', required=False, help='Number of items to list') +@session_cmd.command('list') +@click.pass_context +def list_sessions(ctx, protocol: str, host: str, port: str, token: str = None, n_max: int = None): + """ + Return: + ------ + - count: number of sessions + - result: list of sessions + + """ + url = get_api_url(protocol=protocol, host=host, port=port, endpoint='sessions') + headers = {} + + if n_max: + headers['X-Limit'] = n_max + + _token = get_token(token) + + if _token: + headers['Authorization'] = _token + + click.echo(f'\nListing sessions: {url}\n') + click.echo(f'Headers: {headers}') + + try: + response = requests.get(url, headers=headers) + print_response(response, 'sessions') + except requests.exceptions.ConnectionError: + click.echo(f'Error: Could not connect to {url}') diff --git a/fedn/cli/shared.py b/fedn/cli/shared.py new file mode 100644 index 000000000..81e5ebd07 --- /dev/null +++ b/fedn/cli/shared.py @@ -0,0 +1,104 @@ +import os + +import click +import yaml + +from fedn.common.log_config import logger + +CONTROLLER_DEFAULTS = { + 'protocol': 'http', + 'host': 'localhost', + 'port': 8092, + 'debug': False +} + +COMBINER_DEFAULTS = { + 'discover_host': 'localhost', + 'discover_port': 8092, + 'host': 'localhost', + 'port': 12080, + "name": "combiner", + "max_clients": 30 +} + +CLIENT_DEFAULTS = { + 'discover_host': 'localhost', + 'discover_port': 8092, +} + +API_VERSION = 'v1' + + +def apply_config(path: str, config: dict): + """Parse client config from file. + + Override configs from the CLI with settings in config file. + + :param config: Client config (dict). + """ + with open(path, 'r') as file: + try: + settings = dict(yaml.safe_load(file)) + except Exception: + logger.error('Failed to read config from settings file, exiting.') + return + + for key, val in settings.items(): + config[key] = val + + +def get_api_url(protocol: str, host: str, port: str, endpoint: str) -> str: + _url = os.environ.get('FEDN_CONTROLLER_URL') + + if _url: + return f'{_url}/api/{API_VERSION}/{endpoint}/' + + _protocol = protocol or os.environ.get('FEDN_CONTROLLER_PROTOCOL') or CONTROLLER_DEFAULTS['protocol'] + _host = host or os.environ.get('FEDN_CONTROLLER_HOST') or CONTROLLER_DEFAULTS['host'] + _port = port or os.environ.get('FEDN_CONTROLLER_PORT') or CONTROLLER_DEFAULTS['port'] + + return f'{_protocol}://{_host}:{_port}/api/{API_VERSION}/{endpoint}/' + + +def get_token(token: str) -> str: + _token = token or os.environ.get("FEDN_AUTH_TOKEN", None) + + if _token is None: + return None + + scheme = os.environ.get("FEDN_AUTH_SCHEME", "Bearer") + + return f"{scheme} {_token}" + + +def get_client_package_dir(path: str) -> str: + return path or os.environ.get('FEDN_PACKAGE_DIR', None) + + +# Print response from api (list of entities) +def print_response(response, entity_name: str): + """ + Prints the api response to the cli. + :param response: + type: array + description: list of entities + :param entity_name: + type: string + description: name of entity + return: None + """ + if response.status_code == 200: + json_data = response.json() + count, result = json_data.values() + click.echo(f'Found {count} {entity_name}') + click.echo('\n---------------------------------\n') + for obj in result: + click.echo('{') + for k, v in obj.items(): + click.echo(f'\t{k}: {v}') + click.echo('}') + elif response.status_code == 500: + json_data = response.json() + click.echo(f'Error: {json_data["message"]}') + else: + click.echo(f'Error: {response.status_code}') diff --git a/fedn/cli/status_cmd.py b/fedn/cli/status_cmd.py new file mode 100644 index 000000000..457fc9c00 --- /dev/null +++ b/fedn/cli/status_cmd.py @@ -0,0 +1,51 @@ +import click +import requests + +from .main import main +from .shared import CONTROLLER_DEFAULTS, get_api_url, get_token, print_response + + +@main.group('status') +@click.pass_context +def status_cmd(ctx): + """ + + :param ctx: + """ + pass + + +@click.option('-p', '--protocol', required=False, default=CONTROLLER_DEFAULTS['protocol'], help='Communication protocol of controller (api)') +@click.option('-H', '--host', required=False, default=CONTROLLER_DEFAULTS['host'], help='Hostname of controller (api)') +@click.option('-P', '--port', required=False, default=CONTROLLER_DEFAULTS['port'], help='Port of controller (api)') +@click.option('-t', '--token', required=False, help='Authentication token') +@click.option('--n_max', required=False, help='Number of items to list') +@status_cmd.command('list') +@click.pass_context +def list_statuses(ctx, protocol: str, host: str, port: str, token: str = None, n_max: int = None): + """ + Return: + ------ + - count: number of statuses + - result: list of statuses + + """ + url = get_api_url(protocol=protocol, host=host, port=port, endpoint='statuses') + headers = {} + + if n_max: + headers['X-Limit'] = n_max + + _token = get_token(token) + + if _token: + headers['Authorization'] = _token + + click.echo(f'\nListing statuses: {url}\n') + click.echo(f'Headers: {headers}') + + try: + response = requests.get(url, headers=headers) + print_response(response, 'statuses') + except requests.exceptions.ConnectionError: + click.echo(f'Error: Could not connect to {url}') diff --git a/fedn/cli/validation_cmd.py b/fedn/cli/validation_cmd.py new file mode 100644 index 000000000..3707f9bb8 --- /dev/null +++ b/fedn/cli/validation_cmd.py @@ -0,0 +1,51 @@ +import click +import requests + +from .main import main +from .shared import CONTROLLER_DEFAULTS, get_api_url, get_token, print_response + + +@main.group('validation') +@click.pass_context +def validation_cmd(ctx): + """ + + :param ctx: + """ + pass + + +@click.option('-p', '--protocol', required=False, default=CONTROLLER_DEFAULTS['protocol'], help='Communication protocol of controller (api)') +@click.option('-H', '--host', required=False, default=CONTROLLER_DEFAULTS['host'], help='Hostname of controller (api)') +@click.option('-P', '--port', required=False, default=CONTROLLER_DEFAULTS['port'], help='Port of controller (api)') +@click.option('-t', '--token', required=False, help='Authentication token') +@click.option('--n_max', required=False, help='Number of items to list') +@validation_cmd.command('list') +@click.pass_context +def list_validations(ctx, protocol: str, host: str, port: str, token: str = None, n_max: int = None): + """ + Return: + ------ + - count: number of validations + - result: list of validations + + """ + url = get_api_url(protocol=protocol, host=host, port=port, endpoint='validations') + headers = {} + + if n_max: + headers['X-Limit'] = n_max + + _token = get_token(token) + + if _token: + headers['Authorization'] = _token + + click.echo(f'\nListing validations: {url}\n') + click.echo(f'Headers: {headers}') + + try: + response = requests.get(url, headers=headers) + print_response(response, 'validations') + except requests.exceptions.ConnectionError: + click.echo(f'Error: Could not connect to {url}')