From b963f4b5f9f7ddff7c2279c2b9586ac9dd6657fe Mon Sep 17 00:00:00 2001 From: sowmyasri Date: Wed, 17 Jul 2024 11:03:34 +0200 Subject: [PATCH 01/17] flag for not deleting the venv --- fedn/cli/run_cmd.py | 27 ++++++++++++++------------- 1 file changed, 14 insertions(+), 13 deletions(-) diff --git a/fedn/cli/run_cmd.py b/fedn/cli/run_cmd.py index 20836cf96..26c9e195d 100644 --- a/fedn/cli/run_cmd.py +++ b/fedn/cli/run_cmd.py @@ -47,8 +47,9 @@ def run_cmd(ctx): @run_cmd.command("startup") @click.option("-p", "--path", required=True, help="Path to package directory containing fedn.yaml") +@click.option("-p", "--venv", default=True,is_flag=True,required=False, help="flag if set to False doesn't remove venv") @click.pass_context -def startup_cmd(ctx, path): +def startup_cmd(ctx, path,venv): """Execute 'startup' entrypoint in fedn.yaml. :param ctx: @@ -70,16 +71,16 @@ def startup_cmd(ctx, path): dispatcher = Dispatcher(config, path) _ = dispatcher._get_or_create_python_env() dispatcher.run_cmd("startup") - - # delete the virtualenv - if dispatcher.python_env_path: - logger.info(f"Removing virtualenv {dispatcher.python_env_path}") - shutil.rmtree(dispatcher.python_env_path) + if venv: + # delete the virtualenv + if dispatcher.python_env_path: + logger.info(f"Removing virtualenv {dispatcher.python_env_path}") + shutil.rmtree(dispatcher.python_env_path) @run_cmd.command("build") -@click.option("-p", "--path", required=True, help="Path to package directory containing fedn.yaml") +@click.option("-p", "--venv", default=True,is_flag=True,required=False, help="flag if set to False doesn't remove venv") @click.pass_context -def build_cmd(ctx, path): +def build_cmd(ctx, path,venv): """Execute 'build' entrypoint in fedn.yaml. :param ctx: @@ -101,11 +102,11 @@ def build_cmd(ctx, path): dispatcher = Dispatcher(config, path) _ = dispatcher._get_or_create_python_env() dispatcher.run_cmd("build") - - # delete the virtualenv - if dispatcher.python_env_path: - logger.info(f"Removing virtualenv {dispatcher.python_env_path}") - shutil.rmtree(dispatcher.python_env_path) + if venv: + # delete the virtualenv + if dispatcher.python_env_path: + logger.info(f"Removing virtualenv {dispatcher.python_env_path}") + shutil.rmtree(dispatcher.python_env_path) @run_cmd.command("client") From 060907d0633504597c7043961e2f89e027abc0a8 Mon Sep 17 00:00:00 2001 From: sowmyasri Date: Wed, 17 Jul 2024 14:38:30 +0200 Subject: [PATCH 02/17] added flag to reuse the existing env without deleting , default is to delete --- fedn/cli/run_cmd.py | 30 ++++++++++++++++-------------- 1 file changed, 16 insertions(+), 14 deletions(-) diff --git a/fedn/cli/run_cmd.py b/fedn/cli/run_cmd.py index b7441417d..062f58c25 100644 --- a/fedn/cli/run_cmd.py +++ b/fedn/cli/run_cmd.py @@ -47,8 +47,9 @@ def run_cmd(ctx): @click.option("-p", "--path", required=True, help="Path to package directory containing fedn.yaml") @click.option("-i", "--input", required=True, help="Path to input model" ) @click.option("-o", "--output", required=True,help="Path to write the output JSON containing validation metrics") +@click.option("-v", "--venv", default=True,type=bool,required=False, help="flag if set to False doesn't remove venv") @click.pass_context -def validate_cmd(ctx, path,input,output): +def validate_cmd(ctx, path,input,output,venv): """Execute 'validate' entrypoint in fedn.yaml. :param ctx: @@ -70,17 +71,18 @@ def validate_cmd(ctx, path,input,output): dispatcher = Dispatcher(config, path) _ = dispatcher._get_or_create_python_env() dispatcher.run_cmd("validate {} {}".format(input, output)) - - # delete the virtualenv - if dispatcher.python_env_path: - logger.info(f"Removing virtualenv {dispatcher.python_env_path}") - shutil.rmtree(dispatcher.python_env_path) + if venv: + # delete the virtualenv + if dispatcher.python_env_path: + logger.info(f"Removing virtualenv {dispatcher.python_env_path}") + shutil.rmtree(dispatcher.python_env_path) @run_cmd.command("train") @click.option("-p", "--path", required=True, help="Path to package directory containing fedn.yaml") @click.option("-i", "--input", required=True, help="Path to input model parameters" ) @click.option("-o", "--output", required=True,help="Path to write the updated model parameters ") +@click.option("-v", "--venv", default=True,type=bool,required=False, help="flag if set to False doesn't remove venv") @click.pass_context -def train_cmd(ctx, path,input,output): +def train_cmd(ctx, path,input,output,venv): """Execute 'train' entrypoint in fedn.yaml. :param ctx: @@ -102,14 +104,14 @@ def train_cmd(ctx, path,input,output): dispatcher = Dispatcher(config, path) _ = dispatcher._get_or_create_python_env() dispatcher.run_cmd("train {} {}".format(input, output)) - - # delete the virtualenv - if dispatcher.python_env_path: - logger.info(f"Removing virtualenv {dispatcher.python_env_path}") - shutil.rmtree(dispatcher.python_env_path) + if venv: + # delete the virtualenv + if dispatcher.python_env_path: + logger.info(f"Removing virtualenv {dispatcher.python_env_path}") + shutil.rmtree(dispatcher.python_env_path) @run_cmd.command("startup") @click.option("-p", "--path", required=True, help="Path to package directory containing fedn.yaml") -@click.option("-p", "--venv", default=True,is_flag=True,required=False, help="flag if set to False doesn't remove venv") +@click.option("-v", "--venv", default=True,type=bool,required=False, help="flag if set to False doesn't remove venv") @click.pass_context def startup_cmd(ctx, path,venv): """Execute 'startup' entrypoint in fedn.yaml. @@ -140,7 +142,7 @@ def startup_cmd(ctx, path,venv): shutil.rmtree(dispatcher.python_env_path) @run_cmd.command("build") -@click.option("-p", "--venv", default=True,is_flag=True,required=False, help="flag if set to False doesn't remove venv") +@click.option("-v", "--venv", default=True,type=bool,required=False, help="flag if set to False doesn't remove venv") @click.pass_context def build_cmd(ctx, path,venv): """Execute 'build' entrypoint in fedn.yaml. From 55e179affecd33a8f0b78891dd3fe81774172336 Mon Sep 17 00:00:00 2001 From: sowmyasri Date: Fri, 19 Jul 2024 10:26:19 +0200 Subject: [PATCH 03/17] fixed the build command and added the path flag --- fedn/cli/run_cmd.py | 1 + 1 file changed, 1 insertion(+) diff --git a/fedn/cli/run_cmd.py b/fedn/cli/run_cmd.py index 062f58c25..c94680ee3 100644 --- a/fedn/cli/run_cmd.py +++ b/fedn/cli/run_cmd.py @@ -142,6 +142,7 @@ def startup_cmd(ctx, path,venv): shutil.rmtree(dispatcher.python_env_path) @run_cmd.command("build") +@click.option("-p", "--path", required=True, help="Path to package directory containing fedn.yaml") @click.option("-v", "--venv", default=True,type=bool,required=False, help="flag if set to False doesn't remove venv") @click.pass_context def build_cmd(ctx, path,venv): From da5654d005c051414d44c13c802a680023eb0051 Mon Sep 17 00:00:00 2001 From: sowmyasri Date: Fri, 19 Jul 2024 14:14:30 +0200 Subject: [PATCH 04/17] changed flag to remove-venv logging the behaviour of venv not found --- fedn/cli/run_cmd.py | 36 ++++++++++++++++++++++-------------- 1 file changed, 22 insertions(+), 14 deletions(-) diff --git a/fedn/cli/run_cmd.py b/fedn/cli/run_cmd.py index c94680ee3..5c0fc797e 100644 --- a/fedn/cli/run_cmd.py +++ b/fedn/cli/run_cmd.py @@ -46,10 +46,10 @@ def run_cmd(ctx): @run_cmd.command("validate") @click.option("-p", "--path", required=True, help="Path to package directory containing fedn.yaml") @click.option("-i", "--input", required=True, help="Path to input model" ) -@click.option("-o", "--output", required=True,help="Path to write the output JSON containing validation metrics") -@click.option("-v", "--venv", default=True,type=bool,required=False, help="flag if set to False doesn't remove venv") +@click.option("-o", "--output", required=True, help="Path to write the output JSON containing validation metrics") +@click.option("-v", "--remove-venv", default=True, type=bool, required=False, help="flag if set to False doesn't remove venv") @click.pass_context -def validate_cmd(ctx, path,input,output,venv): +def validate_cmd(ctx, path,input,output,remove_venv): """Execute 'validate' entrypoint in fedn.yaml. :param ctx: @@ -71,18 +71,20 @@ def validate_cmd(ctx, path,input,output,venv): dispatcher = Dispatcher(config, path) _ = dispatcher._get_or_create_python_env() dispatcher.run_cmd("validate {} {}".format(input, output)) - if venv: + if remove_venv: # delete the virtualenv if dispatcher.python_env_path: logger.info(f"Removing virtualenv {dispatcher.python_env_path}") shutil.rmtree(dispatcher.python_env_path) + else: + logger.warning("No virtualenv found to remove.") @run_cmd.command("train") @click.option("-p", "--path", required=True, help="Path to package directory containing fedn.yaml") @click.option("-i", "--input", required=True, help="Path to input model parameters" ) -@click.option("-o", "--output", required=True,help="Path to write the updated model parameters ") -@click.option("-v", "--venv", default=True,type=bool,required=False, help="flag if set to False doesn't remove venv") +@click.option("-o", "--output", required=True, help="Path to write the updated model parameters ") +@click.option("-v", "--remove-venv", default=True, type=bool, required=False, help="flag if set to False doesn't remove venv") @click.pass_context -def train_cmd(ctx, path,input,output,venv): +def train_cmd(ctx, path,input,output,remove_venv): """Execute 'train' entrypoint in fedn.yaml. :param ctx: @@ -104,16 +106,18 @@ def train_cmd(ctx, path,input,output,venv): dispatcher = Dispatcher(config, path) _ = dispatcher._get_or_create_python_env() dispatcher.run_cmd("train {} {}".format(input, output)) - if venv: + if remove_venv: # delete the virtualenv if dispatcher.python_env_path: logger.info(f"Removing virtualenv {dispatcher.python_env_path}") shutil.rmtree(dispatcher.python_env_path) + else: + logger.warning("No virtualenv found to remove.") @run_cmd.command("startup") @click.option("-p", "--path", required=True, help="Path to package directory containing fedn.yaml") -@click.option("-v", "--venv", default=True,type=bool,required=False, help="flag if set to False doesn't remove venv") +@click.option("-v", "--remove-venv", default=True, type=bool, required=False, help="flag if set to False doesn't remove venv") @click.pass_context -def startup_cmd(ctx, path,venv): +def startup_cmd(ctx, path,remove_venv): """Execute 'startup' entrypoint in fedn.yaml. :param ctx: @@ -135,17 +139,19 @@ def startup_cmd(ctx, path,venv): dispatcher = Dispatcher(config, path) _ = dispatcher._get_or_create_python_env() dispatcher.run_cmd("startup") - if venv: + if remove_venv: # delete the virtualenv if dispatcher.python_env_path: logger.info(f"Removing virtualenv {dispatcher.python_env_path}") shutil.rmtree(dispatcher.python_env_path) + else: + logger.warning("No virtualenv found to remove.") @run_cmd.command("build") @click.option("-p", "--path", required=True, help="Path to package directory containing fedn.yaml") -@click.option("-v", "--venv", default=True,type=bool,required=False, help="flag if set to False doesn't remove venv") +@click.option("-v", "--remove-venv", default=True, type=bool, required=False, help="flag if set to False doesn't remove venv") @click.pass_context -def build_cmd(ctx, path,venv): +def build_cmd(ctx, path,remove_venv): """Execute 'build' entrypoint in fedn.yaml. :param ctx: @@ -167,11 +173,13 @@ def build_cmd(ctx, path,venv): dispatcher = Dispatcher(config, path) _ = dispatcher._get_or_create_python_env() dispatcher.run_cmd("build") - if venv: + if remove_venv: # delete the virtualenv if dispatcher.python_env_path: logger.info(f"Removing virtualenv {dispatcher.python_env_path}") shutil.rmtree(dispatcher.python_env_path) + else: + logger.warning("No virtualenv found to remove.") @run_cmd.command("client") From a67f1df9410cea9422fed67c59cc6e897936898f Mon Sep 17 00:00:00 2001 From: sowmyasri Date: Fri, 26 Jul 2024 13:28:01 +0200 Subject: [PATCH 05/17] wrapped check yaml exist to separate method changed the import test cases for CLI command train,validate,startup,controller start and version --- fedn/cli/client_cmd.py | 4 +- fedn/cli/controller_cmd.py | 2 +- fedn/cli/run_cmd.py | 35 +++--- fedn/cli/tests/tests.py | 227 ++++++++++++++++++++++++++++++++++++- 4 files changed, 243 insertions(+), 25 deletions(-) diff --git a/fedn/cli/client_cmd.py b/fedn/cli/client_cmd.py index 80b0b3353..bb71003d7 100644 --- a/fedn/cli/client_cmd.py +++ b/fedn/cli/client_cmd.py @@ -6,8 +6,8 @@ from fedn.common.exceptions import InvalidClientConfig from fedn.network.clients.client import Client -from .main import main -from .shared import CONTROLLER_DEFAULTS, apply_config, get_api_url, get_token, print_response +from fedn.cli.main import main +from fedn.cli.shared import CONTROLLER_DEFAULTS, apply_config, get_api_url, get_token, print_response def validate_client_config(config): diff --git a/fedn/cli/controller_cmd.py b/fedn/cli/controller_cmd.py index ab8727b27..443628c9a 100644 --- a/fedn/cli/controller_cmd.py +++ b/fedn/cli/controller_cmd.py @@ -1,6 +1,6 @@ import click -from .main import main +from fedn.cli.main import main @main.group("controller") diff --git a/fedn/cli/run_cmd.py b/fedn/cli/run_cmd.py index 5c0fc797e..9191c0d55 100644 --- a/fedn/cli/run_cmd.py +++ b/fedn/cli/run_cmd.py @@ -10,9 +10,9 @@ from fedn.network.combiner.combiner import Combiner from fedn.utils.dispatcher import Dispatcher, _read_yaml_file -from .client_cmd import validate_client_config -from .main import main -from .shared import apply_config +from fedn.cli.client_cmd import validate_client_config +from fedn.cli.main import main +from fedn.cli.shared import apply_config def get_statestore_config_from_file(init): @@ -36,7 +36,14 @@ def check_helper_config_file(config): exit(-1) return helper - +def check_yaml_exists(path): + """Check if fedn.yaml exists in the given path.""" + yaml_file = os.path.join(path, "fedn.yaml") + if not os.path.exists(yaml_file): + logger.error(f"Could not find fedn.yaml in {path}") + click.echo(f"Could not find fedn.yaml in {path}") + exit(-1) + return yaml_file @main.group("run") @click.pass_context def run_cmd(ctx): @@ -57,10 +64,7 @@ def validate_cmd(ctx, path,input,output,remove_venv): :type path: str """ path = os.path.abspath(path) - yaml_file = os.path.join(path, "fedn.yaml") - if not os.path.exists(yaml_file): - logger.error(f"Could not find fedn.yaml in {path}") - exit(-1) + yaml_file = check_yaml_exists(path) config = _read_yaml_file(yaml_file) # Check that validate is defined in fedn.yaml under entry_points @@ -92,10 +96,7 @@ def train_cmd(ctx, path,input,output,remove_venv): :type path: str """ path = os.path.abspath(path) - yaml_file = os.path.join(path, "fedn.yaml") - if not os.path.exists(yaml_file): - logger.error(f"Could not find fedn.yaml in {path}") - exit(-1) + yaml_file = check_yaml_exists(path) config = _read_yaml_file(yaml_file) # Check that train is defined in fedn.yaml under entry_points @@ -125,10 +126,7 @@ def startup_cmd(ctx, path,remove_venv): :type path: str """ path = os.path.abspath(path) - yaml_file = os.path.join(path, "fedn.yaml") - if not os.path.exists(yaml_file): - logger.error(f"Could not find fedn.yaml in {path}") - exit(-1) + yaml_file = check_yaml_exists(path) config = _read_yaml_file(yaml_file) # Check that startup is defined in fedn.yaml under entry_points @@ -159,10 +157,7 @@ def build_cmd(ctx, path,remove_venv): :type path: str """ path = os.path.abspath(path) - yaml_file = os.path.join(path, "fedn.yaml") - if not os.path.exists(yaml_file): - logger.error(f"Could not find fedn.yaml in {path}") - exit(-1) + yaml_file = check_yaml_exists(path) config = _read_yaml_file(yaml_file) # Check that build is defined in fedn.yaml under entry_points diff --git a/fedn/cli/tests/tests.py b/fedn/cli/tests/tests.py index dffe90a2c..7cf6878c4 100644 --- a/fedn/cli/tests/tests.py +++ b/fedn/cli/tests/tests.py @@ -1,9 +1,17 @@ import unittest - +import sys +import os +import fedn +from unittest.mock import patch from click.testing import CliRunner from run_cmd import check_helper_config_file +from run_cmd import run_cmd,check_yaml_exists,logger +import click +from main import main +from fedn.network.api.server import start_server_api +from controller_cmd import main, controller_cmd - +MOCK_VERSION = "0.11.1" class TestReducerCLI(unittest.TestCase): def setUp(self): @@ -56,6 +64,221 @@ def test_get_statestore_config_from_file(self): # self.assertEqual(result.output, "--remote was set to False, but no helper was found in --init settings file: settings.yaml\n") # self.assertEqual(result.exit_code, -1) + #testcase for --version in fedn + @patch('main.get_version') + def test_version_output(self, mock_get_version): + # Mock the get_version function to return a predefined version + mock_get_version.return_value = MOCK_VERSION + + runner = CliRunner() + result = runner.invoke(main, ['--version']) + + # Check that the command exits with a status code of 0 + self.assertEqual(result.exit_code, 0) + + # Check that the output contains the mocked version string + expected_output = f"main, version {MOCK_VERSION}\n" + self.assertEqual(result.output, expected_output) + + #train command unit test cases + #To test check yaml function + @patch('run_cmd.os.path.exists') + @patch('run_cmd.click.echo') + def test_yaml_exists(self, mock_click_echo, mock_exists): + path = '/fake/path' + mock_exists.return_value = True + + # Call the function + result = check_yaml_exists(path) + # Assertions + mock_exists.assert_called_once_with(os.path.join(path, 'fedn.yaml')) + self.assertEqual(result, os.path.join(path, 'fedn.yaml')) + mock_click_echo.assert_not_called() + #test missing fedn yaml file + @patch('run_cmd.os.path.exists') + def test_missing_fedn_yaml(self, mock_exists): + mock_exists.return_value = False + result = self.runner.invoke(run_cmd, [ + 'train', + '--path', 'fedn/examples/mnist-pytorch/client', + '--input', 'client.npz', + '--output', 'client' + ]) + self.assertEqual(result.exit_code, -1) + self.assertIn("", result.output) + + #train cmd missing in fedn yaml file + @patch('run_cmd._read_yaml_file') + @patch('run_cmd.logger') + @patch('run_cmd.exit') + @patch('run_cmd.check_yaml_exists') + def test_train_not_defined(self, mock_check_yaml_exists, mock_exit, mock_logger, mock_read_yaml_file): + # Setup the mock to simulate fedn.yaml content without "train" entry point + mock_read_yaml_file.return_value = { + "entry_points": { + "vaidate": "some_train_command" + } + } + mock_check_yaml_exists.return_value = '/fake/path/fedn.yaml' + result = self.runner.invoke(run_cmd, [ + 'train', + '--path', '/fake/path', + '--input', 'input', + '--output', 'output', + '--remove-venv', 'True' + ]) + mock_logger.error.assert_called_once_with("No train command defined in fedn.yaml") + #print("hereeeeee",mock_logger.error.call_count) + log_messages = [call[0][0] for call in mock_logger.error.call_args_list] + #print("Captured log messages:", log_messages) + mock_exit.assert_called_once_with(-1) + + #to test with venv flag as false + @patch('run_cmd.os.path.exists') + @patch('run_cmd.logger') + @patch('run_cmd.Dispatcher') + def test_train_cmd_with_venv_false(self, MockDispatcher,mock_exists,mock_logger): + mock_exists.return_value = True + mock_dispatcher = MockDispatcher.return_value + mock_dispatcher.run_cmd.return_value = None + result = self.runner.invoke(run_cmd, [ + 'train', + '--path', '../../.././fedn/examples/mnist-pytorch/client', + '--input', 'client.npz', + '--output', 'client', + '--remove-venv', 'False' + ]) + + self.assertEqual(result.exit_code, 0) + mock_dispatcher.run_cmd.assert_called_once_with("train client.npz client") + #print(mock_dispatcher.run_cmd.call_count) + +#Validate cmd test cases + @patch('run_cmd._read_yaml_file') + @patch('run_cmd.logger') + @patch('run_cmd.exit') + @patch('run_cmd.check_yaml_exists') + def test_validate_not_defined(self, mock_check_yaml_exists, mock_exit, mock_logger, mock_read_yaml_file): + mock_read_yaml_file.return_value = { + "entry_points": { + "train": "some_train_command" + } + } + mock_check_yaml_exists.return_value = '/fake/path/fedn.yaml' + result = self.runner.invoke(run_cmd, [ + 'validate', + '--path', '/fake/path', + '--input', 'input', + '--output', 'output', + '--remove-venv', 'True' + ]) + + + # Verify that the error was logged + mock_logger.error.assert_called_once_with("No validate command defined in fedn.yaml") + #log_messages = [call[0][0] for call in mock_logger.error.call_args_list] + #print("Captured log messages:", log_messages) + mock_exit.assert_called_once_with(-1) + + #test missing fedn yaml file + @patch('run_cmd.os.path.exists') + def test_missing_fedn_yaml(self, mock_exists): + mock_exists.return_value = False + result = self.runner.invoke(run_cmd, [ + 'vaidate', + '--path', 'fedn/examples/mnist-pytorch/client', + '--input', 'client.npz', + '--output', 'client' + ]) + self.assertEqual(result.exit_code, -1) + self.assertIn("", result.output) + + #Test validate cmd with venv false + @patch('run_cmd.os.path.exists') + @patch('run_cmd.logger') + @patch('run_cmd.Dispatcher') + def test_validate_cmd_with_venv_false(self, MockDispatcher,mock_exists,mock_logger): + mock_exists.return_value = True + mock_dispatcher = MockDispatcher.return_value + mock_dispatcher.run_cmd.return_value = None + result = self.runner.invoke(run_cmd, [ + 'validate', + '--path', '../../.././fedn/examples/mnist-pytorch/client', + '--input', 'client.npz', + '--output', 'client', + '--remove-venv', 'False' + ]) + + self.assertEqual(result.exit_code, 0) + mock_dispatcher.run_cmd.assert_called_once_with("validate client.npz client") + #print(mock_dispatcher.run_cmd.call_count) + +#build cmd test cases + @patch('run_cmd._read_yaml_file') + @patch('run_cmd.logger') + @patch('run_cmd.exit') + @patch('run_cmd.check_yaml_exists') + def test_startup_not_defined(self, mock_check_yaml_exists, mock_exit, mock_logger, mock_read_yaml_file): + mock_read_yaml_file.return_value = { + "entry_points": { + "train": "some_train_command" + } + } + mock_check_yaml_exists.return_value = '/fake/path/fedn.yaml' + runner = CliRunner() + result = runner.invoke(run_cmd, [ + 'startup', + '--path', '/fake/path', + '--remove-venv', 'True' + ]) + + + # Verify that the error was logged + mock_logger.error.assert_called_once_with("No startup command defined in fedn.yaml") + log_messages = [call[0][0] for call in mock_logger.error.call_args_list] + #print("Captured log messages:", log_messages) + mock_exit.assert_called_once_with(-1) + + #test missing fedn yaml file + @patch('run_cmd.os.path.exists') + def test_missing_fedn_yaml(self, mock_exists): + mock_exists.return_value = False + result = self.runner.invoke(run_cmd, [ + 'startup', + '--path', 'fedn/examples/mnist-pytorch/client' + ]) + self.assertEqual(result.exit_code, -1) + self.assertIn("", result.output) + + @patch('run_cmd.os.path.exists') + @patch('run_cmd.logger') + @patch('run_cmd.Dispatcher') + def test_startup_cmd_with_venv_false(self, MockDispatcher,mock_exists,mock_logger): + mock_exists.return_value = True + mock_dispatcher = MockDispatcher.return_value + mock_dispatcher.run_cmd.return_value = None + result = self.runner.invoke(run_cmd, [ + 'startup', + '--path', '../../.././fedn/examples/mnist-pytorch/client', + '--remove-venv', 'False' + ]) + + self.assertEqual(result.exit_code, 0) + mock_dispatcher.run_cmd.assert_called_once_with("startup") + #print(mock_dispatcher.run_cmd.call_count) + + #to test controller start + @patch('fedn.network.api.server.start_server_api') + def test_controller_start(self, mock_start_server_api): + runner = CliRunner() + result = runner.invoke(main, ['controller', 'start']) + + # Check that the command exits with a status code of 0 + self.assertEqual(result.exit_code, 0) + + # Check that the start_server_api function was called + mock_start_server_api.assert_called_once() + #print("hereeee",mock_start_server_api.call_count) def test_check_helper_config_file(self): self.assertEqual(check_helper_config_file( From 5b2120088bdb3f6bdd1b9a74222a02bf6c381538 Mon Sep 17 00:00:00 2001 From: sowmyasri Date: Wed, 31 Jul 2024 11:21:59 +0200 Subject: [PATCH 06/17] added Gunicorn server as production server, if the debug is False in settings yaml file --- fedn/network/api/gunicorn_app.py | 24 ++++++++++++++++++++++++ fedn/network/api/server.py | 16 +++++++++------- 2 files changed, 33 insertions(+), 7 deletions(-) create mode 100644 fedn/network/api/gunicorn_app.py diff --git a/fedn/network/api/gunicorn_app.py b/fedn/network/api/gunicorn_app.py new file mode 100644 index 000000000..ee41d25f9 --- /dev/null +++ b/fedn/network/api/gunicorn_app.py @@ -0,0 +1,24 @@ +from gunicorn.app.base import BaseApplication +import os +class GunicornApp(BaseApplication): + def __init__(self, app, options=None): + self.options = options or {} + self.application = app + super().__init__() + + def load_config(self): + config = {key: value for key, value in self.options.items() + if key in self.cfg.settings and value is not None} + for key, value in config.items(): + self.cfg.set(key.lower(), value) + + def load(self): + return self.application + +def run_gunicorn(app, workers=4): + workers=os.cpu_count() + options = { + 'bind': '127.0.0.1:8000', # Specify the bind address and port here + 'workers': workers, + } + GunicornApp(app, options).run() \ No newline at end of file diff --git a/fedn/network/api/server.py b/fedn/network/api/server.py index d56c3ab0b..3ab8ecf3e 100644 --- a/fedn/network/api/server.py +++ b/fedn/network/api/server.py @@ -7,6 +7,7 @@ from fedn.network.api.interface import API from fedn.network.api.shared import control, statestore from fedn.network.api.v1 import _routes +from fedn.network.api.gunicorn_app import run_gunicorn custom_url_prefix = os.environ.get("FEDN_CUSTOM_URL_PREFIX", False) # statestore_config,modelstorage_config,network_id,control=set_statestore_config() @@ -628,12 +629,13 @@ def list_combiners_data(): def start_server_api(): - config = get_controller_config() - port = config["port"] - debug = config["debug"] - host = "0.0.0.0" - app.run(debug=debug, port=port, host=host) - - + config = get_controller_config() + port = config["port"] + host = "0.0.0.0" + debug = config["debug"] + if debug: + app.run(debug=debug, port=port, host=host) + else: + run_gunicorn(app) if __name__ == "__main__": start_server_api() From 9961836e62ef0e43af01a342aa103cfef81ba018 Mon Sep 17 00:00:00 2001 From: sowmyasri Date: Wed, 31 Jul 2024 11:26:58 +0200 Subject: [PATCH 07/17] ruff double quotes error corrected --- fedn/network/api/gunicorn_app.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/fedn/network/api/gunicorn_app.py b/fedn/network/api/gunicorn_app.py index ee41d25f9..c5ffe22dc 100644 --- a/fedn/network/api/gunicorn_app.py +++ b/fedn/network/api/gunicorn_app.py @@ -18,7 +18,7 @@ def load(self): def run_gunicorn(app, workers=4): workers=os.cpu_count() options = { - 'bind': '127.0.0.1:8000', # Specify the bind address and port here - 'workers': workers, + "bind": "127.0.0.1:8000", # Specify the bind address and port here + "workers": workers, } - GunicornApp(app, options).run() \ No newline at end of file + GunicornApp(app, options).run() From 69795a1c33c60f86e4c9399bd322cd39bff80f41 Mon Sep 17 00:00:00 2001 From: sowmyasri Date: Wed, 31 Jul 2024 15:01:23 +0200 Subject: [PATCH 08/17] fixed the function exxecution during import --- fedn/network/api/gunicorn_app.py | 2 ++ fedn/network/api/server.py | 4 ++-- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/fedn/network/api/gunicorn_app.py b/fedn/network/api/gunicorn_app.py index c5ffe22dc..f6feb3803 100644 --- a/fedn/network/api/gunicorn_app.py +++ b/fedn/network/api/gunicorn_app.py @@ -22,3 +22,5 @@ def run_gunicorn(app, workers=4): "workers": workers, } GunicornApp(app, options).run() +if __name__ == "main": + run_gunicorn() diff --git a/fedn/network/api/server.py b/fedn/network/api/server.py index 3ab8ecf3e..2774f34d3 100644 --- a/fedn/network/api/server.py +++ b/fedn/network/api/server.py @@ -7,7 +7,7 @@ from fedn.network.api.interface import API from fedn.network.api.shared import control, statestore from fedn.network.api.v1 import _routes -from fedn.network.api.gunicorn_app import run_gunicorn +import gunicorn_app custom_url_prefix = os.environ.get("FEDN_CUSTOM_URL_PREFIX", False) # statestore_config,modelstorage_config,network_id,control=set_statestore_config() @@ -636,6 +636,6 @@ def start_server_api(): if debug: app.run(debug=debug, port=port, host=host) else: - run_gunicorn(app) + gunicorn_app.run_gunicorn(app) if __name__ == "__main__": start_server_api() From 488d9f27cde5d84d02901eee23b39f600b9282f9 Mon Sep 17 00:00:00 2001 From: sowmyasri Date: Wed, 31 Jul 2024 15:36:29 +0200 Subject: [PATCH 09/17] added gunicorn dependency in pyproject.toml --- pyproject.toml | 1 + 1 file changed, 1 insertion(+) diff --git a/pyproject.toml b/pyproject.toml index 3806fee56..e1d684b7e 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -31,6 +31,7 @@ requires-python = '>=3.8,<3.13' dependencies = [ "requests", "urllib3>=1.26.4", + "gunicorn>=20.0.4,<21.0.0", "minio", "grpcio~=1.60.0", "grpcio-tools~=1.60.0", From e8eb846bc52afb0182ed93aa5c95c4471a5574a7 Mon Sep 17 00:00:00 2001 From: sowmyasri Date: Wed, 31 Jul 2024 16:04:46 +0200 Subject: [PATCH 10/17] fixed the import error --- fedn/network/api/server.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fedn/network/api/server.py b/fedn/network/api/server.py index 2774f34d3..2c7de9cc3 100644 --- a/fedn/network/api/server.py +++ b/fedn/network/api/server.py @@ -7,7 +7,7 @@ from fedn.network.api.interface import API from fedn.network.api.shared import control, statestore from fedn.network.api.v1 import _routes -import gunicorn_app +from fedn.network.api import gunicorn_app custom_url_prefix = os.environ.get("FEDN_CUSTOM_URL_PREFIX", False) # statestore_config,modelstorage_config,network_id,control=set_statestore_config() From 41d0db712fbb646f8d15090aa3e86b102ecc80b0 Mon Sep 17 00:00:00 2001 From: sowmyasri Date: Mon, 12 Aug 2024 15:37:38 +0200 Subject: [PATCH 11/17] updated the code with the changes suggested --- fedn/cli/run_cmd.py | 53 ++++++++++++-------------------- fedn/network/api/gunicorn_app.py | 8 ++--- fedn/network/api/server.py | 3 +- pyproject.toml | 2 +- 4 files changed, 26 insertions(+), 40 deletions(-) diff --git a/fedn/cli/run_cmd.py b/fedn/cli/run_cmd.py index 9191c0d55..b5afa0fed 100644 --- a/fedn/cli/run_cmd.py +++ b/fedn/cli/run_cmd.py @@ -14,7 +14,6 @@ from fedn.cli.main import main from fedn.cli.shared import apply_config - def get_statestore_config_from_file(init): """:param init: :return: @@ -44,6 +43,14 @@ def check_yaml_exists(path): click.echo(f"Could not find fedn.yaml in {path}") exit(-1) return yaml_file +def delete_virtual_environment(remove_venv,dispatcher): + if remove_venv: + # delete the virtualenv + if dispatcher.python_env_path: + logger.info(f"Removing virtualenv {dispatcher.python_env_path}") + shutil.rmtree(dispatcher.python_env_path) + else: + logger.warning("No virtualenv found to remove.") @main.group("run") @click.pass_context def run_cmd(ctx): @@ -54,7 +61,7 @@ def run_cmd(ctx): @click.option("-p", "--path", required=True, help="Path to package directory containing fedn.yaml") @click.option("-i", "--input", required=True, help="Path to input model" ) @click.option("-o", "--output", required=True, help="Path to write the output JSON containing validation metrics") -@click.option("-v", "--remove-venv", default=True, type=bool, required=False, help="flag if set to False doesn't remove venv") +@click.option("-v", "--remove-venv", is_flag=True, default=True,required=False, help="flag if set to False doesn't remove venv") @click.pass_context def validate_cmd(ctx, path,input,output,remove_venv): """Execute 'validate' entrypoint in fedn.yaml. @@ -75,18 +82,12 @@ def validate_cmd(ctx, path,input,output,remove_venv): dispatcher = Dispatcher(config, path) _ = dispatcher._get_or_create_python_env() dispatcher.run_cmd("validate {} {}".format(input, output)) - if remove_venv: - # delete the virtualenv - if dispatcher.python_env_path: - logger.info(f"Removing virtualenv {dispatcher.python_env_path}") - shutil.rmtree(dispatcher.python_env_path) - else: - logger.warning("No virtualenv found to remove.") + delete_virtual_environment(remove_venv,dispatcher) @run_cmd.command("train") @click.option("-p", "--path", required=True, help="Path to package directory containing fedn.yaml") @click.option("-i", "--input", required=True, help="Path to input model parameters" ) @click.option("-o", "--output", required=True, help="Path to write the updated model parameters ") -@click.option("-v", "--remove-venv", default=True, type=bool, required=False, help="flag if set to False doesn't remove venv") +@click.option("-v", "--remove-venv", is_flag=True, default=True,required=False, help="flag if set to False doesn't remove venv") @click.pass_context def train_cmd(ctx, path,input,output,remove_venv): """Execute 'train' entrypoint in fedn.yaml. @@ -107,16 +108,11 @@ def train_cmd(ctx, path,input,output,remove_venv): dispatcher = Dispatcher(config, path) _ = dispatcher._get_or_create_python_env() dispatcher.run_cmd("train {} {}".format(input, output)) - if remove_venv: - # delete the virtualenv - if dispatcher.python_env_path: - logger.info(f"Removing virtualenv {dispatcher.python_env_path}") - shutil.rmtree(dispatcher.python_env_path) - else: - logger.warning("No virtualenv found to remove.") + delete_virtual_environment(remove_venv,dispatcher) + @run_cmd.command("startup") @click.option("-p", "--path", required=True, help="Path to package directory containing fedn.yaml") -@click.option("-v", "--remove-venv", default=True, type=bool, required=False, help="flag if set to False doesn't remove venv") +@click.option("-v", "--remove-venv", is_flag=True, default=True,required=False, help="flag if set to False doesn't remove venv") @click.pass_context def startup_cmd(ctx, path,remove_venv): """Execute 'startup' entrypoint in fedn.yaml. @@ -137,17 +133,12 @@ def startup_cmd(ctx, path,remove_venv): dispatcher = Dispatcher(config, path) _ = dispatcher._get_or_create_python_env() dispatcher.run_cmd("startup") - if remove_venv: - # delete the virtualenv - if dispatcher.python_env_path: - logger.info(f"Removing virtualenv {dispatcher.python_env_path}") - shutil.rmtree(dispatcher.python_env_path) - else: - logger.warning("No virtualenv found to remove.") + delete_virtual_environment(remove_venv,dispatcher) + @run_cmd.command("build") @click.option("-p", "--path", required=True, help="Path to package directory containing fedn.yaml") -@click.option("-v", "--remove-venv", default=True, type=bool, required=False, help="flag if set to False doesn't remove venv") +@click.option("-v", "--remove-venv", is_flag=True, default=True,required=False, help="flag if set to False doesn't remove venv") @click.pass_context def build_cmd(ctx, path,remove_venv): """Execute 'build' entrypoint in fedn.yaml. @@ -168,13 +159,9 @@ def build_cmd(ctx, path,remove_venv): dispatcher = Dispatcher(config, path) _ = dispatcher._get_or_create_python_env() dispatcher.run_cmd("build") - if remove_venv: - # delete the virtualenv - if dispatcher.python_env_path: - logger.info(f"Removing virtualenv {dispatcher.python_env_path}") - shutil.rmtree(dispatcher.python_env_path) - else: - logger.warning("No virtualenv found to remove.") + print(remove_venv) + delete_virtual_environment(remove_venv,dispatcher) + @run_cmd.command("client") diff --git a/fedn/network/api/gunicorn_app.py b/fedn/network/api/gunicorn_app.py index f6feb3803..f7c92b768 100644 --- a/fedn/network/api/gunicorn_app.py +++ b/fedn/network/api/gunicorn_app.py @@ -15,12 +15,10 @@ def load_config(self): def load(self): return self.application -def run_gunicorn(app, workers=4): - workers=os.cpu_count() +def run_gunicorn(app, host,port,workers=4): + bind_address=str(host)+":"+str(port) options = { - "bind": "127.0.0.1:8000", # Specify the bind address and port here + "bind": bind_address, # Specify the bind address and port here "workers": workers, } GunicornApp(app, options).run() -if __name__ == "main": - run_gunicorn() diff --git a/fedn/network/api/server.py b/fedn/network/api/server.py index 2c7de9cc3..0c9806d86 100644 --- a/fedn/network/api/server.py +++ b/fedn/network/api/server.py @@ -636,6 +636,7 @@ def start_server_api(): if debug: app.run(debug=debug, port=port, host=host) else: - gunicorn_app.run_gunicorn(app) + workers=os.cpu_count() + gunicorn_app.run_gunicorn(app,host,port,workers) if __name__ == "__main__": start_server_api() diff --git a/pyproject.toml b/pyproject.toml index e1d684b7e..43d3e87b9 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -31,7 +31,7 @@ requires-python = '>=3.8,<3.13' dependencies = [ "requests", "urllib3>=1.26.4", - "gunicorn>=20.0.4,<21.0.0", + "gunicorn>=20.0.4", "minio", "grpcio~=1.60.0", "grpcio-tools~=1.60.0", From aa947c79f5c4cee21a0537e9e7e0cb738dccacdd Mon Sep 17 00:00:00 2001 From: sowmyasri Date: Mon, 12 Aug 2024 15:39:48 +0200 Subject: [PATCH 12/17] removed unused package --- fedn/network/api/gunicorn_app.py | 1 - 1 file changed, 1 deletion(-) diff --git a/fedn/network/api/gunicorn_app.py b/fedn/network/api/gunicorn_app.py index f7c92b768..d4e822629 100644 --- a/fedn/network/api/gunicorn_app.py +++ b/fedn/network/api/gunicorn_app.py @@ -1,5 +1,4 @@ from gunicorn.app.base import BaseApplication -import os class GunicornApp(BaseApplication): def __init__(self, app, options=None): self.options = options or {} From 1d353aeb6e0dc7d5c77b4eb3890725a00f6eef2b Mon Sep 17 00:00:00 2001 From: sowmyasri Date: Wed, 14 Aug 2024 13:15:02 +0200 Subject: [PATCH 13/17] changed the virtual-env flag to keep-venv and some code clean up --- fedn/cli/run_cmd.py | 30 ++++++++++++++---------------- fedn/network/api/gunicorn_app.py | 2 +- fedn/network/api/server.py | 2 +- 3 files changed, 16 insertions(+), 18 deletions(-) diff --git a/fedn/cli/run_cmd.py b/fedn/cli/run_cmd.py index b5afa0fed..ae33c566c 100644 --- a/fedn/cli/run_cmd.py +++ b/fedn/cli/run_cmd.py @@ -43,8 +43,8 @@ def check_yaml_exists(path): click.echo(f"Could not find fedn.yaml in {path}") exit(-1) return yaml_file -def delete_virtual_environment(remove_venv,dispatcher): - if remove_venv: +def delete_virtual_environment(keep_venv, dispatcher): + if keep_venv: # delete the virtualenv if dispatcher.python_env_path: logger.info(f"Removing virtualenv {dispatcher.python_env_path}") @@ -61,9 +61,9 @@ def run_cmd(ctx): @click.option("-p", "--path", required=True, help="Path to package directory containing fedn.yaml") @click.option("-i", "--input", required=True, help="Path to input model" ) @click.option("-o", "--output", required=True, help="Path to write the output JSON containing validation metrics") -@click.option("-v", "--remove-venv", is_flag=True, default=True,required=False, help="flag if set to False doesn't remove venv") +@click.option("-v", "--keep-venv", is_flag=True, default=True, required=False, help="flag if set to False doesn't remove venv") @click.pass_context -def validate_cmd(ctx, path,input,output,remove_venv): +def validate_cmd(ctx, path, input, output, keep_venv): """Execute 'validate' entrypoint in fedn.yaml. :param ctx: @@ -82,14 +82,14 @@ def validate_cmd(ctx, path,input,output,remove_venv): dispatcher = Dispatcher(config, path) _ = dispatcher._get_or_create_python_env() dispatcher.run_cmd("validate {} {}".format(input, output)) - delete_virtual_environment(remove_venv,dispatcher) + delete_virtual_environment(keep_venv, dispatcher) @run_cmd.command("train") @click.option("-p", "--path", required=True, help="Path to package directory containing fedn.yaml") @click.option("-i", "--input", required=True, help="Path to input model parameters" ) @click.option("-o", "--output", required=True, help="Path to write the updated model parameters ") -@click.option("-v", "--remove-venv", is_flag=True, default=True,required=False, help="flag if set to False doesn't remove venv") +@click.option("-v", "--keep-venv", is_flag=True, default=True, required=False, help="flag if set to False doesn't remove venv") @click.pass_context -def train_cmd(ctx, path,input,output,remove_venv): +def train_cmd(ctx, path, input, output, keep_venv): """Execute 'train' entrypoint in fedn.yaml. :param ctx: @@ -108,13 +108,13 @@ def train_cmd(ctx, path,input,output,remove_venv): dispatcher = Dispatcher(config, path) _ = dispatcher._get_or_create_python_env() dispatcher.run_cmd("train {} {}".format(input, output)) - delete_virtual_environment(remove_venv,dispatcher) + delete_virtual_environment(keep_venv, dispatcher) @run_cmd.command("startup") @click.option("-p", "--path", required=True, help="Path to package directory containing fedn.yaml") -@click.option("-v", "--remove-venv", is_flag=True, default=True,required=False, help="flag if set to False doesn't remove venv") +@click.option("-v", "--keep-venv", is_flag=True, default=True, required=False, help="flag if set to False doesn't remove venv") @click.pass_context -def startup_cmd(ctx, path,remove_venv): +def startup_cmd(ctx, path, keep_venv): """Execute 'startup' entrypoint in fedn.yaml. :param ctx: @@ -129,18 +129,17 @@ def startup_cmd(ctx, path,remove_venv): if "startup" not in config["entry_points"]: logger.error("No startup command defined in fedn.yaml") exit(-1) - dispatcher = Dispatcher(config, path) _ = dispatcher._get_or_create_python_env() dispatcher.run_cmd("startup") - delete_virtual_environment(remove_venv,dispatcher) + delete_virtual_environment(keep_venv, dispatcher) @run_cmd.command("build") @click.option("-p", "--path", required=True, help="Path to package directory containing fedn.yaml") -@click.option("-v", "--remove-venv", is_flag=True, default=True,required=False, help="flag if set to False doesn't remove venv") +@click.option("-v", "--keep-venv", is_flag=True, default=True, required=False, help="flag if set to False doesn't remove venv") @click.pass_context -def build_cmd(ctx, path,remove_venv): +def build_cmd(ctx, path, keep_venv): """Execute 'build' entrypoint in fedn.yaml. :param ctx: @@ -159,8 +158,7 @@ def build_cmd(ctx, path,remove_venv): dispatcher = Dispatcher(config, path) _ = dispatcher._get_or_create_python_env() dispatcher.run_cmd("build") - print(remove_venv) - delete_virtual_environment(remove_venv,dispatcher) + delete_virtual_environment(keep_venv, dispatcher) diff --git a/fedn/network/api/gunicorn_app.py b/fedn/network/api/gunicorn_app.py index d4e822629..8c5cb0c30 100644 --- a/fedn/network/api/gunicorn_app.py +++ b/fedn/network/api/gunicorn_app.py @@ -15,7 +15,7 @@ def load(self): return self.application def run_gunicorn(app, host,port,workers=4): - bind_address=str(host)+":"+str(port) + bind_address = f"{host}:{port}" options = { "bind": bind_address, # Specify the bind address and port here "workers": workers, diff --git a/fedn/network/api/server.py b/fedn/network/api/server.py index 0c9806d86..5ede51116 100644 --- a/fedn/network/api/server.py +++ b/fedn/network/api/server.py @@ -637,6 +637,6 @@ def start_server_api(): app.run(debug=debug, port=port, host=host) else: workers=os.cpu_count() - gunicorn_app.run_gunicorn(app,host,port,workers) + gunicorn_app.run_gunicorn(app, host, port, workers) if __name__ == "__main__": start_server_api() From 37c9bd1afc2bc2d22ff009de980de161d9da9cef Mon Sep 17 00:00:00 2001 From: sowmyasri Date: Sun, 18 Aug 2024 21:56:51 +0200 Subject: [PATCH 14/17] run_cmd updated with keep-venv flag --- fedn/cli/run_cmd.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/fedn/cli/run_cmd.py b/fedn/cli/run_cmd.py index ae33c566c..93323b575 100644 --- a/fedn/cli/run_cmd.py +++ b/fedn/cli/run_cmd.py @@ -44,7 +44,7 @@ def check_yaml_exists(path): exit(-1) return yaml_file def delete_virtual_environment(keep_venv, dispatcher): - if keep_venv: + if not keep_venv: # delete the virtualenv if dispatcher.python_env_path: logger.info(f"Removing virtualenv {dispatcher.python_env_path}") @@ -61,7 +61,7 @@ def run_cmd(ctx): @click.option("-p", "--path", required=True, help="Path to package directory containing fedn.yaml") @click.option("-i", "--input", required=True, help="Path to input model" ) @click.option("-o", "--output", required=True, help="Path to write the output JSON containing validation metrics") -@click.option("-v", "--keep-venv", is_flag=True, default=True, required=False, help="flag if set to False doesn't remove venv") +@click.option("-v", "--keep-venv", is_flag=True, required=False, help="flag if set to False doesn't remove venv") @click.pass_context def validate_cmd(ctx, path, input, output, keep_venv): """Execute 'validate' entrypoint in fedn.yaml. @@ -87,7 +87,7 @@ def validate_cmd(ctx, path, input, output, keep_venv): @click.option("-p", "--path", required=True, help="Path to package directory containing fedn.yaml") @click.option("-i", "--input", required=True, help="Path to input model parameters" ) @click.option("-o", "--output", required=True, help="Path to write the updated model parameters ") -@click.option("-v", "--keep-venv", is_flag=True, default=True, required=False, help="flag if set to False doesn't remove venv") +@click.option("-v", "--keep-venv", is_flag=True, required=False, help="flag if set to False doesn't remove venv") @click.pass_context def train_cmd(ctx, path, input, output, keep_venv): """Execute 'train' entrypoint in fedn.yaml. @@ -112,7 +112,7 @@ def train_cmd(ctx, path, input, output, keep_venv): @run_cmd.command("startup") @click.option("-p", "--path", required=True, help="Path to package directory containing fedn.yaml") -@click.option("-v", "--keep-venv", is_flag=True, default=True, required=False, help="flag if set to False doesn't remove venv") +@click.option("-v", "--keep-venv", is_flag=True, required=False, help="flag if set to False doesn't remove venv") @click.pass_context def startup_cmd(ctx, path, keep_venv): """Execute 'startup' entrypoint in fedn.yaml. @@ -137,7 +137,7 @@ def startup_cmd(ctx, path, keep_venv): @run_cmd.command("build") @click.option("-p", "--path", required=True, help="Path to package directory containing fedn.yaml") -@click.option("-v", "--keep-venv", is_flag=True, default=True, required=False, help="flag if set to False doesn't remove venv") +@click.option("-v", "--keep-venv", is_flag=True, required=False, help="flag if set to False doesn't remove venv") @click.pass_context def build_cmd(ctx, path, keep_venv): """Execute 'build' entrypoint in fedn.yaml. From e67db3021be96de33d31d3fc428c766703b953e8 Mon Sep 17 00:00:00 2001 From: sowmyasri Date: Mon, 19 Aug 2024 11:10:38 +0200 Subject: [PATCH 15/17] code clean up --- fedn/cli/run_cmd.py | 37 ++++++++++++++++++------------------- 1 file changed, 18 insertions(+), 19 deletions(-) diff --git a/fedn/cli/run_cmd.py b/fedn/cli/run_cmd.py index 93323b575..fb6179b7c 100644 --- a/fedn/cli/run_cmd.py +++ b/fedn/cli/run_cmd.py @@ -43,14 +43,12 @@ def check_yaml_exists(path): click.echo(f"Could not find fedn.yaml in {path}") exit(-1) return yaml_file -def delete_virtual_environment(keep_venv, dispatcher): - if not keep_venv: - # delete the virtualenv - if dispatcher.python_env_path: - logger.info(f"Removing virtualenv {dispatcher.python_env_path}") - shutil.rmtree(dispatcher.python_env_path) - else: - logger.warning("No virtualenv found to remove.") +def delete_virtual_environment(dispatcher): + if dispatcher.python_env_path: + logger.info(f"Removing virtualenv {dispatcher.python_env_path}") + shutil.rmtree(dispatcher.python_env_path) + else: + logger.warning("No virtualenv found to remove.") @main.group("run") @click.pass_context def run_cmd(ctx): @@ -61,7 +59,7 @@ def run_cmd(ctx): @click.option("-p", "--path", required=True, help="Path to package directory containing fedn.yaml") @click.option("-i", "--input", required=True, help="Path to input model" ) @click.option("-o", "--output", required=True, help="Path to write the output JSON containing validation metrics") -@click.option("-v", "--keep-venv", is_flag=True, required=False, help="flag if set to False doesn't remove venv") +@click.option("-v", "--keep-venv", is_flag=True, required=False, help="flag if set doesn't remove venv") @click.pass_context def validate_cmd(ctx, path, input, output, keep_venv): """Execute 'validate' entrypoint in fedn.yaml. @@ -82,12 +80,13 @@ def validate_cmd(ctx, path, input, output, keep_venv): dispatcher = Dispatcher(config, path) _ = dispatcher._get_or_create_python_env() dispatcher.run_cmd("validate {} {}".format(input, output)) - delete_virtual_environment(keep_venv, dispatcher) + if not keep_venv: + delete_virtual_environment(dispatcher) @run_cmd.command("train") @click.option("-p", "--path", required=True, help="Path to package directory containing fedn.yaml") @click.option("-i", "--input", required=True, help="Path to input model parameters" ) @click.option("-o", "--output", required=True, help="Path to write the updated model parameters ") -@click.option("-v", "--keep-venv", is_flag=True, required=False, help="flag if set to False doesn't remove venv") +@click.option("-v", "--keep-venv", is_flag=True, required=False, help="flag if set doesn't remove venv") @click.pass_context def train_cmd(ctx, path, input, output, keep_venv): """Execute 'train' entrypoint in fedn.yaml. @@ -108,11 +107,11 @@ def train_cmd(ctx, path, input, output, keep_venv): dispatcher = Dispatcher(config, path) _ = dispatcher._get_or_create_python_env() dispatcher.run_cmd("train {} {}".format(input, output)) - delete_virtual_environment(keep_venv, dispatcher) - + if not keep_venv: + delete_virtual_environment(dispatcher) @run_cmd.command("startup") @click.option("-p", "--path", required=True, help="Path to package directory containing fedn.yaml") -@click.option("-v", "--keep-venv", is_flag=True, required=False, help="flag if set to False doesn't remove venv") +@click.option("-v", "--keep-venv", is_flag=True, required=False, help="flag if set doesn't remove venv") @click.pass_context def startup_cmd(ctx, path, keep_venv): """Execute 'startup' entrypoint in fedn.yaml. @@ -132,12 +131,12 @@ def startup_cmd(ctx, path, keep_venv): dispatcher = Dispatcher(config, path) _ = dispatcher._get_or_create_python_env() dispatcher.run_cmd("startup") - delete_virtual_environment(keep_venv, dispatcher) - + if not keep_venv: + delete_virtual_environment(dispatcher) @run_cmd.command("build") @click.option("-p", "--path", required=True, help="Path to package directory containing fedn.yaml") -@click.option("-v", "--keep-venv", is_flag=True, required=False, help="flag if set to False doesn't remove venv") +@click.option("-v", "--keep-venv", is_flag=True, required=False, help="flag if set doesn't remove venv") @click.pass_context def build_cmd(ctx, path, keep_venv): """Execute 'build' entrypoint in fedn.yaml. @@ -158,8 +157,8 @@ def build_cmd(ctx, path, keep_venv): dispatcher = Dispatcher(config, path) _ = dispatcher._get_or_create_python_env() dispatcher.run_cmd("build") - delete_virtual_environment(keep_venv, dispatcher) - + if not keep_venv: + delete_virtual_environment(dispatcher) @run_cmd.command("client") From 317904cfb8a25fcdef398824f6c91b6fbf17a29d Mon Sep 17 00:00:00 2001 From: sowmyasri Date: Mon, 19 Aug 2024 11:22:23 +0200 Subject: [PATCH 16/17] updated help command --- fedn/cli/run_cmd.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/fedn/cli/run_cmd.py b/fedn/cli/run_cmd.py index ca74caebc..731f83624 100644 --- a/fedn/cli/run_cmd.py +++ b/fedn/cli/run_cmd.py @@ -59,7 +59,7 @@ def run_cmd(ctx): @click.option("-p", "--path", required=True, help="Path to package directory containing fedn.yaml") @click.option("-i", "--input", required=True, help="Path to input model" ) @click.option("-o", "--output", required=True, help="Path to write the output JSON containing validation metrics") -@click.option("-v", "--keep-venv", is_flag=True, required=False, help="flag if set doesn't remove venv") +@click.option("-v", "--keep-venv", is_flag=True, required=False, help="Use flag to keep the python virtual environment (python_env in fedn.yaml)") @click.pass_context def validate_cmd(ctx, path, input, output, keep_venv): """Execute 'validate' entrypoint in fedn.yaml. @@ -86,7 +86,7 @@ def validate_cmd(ctx, path, input, output, keep_venv): @click.option("-p", "--path", required=True, help="Path to package directory containing fedn.yaml") @click.option("-i", "--input", required=True, help="Path to input model parameters" ) @click.option("-o", "--output", required=True, help="Path to write the updated model parameters ") -@click.option("-v", "--keep-venv", is_flag=True, required=False, help="flag if set doesn't remove venv") +@click.option("-v", "--keep-venv", is_flag=True, required=False, help="Use flag to keep the python virtual environment (python_env in fedn.yaml)") @click.pass_context def train_cmd(ctx, path, input, output, keep_venv): """Execute 'train' entrypoint in fedn.yaml. @@ -111,7 +111,7 @@ def train_cmd(ctx, path, input, output, keep_venv): delete_virtual_environment(dispatcher) @run_cmd.command("startup") @click.option("-p", "--path", required=True, help="Path to package directory containing fedn.yaml") -@click.option("-v", "--keep-venv", is_flag=True, required=False, help="flag if set doesn't remove venv") +@click.option("-v", "--keep-venv", is_flag=True, required=False, help="Use flag to keep the python virtual environment (python_env in fedn.yaml)") @click.pass_context def startup_cmd(ctx, path, keep_venv): """Execute 'startup' entrypoint in fedn.yaml. @@ -136,7 +136,7 @@ def startup_cmd(ctx, path, keep_venv): @run_cmd.command("build") @click.option("-p", "--path", required=True, help="Path to package directory containing fedn.yaml") -@click.option("-v", "--keep-venv", is_flag=True, required=False, help="flag if set doesn't remove venv") +@click.option("-v", "--keep-venv", is_flag=True, required=False, help="Use flag to keep the python virtual environment (python_env in fedn.yaml)") @click.pass_context def build_cmd(ctx, path, keep_venv): """Execute 'build' entrypoint in fedn.yaml. From 7847b70b8c67b57453f355259b2b8e618b179ea7 Mon Sep 17 00:00:00 2001 From: sowmyasri Date: Mon, 2 Sep 2024 17:14:43 +0200 Subject: [PATCH 17/17] fixed indent issue --- fedn/network/api/server.py | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/fedn/network/api/server.py b/fedn/network/api/server.py index 5ede51116..ac87ab7b1 100644 --- a/fedn/network/api/server.py +++ b/fedn/network/api/server.py @@ -629,14 +629,14 @@ def list_combiners_data(): def start_server_api(): - config = get_controller_config() - port = config["port"] - host = "0.0.0.0" - debug = config["debug"] - if debug: - app.run(debug=debug, port=port, host=host) - else: - workers=os.cpu_count() - gunicorn_app.run_gunicorn(app, host, port, workers) + config = get_controller_config() + port = config["port"] + host = "0.0.0.0" + debug = config["debug"] + if debug: + app.run(debug=debug, port=port, host=host) + else: + workers=os.cpu_count() + gunicorn_app.run_gunicorn(app, host, port, workers) if __name__ == "__main__": start_server_api()