From 49e53d21f6cc6ff1c67d9d0b72df705b8e12f11a Mon Sep 17 00:00:00 2001 From: sowmyasris Date: Thu, 11 Jul 2024 10:06:41 +0200 Subject: [PATCH] Feature/SK-932 | Add fedn controller start (#651) --- .github/workflows/code-checks.yaml | 1 + docker-compose.yaml | 2 +- fedn/cli/__init__.py | 1 + fedn/cli/controller_cmd.py | 18 ++++++++++++++ fedn/cli/main.py | 9 ++----- fedn/common/config.py | 14 ++++++++--- fedn/common/settings-controller.yaml.template | 24 +++++++++++++++++++ fedn/network/api/server.py | 11 +++++++-- fedn/network/api/shared.py | 1 - fedn/network/api/v1/shared.py | 3 +-- fedn/utils/dist.py | 17 +++++++++++++ 11 files changed, 85 insertions(+), 16 deletions(-) create mode 100644 fedn/cli/controller_cmd.py create mode 100644 fedn/common/settings-controller.yaml.template create mode 100644 fedn/utils/dist.py diff --git a/.github/workflows/code-checks.yaml b/.github/workflows/code-checks.yaml index dbe378c63..8c48a3015 100644 --- a/.github/workflows/code-checks.yaml +++ b/.github/workflows/code-checks.yaml @@ -25,6 +25,7 @@ jobs: --exclude-dir='docs' --exclude-dir='flower-client' --exclude='tests.py' + --exclude='controller_cmd.py' --exclude='README.rst' '^[ \t]+(import|from) ' -I . diff --git a/docker-compose.yaml b/docker-compose.yaml index 85386c6da..c3620e79d 100644 --- a/docker-compose.yaml +++ b/docker-compose.yaml @@ -78,7 +78,7 @@ services: - mongo entrypoint: [ "sh", "-c" ] command: - - "/venv/bin/pip install --no-cache-dir -e . && /venv/bin/python fedn/network/api/server.py" + - "/venv/bin/pip install --no-cache-dir -e . && /venv/bin/fedn controller start" ports: - 8092:8092 diff --git a/fedn/cli/__init__.py b/fedn/cli/__init__.py index 137fc9b9c..bcd27dc53 100644 --- a/fedn/cli/__init__.py +++ b/fedn/cli/__init__.py @@ -9,3 +9,4 @@ from .session_cmd import session_cmd # noqa: F401 from .status_cmd import status_cmd # noqa: F401 from .validation_cmd import validation_cmd # noqa: F401 +from .controller_cmd import controller_cmd # noqa: F401 diff --git a/fedn/cli/controller_cmd.py b/fedn/cli/controller_cmd.py new file mode 100644 index 000000000..ab8727b27 --- /dev/null +++ b/fedn/cli/controller_cmd.py @@ -0,0 +1,18 @@ +import click + +from .main import main + + +@main.group("controller") +@click.pass_context +def controller_cmd(ctx): + """:param ctx:""" + pass + + +@controller_cmd.command("start") +@click.pass_context +def controller_cmd(ctx): + from fedn.network.api.server import start_server_api + + start_server_api() diff --git a/fedn/cli/main.py b/fedn/cli/main.py index 0d5660c0b..ab1dd448e 100644 --- a/fedn/cli/main.py +++ b/fedn/cli/main.py @@ -1,5 +1,4 @@ -import importlib.metadata - +from fedn.utils.dist import get_version import click CONTEXT_SETTINGS = dict( @@ -7,11 +6,7 @@ help_option_names=["-h", "--help"], ) -# Dynamically get the version of the package -try: - version = importlib.metadata.version("fedn") -except importlib.metadata.PackageNotFoundError: - version = "unknown" +version=get_version("fedn") @click.group(context_settings=CONTEXT_SETTINGS) diff --git a/fedn/common/config.py b/fedn/common/config.py index 94b346d65..23d873ff7 100644 --- a/fedn/common/config.py +++ b/fedn/common/config.py @@ -2,6 +2,8 @@ import yaml +from fedn.utils.dist import get_package_path + SECRET_KEY = os.environ.get("FEDN_JWT_SECRET_KEY", False) FEDN_JWT_CUSTOM_CLAIM_KEY = os.environ.get("FEDN_JWT_CUSTOM_CLAIM_KEY", False) FEDN_JWT_CUSTOM_CLAIM_VALUE = os.environ.get("FEDN_JWT_CUSTOM_CLAIM_VALUE", False) @@ -20,9 +22,15 @@ def get_environment_config(): """Get the configuration from environment variables.""" global STATESTORE_CONFIG global MODELSTORAGE_CONFIG - - STATESTORE_CONFIG = os.environ.get("STATESTORE_CONFIG", "/workspaces/fedn/config/settings-reducer.yaml.template") - MODELSTORAGE_CONFIG = os.environ.get("MODELSTORAGE_CONFIG", "/workspaces/fedn/config/settings-reducer.yaml.template") + if not os.environ.get("STATESTORE_CONFIG", False): + STATESTORE_CONFIG = get_package_path() + "/common/settings-controller.yaml.template" + else: + STATESTORE_CONFIG = os.environ.get("STATESTORE_CONFIG") + + if not os.environ.get("MODELSTORAGE_CONFIG", False): + MODELSTORAGE_CONFIG = get_package_path() + "/common/settings-controller.yaml.template" + else: + MODELSTORAGE_CONFIG = os.environ.get("MODELSTORAGE_CONFIG") def get_statestore_config(file=None): diff --git a/fedn/common/settings-controller.yaml.template b/fedn/common/settings-controller.yaml.template new file mode 100644 index 000000000..a5266a38b --- /dev/null +++ b/fedn/common/settings-controller.yaml.template @@ -0,0 +1,24 @@ +network_id: fedn-network +controller: + host: localhost + port: 8092 + debug: True + +statestore: + type: MongoDB + mongo_config: + username: fedn_admin + password: password + host: localhost + port: 6534 + +storage: + storage_type: S3 + storage_config: + storage_hostname: localhost + storage_port: 9000 + storage_access_key: fedn_admin + storage_secret_key: password + storage_bucket: fedn-models + context_bucket: fedn-context + storage_secure_mode: False diff --git a/fedn/network/api/server.py b/fedn/network/api/server.py index c9e54ff87..d56c3ab0b 100644 --- a/fedn/network/api/server.py +++ b/fedn/network/api/server.py @@ -9,6 +9,7 @@ from fedn.network.api.v1 import _routes custom_url_prefix = os.environ.get("FEDN_CUSTOM_URL_PREFIX", False) +# statestore_config,modelstorage_config,network_id,control=set_statestore_config() api = API(statestore, control) app = Flask(__name__) for bp in _routes: @@ -625,8 +626,14 @@ def list_combiners_data(): if custom_url_prefix: app.add_url_rule(f"{custom_url_prefix}/list_combiners_data", view_func=list_combiners_data, methods=["POST"]) -if __name__ == "__main__": + +def start_server_api(): config = get_controller_config() port = config["port"] debug = config["debug"] - app.run(debug=debug, port=port, host="0.0.0.0") + host = "0.0.0.0" + app.run(debug=debug, port=port, host=host) + + +if __name__ == "__main__": + start_server_api() diff --git a/fedn/network/api/shared.py b/fedn/network/api/shared.py index fc8d4ae57..9e0e5acbd 100644 --- a/fedn/network/api/shared.py +++ b/fedn/network/api/shared.py @@ -5,7 +5,6 @@ statestore_config = get_statestore_config() modelstorage_config = get_modelstorage_config() network_id = get_network_config() - statestore = MongoStateStore(network_id, statestore_config["mongo_config"]) statestore.set_storage_backend(modelstorage_config) control = Control(statestore=statestore) diff --git a/fedn/network/api/v1/shared.py b/fedn/network/api/v1/shared.py index b7ae170af..a27a6f637 100644 --- a/fedn/network/api/v1/shared.py +++ b/fedn/network/api/v1/shared.py @@ -3,10 +3,9 @@ import pymongo from pymongo.database import Database -from fedn.network.api.shared import statestore_config, network_id +from fedn.network.api.shared import statestore_config,network_id api_version = "v1" - mc = pymongo.MongoClient(**statestore_config["mongo_config"]) mc.server_info() mdb: Database = mc[network_id] diff --git a/fedn/utils/dist.py b/fedn/utils/dist.py new file mode 100644 index 000000000..e5fa7192b --- /dev/null +++ b/fedn/utils/dist.py @@ -0,0 +1,17 @@ +import importlib.metadata + +import fedn + + +def get_version(pacakge): + # Dynamically get the version of the package + try: + version = importlib.metadata.version("fedn") + except importlib.metadata.PackageNotFoundError: + version = "unknown" + return version + + +def get_package_path(): + # Get the path of the package + return fedn.__path__[0]