From 94971dd86ef3a821206d17a72be7e70e4e46a6a0 Mon Sep 17 00:00:00 2001 From: Fredrik Wrede Date: Wed, 4 Oct 2023 09:56:43 +0200 Subject: [PATCH] Feature/SK-545 | Statestore init from api-server lack proper setup (#479) --- docker-compose.yaml | 1 + fedn/cli/run_cmd.py | 2 +- fedn/fedn/common/config.py | 44 +++++++++++---- fedn/fedn/network/api/server.py | 8 ++- fedn/fedn/network/controller/controlbase.py | 9 +++ .../network/statestore/mongostatestore.py | 56 ++----------------- 6 files changed, 54 insertions(+), 66 deletions(-) diff --git a/docker-compose.yaml b/docker-compose.yaml index 3fa75ff3a..c8d3aff15 100644 --- a/docker-compose.yaml +++ b/docker-compose.yaml @@ -85,6 +85,7 @@ services: - PROJECT=project - FLASK_DEBUG=1 - STATESTORE_CONFIG=/app/config/settings-reducer.yaml + - MODELSTORAGE_CONFIG=/app/config/settings-reducer.yaml build: context: . args: diff --git a/fedn/cli/run_cmd.py b/fedn/cli/run_cmd.py index b6f09cab9..119b8de45 100644 --- a/fedn/cli/run_cmd.py +++ b/fedn/cli/run_cmd.py @@ -190,7 +190,7 @@ def dashboard_cmd(ctx, host, port, secret_key, local_package, name, init): statestore_config = fedn_config['statestore'] if statestore_config['type'] == 'MongoDB': statestore = MongoStateStore( - network_id, statestore_config['mongo_config'], defaults=config['init']) + network_id, statestore_config['mongo_config'], fedn_config['storage']) else: print("Unsupported statestore type, exiting. ", flush=True) exit(-1) diff --git a/fedn/fedn/common/config.py b/fedn/fedn/common/config.py index bf7321417..f6c827d0d 100644 --- a/fedn/fedn/common/config.py +++ b/fedn/fedn/common/config.py @@ -2,18 +2,33 @@ import yaml -STATESTORE_CONFIG = os.environ.get('STATESTORE_CONFIG', '/workspaces/fedn/config/settings-reducer.yaml.template') -MODELSTORAGE_CONFIG = os.environ.get('MODELSTORAGE_CONFIG', '/workspaces/fedn/config/settings-reducer.yaml.template') +global STATESTORE_CONFIG +global MODELSTORAGE_CONFIG -def get_statestore_config(file=STATESTORE_CONFIG): +def get_environment_config(): + """ Get the configuration from environment variables. + """ + global STATESTORE_CONFIG + global MODELSTORAGE_CONFIG + + STATESTORE_CONFIG = os.environ.get('STATESTORE_CONFIG', + '/workspaces/fedn/config/settings-reducer.yaml.template') + MODELSTORAGE_CONFIG = os.environ.get('MODELSTORAGE_CONFIG', + '/workspaces/fedn/config/settings-reducer.yaml.template') + + +def get_statestore_config(file=None): """ Get the statestore configuration from file. - :param file: The statestore configuration file (yaml) path. + :param file: The statestore configuration file (yaml) path (optional). :type file: str :return: The statestore configuration as a dict. :rtype: dict """ + if file is None: + get_environment_config() + file = STATESTORE_CONFIG with open(file, 'r') as config_file: try: settings = dict(yaml.safe_load(config_file)) @@ -22,14 +37,17 @@ def get_statestore_config(file=STATESTORE_CONFIG): return settings["statestore"] -def get_modelstorage_config(file=MODELSTORAGE_CONFIG): +def get_modelstorage_config(file=None): """ Get the model storage configuration from file. - :param file: The model storage configuration file (yaml) path. + :param file: The model storage configuration file (yaml) path (optional). :type file: str :return: The model storage configuration as a dict. :rtype: dict """ + if file is None: + get_environment_config() + file = MODELSTORAGE_CONFIG with open(file, 'r') as config_file: try: settings = dict(yaml.safe_load(config_file)) @@ -38,14 +56,17 @@ def get_modelstorage_config(file=MODELSTORAGE_CONFIG): return settings["storage"] -def get_network_config(file=STATESTORE_CONFIG): +def get_network_config(file=None): """ Get the network configuration from file. - :param file: The network configuration file (yaml) path. + :param file: The network configuration file (yaml) path (optional). :type file: str :return: The network id. :rtype: str """ + if file is None: + get_environment_config() + file = STATESTORE_CONFIG with open(file, 'r') as config_file: try: settings = dict(yaml.safe_load(config_file)) @@ -54,14 +75,17 @@ def get_network_config(file=STATESTORE_CONFIG): return settings["network_id"] -def get_controller_config(file=STATESTORE_CONFIG): +def get_controller_config(file=None): """ Get the controller configuration from file. - :param file: The controller configuration file (yaml) path. + :param file: The controller configuration file (yaml) path (optional). :type file: str :return: The controller configuration as a dict. :rtype: dict """ + if file is None: + get_environment_config() + file = STATESTORE_CONFIG with open(file, 'r') as config_file: try: settings = dict(yaml.safe_load(config_file)) diff --git a/fedn/fedn/network/api/server.py b/fedn/fedn/network/api/server.py index f841a49e8..4e0e93775 100644 --- a/fedn/fedn/network/api/server.py +++ b/fedn/fedn/network/api/server.py @@ -1,16 +1,18 @@ from flask import Flask, jsonify, request -from fedn.common.config import (get_controller_config, get_network_config, - get_statestore_config) +from fedn.common.config import (get_controller_config, get_modelstorage_config, + get_network_config, get_statestore_config) from fedn.network.api.interface import API from fedn.network.controller.control import Control from fedn.network.statestore.mongostatestore import MongoStateStore statestore_config = get_statestore_config() network_id = get_network_config() +modelstorage_config = get_modelstorage_config() statestore = MongoStateStore( network_id, - statestore_config['mongo_config'] + statestore_config['mongo_config'], + modelstorage_config ) control = Control(statestore=statestore) api = API(statestore, control) diff --git a/fedn/fedn/network/controller/controlbase.py b/fedn/fedn/network/controller/controlbase.py index f033b0b29..e38d31e38 100644 --- a/fedn/fedn/network/controller/controlbase.py +++ b/fedn/fedn/network/controller/controlbase.py @@ -10,6 +10,9 @@ from fedn.network.combiner.interfaces import CombinerUnavailableError from fedn.network.state import ReducerState +# Maximum number of tries to connect to statestore and retrieve storage configuration +MAX_TRIES_BACKEND = os.getenv('MAX_TRIES_BACKEND', 10) + class UnsupportedStorageBackend(Exception): pass @@ -42,12 +45,18 @@ def __init__(self, statestore): try: not_ready = True + tries = 0 while not_ready: storage_config = self.statestore.get_storage_backend() if storage_config: not_ready = False else: + print( + "REDUCER CONTROL: Storage backend not configured, waiting...", flush=True) sleep(5) + tries += 1 + if tries > MAX_TRIES_BACKEND: + raise Exception except Exception: print( "REDUCER CONTROL: Failed to retrive storage configuration, exiting.", flush=True) diff --git a/fedn/fedn/network/statestore/mongostatestore.py b/fedn/fedn/network/statestore/mongostatestore.py index 41932c5a5..f991701d4 100644 --- a/fedn/fedn/network/statestore/mongostatestore.py +++ b/fedn/fedn/network/statestore/mongostatestore.py @@ -2,7 +2,6 @@ from datetime import datetime import pymongo -import yaml from fedn.common.storage.db.mongo import connect_to_mongodb from fedn.network.state import ReducerStateToString, StringToReducerState @@ -21,7 +20,7 @@ class MongoStateStore(StateStoreBase): :type defaults: dict """ - def __init__(self, network_id, config, defaults=None): + def __init__(self, network_id, config, model_storage_config): """ Constructor.""" self.__inited = False try: @@ -58,56 +57,9 @@ def __init__(self, network_id, config, defaults=None): self.clients = None raise - if defaults: - with open(defaults, 'r') as file: - try: - settings = dict(yaml.safe_load(file)) - print(settings, flush=True) - - # Control settings - if "control" in settings and settings["control"]: - control = settings['control'] - try: - self.transition(str(control['state'])) - except KeyError: - self.transition("idle") - - if "model" in control: - if not self.get_latest(): - self.set_latest(str(control['model'])) - else: - print( - "Model trail already initialized - refusing to overwrite from config. Purge model trail if you want to reseed the system.", - flush=True) - - if "context" in control: - print("Setting filepath to {}".format( - control['context']), flush=True) - # TODO Fix the ugly latering of indirection due to a bug in secure_filename returning an object with filename as attribute - # TODO fix with unboxing of value before storing and where consuming. - self.control.config.update_one({'key': 'package'}, - {'$set': {'filename': control['context']}}, True) - if "helper" in control: - # self.set_framework(control['helper']) - pass - - round_config = {'timeout': 180, 'validate': True} - try: - round_config['timeout'] = control['timeout'] - except Exception: - pass - - try: - round_config['validate'] = control['validate'] - except Exception: - pass - - # Storage settings - self.set_storage_backend(settings['storage']) - - self.__inited = True - except yaml.YAMLError as e: - print(e) + # Storage settings + self.set_storage_backend(model_storage_config) + self.__inited = True def is_inited(self): """ Check if the statestore is intialized.