diff --git a/.vscode/settings.json b/.vscode/settings.json index 07cfc57ae..64a8a26af 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -3,6 +3,13 @@ "editor.codeActionsOnSave": { "source.organizeImports": true }, + "[python]": { + "editor.defaultFormatter": "ms-python.autopep8", + "editor.formatOnSave": true, + "editor.codeActionsOnSave": { + "source.organizeImports": true + }, + }, "python.linting.enabled": true, "python.linting.flake8Enabled": true, } \ No newline at end of file diff --git a/fedn/cli/run_cmd.py b/fedn/cli/run_cmd.py index cc41177f9..5f9d93a05 100644 --- a/fedn/cli/run_cmd.py +++ b/fedn/cli/run_cmd.py @@ -183,7 +183,7 @@ def dashboard_cmd(ctx, host, port, secret_key, local_package, name, init): # Read settings from config file try: fedn_config = get_statestore_config_from_file(config['init']) - except Exception as e: + except Exception: print('Failed to read config from settings file, trying default values.', flush=True) fedn_config = get_default_config() @@ -296,4 +296,4 @@ def controller_cmd(ctx): """ """ controller = Controller() - controller.run() \ No newline at end of file + controller.run() diff --git a/fedn/fedn/common/config.py b/fedn/fedn/common/config.py index 6d57205fa..545e13cab 100644 --- a/fedn/fedn/common/config.py +++ b/fedn/fedn/common/config.py @@ -2,11 +2,10 @@ import yaml -from fedn.common.log_config import logger - global STATESTORE_CONFIG global MODELSTORAGE_CONFIG + def get_default_config(): statestore_config = { "statestore": { @@ -45,6 +44,7 @@ def get_default_config(): fedn_config.update(storage_config) return fedn_config + def get_environment_config(): """ Get the configuration from environment variables. """ @@ -65,7 +65,7 @@ def get_statestore_config(file=None): """ if file is None: get_environment_config() - if STATESTORE_CONFIG: + if STATESTORE_CONFIG: file = STATESTORE_CONFIG else: fedn_config = get_default_config() diff --git a/fedn/fedn/common/log_config.py b/fedn/fedn/common/log_config.py index fbc3541cd..a9c0f24e9 100644 --- a/fedn/fedn/common/log_config.py +++ b/fedn/fedn/common/log_config.py @@ -21,6 +21,7 @@ except ImportError: telemetry_enabled = False + def get_system_info(): system_info = [ ["os.name", os.name], @@ -34,6 +35,7 @@ def get_system_info(): ] return system_info + # Configure the tracer to export traces to Jaeger resource = Resource.create({ResourceAttributes.SERVICE_NAME: "FEDn Client"}) tracer_provider = TracerProvider(resource=resource) @@ -53,7 +55,6 @@ def get_system_info(): tracer = None - urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning) logging.getLogger("urllib3").setLevel(logging.ERROR) @@ -64,6 +65,7 @@ def get_system_info(): formatter = logging.Formatter('%(asctime)s [%(levelname)s] %(message)s', datefmt='%Y-%m-%d %H:%M:%S') handler.setFormatter(formatter) + def add_trace(name=""): def decorator(func): @wraps(func) @@ -71,7 +73,7 @@ def wrapper(*args, **kwargs): self = args[0] name = func.__name__ if tracer: - + with tracer.start_as_current_span(name) as span: # print("name={}....{}".format(name, attributes)) if self.trace_attribs: @@ -87,19 +89,23 @@ def wrapper(*args, **kwargs): return wrapper return decorator + def get_tracer(): global tracer return tracer + def enable_tracing(): global tracer tracer = trace.get_tracer(__name__) + def log_remote(server='localhost:8000', path='/log'): http_handler = logging.handlers.HTTPHandler(server, '/log', method='POST') http_handler.setLevel(logging.WARNING) logger.addHandler(http_handler) + def set_log_level_from_string(level_str): """ Set the log level based on a string input. diff --git a/fedn/fedn/common/storage/db/mongo.py b/fedn/fedn/common/storage/db/mongo.py index f0666f886..e7cbf5b8f 100644 --- a/fedn/fedn/common/storage/db/mongo.py +++ b/fedn/fedn/common/storage/db/mongo.py @@ -1,5 +1,6 @@ import pymongo + def connect_to_mongodb(config, network_id): """ diff --git a/fedn/fedn/common/storage/filesystem/filesystem.py b/fedn/fedn/common/storage/filesystem/filesystem.py index 23ec48455..5088892fb 100644 --- a/fedn/fedn/common/storage/filesystem/filesystem.py +++ b/fedn/fedn/common/storage/filesystem/filesystem.py @@ -1,6 +1,7 @@ import os import uuid + class LocalFileSystemModelRepository: def __init__(self, directory='./'): self.directory = directory @@ -63,4 +64,4 @@ def delete_compute_package(self, compute_package): if os.path.exists(package_path): os.remove(package_path) else: - raise FileNotFoundError(f"Compute package {compute_package} not found.") \ No newline at end of file + raise FileNotFoundError(f"Compute package {compute_package} not found.") diff --git a/fedn/fedn/network/api/interface.py b/fedn/fedn/network/api/interface.py index 59e4934dc..425f37762 100644 --- a/fedn/fedn/network/api/interface.py +++ b/fedn/fedn/network/api/interface.py @@ -314,7 +314,7 @@ def download_compute_package(self, name): return send_from_directory( self.local_path, name, as_attachment=True ) - except Exception as err: + except Exception: try: data = self.control.get_compute_package(name) file_path = os.path.join(self.local_path, name) @@ -323,7 +323,7 @@ def download_compute_package(self, name): return send_from_directory( self.local_path, name, as_attachment=True ) - except Exception as err: + except Exception: raise finally: mutex.release() diff --git a/fedn/fedn/network/api/server.py b/fedn/fedn/network/api/server.py index 90f50096a..01ede8f65 100644 --- a/fedn/fedn/network/api/server.py +++ b/fedn/fedn/network/api/server.py @@ -1,7 +1,8 @@ from flask import Flask, jsonify, request -from fedn.common.config import (get_controller_config, get_modelstorage_config, - get_network_config, get_statestore_config, get_default_config) +from fedn.common.config import (get_controller_config, get_default_config, + get_modelstorage_config, get_network_config, + get_statestore_config) from fedn.common.log_config import logger from fedn.network.api.interface import API from fedn.network.controller.control import Control @@ -13,19 +14,19 @@ def __init__(self): self.app = Flask(__name__) try: statestore_config = get_statestore_config() - except FileNotFoundError as err: + except (FileNotFoundError, TypeError): logger.debug("No statestore config, using default values.") fedn_config = get_default_config() statestore_config = fedn_config['statestore'] try: network_id = get_network_config() - except (FileNotFoundError, TypeError) as err: + except (FileNotFoundError, TypeError): logger.debug("No network config found, using default values.") fedn_config = get_default_config() network_id = fedn_config['network_id'] try: modelstorage_config = get_modelstorage_config() - except (FileNotFoundError, TypeError) as err: + except (FileNotFoundError, TypeError): logger.debug("No model storage config found, using default values.") fedn_config = get_default_config() modelstorage_config = fedn_config['storage_config'] @@ -36,7 +37,7 @@ def __init__(self): self.api = API(statestore, control) self.setup_routes() - + def setup_routes(self): @self.app.route("/get_model_trail", methods=["GET"]) def get_model_trail(): @@ -48,7 +49,6 @@ def get_model_trail(): """ return self.api.get_model_trail() - @self.app.route("/list_models", methods=["GET"]) def list_models(): """Get models from the statestore. @@ -68,7 +68,6 @@ def list_models(): return self.api.get_models(session_id, limit, skip) - @self.app.route("/delete_model_trail", methods=["GET", "POST"]) def delete_model_trail(): """Delete the model trail for a given session. @@ -79,7 +78,6 @@ def delete_model_trail(): """ return jsonify({"message": "Not implemented"}), 501 - @self.app.route("/list_clients", methods=["GET"]) def list_clients(): """Get all clients from the statestore. @@ -93,7 +91,6 @@ def list_clients(): return self.api.get_clients(limit, skip, status) - @self.app.route("/get_active_clients", methods=["GET"]) def get_active_clients(): """Get all active clients from the statestore. @@ -110,7 +107,6 @@ def get_active_clients(): ) return self.api.get_active_clients(combiner_id) - @self.app.route("/list_combiners", methods=["GET"]) def list_combiners(): """Get all combiners in the network. @@ -123,7 +119,6 @@ def list_combiners(): return self.api.get_all_combiners(limit, skip) - @self.app.route("/get_combiner", methods=["GET"]) def get_combiner(): """Get a combiner from the statestore. @@ -140,7 +135,6 @@ def get_combiner(): ) return self.api.get_combiner(combiner_id) - @self.app.route("/list_rounds", methods=["GET"]) def list_rounds(): """Get all rounds from the statestore. @@ -149,7 +143,6 @@ def list_rounds(): """ return self.api.get_all_rounds() - @self.app.route("/get_round", methods=["GET"]) def get_round(): """Get a round from the statestore. @@ -163,7 +156,6 @@ def get_round(): return jsonify({"success": False, "message": "Missing round id."}), 400 return self.api.get_round(round_id) - @self.app.route("/start_session", methods=["GET", "POST"]) def start_session(): """Start a new session. @@ -173,7 +165,6 @@ def start_session(): json_data = request.get_json() return self.api.start_session(**json_data) - @self.app.route("/list_sessions", methods=["GET"]) def list_sessions(): """Get all sessions from the statestore. @@ -185,7 +176,6 @@ def list_sessions(): return self.api.get_all_sessions(limit, skip) - @self.app.route("/get_session", methods=["GET"]) def get_session(): """Get a session from the statestore. @@ -202,7 +192,6 @@ def get_session(): ) return self.api.get_session(session_id) - @self.app.route("/set_package", methods=["POST"]) def set_package(): """ Set the compute package in the statestore. @@ -229,7 +218,6 @@ def set_package(): return jsonify({"success": False, "message": "Missing file."}), 400 return self.api.set_compute_package(file=file, helper_type=helper_type) - @self.app.route("/get_package", methods=["GET"]) def get_package(): """Get the compute package from the statestore. @@ -238,7 +226,6 @@ def get_package(): """ return self.api.get_compute_package() - @self.app.route("/download_package", methods=["GET"]) def download_package(): """Download the compute package. @@ -248,13 +235,11 @@ def download_package(): name = request.args.get("name", None) return self.api.download_compute_package(name) - @self.app.route("/get_package_checksum", methods=["GET"]) def get_package_checksum(): name = request.args.get("name", None) return self.api.get_checksum(name) - @self.app.route("/get_latest_model", methods=["GET"]) def get_latest_model(): """Get the latest model from the statestore. @@ -263,10 +248,8 @@ def get_latest_model(): """ return self.api.get_latest_model() - # Get initial model endpoint - @self.app.route("/get_initial_model", methods=["GET"]) def get_initial_model(): """Get the initial model from the statestore. @@ -275,7 +258,6 @@ def get_initial_model(): """ return self.api.get_initial_model() - @self.app.route("/set_initial_model", methods=["POST"]) def set_initial_model(): """Set the initial model in the statestore and upload to model repository. @@ -295,7 +277,6 @@ def set_initial_model(): return jsonify({"success": False, "message": "Missing file."}), 400 return self.api.set_initial_model(file) - @self.app.route("/get_controller_status", methods=["GET"]) def get_controller_status(): """Get the status of the controller. @@ -304,7 +285,6 @@ def get_controller_status(): """ return self.api.get_controller_status() - @self.app.route("/get_client_config", methods=["GET"]) def get_client_config(): """Get the client configuration. @@ -314,7 +294,6 @@ def get_client_config(): checksum = request.args.get("checksum", True) return self.api.get_client_config(checksum) - @self.app.route("/get_events", methods=["GET"]) def get_events(): """Get the events from the statestore. @@ -326,7 +305,6 @@ def get_events(): return self.api.get_events(**kwargs) - @self.app.route("/list_validations", methods=["GET"]) def list_validations(): """Get all validations from the statestore. @@ -337,7 +315,6 @@ def list_validations(): kwargs = request.args.to_dict() return self.api.get_all_validations(**kwargs) - @self.app.route("/add_combiner", methods=["POST"]) def add_combiner(): """Add a combiner to the network. @@ -352,7 +329,6 @@ def add_combiner(): return jsonify({"success": False, "message": str(e)}), 400 return response - @self.app.route("/add_client", methods=["POST"]) def add_client(): """Add a client to the network. @@ -368,7 +344,6 @@ def add_client(): return jsonify({"success": False, "message": str(e)}), 400 return response - @self.app.route("/list_combiners_data", methods=["POST"]) def list_combiners_data(): """List data from combiners. @@ -387,7 +362,6 @@ def list_combiners_data(): return jsonify({"success": False, "message": str(e)}), 400 return response - @self.app.route("/get_plot_data", methods=["GET"]) def get_plot_data(): """Get plot data from the statestore. @@ -404,7 +378,7 @@ def get_plot_data(): def run(self): try: config = get_controller_config() - except (FileNotFoundError, TypeError) as err: + except (FileNotFoundError, TypeError): logger.debug("Found no controller config, using default values.") fedn_config = get_default_config() config = fedn_config['controller'] @@ -412,6 +386,7 @@ def run(self): debug = config["debug"] self.app.run(debug=debug, port=port, host="0.0.0.0") + if __name__ == "__main__": controller = Controller() controller.run() diff --git a/fedn/fedn/network/clients/package.py b/fedn/fedn/network/clients/package.py index 61fdc80ed..329032d20 100644 --- a/fedn/fedn/network/clients/package.py +++ b/fedn/fedn/network/clients/package.py @@ -3,8 +3,8 @@ # import cgi import os -import tarfile import sys +import tarfile from distutils.dir_util import copy_tree import requests diff --git a/fedn/fedn/network/combiner/server.py b/fedn/fedn/network/combiner/server.py index ddb4d5d37..33a3fa2d8 100644 --- a/fedn/fedn/network/combiner/server.py +++ b/fedn/fedn/network/combiner/server.py @@ -13,8 +13,9 @@ import fedn.common.net.grpc.fedn_pb2 as fedn import fedn.common.net.grpc.fedn_pb2_grpc as rpc from fedn.common.net.grpc.server import Server +from fedn.common.storage.filesystem.filesystem import \ + LocalFileSystemModelRepository from fedn.common.storage.s3.s3repo import S3ModelRepository -from fedn.common.storage.filesystem.filesystem import LocalFileSystemModelRepository from fedn.common.tracer.mongotracer import MongoTracer from fedn.network.combiner.connect import ConnectorCombiner, Status from fedn.network.combiner.modelservice import ModelService diff --git a/fedn/fedn/network/controller/controlbase.py b/fedn/fedn/network/controller/controlbase.py index 5012e26e5..48e9f43c1 100644 --- a/fedn/fedn/network/controller/controlbase.py +++ b/fedn/fedn/network/controller/controlbase.py @@ -4,8 +4,9 @@ from time import sleep import fedn.utils.helpers +from fedn.common.storage.filesystem.filesystem import \ + LocalFileSystemModelRepository from fedn.common.storage.s3.s3repo import S3ModelRepository -from fedn.common.storage.filesystem.filesystem import LocalFileSystemModelRepository from fedn.common.tracer.mongotracer import MongoTracer from fedn.network.api.network import Network from fedn.network.combiner.interfaces import CombinerUnavailableError