Merge master and fix conflicts

.github/workflows/build-containers.yaml

@@ -1,6 +1,7 @@
 name: "build containers"
 
 on:
+  workflow_dispatch:
   push:
     branches:
       - master

.github/workflows/code-checks.yaml

@@ -25,6 +25,7 @@ jobs:
           --exclude-dir='docs'
           --exclude-dir='flower-client'
           --exclude='tests.py'
+          --exclude='controller_cmd.py'
           --exclude='README.rst'
           '^[ \t]+(import|from) ' -I . 
 

.github/workflows/push-to-pypi.yaml

@@ -1,8 +1,9 @@
 name: Publish Python distribution to PyPI
 
 on:
+  workflow_dispatch:
   release:
-    types: [created]
+    types: published
 
 jobs:
   build-and-publish:

docker-compose.yaml

@@ -78,7 +78,7 @@ services:
       - mongo
     entrypoint: [ "sh", "-c" ]
     command:
-      - "/venv/bin/pip install --no-cache-dir -e . && /venv/bin/python fedn/network/api/server.py"
+      - "/venv/bin/pip install --no-cache-dir -e . && /venv/bin/fedn controller start"
     ports:
       - 8092:8092
 

docs/conf.py

@@ -12,7 +12,7 @@
 author = "Scaleout Systems AB"
 
 # The full version, including alpha/beta/rc tags
-release = "0.11.0"
+release = "0.11.1"
 
 # Add any Sphinx extension module names here, as strings
 extensions = [

fedn/cli/__init__.py

@@ -9,3 +9,4 @@
 from .session_cmd import session_cmd  # noqa: F401
 from .status_cmd import status_cmd  # noqa: F401
 from .validation_cmd import validation_cmd  # noqa: F401
+from .controller_cmd import controller_cmd  # noqa: F401

fedn/cli/controller_cmd.py

@@ -0,0 +1,18 @@
+import click
+
+from .main import main
+
+
+@main.group("controller")
+@click.pass_context
+def controller_cmd(ctx):
+    """:param ctx:"""
+    pass
+
+
+@controller_cmd.command("start")
+@click.pass_context
+def controller_cmd(ctx):
+    from fedn.network.api.server import start_server_api
+
+    start_server_api()

fedn/cli/main.py

@@ -1,17 +1,12 @@
-import importlib.metadata
-
+from fedn.utils.dist import get_version
 import click
 
 CONTEXT_SETTINGS = dict(
     # Support -h as a shortcut for --help
     help_option_names=["-h", "--help"],
 )
 
-# Dynamically get the version of the package
-try:
-    version = importlib.metadata.version("fedn")
-except importlib.metadata.PackageNotFoundError:
-    version = "unknown"
+version=get_version("fedn")
 
 
 @click.group(context_settings=CONTEXT_SETTINGS)

fedn/common/config.py

@@ -2,6 +2,8 @@
 
 import yaml
 
+from fedn.utils.dist import get_package_path
+
 SECRET_KEY = os.environ.get("FEDN_JWT_SECRET_KEY", False)
 FEDN_JWT_CUSTOM_CLAIM_KEY = os.environ.get("FEDN_JWT_CUSTOM_CLAIM_KEY", False)
 FEDN_JWT_CUSTOM_CLAIM_VALUE = os.environ.get("FEDN_JWT_CUSTOM_CLAIM_VALUE", False)
@@ -20,9 +22,15 @@ def get_environment_config():
     """Get the configuration from environment variables."""
     global STATESTORE_CONFIG
     global MODELSTORAGE_CONFIG
-
-    STATESTORE_CONFIG = os.environ.get("STATESTORE_CONFIG", "/workspaces/fedn/config/settings-reducer.yaml.template")
-    MODELSTORAGE_CONFIG = os.environ.get("MODELSTORAGE_CONFIG", "/workspaces/fedn/config/settings-reducer.yaml.template")
+    if not os.environ.get("STATESTORE_CONFIG", False):
+        STATESTORE_CONFIG = get_package_path() + "/common/settings-controller.yaml.template"
+    else:
+        STATESTORE_CONFIG = os.environ.get("STATESTORE_CONFIG")
+
+    if not os.environ.get("MODELSTORAGE_CONFIG", False):
+        MODELSTORAGE_CONFIG = get_package_path() + "/common/settings-controller.yaml.template"
+    else:
+        MODELSTORAGE_CONFIG = os.environ.get("MODELSTORAGE_CONFIG")
 
 
 def get_statestore_config(file=None):

fedn/common/settings-controller.yaml.template

@@ -0,0 +1,24 @@
+network_id: fedn-network
+controller:
+  host: localhost
+  port: 8092
+  debug: True
+
+statestore:
+  type: MongoDB
+  mongo_config:
+    username: fedn_admin
+    password: password
+    host: localhost
+    port: 6534
+
+storage:
+  storage_type: S3
+  storage_config:
+    storage_hostname: localhost
+    storage_port: 9000
+    storage_access_key: fedn_admin
+    storage_secret_key: password
+    storage_bucket: fedn-models
+    context_bucket: fedn-context
+    storage_secure_mode: False 

fedn/network/api/server.py

@@ -9,6 +9,7 @@
 from fedn.network.api.v1 import _routes
 
 custom_url_prefix = os.environ.get("FEDN_CUSTOM_URL_PREFIX", False)
+# statestore_config,modelstorage_config,network_id,control=set_statestore_config()
 api = API(statestore, control)
 app = Flask(__name__)
 for bp in _routes:
@@ -625,8 +626,14 @@ def list_combiners_data():
 if custom_url_prefix:
     app.add_url_rule(f"{custom_url_prefix}/list_combiners_data", view_func=list_combiners_data, methods=["POST"])
 
-if __name__ == "__main__":
+
+def start_server_api():
     config = get_controller_config()
     port = config["port"]
     debug = config["debug"]
-    app.run(debug=debug, port=port, host="0.0.0.0")
+    host = "0.0.0.0"
+    app.run(debug=debug, port=port, host=host)
+
+
+if __name__ == "__main__":
+    start_server_api()

fedn/network/api/shared.py

@@ -5,7 +5,6 @@
 statestore_config = get_statestore_config()
 modelstorage_config = get_modelstorage_config()
 network_id = get_network_config()
-
 statestore = MongoStateStore(network_id, statestore_config["mongo_config"])
 statestore.set_storage_backend(modelstorage_config)
 control = Control(statestore=statestore)

fedn/network/api/v1/shared.py

@@ -3,10 +3,9 @@
 import pymongo
 from pymongo.database import Database
 
-from fedn.network.api.shared import statestore_config, network_id
+from fedn.network.api.shared import statestore_config,network_id
 
 api_version = "v1"
-
 mc = pymongo.MongoClient(**statestore_config["mongo_config"])
 mc.server_info()
 mdb: Database = mc[network_id]

fedn/network/combiner/combiner.py

@@ -127,7 +127,10 @@ def __init__(self, config):
         # Set the status to offline for previous clients.
         previous_clients = self.statestore.clients.find({"combiner": config["name"]})
         for client in previous_clients:
-            self.statestore.set_client({"name": client["name"], "status": "offline", "client_id": client["client_id"]})
+            try:
+                self.statestore.set_client({"name": client["name"], "status": "offline", "client_id": client["client_id"]})
+            except KeyError:
+                self.statestore.set_client({"name": client["name"], "status": "offline"})
 
         self.modelservice = ModelService()
 

fedn/network/storage/statestore/mongostatestore.py

@@ -738,7 +738,13 @@ def set_client(self, client_data):
         :return:
         """
         client_data["updated_at"] = str(datetime.now())
-        self.clients.update_one({"client_id": client_data["client_id"]}, {"$set": client_data}, True)
+        try:
+            self.clients.update_one({"client_id": client_data["client_id"]}, {"$set": client_data}, True)
+        except KeyError:
+            # If client_id is not present, use name as identifier, for backwards compatibility
+            id = str(uuid.uuid4())
+            client_data["client_id"] = id
+            self.clients.update_one({"name": client_data["name"]}, {"$set": client_data}, True)
 
     def get_client(self, client_id):
         """Get client by client_id.

fedn/utils/dist.py

@@ -0,0 +1,17 @@
+import importlib.metadata
+
+import fedn
+
+
+def get_version(pacakge):
+    # Dynamically get the version of the package
+    try:
+        version = importlib.metadata.version("fedn")
+    except importlib.metadata.PackageNotFoundError:
+        version = "unknown"
+    return version
+
+
+def get_package_path():
+    # Get the path of the package
+    return fedn.__path__[0]

pyproject.toml

@@ -6,7 +6,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "fedn"
-version = "0.11.0"
+version = "0.11.1"
 description = "Scaleout Federated Learning"
 authors = [{ name = "Scaleout Systems AB", email = "[email protected]" }]
 readme = "README.rst"