From 19810d35096d03e85d0451fd09b8cae5d6b7cfec Mon Sep 17 00:00:00 2001 From: Fredrik Wrede Date: Thu, 12 Sep 2024 15:22:53 +0000 Subject: [PATCH 1/3] fix --- examples/FedSimSiam/client/python_env.yaml | 11 ++++++++--- examples/flower-client/client/python_env.yaml | 11 ++++++++--- examples/huggingface/client/python_env.yaml | 10 +++++++--- examples/mnist-pytorch/client/python_env.yaml | 9 +++++++-- examples/monai-2D-mednist/client/python_env.yaml | 10 +++++++--- 5 files changed, 37 insertions(+), 14 deletions(-) diff --git a/examples/FedSimSiam/client/python_env.yaml b/examples/FedSimSiam/client/python_env.yaml index 82aba8729..9c084dbcb 100644 --- a/examples/FedSimSiam/client/python_env.yaml +++ b/examples/FedSimSiam/client/python_env.yaml @@ -4,6 +4,11 @@ build_dependencies: - setuptools - wheel==0.37.1 dependencies: - - torch==2.2.0 - - torchvision==0.17.0 - - fedn==0.9.0 + - torch==2.4.1; (sys_platform == "darwin" and platform_machine == "arm64") or (sys_platform == "win" or sys_platform == "linux") + # PyTorch macOS x86 builds deprecation + - torch==2.2.2; sys_platform == "darwin" and platform_machine == "x86_64" + - torchvision==0.19.1; (sys_platform == "darwin" and platform_machine == "arm64") or (sys_platform == "win" or sys_platform == "linux") + - torchvision==0.17.2; sys_platform == "darwin" and platform_machine == "x86_64" + - numpy==2.0.2; (sys_platform == "darwin" and platform_machine == "arm64") or (sys_platform == "win" or sys_platform == "linux") + - numpy==1.26.4; sys_platform == "darwin" and platform_machine == "x86_64" + - fedn diff --git a/examples/flower-client/client/python_env.yaml b/examples/flower-client/client/python_env.yaml index 984a2e96d..02e024a07 100644 --- a/examples/flower-client/client/python_env.yaml +++ b/examples/flower-client/client/python_env.yaml @@ -4,8 +4,13 @@ build_dependencies: - setuptools - wheel==0.37.1 dependencies: - - torch==2.2.1 - - torchvision==0.17.1 + - fedn[flower] + - torch==2.4.1; (sys_platform == "darwin" and platform_machine == "arm64") or (sys_platform == "win" or sys_platform == "linux") + # PyTorch macOS x86 builds deprecation + - torch==2.2.2; sys_platform == "darwin" and platform_machine == "x86_64" + - torchvision==0.19.1; (sys_platform == "darwin" and platform_machine == "arm64") or (sys_platform == "win" or sys_platform == "linux") + - torchvision==0.17.2; sys_platform == "darwin" and platform_machine == "x86_64" + - numpy==2.0.2; (sys_platform == "darwin" and platform_machine == "arm64") or (sys_platform == "win" or sys_platform == "linux") + - numpy==1.26.4; sys_platform == "darwin" and platform_machine == "x86_64" - fire==0.3.1 - - fedn[flower]==0.9.0 - flwr-datasets[vision]==0.1.0 \ No newline at end of file diff --git a/examples/huggingface/client/python_env.yaml b/examples/huggingface/client/python_env.yaml index 188b4bb12..f57851f35 100644 --- a/examples/huggingface/client/python_env.yaml +++ b/examples/huggingface/client/python_env.yaml @@ -4,9 +4,13 @@ build_dependencies: - setuptools - wheel dependencies: - - numpy==2.0.2 - - torch==2.4.1 - - torchvision==0.19.1 + - torch==2.4.1; (sys_platform == "darwin" and platform_machine == "arm64") or (sys_platform == "win" or sys_platform == "linux") + # PyTorch macOS x86 builds deprecation + - torch==2.2.2; sys_platform == "darwin" and platform_machine == "x86_64" + - torchvision==0.19.1; (sys_platform == "darwin" and platform_machine == "arm64") or (sys_platform == "win" or sys_platform == "linux") + - torchvision==0.17.2; sys_platform == "darwin" and platform_machine == "x86_64" + - numpy==2.0.2; (sys_platform == "darwin" and platform_machine == "arm64") or (sys_platform == "win" or sys_platform == "linux") + - numpy==1.26.4; sys_platform == "darwin" and platform_machine == "x86_64" - transformers - datasets - fedn diff --git a/examples/mnist-pytorch/client/python_env.yaml b/examples/mnist-pytorch/client/python_env.yaml index f43d2353d..7a5fd853f 100644 --- a/examples/mnist-pytorch/client/python_env.yaml +++ b/examples/mnist-pytorch/client/python_env.yaml @@ -4,6 +4,11 @@ build_dependencies: - setuptools - wheel dependencies: - - torch - - torchvision - fedn + - torch==2.4.1; (sys_platform == "darwin" and platform_machine == "arm64") or (sys_platform == "win" or sys_platform == "linux") + # PyTorch macOS x86 builds deprecation + - torch==2.2.2; sys_platform == "darwin" and platform_machine == "x86_64" + - torchvision==0.19.1; (sys_platform == "darwin" and platform_machine == "arm64") or (sys_platform == "win" or sys_platform == "linux") + - torchvision==0.17.2; sys_platform == "darwin" and platform_machine == "x86_64" + - numpy==2.0.2; (sys_platform == "darwin" and platform_machine == "arm64") or (sys_platform == "win" or sys_platform == "linux") + - numpy==1.26.4; sys_platform == "darwin" and platform_machine == "x86_64" diff --git a/examples/monai-2D-mednist/client/python_env.yaml b/examples/monai-2D-mednist/client/python_env.yaml index ec39b5084..02316dbb0 100644 --- a/examples/monai-2D-mednist/client/python_env.yaml +++ b/examples/monai-2D-mednist/client/python_env.yaml @@ -4,9 +4,13 @@ build_dependencies: - setuptools - wheel dependencies: - - torch==2.2.1 - - torchvision==0.17.1 - fedn + - torch==2.4.1; (sys_platform == "darwin" and platform_machine == "arm64") or (sys_platform == "win" or sys_platform == "linux") + # PyTorch macOS x86 builds deprecation + - torch==2.2.2; sys_platform == "darwin" and platform_machine == "x86_64" + - torchvision==0.19.1; (sys_platform == "darwin" and platform_machine == "arm64") or (sys_platform == "win" or sys_platform == "linux") + - torchvision==0.17.2; sys_platform == "darwin" and platform_machine == "x86_64" + - numpy==2.0.2; (sys_platform == "darwin" and platform_machine == "arm64") or (sys_platform == "win" or sys_platform == "linux") + - numpy==1.26.4; sys_platform == "darwin" and platform_machine == "x86_64" - monai-weekly[pillow, tqdm] - - numpy==1.26.4 - scikit-learn From 3178f5b5906b93c5203cbff76b1be0f997645138 Mon Sep 17 00:00:00 2001 From: Fredrik Wrede Date: Thu, 12 Sep 2024 15:46:31 +0000 Subject: [PATCH 2/3] support py 3.8 --- examples/FedSimSiam/client/python_env.yaml | 5 +++-- examples/flower-client/client/python_env.yaml | 5 +++-- examples/huggingface/client/python_env.yaml | 5 +++-- examples/mnist-pytorch-DPSGD/client/python_env.yaml | 7 ++++--- examples/mnist-pytorch/client/python_env.yaml | 5 +++-- examples/monai-2D-mednist/client/python_env.yaml | 5 +++-- 6 files changed, 19 insertions(+), 13 deletions(-) diff --git a/examples/FedSimSiam/client/python_env.yaml b/examples/FedSimSiam/client/python_env.yaml index 9c084dbcb..f4cdc5c6f 100644 --- a/examples/FedSimSiam/client/python_env.yaml +++ b/examples/FedSimSiam/client/python_env.yaml @@ -9,6 +9,7 @@ dependencies: - torch==2.2.2; sys_platform == "darwin" and platform_machine == "x86_64" - torchvision==0.19.1; (sys_platform == "darwin" and platform_machine == "arm64") or (sys_platform == "win" or sys_platform == "linux") - torchvision==0.17.2; sys_platform == "darwin" and platform_machine == "x86_64" - - numpy==2.0.2; (sys_platform == "darwin" and platform_machine == "arm64") or (sys_platform == "win" or sys_platform == "linux") - - numpy==1.26.4; sys_platform == "darwin" and platform_machine == "x86_64" + - numpy==2.0.2; (sys_platform == "darwin" and platform_machine == "arm64" and python_version >= "3.9") or (sys_platform == "win" or sys_platform == "linux" and python_version >= "3.9") + - numpy==1.26.4; (sys_platform == "darwin" and platform_machine == "x86_64" and python_version >= "3.9") + - numpy==1.24.4; python_version == "3.8" - fedn diff --git a/examples/flower-client/client/python_env.yaml b/examples/flower-client/client/python_env.yaml index 02e024a07..16879da90 100644 --- a/examples/flower-client/client/python_env.yaml +++ b/examples/flower-client/client/python_env.yaml @@ -10,7 +10,8 @@ dependencies: - torch==2.2.2; sys_platform == "darwin" and platform_machine == "x86_64" - torchvision==0.19.1; (sys_platform == "darwin" and platform_machine == "arm64") or (sys_platform == "win" or sys_platform == "linux") - torchvision==0.17.2; sys_platform == "darwin" and platform_machine == "x86_64" - - numpy==2.0.2; (sys_platform == "darwin" and platform_machine == "arm64") or (sys_platform == "win" or sys_platform == "linux") - - numpy==1.26.4; sys_platform == "darwin" and platform_machine == "x86_64" + - numpy==2.0.2; (sys_platform == "darwin" and platform_machine == "arm64" and python_version >= "3.9") or (sys_platform == "win" or sys_platform == "linux" and python_version >= "3.9") + - numpy==1.26.4; (sys_platform == "darwin" and platform_machine == "x86_64" and python_version >= "3.9") + - numpy==1.24.4; python_version == "3.8" - fire==0.3.1 - flwr-datasets[vision]==0.1.0 \ No newline at end of file diff --git a/examples/huggingface/client/python_env.yaml b/examples/huggingface/client/python_env.yaml index f57851f35..23052db0a 100644 --- a/examples/huggingface/client/python_env.yaml +++ b/examples/huggingface/client/python_env.yaml @@ -9,8 +9,9 @@ dependencies: - torch==2.2.2; sys_platform == "darwin" and platform_machine == "x86_64" - torchvision==0.19.1; (sys_platform == "darwin" and platform_machine == "arm64") or (sys_platform == "win" or sys_platform == "linux") - torchvision==0.17.2; sys_platform == "darwin" and platform_machine == "x86_64" - - numpy==2.0.2; (sys_platform == "darwin" and platform_machine == "arm64") or (sys_platform == "win" or sys_platform == "linux") - - numpy==1.26.4; sys_platform == "darwin" and platform_machine == "x86_64" + - numpy==2.0.2; (sys_platform == "darwin" and platform_machine == "arm64" and python_version >= "3.9") or (sys_platform == "win" or sys_platform == "linux" and python_version >= "3.9") + - numpy==1.26.4; (sys_platform == "darwin" and platform_machine == "x86_64" and python_version >= "3.9") + - numpy==1.24.4; python_version == "3.8" - transformers - datasets - fedn diff --git a/examples/mnist-pytorch-DPSGD/client/python_env.yaml b/examples/mnist-pytorch-DPSGD/client/python_env.yaml index a8175e20c..c9341a6ef 100644 --- a/examples/mnist-pytorch-DPSGD/client/python_env.yaml +++ b/examples/mnist-pytorch-DPSGD/client/python_env.yaml @@ -5,11 +5,12 @@ build_dependencies: - wheel dependencies: - fedn - - torch==2.4.1; (sys_platform == "darwin" and platform_machine == "arm64") or (sys_platform == "win" or sys_platform == "linux") + - torch==2.4.1; (sys_platform == "darwin" and platform_machine == "arm64") or (sys_platform == "win" or sys_platform == "linux") # PyTorch macOS x86 builds deprecation - torch==2.2.2; sys_platform == "darwin" and platform_machine == "x86_64" - torchvision==0.19.1; (sys_platform == "darwin" and platform_machine == "arm64") or (sys_platform == "win" or sys_platform == "linux") - torchvision==0.17.2; sys_platform == "darwin" and platform_machine == "x86_64" - - numpy==2.0.2; (sys_platform == "darwin" and platform_machine == "arm64") or (sys_platform == "win" or sys_platform == "linux") - - numpy==1.26.4; sys_platform == "darwin" and platform_machine == "x86_64" + - numpy==2.0.2; (sys_platform == "darwin" and platform_machine == "arm64" and python_version >= "3.9") or (sys_platform == "win" or sys_platform == "linux" and python_version >= "3.9") + - numpy==1.26.4; (sys_platform == "darwin" and platform_machine == "x86_64" and python_version >= "3.9") + - numpy==1.24.4; python_version == "3.8" - opacus diff --git a/examples/mnist-pytorch/client/python_env.yaml b/examples/mnist-pytorch/client/python_env.yaml index 7a5fd853f..642ed1cf0 100644 --- a/examples/mnist-pytorch/client/python_env.yaml +++ b/examples/mnist-pytorch/client/python_env.yaml @@ -10,5 +10,6 @@ dependencies: - torch==2.2.2; sys_platform == "darwin" and platform_machine == "x86_64" - torchvision==0.19.1; (sys_platform == "darwin" and platform_machine == "arm64") or (sys_platform == "win" or sys_platform == "linux") - torchvision==0.17.2; sys_platform == "darwin" and platform_machine == "x86_64" - - numpy==2.0.2; (sys_platform == "darwin" and platform_machine == "arm64") or (sys_platform == "win" or sys_platform == "linux") - - numpy==1.26.4; sys_platform == "darwin" and platform_machine == "x86_64" + - numpy==2.0.2; (sys_platform == "darwin" and platform_machine == "arm64" and python_version >= "3.9") or (sys_platform == "win" or sys_platform == "linux" and python_version >= "3.9") + - numpy==1.26.4; (sys_platform == "darwin" and platform_machine == "x86_64" and python_version >= "3.9") + - numpy==1.24.4; python_version == "3.8" diff --git a/examples/monai-2D-mednist/client/python_env.yaml b/examples/monai-2D-mednist/client/python_env.yaml index 02316dbb0..49d371872 100644 --- a/examples/monai-2D-mednist/client/python_env.yaml +++ b/examples/monai-2D-mednist/client/python_env.yaml @@ -10,7 +10,8 @@ dependencies: - torch==2.2.2; sys_platform == "darwin" and platform_machine == "x86_64" - torchvision==0.19.1; (sys_platform == "darwin" and platform_machine == "arm64") or (sys_platform == "win" or sys_platform == "linux") - torchvision==0.17.2; sys_platform == "darwin" and platform_machine == "x86_64" - - numpy==2.0.2; (sys_platform == "darwin" and platform_machine == "arm64") or (sys_platform == "win" or sys_platform == "linux") - - numpy==1.26.4; sys_platform == "darwin" and platform_machine == "x86_64" + - numpy==2.0.2; (sys_platform == "darwin" and platform_machine == "arm64" and python_version >= "3.9") or (sys_platform == "win" or sys_platform == "linux" and python_version >= "3.9") + - numpy==1.26.4; (sys_platform == "darwin" and platform_machine == "x86_64" and python_version >= "3.9") + - numpy==1.24.4; python_version == "3.8" - monai-weekly[pillow, tqdm] - scikit-learn From 76a8f190fc51694a405067c6298eadf09bcc11d6 Mon Sep 17 00:00:00 2001 From: Fredrik Wrede Date: Fri, 13 Sep 2024 12:13:09 +0000 Subject: [PATCH 3/3] fix race condition in bucket creation --- fedn/network/combiner/shared.py | 2 +- fedn/network/storage/s3/repository.py | 9 +++++---- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/fedn/network/combiner/shared.py b/fedn/network/combiner/shared.py index 5e5ee114c..e1dd6854f 100644 --- a/fedn/network/combiner/shared.py +++ b/fedn/network/combiner/shared.py @@ -8,6 +8,6 @@ network_id = get_network_config() statestore = MongoStateStore(network_id, statestore_config["mongo_config"]) -repository = Repository(modelstorage_config["storage_config"]) +repository = Repository(modelstorage_config["storage_config"], init_buckets=False) modelservice = ModelService() diff --git a/fedn/network/storage/s3/repository.py b/fedn/network/storage/s3/repository.py index 2a5ee3449..0b08fe7b9 100644 --- a/fedn/network/storage/s3/repository.py +++ b/fedn/network/storage/s3/repository.py @@ -8,7 +8,7 @@ class Repository: """Interface for storing model objects and compute packages in S3 compatible storage.""" - def __init__(self, config): + def __init__(self, config, init_buckets=True): self.model_bucket = config["storage_bucket"] self.context_bucket = config["context_bucket"] try: @@ -19,9 +19,10 @@ def __init__(self, config): # TODO: Make a plug-in solution self.client = MINIORepository(config) - self.client.create_bucket(self.context_bucket) - self.client.create_bucket(self.model_bucket) - self.client.create_bucket(self.inference_bucket) + if init_buckets: + self.client.create_bucket(self.context_bucket) + self.client.create_bucket(self.model_bucket) + self.client.create_bucket(self.inference_bucket) def get_model(self, model_id): """Retrieve a model with id model_id.