Fixed a race condition issue during the server start. (#2235)

Co-authored-by: Chester Chen <[email protected]>
NVIDIA · Dec 21, 2023 · d67baf4 · d67baf4
1 parent 2a5b233
commit d67baf4
Show file tree

Hide file tree

Showing 2 changed files with 5 additions and 0 deletions.
diff --git a/nvflare/private/fed/app/deployer/server_deployer.py b/nvflare/private/fed/app/deployer/server_deployer.py
@@ -22,6 +22,7 @@
 from nvflare.private.fed.server.job_runner import JobRunner
 from nvflare.private.fed.server.run_manager import RunManager
 from nvflare.private.fed.server.server_cmd_modules import ServerCommandModules
+from nvflare.private.fed.server.server_status import ServerStatus
 
 
 class ServerDeployer:
@@ -121,6 +122,7 @@ def deploy(self, args):
             services.engine.fire_event(EventType.SYSTEM_BOOTSTRAP, fl_ctx)
 
             threading.Thread(target=self._start_job_runner, args=[job_runner, fl_ctx]).start()
+            services.status = ServerStatus.STARTED
 
             services.engine.fire_event(EventType.SYSTEM_START, fl_ctx)
             print("deployed FL server trainer.")

diff --git a/nvflare/private/fed/server/fed_server.py b/nvflare/private/fed/server/fed_server.py
@@ -779,6 +779,9 @@ def _init_agent(self, args=None):
         return self.overseer_agent
 
     def _check_server_state(self, overseer_agent):
+        if self.status != ServerStatus.STARTED:
+            return
+
         if overseer_agent.is_shutdown():
             self.engine.shutdown_server()
             return