request = <SubRequest 'ray_cluster_4cpu' for <Function test_experiment_model_resume_distributed[horovod]>>

    @pytest.fixture(scope="module")
    def ray_cluster_4cpu(request):
        with _ray_start(request, num_cpus=4):
>           yield

tests/conftest.py:150: 
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 
/opt/hostedtoolcache/Python/3.8.17/x64/lib/python3.8/contextlib.py:120: in __exit__
    next(self.gen)
tests/conftest.py:194: in _ray_start
    ray.shutdown()
/opt/hostedtoolcache/Python/3.8.17/x64/lib/python3.8/site-packages/ray/_private/client_mode_hook.py:105: in wrapper
    return func(*args, **kwargs)
/opt/hostedtoolcache/Python/3.8.17/x64/lib/python3.8/site-packages/ray/_private/worker.py:1626: in shutdown
    _global_node.kill_all_processes(check_alive=False, allow_graceful=True)
/opt/hostedtoolcache/Python/3.8.17/x64/lib/python3.8/site-packages/ray/_private/node.py:1358: in kill_all_processes
    self._kill_process_type(
/opt/hostedtoolcache/Python/3.8.17/x64/lib/python3.8/site-packages/ray/_private/node.py:1158: in _kill_process_type
    self._kill_process_impl(
/opt/hostedtoolcache/Python/3.8.17/x64/lib/python3.8/site-packages/ray/_private/node.py:1214: in _kill_process_impl
    process.wait(timeout_seconds)
/opt/hostedtoolcache/Python/3.8.17/x64/lib/python3.8/subprocess.py:1083: in wait
    return self._wait(timeout=timeout)
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 

self = <ray._private.services.ConsolePopen object at 0x7f11a18fab20>
timeout = 1

    def _wait(self, timeout):
        """Internal implementation of wait() on POSIX."""
        if self.returncode is not None:
            return self.returncode
    
        if timeout is not None:
            endtime = _time() + timeout
            # Enter a busy loop if we have a timeout.  This busy loop was
            # cribbed from Lib/threading.py in Thread.wait() at r71065.
            delay = 0.0005 # 500 us -> initial delay of 1 ms
            while True:
                if self._waitpid_lock.acquire(False):
                    try:
                        if self.returncode is not None:
                            break  # Another thread waited.
                        (pid, sts) = self._try_wait(os.WNOHANG)
                        assert pid == self.pid or pid == 0
                        if pid == self.pid:
                            self._handle_exitstatus(sts)
                            break
                    finally:
                        self._waitpid_lock.release()
                remaining = self._remaining_time(endtime)
                if remaining <= 0:
                    raise TimeoutExpired(self.args, timeout)
                delay = min(delay * 2, remaining, .05)
>               time.sleep(delay)
E               Failed: Timeout >300.0s

/opt/hostedtoolcache/Python/3.8.17/x64/lib/python3.8/subprocess.py:1816: Failed