diff --git a/homeassistant/components/zha/__init__.py b/homeassistant/components/zha/__init__.py
index 340e0db40a696b..1eb3369c1bef9c 100644
--- a/homeassistant/components/zha/__init__.py
+++ b/homeassistant/components/zha/__init__.py
@@ -37,8 +37,6 @@
     DOMAIN,
     PLATFORMS,
     SIGNAL_ADD_ENTITIES,
-    STARTUP_FAILURE_DELAY_S,
-    STARTUP_RETRIES,
     RadioType,
 )
 from .core.device import get_device_automation_triggers
@@ -161,49 +159,40 @@ async def async_setup_entry(hass: HomeAssistant, config_entry: ConfigEntry) -> b
 
     _LOGGER.debug("Trigger cache: %s", zha_data.device_trigger_cache)
 
-    # Retry setup a few times before giving up to deal with missing serial ports in VMs
-    for attempt in range(STARTUP_RETRIES):
-        try:
-            zha_gateway = await ZHAGateway.async_from_config(
-                hass=hass,
-                config=zha_data.yaml_config,
-                config_entry=config_entry,
-            )
-            break
-        except NetworkSettingsInconsistent as exc:
-            await warn_on_inconsistent_network_settings(
-                hass,
-                config_entry=config_entry,
-                old_state=exc.old_state,
-                new_state=exc.new_state,
-            )
-            raise ConfigEntryError(
-                "Network settings do not match most recent backup"
-            ) from exc
-        except TransientConnectionError as exc:
-            raise ConfigEntryNotReady from exc
-        except Exception as exc:
-            _LOGGER.debug(
-                "Couldn't start coordinator (attempt %s of %s)",
-                attempt + 1,
-                STARTUP_RETRIES,
-                exc_info=exc,
-            )
-
-            if attempt < STARTUP_RETRIES - 1:
-                await asyncio.sleep(STARTUP_FAILURE_DELAY_S)
-                continue
-
-            if RadioType[config_entry.data[CONF_RADIO_TYPE]] == RadioType.ezsp:
-                try:
-                    # Ignore all exceptions during probing, they shouldn't halt setup
-                    await warn_on_wrong_silabs_firmware(
-                        hass, config_entry.data[CONF_DEVICE][CONF_DEVICE_PATH]
-                    )
-                except AlreadyRunningEZSP as ezsp_exc:
-                    raise ConfigEntryNotReady from ezsp_exc
-
-            raise
+    try:
+        zha_gateway = await ZHAGateway.async_from_config(
+            hass=hass,
+            config=zha_data.yaml_config,
+            config_entry=config_entry,
+        )
+    except NetworkSettingsInconsistent as exc:
+        await warn_on_inconsistent_network_settings(
+            hass,
+            config_entry=config_entry,
+            old_state=exc.old_state,
+            new_state=exc.new_state,
+        )
+        raise ConfigEntryError(
+            "Network settings do not match most recent backup"
+        ) from exc
+    except TransientConnectionError as exc:
+        raise ConfigEntryNotReady from exc
+    except Exception as exc:
+        _LOGGER.debug("Failed to set up ZHA", exc_info=exc)
+        device_path = config_entry.data[CONF_DEVICE][CONF_DEVICE_PATH]
+
+        if (
+            not device_path.startswith("socket://")
+            and RadioType[config_entry.data[CONF_RADIO_TYPE]] == RadioType.ezsp
+        ):
+            try:
+                # Ignore all exceptions during probing, they shouldn't halt setup
+                if await warn_on_wrong_silabs_firmware(hass, device_path):
+                    raise ConfigEntryError("Incorrect firmware installed") from exc
+            except AlreadyRunningEZSP as ezsp_exc:
+                raise ConfigEntryNotReady from ezsp_exc
+
+        raise ConfigEntryNotReady from exc
 
     repairs.async_delete_blocking_issues(hass)
 
diff --git a/homeassistant/components/zha/core/const.py b/homeassistant/components/zha/core/const.py
index f89ed8d9a52913..ecbd347a6211c3 100644
--- a/homeassistant/components/zha/core/const.py
+++ b/homeassistant/components/zha/core/const.py
@@ -409,9 +409,6 @@ class Strobe(t.enum8):
     Strobe = 0x01
 
 
-STARTUP_FAILURE_DELAY_S = 3
-STARTUP_RETRIES = 3
-
 EZSP_OVERWRITE_EUI64 = (
     "i_understand_i_can_update_eui64_only_once_and_i_still_want_to_do_it"
 )
diff --git a/tests/components/zha/conftest.py b/tests/components/zha/conftest.py
index 1b3a536007ad55..55405d0a51c812 100644
--- a/tests/components/zha/conftest.py
+++ b/tests/components/zha/conftest.py
@@ -46,7 +46,7 @@ def disable_request_retry_delay():
     with patch(
         "homeassistant.components.zha.core.cluster_handlers.RETRYABLE_REQUEST_DECORATOR",
         zigpy.util.retryable_request(tries=3, delay=0),
-    ), patch("homeassistant.components.zha.STARTUP_FAILURE_DELAY_S", 0.01):
+    ):
         yield
 
 
diff --git a/tests/components/zha/test_repairs.py b/tests/components/zha/test_repairs.py
index d168e2e57b12a3..0efff5ecb526f1 100644
--- a/tests/components/zha/test_repairs.py
+++ b/tests/components/zha/test_repairs.py
@@ -95,7 +95,6 @@ def test_detect_radio_hardware_failure(hass: HomeAssistant) -> None:
         assert _detect_radio_hardware(hass, SKYCONNECT_DEVICE) == HardwareType.OTHER
 
 
-@patch("homeassistant.components.zha.STARTUP_RETRIES", new=1)
 @pytest.mark.parametrize(
     ("detected_hardware", "expected_learn_more_url"),
     [
@@ -176,7 +175,7 @@ async def test_multipan_firmware_no_repair_on_probe_failure(
         await hass.config_entries.async_setup(config_entry.entry_id)
         await hass.async_block_till_done()
 
-        assert config_entry.state == ConfigEntryState.SETUP_ERROR
+        assert config_entry.state == ConfigEntryState.SETUP_RETRY
 
     await hass.config_entries.async_unload(config_entry.entry_id)
 
@@ -189,7 +188,6 @@ async def test_multipan_firmware_no_repair_on_probe_failure(
     assert issue is None
 
 
-@patch("homeassistant.components.zha.STARTUP_RETRIES", new=1)
 async def test_multipan_firmware_retry_on_probe_ezsp(
     hass: HomeAssistant,
     config_entry: MockConfigEntry,