Feature/mvds00/test axon subprocess error checking

tests/e2e_tests/multistep/test_axon.py

@@ -1,5 +1,6 @@
 import asyncio
 import sys
+import textwrap
 
 import pytest
 
@@ -29,6 +30,8 @@
 
 """
 
+INDENT = "   "
+
 
 @pytest.mark.asyncio
 async def test_axon(local_chain):
@@ -68,35 +71,52 @@ async def test_axon(local_chain):
 
     # register miner
     # "python neurons/miner.py --netuid 1 --subtensor.chain_endpoint ws://localhost:9945 --wallet.name wallet.name --wallet.hotkey wallet.hotkey.ss58_address"
-    cmd = " ".join(
-        [
-            f"{sys.executable}",
-            f'"{template_path}{templates_repo}/neurons/miner.py"',
-            "--no_prompt",
-            "--netuid",
-            str(netuid),
-            "--subtensor.network",
-            "local",
-            "--subtensor.chain_endpoint",
-            "ws://localhost:9945",
-            "--wallet.path",
-            wallet.path,
-            "--wallet.name",
-            wallet.name,
-            "--wallet.hotkey",
-            "default",
-        ]
-    )
-
-    axon_process = await asyncio.create_subprocess_shell(
-        cmd,
+    args = [
+        f"{template_path}{templates_repo}/neurons/miner.py",
+        "--no_prompt",
+        "--netuid",
+        str(netuid),
+        "--subtensor.network",
+        "local",
+        "--subtensor.chain_endpoint",
+        "ws://localhost:9945",
+        "--wallet.path",
+        wallet.path,
+        "--wallet.name",
+        wallet.name,
+        "--wallet.hotkey",
+        "default",
+    ]
+
+    cmd = sys.executable + " " + " ".join(args)
+
+    axon_process = await asyncio.create_subprocess_exec(
+        sys.executable,
+        *args,
         stdout=asyncio.subprocess.PIPE,
         stderr=asyncio.subprocess.PIPE,
+        env={"NEST_ASYNCIO": "0"},
     )
-    logging.info("Neuron Alice is now mining")
+    await asyncio.sleep(1)  # wait for the miner to start up or fail
+    if axon_process.returncode != None:
+        stdout, stderr = await axon_process.communicate()
+        stdout = textwrap.indent(stdout.decode("UTF8"), INDENT)
+        stderr = textwrap.indent(stderr.decode("UTF8"), INDENT)
+        assert (
+            axon_process.returncode == None
+        ), f"miner process stopped immediately or did not start:\n{cmd}\nstdout:\n{stdout}\nstderr:\n{stderr}"
+
+    logging.info(f"Neuron Alice is now mining: {axon_process}")
     await asyncio.sleep(
         5
     )  # wait for 5 seconds for the metagraph to refresh with latest data
+    if axon_process.returncode != None:
+        stdout, stderr = await axon_process.communicate()
+        stdout = textwrap.indent(stdout.decode("UTF8"), INDENT)
+        stderr = textwrap.indent(stderr.decode("UTF8"), INDENT)
+        assert (
+            axon_process.returncode == None
+        ), f"miner process stopped unexpectedly:\n{cmd}\nstdout:\n{stdout}\nstderr:\n{stderr}"
 
     # refresh metagraph
     metagraph = bittensor.metagraph(netuid=netuid, network="ws://localhost:9945")
@@ -109,4 +129,29 @@ async def test_axon(local_chain):
     assert updated_axon.port == 8091
     assert updated_axon.hotkey == alice_keypair.ss58_address
     assert updated_axon.coldkey == alice_keypair.ss58_address
+
+    # Terminate miner and make sure nothing bad happened.
+    if axon_process.returncode == None:
+        logging.info("terminating miner")
+        try:
+            # try/except to prevent exception in case the process just ends now
+            axon_process.terminate()
+        except:
+            pass
+    await asyncio.sleep(1)
+
+    if axon_process.returncode == None:
+        logging.info("killing miner")
+        try:
+            # try/except to prevent exception in case the process just ends now
+            axon_process.kill()
+        except:
+            pass
+
+    stdout, stderr = await axon_process.communicate()
+    stdout = textwrap.indent(stdout.decode("UTF8"), INDENT)
+    stderr = textwrap.indent(stderr.decode("UTF8"), INDENT)
+    assert (
+        "Exception" not in stderr
+    ), f"miner output indicates issue:\nstdout:\n{stdout}\nstderr:\n{stderr}"
     logging.info("Passed test_axon")