diff --git a/llmfoundry/__init__.py b/llmfoundry/__init__.py
index 8dbd180c0a..b851aaa559 100644
--- a/llmfoundry/__init__.py
+++ b/llmfoundry/__init__.py
@@ -50,6 +50,7 @@
     tokenizers,
     utils,
 )
+from llmfoundry._version import __version__
 from llmfoundry.data import StreamingFinetuningDataset, StreamingTextDataset
 from llmfoundry.eval import InContextLearningDataset, InContextLearningMetric
 from llmfoundry.models.hf import ComposerHFCausalLM
@@ -63,6 +64,7 @@
 from llmfoundry.optim import DecoupledLionW
 
 __all__ = [
+    '__version__',
     'StreamingFinetuningDataset',
     'StreamingTextDataset',
     'InContextLearningDataset',
@@ -87,5 +89,3 @@
     'tokenizers',
     'utils',
 ]
-
-__version__ = '0.11.0.dev0'
diff --git a/llmfoundry/_version.py b/llmfoundry/_version.py
new file mode 100644
index 0000000000..4c11746b43
--- /dev/null
+++ b/llmfoundry/_version.py
@@ -0,0 +1,6 @@
+# Copyright 2024 MosaicML LLM Foundry authors
+# SPDX-License-Identifier: Apache-2.0
+
+"""The LLM Foundry Version."""
+
+__version__ = '0.11.0.dev'
diff --git a/llmfoundry/callbacks/async_eval_callback.py b/llmfoundry/callbacks/async_eval_callback.py
index 646d86c8d3..1b3c31e861 100644
--- a/llmfoundry/callbacks/async_eval_callback.py
+++ b/llmfoundry/callbacks/async_eval_callback.py
@@ -557,7 +557,8 @@ def launch_run(self, checkpoint: str, current_interval: Time) -> Run:
                 installation_path = i['path']
 
         if not found_llm_foundry:
-            from llmfoundry import __version__ as latest_foundry_version
+            from llmfoundry._version import \
+                __version__ as latest_foundry_version
 
             # If github integration is not found, foundry is likely installed
             # through the run command. In this case, we'll add the integration
diff --git a/pyproject.toml b/pyproject.toml
index 53007cafaf..e5c931f4c5 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -79,7 +79,7 @@ reportMissingImports = "none"
 # Pytest
 [tool.pytest.ini_options]
 # By default, skip gpu tests
-addopts = "--tb=short -m 'not gpu'"
+addopts = "--tb=short -m 'not gpu' --color=yes"
 
 markers = [
     # For distributed testing
diff --git a/setup.py b/setup.py
index 494db8d62a..185d3970f7 100644
--- a/setup.py
+++ b/setup.py
@@ -5,7 +5,7 @@
 
 import copy
 import os
-import re
+from typing import Any, Dict, Mapping
 
 import setuptools
 from setuptools import setup
@@ -15,17 +15,15 @@
 _REPO_REAL_PATH = os.path.dirname(os.path.realpath(__file__))
 _PACKAGE_REAL_PATH = os.path.join(_REPO_REAL_PATH, _PACKAGE_DIR)
 
-# Read the repo version
+# Read the llm-foundry version
 # We can't use `.__version__` from the library since it's not installed yet
-with open(os.path.join(_PACKAGE_REAL_PATH, '__init__.py')) as f:
+version_path = os.path.join(_PACKAGE_REAL_PATH, '_version.py')
+with open(version_path, encoding='utf-8') as f:
+    version_globals: Dict[str, Any] = {}
+    version_locals: Mapping[str, object] = {}
     content = f.read()
-# regex: '__version__', whitespace?, '=', whitespace, quote, version, quote
-# we put parens around the version so that it becomes elem 1 of the match
-expr = re.compile(
-    r"""^__version__\s*=\s*['"]([0-9]+\.[0-9]+\.[0-9]+(?:\.\w+)?)['"]""",
-    re.MULTILINE,
-)
-repo_version = expr.findall(content)[0]
+    exec(content, version_globals, version_locals)
+    repo_version = str(version_locals['__version__'])
 
 # Use repo README for PyPi description
 with open('README.md', 'r', encoding='utf-8') as fh:
@@ -56,7 +54,7 @@
 install_requires = [
     'mosaicml[libcloud,wandb,oci,gcs,mlflow]>=0.23.4,<0.24',
     'mlflow>=2.14.1,<2.15',
-    'accelerate>=0.25,<0.33',  # for HF inference `device_map`
+    'accelerate>=0.25,<0.34',  # for HF inference `device_map`
     'transformers>=4.43.2,<4.44',
     'mosaicml-streaming>=0.8.0,<0.9',
     'torch>=2.3.0,<2.4',
@@ -70,7 +68,7 @@
     'onnx==1.16.1',
     'onnxruntime==1.18.1',
     'boto3>=1.21.45,<2',
-    'huggingface-hub>=0.19.0,<0.24',
+    'huggingface-hub>=0.19.0,<0.25',
     'beautifulsoup4>=4.12.2,<5',  # required for model download utils
     'tenacity>=8.2.3,<9',
     'catalogue>=2,<3',