diff --git a/import_tracker/__main__.py b/import_tracker/__main__.py
index 6c2332e..f6efd79 100644
--- a/import_tracker/__main__.py
+++ b/import_tracker/__main__.py
@@ -29,6 +29,7 @@
 # Local
 from .constants import THIS_PACKAGE
 from .import_tracker import track_module
+from .lazy_import_errors import enable_tracking_mode
 from .log import log
 
 ## Implementation Details ######################################################
@@ -390,6 +391,10 @@ def main():
     )
     args = parser.parse_args()
 
+    # Mark the environment as tracking mode so that any lazy import errors are
+    # disabled
+    enable_tracking_mode()
+
     # Set the level on the shared logger
     log_level = getattr(logging, args.log_level.upper(), None)
     if log_level is None:
diff --git a/import_tracker/lazy_import_errors.py b/import_tracker/lazy_import_errors.py
index da01138..31ced5f 100644
--- a/import_tracker/lazy_import_errors.py
+++ b/import_tracker/lazy_import_errors.py
@@ -32,6 +32,18 @@ def lazy_import_errors():
 
 ## Implementation Details ######################################################
 
+_TRACKING_MODE = False
+
+
+def enable_tracking_mode():
+    """This is used by the main function to disable all lazy import errors when
+    running as a standalone script to track the modules in a library.
+
+    This function should NOT be exposed as a public API
+    """
+    global _TRACKING_MODE
+    _TRACKING_MODE = True
+
 
 class _LazyImportErrorCtx(AbstractContextManager):
     """This class implements the Context Manager version of lazy_import_errors"""
@@ -41,7 +53,11 @@ def __init__(self):
         acts as the context manager, so the __enter__ implementation lives in
         the constructor.
         """
-        if sys.meta_path and not isinstance(sys.meta_path[-1], _LazyErrorMetaFinder):
+        if (
+            not _TRACKING_MODE
+            and sys.meta_path
+            and not isinstance(sys.meta_path[-1], _LazyErrorMetaFinder)
+        ):
             sys.meta_path.append(_LazyErrorMetaFinder())
 
     @staticmethod
diff --git a/test/sample_libs/lazy_import_errors/__init__.py b/test/sample_libs/lazy_import_errors/__init__.py
new file mode 100644
index 0000000..01a9aa1
--- /dev/null
+++ b/test/sample_libs/lazy_import_errors/__init__.py
@@ -0,0 +1,10 @@
+"""
+This sample module uses import tracker's lazy_import_errors
+"""
+
+# Local
+import import_tracker
+
+with import_tracker.lazy_import_errors():
+    # Local
+    from . import foo
diff --git a/test/sample_libs/lazy_import_errors/foo.py b/test/sample_libs/lazy_import_errors/foo.py
new file mode 100644
index 0000000..7aa9d19
--- /dev/null
+++ b/test/sample_libs/lazy_import_errors/foo.py
@@ -0,0 +1,9 @@
+"""
+This module will be loaded with lazy errors
+"""
+
+try:
+    # Third Party
+    import not_there
+except:
+    not_there = "NOT THERE"
diff --git a/test/test_main.py b/test/test_main.py
index 535a7da..608744c 100644
--- a/test/test_main.py
+++ b/test/test_main.py
@@ -126,3 +126,15 @@ def test_sibling_import(capsys):
     assert (set(parsed_out["inter_mod_deps.submod5"])) == {
         "yaml",
     }
+
+
+def test_lib_with_lazy_imports(capsys):
+    """Make sure that a library which uses import_tracker's lazy import errors
+    and has "traditional" conditional dependencies does not blow up when tracked
+    """
+    with cli_args("--name", "lazy_import_errors"):
+        main()
+    captured = capsys.readouterr()
+    assert captured.out
+    parsed_out = json.loads(captured.out)
+    assert "lazy_import_errors" in parsed_out