diff --git a/code/.bumpversion.cfg b/code/.bumpversion.cfg
index 8bb7f433e..f3af589e8 100644
--- a/code/.bumpversion.cfg
+++ b/code/.bumpversion.cfg
@@ -3,7 +3,7 @@ current_version = 0.8.0
 commit = False
 tag = False
 parse = (?P<major>\d+)\.(?P<minor>\d+)\.(?P<patch>\d+)(-dev(?P<dev>\d+))?
-serialize = 
+serialize =
 	{major}.{minor}.{patch}-dev{dev}
 	{major}.{minor}.{patch}
 
diff --git a/lib/esbonio/.bumpversion.cfg b/lib/esbonio/.bumpversion.cfg
index 71e602b46..b156ffbc6 100644
--- a/lib/esbonio/.bumpversion.cfg
+++ b/lib/esbonio/.bumpversion.cfg
@@ -3,7 +3,7 @@ current_version = 0.10.0
 commit = False
 tag = False
 parse = (?P<major>\d+)\.(?P<minor>\d+)\.(?P<patch>\d+)(.dev(?P<dev>\d+))?
-serialize = 
+serialize =
 	{major}.{minor}.{patch}.dev{dev}
 	{major}.{minor}.{patch}
 
diff --git a/lib/esbonio/changes/342.fix.rst b/lib/esbonio/changes/342.fix.rst
new file mode 100644
index 000000000..1fad001dc
--- /dev/null
+++ b/lib/esbonio/changes/342.fix.rst
@@ -0,0 +1 @@
+The language server should now correctly handle ``buildDir``, ``confDir`` and ``srcDir`` config values containing paths relative to ``~``
diff --git a/lib/esbonio/esbonio/lsp/sphinx/__init__.py b/lib/esbonio/esbonio/lsp/sphinx/__init__.py
index 5dcfd165e..580f11019 100644
--- a/lib/esbonio/esbonio/lsp/sphinx/__init__.py
+++ b/lib/esbonio/esbonio/lsp/sphinx/__init__.py
@@ -839,7 +839,7 @@ def expand_conf_dir(root_dir: str, conf_dir: str) -> pathlib.Path:
 
     match = PATH_VAR_PATTERN.match(conf_dir)
     if not match or match.group(1) not in {"workspaceRoot", "workspaceFolder"}:
-        return pathlib.Path(conf_dir)
+        return pathlib.Path(conf_dir).expanduser()
 
     conf = pathlib.Path(conf_dir).parts[1:]
     return pathlib.Path(root_dir, *conf).resolve()
@@ -887,7 +887,7 @@ def get_src_dir(
         src = pathlib.Path(src_dir).parts[1:]
         return pathlib.Path(conf_dir, *src).resolve()
 
-    return pathlib.Path(src_dir)
+    return pathlib.Path(src_dir).expanduser()
 
 
 def get_build_dir(
@@ -942,7 +942,11 @@ def get_build_dir(
     build_uri = Uri.from_fs_path(config.build_dir)
     build_dir = Uri.to_fs_path(build_uri)
 
-    return pathlib.Path(build_dir)
+    # But make sure paths starting with '~' are not corrupted
+    if build_dir.startswith("/~"):
+        build_dir = build_dir.replace("/~", "~")
+
+    return pathlib.Path(build_dir).expanduser()
 
 
 cli = setup_cli("esbonio.lsp.sphinx", "Esbonio's Sphinx language server.")
diff --git a/lib/esbonio/tests/test_sphinx.py b/lib/esbonio/tests/test_sphinx.py
index b86da56c2..d23af3f34 100644
--- a/lib/esbonio/tests/test_sphinx.py
+++ b/lib/esbonio/tests/test_sphinx.py
@@ -33,12 +33,227 @@
 from esbonio.lsp import create_language_server
 from esbonio.lsp import ESBONIO_SERVER_PREVIEW
 from esbonio.lsp.sphinx import DEFAULT_MODULES
+from esbonio.lsp.sphinx import expand_conf_dir
+from esbonio.lsp.sphinx import get_build_dir
+from esbonio.lsp.sphinx import get_src_dir
 from esbonio.lsp.sphinx import InitializationOptions
 from esbonio.lsp.sphinx import SphinxConfig
 from esbonio.lsp.sphinx import SphinxLanguageServer
 from esbonio.lsp.testing import ClientServer
 
 
+@py.test.mark.parametrize(
+    "setup, expected",
+    [
+        (
+            (
+                "/path/to/root",
+                "/path/to/config",
+            ),
+            pathlib.Path("/path/to/config"),
+        ),
+        (
+            (
+                "/path/to/root",
+                "~/path/to/config",
+            ),
+            pathlib.Path("~/path/to/config").expanduser(),
+        ),
+        (
+            (
+                "/path/to/root",
+                "${workspaceRoot}/config",
+            ),
+            pathlib.Path("/path/to/root/config"),
+        ),
+        (
+            (
+                "/path/to/root",
+                "${workspaceRoot}/../config",
+            ),
+            pathlib.Path("/path/to/config"),
+        ),
+        (
+            (
+                "/path/to/root",
+                "${workspaceFolder}/config",
+            ),
+            pathlib.Path("/path/to/root/config"),
+        ),
+        (
+            (
+                "/path/to/root",
+                "${workspaceFolder}/../config",
+            ),
+            pathlib.Path("/path/to/config"),
+        ),
+    ],
+)
+def test_expand_conf_dir(setup, expected):
+    """Ensure that the ``expand_conf_dir`` function works as expected."""
+
+    root_uri, conf_dir = setup
+
+    actual = expand_conf_dir(root_uri, conf_dir)
+    assert actual == expected
+
+
+@py.test.mark.parametrize(
+    "setup, expected",
+    [
+        (
+            (
+                "file:///path/to/root",
+                pathlib.Path("/path/to/config"),
+                SphinxConfig(srcDir="/path/to/src"),
+            ),
+            pathlib.Path("/path/to/src"),
+        ),
+        (
+            (
+                "file:///path/to/root",
+                pathlib.Path("/path/to/config"),
+                SphinxConfig(srcDir="~/path/to/src"),
+            ),
+            pathlib.Path("~/path/to/src").expanduser(),
+        ),
+        (
+            (
+                "file:///path/to/root",
+                pathlib.Path("/path/to/config"),
+                SphinxConfig(srcDir="${workspaceRoot}/src"),
+            ),
+            pathlib.Path("/path/to/root/src"),
+        ),
+        (
+            (
+                "file:///path/to/root",
+                pathlib.Path("/path/to/config"),
+                SphinxConfig(srcDir="${workspaceRoot}/../src"),
+            ),
+            pathlib.Path("/path/to/src"),
+        ),
+        (
+            (
+                "file:///path/to/root",
+                pathlib.Path("/path/to/config"),
+                SphinxConfig(srcDir="${workspaceFolder}/src"),
+            ),
+            pathlib.Path("/path/to/root/src"),
+        ),
+        (
+            (
+                "file:///path/to/root",
+                pathlib.Path("/path/to/config"),
+                SphinxConfig(srcDir="${workspaceFolder}/../src"),
+            ),
+            pathlib.Path("/path/to/src"),
+        ),
+        (
+            (
+                "file:///path/to/root",
+                pathlib.Path("/path/to/config"),
+                SphinxConfig(srcDir="${confDir}/src"),
+            ),
+            pathlib.Path("/path/to/config/src"),
+        ),
+        (
+            (
+                "file:///path/to/root",
+                pathlib.Path("/path/to/config"),
+                SphinxConfig(srcDir="${confDir}/../src"),
+            ),
+            pathlib.Path("/path/to/src"),
+        ),
+    ],
+)
+def test_get_src_dir(setup, expected):
+    """Ensure that the ``get_src_dir`` function works as expected."""
+
+    root_uri, conf_dir, sphinx_config = setup
+
+    actual = get_src_dir(root_uri, conf_dir, sphinx_config)
+    assert actual == expected
+
+
+@py.test.mark.parametrize(
+    "setup, expected",
+    [
+        (
+            (
+                "file:///path/to/root",
+                pathlib.Path("/path/to/config"),
+                SphinxConfig(buildDir="/path/to/build"),
+            ),
+            pathlib.Path("/path/to/build"),
+        ),
+        (
+            (
+                "file:///path/to/root",
+                pathlib.Path("/path/to/config"),
+                SphinxConfig(buildDir="~/path/to/build"),
+            ),
+            pathlib.Path("~/path/to/build").expanduser(),
+        ),
+        (
+            (
+                "file:///path/to/root",
+                pathlib.Path("/path/to/config"),
+                SphinxConfig(buildDir="${workspaceRoot}/build"),
+            ),
+            pathlib.Path("/path/to/root/build"),
+        ),
+        (
+            (
+                "file:///path/to/root",
+                pathlib.Path("/path/to/config"),
+                SphinxConfig(buildDir="${workspaceRoot}/../build"),
+            ),
+            pathlib.Path("/path/to/build"),
+        ),
+        (
+            (
+                "file:///path/to/root",
+                pathlib.Path("/path/to/config"),
+                SphinxConfig(buildDir="${workspaceFolder}/build"),
+            ),
+            pathlib.Path("/path/to/root/build"),
+        ),
+        (
+            (
+                "file:///path/to/root",
+                pathlib.Path("/path/to/config"),
+                SphinxConfig(buildDir="${workspaceFolder}/../build"),
+            ),
+            pathlib.Path("/path/to/build"),
+        ),
+        (
+            (
+                "file:///path/to/root",
+                pathlib.Path("/path/to/config"),
+                SphinxConfig(buildDir="${confDir}/build"),
+            ),
+            pathlib.Path("/path/to/config/build"),
+        ),
+        (
+            (
+                "file:///path/to/root",
+                pathlib.Path("/path/to/config"),
+                SphinxConfig(buildDir="${confDir}/../build"),
+            ),
+            pathlib.Path("/path/to/build"),
+        ),
+    ],
+)
+def test_get_build_dir(setup, expected):
+    """Ensure that the ``get_build_dir`` function works as expected."""
+
+    root_uri, conf_dir, sphinx_config = setup
+
+    actual = get_build_dir(root_uri, conf_dir, sphinx_config)
+    assert actual == expected
+
+
 @py.test.fixture(scope="function")
 async def cs():
     """A disposable version of the 'client_server' fixture.