Generate static index html documentation (#8615)

* Include option to generate static index.html

* Added changie

* Using DBT's system load / write file methods for better cross platform
support

* Updated docs tests with dbt.client.systems calls for file reading

* Writing out static_index.html as binary file to prevent line-ending
conversions on Windows. (similar behaviour as index.html)

.changes/unreleased/Features-20230911-144126.yaml

@@ -0,0 +1,6 @@
+kind: Features
+body: Add an option to generate static documentation
+time: 2023-09-11T14:41:26.274655701Z
+custom:
+  Author: mescanne
+  Issue: "8614"

core/dbt/cli/main.py

@@ -269,6 +269,7 @@ def docs(ctx, **kwargs):
 @p.select
 @p.selector
 @p.empty_catalog
+@p.static
 @p.state
 @p.defer_state
 @p.deprecated_state

core/dbt/cli/params.py

@@ -465,6 +465,13 @@
     is_flag=True,
 )
 
+static = click.option(
+    "--static",
+    help="Generate an additional static_index.html with manifest and catalog built-in.",
+    default=False,
+    is_flag=True,
+)
+
 state = click.option(
     "--state",
     envvar="DBT_STATE",

core/dbt/task/generate.py

@@ -5,6 +5,7 @@
 import agate
 
 from dbt.dataclass_schema import ValidationError
+from dbt.clients.system import load_file_contents
 
 from .compile import CompileTask
 
@@ -38,7 +39,9 @@
 import dbt.utils
 import dbt.compilation
 import dbt.exceptions
-
+from dbt.constants import (
+    MANIFEST_FILE_NAME,
+)
 
 CATALOG_FILENAME = "catalog.json"
 
@@ -267,14 +270,32 @@ def run(self) -> CatalogArtifact:
             errors=errors,
         )
 
-        path = os.path.join(self.config.project_target_path, CATALOG_FILENAME)
-        results.write(path)
+        catalog_path = os.path.join(self.config.project_target_path, CATALOG_FILENAME)
+        results.write(catalog_path)
         if self.args.compile:
             write_manifest(self.manifest, self.config.project_target_path)
 
+        if self.args.static:
+
+            # Read manifest.json and catalog.json
+            read_manifest_data = load_file_contents(
+                os.path.join(self.config.project_target_path, MANIFEST_FILE_NAME)
+            )
+            read_catalog_data = load_file_contents(catalog_path)
+
+            # Create new static index file contents
+            index_data = load_file_contents(DOCS_INDEX_FILE_PATH)
+            index_data = index_data.replace('"MANIFEST.JSON INLINE DATA"', read_manifest_data)
+            index_data = index_data.replace('"CATALOG.JSON INLINE DATA"', read_catalog_data)
+
+            # Write out the new index file
+            static_index_path = os.path.join(self.config.project_target_path, "static_index.html")
+            with open(static_index_path, "wb") as static_index_file:
+                static_index_file.write(bytes(index_data, "utf8"))
+
         if exceptions:
             fire_event(WriteCatalogFailure(num_exceptions=len(exceptions)))
-        fire_event(CatalogWritten(path=os.path.abspath(path)))
+        fire_event(CatalogWritten(path=os.path.abspath(catalog_path)))
         return results
 
     def get_node_selector(self) -> ResourceTypeSelector:

tests/functional/docs/test_static.py

@@ -0,0 +1,50 @@
+import pytest
+
+from dbt.clients.system import load_file_contents
+from dbt.include.global_project import DOCS_INDEX_FILE_PATH
+from dbt.tests.util import run_dbt
+import os
+
+
+class TestStaticGenerate:
+    @pytest.fixture(scope="class")
+    def models(self):
+        return {"my_model.sql": "select 1 as fun"}
+
+    def test_static_generated(self, project):
+        run_dbt(["docs", "generate", "--static"])
+
+        source_index_html = load_file_contents(DOCS_INDEX_FILE_PATH)
+
+        target_index_html = load_file_contents(
+            os.path.join(project.project_root, "target", "index.html")
+        )
+
+        # Validate index.html was copied correctly
+        assert len(target_index_html) == len(source_index_html)
+        assert hash(target_index_html) == hash(source_index_html)
+
+        manifest_data = load_file_contents(
+            os.path.join(project.project_root, "target", "manifest.json")
+        )
+
+        catalog_data = load_file_contents(
+            os.path.join(project.project_root, "target", "catalog.json")
+        )
+
+        static_index_html = load_file_contents(
+            os.path.join(project.project_root, "target", "static_index.html")
+        )
+
+        # Calculate expected static_index.html
+        expected_static_index_html = source_index_html
+        expected_static_index_html = expected_static_index_html.replace(
+            '"MANIFEST.JSON INLINE DATA"', manifest_data
+        )
+        expected_static_index_html = expected_static_index_html.replace(
+            '"CATALOG.JSON INLINE DATA"', catalog_data
+        )
+
+        # Validate static_index.html was generated correctly
+        assert len(expected_static_index_html) == len(static_index_html)
+        assert hash(expected_static_index_html) == hash(static_index_html)