Merge pull request #313 from opensafely-core/make-manifest

Add support for generating test manifests.

airlock/file_browser_api.py

@@ -504,6 +504,9 @@ def build_path_tree(
 
 
 def children_sort_key(node: PathItem):
-    """Sort children first by directory, then files."""
+    """Sort children first by directory, then files.
+
+    The name metadata is sorted first, as its special."""
     # this works because True == 1 and False == 0
-    return (node.type == PathType.FILE, node.name())
+    name = node.name()
+    return (name != "metadata", node.type == PathType.FILE, name)

justfile

@@ -162,7 +162,7 @@ test *ARGS: devenv
 
 
 # load example data so there's something to look at in development
-load-example-data: devenv
+load-example-data: devenv && manifests
     #!/usr/bin/env bash
     set -euo pipefail
 
@@ -205,6 +205,10 @@ load-example-data: devenv
     # Configure user details for local login
     cp example-data/dev_users.json "${AIRLOCK_WORK_DIR%/}/${AIRLOCK_DEV_USERS_FILE}"
 
+# generate manifests for local test workspaces
+manifests:
+    cat scripts/manifests.py | $BIN/python manage.py shell
+
 
 # Run the documentation server: to configure the port, append: ---dev-addr localhost:<port>
 docs-serve *ARGS: devenv
@@ -213,3 +217,4 @@ docs-serve *ARGS: devenv
 # Build the documentation
 docs-build *ARGS: devenv
     "$BIN"/mkdocs build --clean {{ ARGS }}
+

scripts/manifests.py

@@ -0,0 +1,10 @@
+from django.conf import settings
+
+from tests import factories
+
+
+workspaces = [w for w in settings.WORKSPACE_DIR.iterdir() if w.is_dir()]
+
+for workspace in workspaces:
+    factories.update_manifest(workspace.name)
+    print(f"Writing manifest.json for workspace {workspace.name}")

tests/factories.py

@@ -1,7 +1,9 @@
+import csv
 import json
 import subprocess
 import tempfile
 import time
+from hashlib import file_digest
 from pathlib import Path
 
 from django.conf import settings
@@ -50,6 +52,81 @@ def ensure_workspace(workspace_or_name):
     raise Exception(f"Invalid workspace: {workspace_or_name})")  # pragma: nocover
 
 
+# get_output_metadata is imported from job-runner
+def get_output_metadata(
+    abspath, level, job_id, job_request, action, commit, excluded, message=None
+):
+    stat = abspath.stat()
+    with abspath.open("rb") as fp:
+        content_hash = file_digest(fp, "sha256").hexdigest()
+
+    rows = cols = None
+    if abspath.suffix == ".csv":
+        with abspath.open() as fp:
+            reader = csv.DictReader(fp)
+            first_row = next(reader, None)
+            if first_row:
+                cols = len(first_row)
+                rows = sum(1 for _ in reader) + 1
+            else:  # pragma: no cover
+                cols = rows = 0
+
+    return {
+        "level": level,
+        "job_id": job_id,
+        "job_request": job_request,
+        "action": action,
+        "commit": commit,
+        "size": stat.st_size,
+        "timestamp": stat.st_mtime,
+        "content_hash": content_hash,
+        "excluded": excluded,
+        "message": message,
+        "row_count": rows,
+        "col_count": cols,
+    }
+
+
+def update_manifest(workspace, files=None):
+    """Write a manifest based on the files currently in the directory.
+
+    Make up action, job ids and commits.
+    """
+    workspace = ensure_workspace(workspace)
+    root = workspace.root()
+    manifest_path = root / "metadata/manifest.json"
+
+    skip_paths = [root / "logs", root / "metadata"]
+
+    if manifest_path.exists():
+        manifest = json.loads(manifest_path.read_text())
+        manifest["workspace"] = workspace.name
+        manifest.setdefault("outputs", {})
+    else:
+        manifest = {"workspace": workspace.name, "repo": None, "outputs": {}}
+
+    if files is None:  # pragma: nocover
+        files = [
+            f.relative_to(root)
+            for f in root.glob("**/*")
+            if f.is_file() and not any(1 for path in skip_paths if path in f.parents)
+        ]
+
+    for i, f in enumerate(files):
+        manifest["outputs"][str(f)] = get_output_metadata(
+            root / f,
+            level="moderately_senstive",
+            job_id=f"job_{i}",
+            job_request=f"job_request_{i}",
+            action=f"action_{i}",
+            commit="abcdefgh" * 5,  # 40 characters,
+            excluded=False,
+        )
+
+    manifest_path.parent.mkdir(exist_ok=True, parents=True)
+    manifest_path.write_text(json.dumps(manifest, indent=2))
+
+
 def create_workspace(name, user=None):
     # create a default user with permission on workspace
     if user is None:  # pragma: nocover
@@ -60,11 +137,13 @@ def create_workspace(name, user=None):
     return bll.get_workspace(name, user)
 
 
-def write_workspace_file(workspace, path, contents=""):
+def write_workspace_file(workspace, path, contents="", manifest=True):
     workspace = ensure_workspace(workspace)
     path = workspace.root() / path
     path.parent.mkdir(parents=True, exist_ok=True)
     path.write_text(contents)
+    if manifest:  # pragma: nocover
+        update_manifest(workspace, [path])
 
 
 def create_repo(workspace, files=None):

tests/unit/test_business_logic.py

@@ -1493,7 +1493,9 @@ def test_group_comment_permissions(bll):
 
 def test_coderepo_from_workspace_bad_json():
     workspace = factories.create_workspace("workspace")
-    factories.write_workspace_file("workspace", "metadata/manifest.json", "")
+    bad_manifest = workspace.root() / "metadata/manifest.json"
+    bad_manifest.parent.mkdir()
+    bad_manifest.write_text("")
 
     with pytest.raises(CodeRepo.RepoNotFound):
         CodeRepo.from_workspace(workspace, "commit")

tests/unit/test_file_browser_api.py

@@ -46,6 +46,8 @@ def test_get_workspace_tree_general(workspace):
     expected = textwrap.dedent(
         """
         workspace*
+          metadata
+            manifest.json
           empty_dir
           some_dir*
             .file.txt
@@ -145,6 +147,7 @@ def test_get_workspace_tree_selected_only_root(workspace):
     expected = textwrap.dedent(
         """
         workspace***
+          metadata
           empty_dir
           some_dir
         """
@@ -422,6 +425,7 @@ def test_workspace_tree_siblings(workspace):
 
     assert tree.siblings() == []
     assert {s.name() for s in tree.get_path("some_dir").siblings()} == {
+        "metadata",
         "empty_dir",
         "some_dir",
     }