Fixed bug so all metadata fields are extracted

NERC-CEH · Oct 11, 2024 · 2ec86cd · 2ec86cd · github-actions · Oct 11, 2024
1 parent 8148da1
commit 2ec86cd
Show file tree

Hide file tree

Showing 4 changed files with 19 additions and 16 deletions.
diff --git a/data/.gitignore b/data/.gitignore
@@ -3,3 +3,4 @@
 /eidc_metadata.json
 /prepared_data.json
 /prepared_eidc_metadata.json
+/extracted_metadata.json
diff --git a/dvc.lock b/dvc.lock
@@ -13,18 +13,18 @@ stages:
       md5: 423dc3a61ede72e1d5c818d74277c0b4
       size: 12140491
   prepare:
-    cmd: python scripts/prepare_data.py data/eidc_metadata.json data/prepared_eidc_metadata.json
+    cmd: python scripts/extract_metadata.py data/eidc_metadata.json data/extracted_metadata.json
     deps:
     - path: data/eidc_metadata.json
       hash: md5
       md5: 423dc3a61ede72e1d5c818d74277c0b4
       size: 12140491
-    - path: scripts/prepare_data.py
+    - path: scripts/extract_metadata.py
       hash: md5
-      md5: bcbf4413aeee83928054d9c6c6c2bacc
-      size: 1224
+      md5: c2fa7d2c4b8f28a6e24536ce0df244fd
+      size: 1296
     outs:
-    - path: data/prepared_eidc_metadata.json
+    - path: data/extracted_metadata.json
       hash: md5
-      md5: 0b4ca8c49da450bc8fec0e92d577466c
-      size: 411936
+      md5: 7d2ae8d6a41a960592f30496eb498af7
+      size: 4578493
diff --git a/dvc.yaml b/dvc.yaml
@@ -6,9 +6,9 @@ stages:
     outs:
     - data/eidc_metadata.json
   prepare:
-    cmd: python scripts/prepare_data.py data/eidc_metadata.json data/prepared_eidc_metadata.json
+    cmd: python scripts/extract_metadata.py data/eidc_metadata.json data/extracted_metadata.json
     deps:
     - data/eidc_metadata.json
-    - scripts/prepare_data.py
+    - scripts/extract_metadata.py
     outs:
-    - data/prepared_eidc_metadata.json
+    - data/extracted_metadata.json
diff --git a/scripts/prepare_data.py → scripts/extract_metadata.py b/scripts/prepare_data.py → scripts/extract_metadata.py
@@ -3,17 +3,19 @@
 from argparse import ArgumentParser
 
 
-METADATA_FIELDS = ["title", "description", "lineage", "title"]
+METADATA_FIELDS = ["title", "description", "lineage"]
 
 
-def extact_eidc_metadata_fields(json_data: Dict, fields: List[str] = METADATA_FIELDS) -> Dict[str,str]:
-    metadata = {}
-    metadata["id"] = json_data["identifier"]
+def extact_eidc_metadata_fields(json_data: Dict, fields: List[str] = METADATA_FIELDS) -> List[Dict[str,str]]:
+    metadatas = []
     for field in fields:
         if json_data[field]:
+            metadata = {}
+            metadata["id"] = json_data["identifier"]
             metadata["field"] = field
             metadata["value"] = json_data[field]
-    return metadata
+            metadatas.append(metadata)
+    return metadatas
 
 
 def parse_eidc_metadata(file_path: str) -> List[Dict[str,str]]:
@@ -22,7 +24,7 @@ def parse_eidc_metadata(file_path: str) -> List[Dict[str,str]]:
         json_data = json.load(f)
         for dataset in json_data["results"]:
             dataset_metadata = extact_eidc_metadata_fields(dataset)
-            data.append(dataset_metadata)
+            data.extend(dataset_metadata)
     return data