Fix broken links for TM5

_quarto.yml

@@ -59,6 +59,7 @@ website:
               - user_data_notebooks/epa-ch4emission-grid-v2express_User_Notebook.ipynb
               - user_data_notebooks/vulcan-ffco2-yeargrid-v4_User_Notebook.ipynb
               - user_data_notebooks/gra2pes-ghg-monthgrid-v1_User_Notebook.ipynb
+              - user_data_notebooks/tm54dvar-ch4flux-monthgrid-v1_User_Notebook.ipynb              
           - section: Natural Greenhouse Gas Sources Emissions and Sinks
             contents:
               - user_data_notebooks/eccodarwin-co2flux-monthgrid-v5_User_Notebook.ipynb
@@ -124,7 +125,8 @@ website:
               - processing_and_verification_reports/ct-ch4-monthgrid-v2023_Processing and Verification Report.qmd
               - processing_and_verification_reports/epa-ch4emission-grid-v2express_Processing and Verification Report.qmd 
               - processing_and_verification_reports/vulcan-ffco2-yeargrid-v4_Processing and Verification Report.qmd  
-              - processing_and_verification_reports/gra2pes-ghg-monthgrid-v1_Processing and Verification Report.qmd
+              - processing_and_verification_reports/tm54dvar-ch4flux-monthgrid-v1_Processing and Verification Report.qmd
+
           - section: Natural Greenhouse Gas Sources Emissions and Sinks
             contents:
               - processing_and_verification_reports/eccodarwin-co2flux-monthgrid-v5_Processing and Verification Report.qmd
@@ -159,6 +161,8 @@ website:
               - data_workflow/epa-ch4emission-grid-v2express_Data_Flow.qmd
               - data_workflow/vulcan-ffco2-yeargrid-v4_Data_Flow.qmd  
               - data_workflow/gra2pes-ghg-monthgrid-v1_Data_Flow.qmd 
+              - data_workflow/m54dvar-ch4flux-monthgrid-v1_Data_Flow.qmd 
+
           - section: Natural Greenhouse Gas Sources Emissions and Sinks
             contents:
               - data_workflow/eccodarwin-co2flux-monthgrid-v5_Data_Flow.qmd

cog_transformation/tm54dvar-ch4flux-monthgrid-v1.ipynb

@@ -0,0 +1,129 @@
+{
+ "cells": [
+  {
+   "cell_type": "raw",
+   "metadata": {},
+   "source": [
+    "---\n",
+    "title: TM5-4DVar Isotopic CH₄ Inverse Fluxes\n",
+    "description: Documentation of data transformation\n",
+    "author: Vishal Gaur\n",
+    "date: August 31, 2023\n",
+    "execute:\n",
+    "  freeze: true\n",
+    "---"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "This script was used to transform the TM5-4DVar Isotopic CH₄ Inverse Fluxes dataset from netCDF to Cloud Optimized GeoTIFF (COG) format for display in the Greenhouse Gas (GHG) Center."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "import os\n",
+    "import xarray\n",
+    "import re\n",
+    "import pandas as pd\n",
+    "import json\n",
+    "import tempfile\n",
+    "import boto3\n",
+    "from datetime import datetime"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "session = boto3.session.Session()\n",
+    "s3_client = session.client(\"s3\")\n",
+    "bucket_name = (\n",
+    "    \"ghgc-data-store-dev\"  # S3 bucket where the COGs are stored after transformation\n",
+    ")\n",
+    "FOLDER_NAME = \"tm5-ch4-inverse-flux\"\n",
+    "\n",
+    "files_processed = pd.DataFrame(\n",
+    "    columns=[\"file_name\", \"COGs_created\"]\n",
+    ")  # A dataframe to keep track of the files that we have transformed into COGs\n",
+    "\n",
+    "# Reading the raw netCDF files from local machine\n",
+    "for name in os.listdir(FOLDER_NAME):\n",
+    "    xds = xarray.open_dataset(f\"{FOLDER_NAME}/{name}\", engine=\"netcdf4\")\n",
+    "    xds = xds.rename({\"latitude\": \"lat\", \"longitude\": \"lon\"})\n",
+    "    xds = xds.assign_coords(lon=(((xds.lon + 180) % 360) - 180)).sortby(\"lon\")\n",
+    "    variable = [var for var in xds.data_vars if \"global\" not in var]\n",
+    "\n",
+    "    for time_increment in range(0, len(xds.months)):\n",
+    "        filename = name.split(\"/ \")[-1]\n",
+    "        filename_elements = re.split(\"[_ .]\", filename)\n",
+    "        start_time = datetime(int(filename_elements[-2]), time_increment + 1, 1)\n",
+    "        for var in variable:\n",
+    "            data = getattr(xds.isel(months=time_increment), var)\n",
+    "            data = data.isel(lat=slice(None, None, -1))\n",
+    "            data.rio.set_spatial_dims(\"lon\", \"lat\", inplace=True)\n",
+    "            data.rio.write_crs(\"epsg:4326\", inplace=True)\n",
+    "\n",
+    "            # # insert date of generated COG into filename\n",
+    "            filename_elements.pop()\n",
+    "            filename_elements[-1] = start_time.strftime(\"%Y%m\")\n",
+    "            filename_elements.insert(2, var)\n",
+    "            cog_filename = \"_\".join(filename_elements)\n",
+    "            # # add extension\n",
+    "            cog_filename = f\"{cog_filename}.tif\"\n",
+    "\n",
+    "            with tempfile.NamedTemporaryFile() as temp_file:\n",
+    "                data.rio.to_raster(\n",
+    "                    temp_file.name,\n",
+    "                    driver=\"COG\",\n",
+    "                )\n",
+    "                s3_client.upload_file(\n",
+    "                    Filename=temp_file.name,\n",
+    "                    Bucket=bucket_name,\n",
+    "                    Key=f\"{FOLDER_NAME}/{cog_filename}\",\n",
+    "                )\n",
+    "\n",
+    "            files_processed = files_processed._append(\n",
+    "                {\"file_name\": name, \"COGs_created\": cog_filename},\n",
+    "                ignore_index=True,\n",
+    "            )\n",
+    "\n",
+    "            print(f\"Generated and saved COG: {cog_filename}\")\n",
+    "\n",
+    "# Generate the json file with the metadata that is present in the netCDF files.\n",
+    "with tempfile.NamedTemporaryFile(mode=\"w+\") as fp:\n",
+    "    json.dump(xds.attrs, fp)\n",
+    "    json.dump({\"data_dimensions\": dict(xds.dims)}, fp)\n",
+    "    json.dump({\"data_variables\": list(xds.data_vars)}, fp)\n",
+    "    fp.flush()\n",
+    "\n",
+    "    s3_client.upload_file(\n",
+    "        Filename=fp.name,\n",
+    "        Bucket=bucket_name,\n",
+    "        Key=f\"{FOLDER_NAME}/metadata.json\",\n",
+    "    )\n",
+    "\n",
+    "# creating the csv file with the names of files transformed.\n",
+    "files_processed.to_csv(\n",
+    "    f\"s3://{bucket_name}/{FOLDER_NAME}/files_converted.csv\",\n",
+    ")\n",
+    "print(\"Done generating COGs\")\n"
+   ]
+  }
+ ],
+ "metadata": {
+  "language_info": {
+   "name": "python"
+  },
+  "orig_nbformat": 4
+ },
+ "nbformat": 4,
+ "nbformat_minor": 2
+}

data_workflow/tm54dvar-ch4flux-monthgrid-v1_Processing and Verification Report.qmd

@@ -0,0 +1,9 @@
+---
+title: TM5-4DVar Isotopic CH₄ Inverse Fluxes
+description: "Global, monthly 1 degree resolution methane emission estimates from microbial, fossil and pyrogenic sources derived using inverse modeling, version 1"
+---
+<object data="./reports/tm54dvar-ch4flux-monthgrid-v1_Processing and Verification Report.pdf" type="application/pdf" width="1024px" height="1000px">
+    <embed src="./reports/tm54dvar-ch4flux-monthgrid-v1_Processing and Verification Report.pdf">
+        <p>This browser does not support PDFs. Please download the PDF to view it: <a href="./reports/tm54dvar-ch4flux-monthgrid-v1_Processing and Verification Report.pdf">Download PDF</a>.</p>
+    </embed>
+</object>

processing_and_verification_reports/tm54dvar-ch4flux-monthgrid-v1_Processing and Verification Report.qmd

@@ -0,0 +1,9 @@
+---
+title: TM5-4DVar Isotopic CH₄ Inverse Fluxes
+description: "Global, monthly 1 degree resolution methane emission estimates from microbial, fossil and pyrogenic sources derived using inverse modeling, version 1"
+---
+<object data="./reports/tm54dvar-ch4flux-monthgrid-v1_Processing and Verification Report.pdf" type="application/pdf" width="1024px" height="1000px">
+    <embed src="./reports/tm54dvar-ch4flux-monthgrid-v1_Processing and Verification Report.pdf">
+        <p>This browser does not support PDFs. Please download the PDF to view it: <a href="./reports/tm54dvar-ch4flux-monthgrid-v1_Processing and Verification Report.pdf">Download PDF</a>.</p>
+    </embed>
+</object>