check-MRI-JRA55-do-1-4-0.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Check JRA55-do 1.4.0 input4MIPS files\n",
    "see\n",
    "- https://github.com/COSIMA/access-om2/issues/120\n",
    "- https://github.com/COSIMA/access-om2/issues/155\n",
    "- https://github.com/COSIMA/libaccessom2/issues/20\n",
    "- https://github.com/COSIMA/libaccessom2/issues/25\n",
    "- https://github.com/COSIMA/libaccessom2/pull/26"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/g/data3/hh5/public/apps/miniconda3/envs/analysis3-19.01/lib/python3.6/site-packages/dask/config.py:168: YAMLLoadWarning: calling yaml.load() without Loader=... is deprecated, as the default Loader is unsafe. Please read https://msg.pyyaml.org/load for full details.\n",
      "  data = yaml.load(f.read()) or {}\n",
      "/g/data3/hh5/public/apps/miniconda3/envs/analysis3-19.01/lib/python3.6/site-packages/distributed/config.py:20: YAMLLoadWarning: calling yaml.load() without Loader=... is deprecated, as the default Loader is unsafe. Please read https://msg.pyyaml.org/load for full details.\n",
      "  defaults = yaml.load(f)\n"
     ]
    }
   ],
   "source": [
    "%matplotlib inline\n",
    "import matplotlib.pyplot as plt\n",
    "import numpy as np\n",
    "import xarray as xr\n",
    "import json\n",
    "import yaml\n",
    "import glob\n",
    "import os"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[['/g/data/qv56/replicas/input4MIPs/CMIP6/OMIP/MRI/MRI-JRA55-do-1-4-0/atmos/3hr/rsds/gr/v20190429/rsds/rsds_input4MIPs_atmosphericState_OMIP_MRI-JRA55-do-1-4-0_gr_{{year}}01010130-{{year}}12312230.nc',\n",
       "  '/g/data/qv56/replicas/input4MIPs/CMIP6/OMIP/MRI/MRI-JRA55-do-1-4-0/atmos/3hr/rlds/gr/v20190429/rlds/rlds_input4MIPs_atmosphericState_OMIP_MRI-JRA55-do-1-4-0_gr_{{year}}01010130-{{year}}12312230.nc',\n",
       "  '/g/data/qv56/replicas/input4MIPs/CMIP6/OMIP/MRI/MRI-JRA55-do-1-4-0/atmos/3hr/prra/gr/v20190429/prra/prra_input4MIPs_atmosphericState_OMIP_MRI-JRA55-do-1-4-0_gr_{{year}}01010130-{{year}}12312230.nc',\n",
       "  '/g/data/qv56/replicas/input4MIPs/CMIP6/OMIP/MRI/MRI-JRA55-do-1-4-0/atmos/3hr/prsn/gr/v20190429/prsn/prsn_input4MIPs_atmosphericState_OMIP_MRI-JRA55-do-1-4-0_gr_{{year}}01010130-{{year}}12312230.nc',\n",
       "  '/g/data/qv56/replicas/input4MIPs/CMIP6/OMIP/MRI/MRI-JRA55-do-1-4-0/atmos/3hrPt/psl/gr/v20190429/psl/psl_input4MIPs_atmosphericState_OMIP_MRI-JRA55-do-1-4-0_gr_{{year}}01010000-{{year}}12312100.nc',\n",
       "  '/g/data/qv56/replicas/input4MIPs/CMIP6/OMIP/MRI/MRI-JRA55-do-1-4-0/land/day/friver/gr/v20190429/friver/friver_input4MIPs_atmosphericState_OMIP_MRI-JRA55-do-1-4-0_gr_{{year}}0101-{{year}}1231.nc',\n",
       "  '/g/data/qv56/replicas/input4MIPs/CMIP6/OMIP/MRI/MRI-JRA55-do-1-4-0/atmos/3hrPt/tas/gr/v20190429/tas/tas_input4MIPs_atmosphericState_OMIP_MRI-JRA55-do-1-4-0_gr_{{year}}01010000-{{year}}12312100.nc',\n",
       "  '/g/data/qv56/replicas/input4MIPs/CMIP6/OMIP/MRI/MRI-JRA55-do-1-4-0/atmos/3hrPt/huss/gr/v20190429/huss/huss_input4MIPs_atmosphericState_OMIP_MRI-JRA55-do-1-4-0_gr_{{year}}01010000-{{year}}12312100.nc',\n",
       "  '/g/data/qv56/replicas/input4MIPs/CMIP6/OMIP/MRI/MRI-JRA55-do-1-4-0/atmos/3hrPt/uas/gr/v20190429/uas/uas_input4MIPs_atmosphericState_OMIP_MRI-JRA55-do-1-4-0_gr_{{year}}01010000-{{year}}12312100.nc',\n",
       "  '/g/data/qv56/replicas/input4MIPs/CMIP6/OMIP/MRI/MRI-JRA55-do-1-4-0/atmos/3hrPt/vas/gr/v20190429/vas/vas_input4MIPs_atmosphericState_OMIP_MRI-JRA55-do-1-4-0_gr_{{year}}01010000-{{year}}12312100.nc']]"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# get JRA55-do forcing fields from forcing.json\n",
    "runs = ['input4MIPs_testing_1deg_jra55v1p4p0_iaf_test_gadi']\n",
    "# runs = ['input4MIPs_testing_1deg_jra55_iaf_control', 'input4MIPs_testing_1deg_jra55_iaf_test_yearp1_qv56']  # runs to compare\n",
    "# runs = ['input4MIPs_testing_1deg_jra55_iaf_control', 'input4MIPs_testing_1deg_jra55_iaf_test_yearp1_qv56_2_hybrid']  # runs to compare\n",
    "forcing = []\n",
    "for r in runs:\n",
    "    fpath = '/g/data/hh5/tmp/cosima/access-om2/'+r+'/output000/atmosphere/forcing.json'\n",
    "    with open(fpath) as json_file:\n",
    "        forcing.append(json.load(json_file))\n",
    "\n",
    "# and also get INPUT from config.yaml \n",
    "configs = []\n",
    "for r in runs:\n",
    "    fpath = '/g/data/hh5/tmp/cosima/access-om2/'+r+'/output000/config.yaml'\n",
    "    with open(fpath) as yaml_file:\n",
    "        configs.append(yaml.load(yaml_file, Loader=yaml.FullLoader))\n",
    "inputs = [run['submodels'][0]['input'] for run in configs]  # NB: assumes atmosphere is 0th submodel\n",
    "\n",
    "# construct full filepaths by combining INPUT from config.yaml and filenames from forcing.json\n",
    "filepaths = []\n",
    "filenames = [[a['filename'] for a in f['inputs']] for f in forcing]\n",
    "for lst,inp in zip(filenames,inputs):\n",
    "    fp = []\n",
    "    for fn in lst:\n",
    "        field = fn[len('INPUT/'):fn.find('_')]\n",
    "        fp.append(fn.replace('INPUT',[i for i in inp if i.find(field)>=0][0]))\n",
    "    filepaths.append(fp)\n",
    "        \n",
    "filepaths  "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## check for forcing files that don't match pattern in forcing.json\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'/g/data/qv56/replicas/input4MIPs/CMIP6/OMIP/MRI/MRI-JRA55-do-1-4-0/atmos/3hr/rsds/gr/v20190429/rsds/rsds_input4MIPs_atmosphericState_OMIP_MRI-JRA55-do-1-4-0_gr_201901010130-201901052230.nc'}\n",
      "{'/g/data/qv56/replicas/input4MIPs/CMIP6/OMIP/MRI/MRI-JRA55-do-1-4-0/atmos/3hr/rlds/gr/v20190429/rlds/rlds_input4MIPs_atmosphericState_OMIP_MRI-JRA55-do-1-4-0_gr_201901010130-201901052230.nc'}\n",
      "{'/g/data/qv56/replicas/input4MIPs/CMIP6/OMIP/MRI/MRI-JRA55-do-1-4-0/atmos/3hr/prra/gr/v20190429/prra/prra_input4MIPs_atmosphericState_OMIP_MRI-JRA55-do-1-4-0_gr_201901010130-201901052230.nc'}\n",
      "{'/g/data/qv56/replicas/input4MIPs/CMIP6/OMIP/MRI/MRI-JRA55-do-1-4-0/atmos/3hr/prsn/gr/v20190429/prsn/prsn_input4MIPs_atmosphericState_OMIP_MRI-JRA55-do-1-4-0_gr_201901010130-201901052230.nc'}\n",
      "{'/g/data/qv56/replicas/input4MIPs/CMIP6/OMIP/MRI/MRI-JRA55-do-1-4-0/atmos/3hrPt/psl/gr/v20190429/psl/psl_input4MIPs_atmosphericState_OMIP_MRI-JRA55-do-1-4-0_gr_201901010000-201901052100.nc'}\n",
      "{'/g/data/qv56/replicas/input4MIPs/CMIP6/OMIP/MRI/MRI-JRA55-do-1-4-0/land/day/friver/gr/v20190429/friver/friver_input4MIPs_atmosphericState_OMIP_MRI-JRA55-do-1-4-0_gr_20190101-20190105.nc'}\n",
      "{'/g/data/qv56/replicas/input4MIPs/CMIP6/OMIP/MRI/MRI-JRA55-do-1-4-0/atmos/3hrPt/tas/gr/v20190429/tas/tas_input4MIPs_atmosphericState_OMIP_MRI-JRA55-do-1-4-0_gr_201901010000-201901052100.nc'}\n",
      "{'/g/data/qv56/replicas/input4MIPs/CMIP6/OMIP/MRI/MRI-JRA55-do-1-4-0/atmos/3hrPt/huss/gr/v20190429/huss/huss_input4MIPs_atmosphericState_OMIP_MRI-JRA55-do-1-4-0_gr_201901010000-201901052100.nc'}\n",
      "{'/g/data/qv56/replicas/input4MIPs/CMIP6/OMIP/MRI/MRI-JRA55-do-1-4-0/atmos/3hrPt/uas/gr/v20190429/uas/uas_input4MIPs_atmosphericState_OMIP_MRI-JRA55-do-1-4-0_gr_201901010000-201901052100.nc'}\n",
      "{'/g/data/qv56/replicas/input4MIPs/CMIP6/OMIP/MRI/MRI-JRA55-do-1-4-0/atmos/3hrPt/vas/gr/v20190429/vas/vas_input4MIPs_atmosphericState_OMIP_MRI-JRA55-do-1-4-0_gr_201901010000-201901052100.nc'}\n"
     ]
    }
   ],
   "source": [
    "for fp in filepaths:\n",
    "    for f in fp:\n",
    "        loadedfiles = set(glob.glob(f.replace('{{year}}','*').replace('{{year+1}}','*')))\n",
    "        allfiles = set(glob.glob(os.path.dirname(f)+'/*'))\n",
    "        print(allfiles-loadedfiles)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python [conda env:analysis3-19.01]",
   "language": "python",
   "name": "conda-env-analysis3-19.01-py"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.7"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}