Skip to content

Commit

Permalink
Merge pull request #358 from blaylockbk/revamp-gfs-template
Browse files Browse the repository at this point in the history 
update GFS and GEFS template
  • Loading branch information
@blaylockbk
blaylockbk authored Aug 7, 2024
2 parents 3fe63a1 + e47ecba commit 047ea49
Show file tree
Hide file tree
Showing 4 changed files with 1,733 additions and 73 deletions.
10 changes: 8 additions & 2 deletions herbie/core.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -515,9 +515,15 @@ def find_idx(self) -> tuple[Optional[Union[Path, str]], Optional[str]]:

@property
def get_remoteFileName(self, source: Optional[str] = None) -> str:
"""Predict remote file name (assumes all sources are named the same)."""
"""Predict remote file name."""
if source is None:
source = list(self.SOURCES)[0]
if hasattr(self, "grib_source") and self.grib_source != "local":
source = self.grib_source
else:
# Just pick the first source in the template
# Note: this might not always be the best approach
# the file names are not consistent between sources.
source = list(self.SOURCES)[0]
return self.SOURCES[source].split("/")[-1]

@property
Expand Down
49 changes: 35 additions & 14 deletions herbie/models/gefs.py
View file
Original file line number Diff line number Diff line change
@@ -1,15 +1,22 @@
## Added by Brian Blaylock
## March 11, 2022

"""
A Herbie template for the GEFS (2017-Present) and GEFS Reforecast (2000-2019)
GRIB2 products.
"""A Herbie template for the GEFS (2017-Present) and GEFS Re-forecast (2000-2019) GRIB2 products."""

"""
from datetime import datetime


class gefs:
"""Global Ensemble Forecast System (GEFS).
Notes
-----
The NODD program provides GEFS data on all major cloud partners
- aws: Archive begins 2017-01-01
- google: Archive begins 2021-01-01
- azure: Last 30 days?
"""

def template(self):
self.DESCRIPTION = "Global Ensemble Forecast System (GEFS)"
self.DETAILS = {
Expand Down Expand Up @@ -47,14 +54,28 @@ def template(self):
self.member = f"p{self.member:02d}"

filedir = f"gefs.{self.date:%Y%m%d/%H}"
filepaths = {
"atmos.5": f"{filedir}/atmos/pgrb2ap5/ge{self.member}.t{self.date:%H}z.pgrb2a.0p50.f{self.fxx:03d}",
"atmos.5b": f"{filedir}/atmos/pgrb2bp5/ge{self.member}.t{self.date:%H}z.pgrb2b.0p50.f{self.fxx:03d}",
"atmos.25": f"{filedir}/atmos/pgrb2sp25/ge{self.member}.t{self.date:%H}z.pgrb2s.0p25.f{self.fxx:03d}",
"wave": f"{filedir}/wave/gridded/gefs.wave.t{self.date:%H}z.{self.member}.global.0p25.f{self.fxx:03d}.grib2",
"chem.5": f"{filedir}/chem/pgrb2ap25/gefs.chem.t{self.date:%H}z.a2d_0p25.f{self.fxx:03d}.grib2",
"chem.25": f"{filedir}/chem/pgrb2ap25/gefs.chem.t{self.date:%H}z.a2d_0p25.f{self.fxx:03d}.grib2",
}

if self.date < datetime(2018, 7, 27):
filepaths = {
"atmos.5": f"{filedir}/ge{self.member}.t{self.date:%H}z.pgrb2af{self.fxx:03d}",
"atmos.5b": f"{filedir}/ge{self.member}.t{self.date:%H}z.pgrb2bf{self.fxx:03d}",
}
elif self.date < datetime(2020, 9, 23):
# Update to GEFS system to put data in directories. Change in form for lead time to fxx.
filepaths = {
"atmos.5": f"{filedir}/pgrb2a/ge{self.member}.t{self.date:%H}z.pgrb2af{self.fxx:02d}",
"atmos.5b": f"{filedir}/pgrb2b/ge{self.member}.t{self.date:%H}z.pgrb2bf{self.fxx:02d}",
}
else:
# Update to GEFS system with wave and chem products. Change in form for lead time to fxxx.
filepaths = {
"atmos.5": f"{filedir}/atmos/pgrb2ap5/ge{self.member}.t{self.date:%H}z.pgrb2a.0p50.f{self.fxx:03d}",
"atmos.5b": f"{filedir}/atmos/pgrb2bp5/ge{self.member}.t{self.date:%H}z.pgrb2b.0p50.f{self.fxx:03d}",
"atmos.25": f"{filedir}/atmos/pgrb2sp25/ge{self.member}.t{self.date:%H}z.pgrb2s.0p25.f{self.fxx:03d}",
"wave": f"{filedir}/wave/gridded/gefs.wave.t{self.date:%H}z.{self.member}.global.0p25.f{self.fxx:03d}.grib2",
"chem.5": f"{filedir}/chem/pgrb2ap25/gefs.chem.t{self.date:%H}z.a2d_0p25.f{self.fxx:03d}.grib2",
"chem.25": f"{filedir}/chem/pgrb2ap25/gefs.chem.t{self.date:%H}z.a2d_0p25.f{self.fxx:03d}.grib2",
}

valid_members = {
"atmos.5": [f"p{i:02d}" for i in range(1, 31)] + ["c00", "spr", "avg"],
Expand Down Expand Up @@ -89,7 +110,7 @@ def template(self):


class gefs_reforecast:
"""Template for GEFS Reforecast data.
"""Template for GEFS Re-forecast data.
These grib files are organized different from other model types.
The files are grouped into variables and clumped by forecast range.
Expand Down
182 changes: 125 additions & 57 deletions herbie/models/gfs.py
View file
Original file line number Diff line number Diff line change
@@ -1,40 +1,37 @@
## Added by Brian Blaylock
## July 26, 2021
"""Herbie template for GFS products."""


class gdas:
def template(self):
self.DESCRIPTION = "Global Data Assimilation System"
self.DETAILS = {
"nomads product description": "https://www.nco.ncep.noaa.gov/pmb/products/gfs/#GDAS",
"google cloud platform": "https://console.cloud.google.com/marketplace/product/noaa-public/gfs?q=search&referrer=search&project=python-232920",
"azure document": "https://github.com/microsoft/AIforEarthDatasets#noaa-global-forecast-system-gfs",
"aws document": "https://registry.opendata.aws/noaa-gfs-bdp-pds",
}
self.PRODUCTS = {
"pgrb2.0p25": "common fields, 0.25 degree resolution",
"pgrb2.1p00": "common fields, 1.00 degree resolution",
}
self.SOURCES = {
"aws": f"https://noaa-gfs-bdp-pds.s3.amazonaws.com/gdas.{self.date:%Y%m%d/%H}/atmos/gdas.t{self.date:%H}z.{self.product}.f{self.fxx:03d}",
"aws-old": f"https://noaa-gfs-bdp-pds.s3.amazonaws.com/gdas.{self.date:%Y%m%d/%H}/gdas.t{self.date:%H}z.{self.product}.f{self.fxx:03d}",
"ftpprd": f"https://ftpprd.ncep.noaa.gov/data/nccf/com/gfs/prod/gdas.{self.date:%Y%m%d/%H}/atmos/gdas.t{self.date:%H}z.{self.product}.f{self.fxx:03d}",
"nomads": f"https://nomads.ncep.noaa.gov/pub/data/nccf/com/gfs/prod/gdas.{self.date:%Y%m%d/%H}/atmos/gdas.t{self.date:%H}z.{self.product}.f{self.fxx:03d}",
"google": f"https://storage.googleapis.com/global-forecast-system/gdas.{self.date:%Y%m%d/%H}/atmos/gdas.t{self.date:%H}z.{self.product}.f{self.fxx:03d}",
"azure": f"https://noaagfs.blob.core.windows.net/gfs/gdas.{self.date:%Y%m%d/%H}/atmos/gfs.t{self.date:%H}z.{self.product}.f{self.fxx:03d}",
}
self.IDX_SUFFIX = [".idx"]
self.LOCALFILE = f"{self.get_remoteFileName}"
from datetime import datetime


class gfs:
"""Global Forecast System Atmosphere Products.
Notes
-----
The NODD program provides GFS data on all major cloud partners
- aws: Archive begins 2021-01-01
- google: Archive begins 2021-01-01
- azure: Last 30 days
The NCAR RDA archive only provides GFS data at 0.25 degree
resolution and these files do not have index files which makes
variable subsetting impossible.
NCEI archive is spotty at best.
"""

def template(self):
self.DESCRIPTION = "Global Forecast System"
self.DETAILS = {
"nomads product description": "https://www.nco.ncep.noaa.gov/pmb/products/gfs",
"google cloud platform": "https://console.cloud.google.com/marketplace/product/noaa-public/gfs?q=search&referrer=search&project=python-232920",
"azure document": "https://github.com/microsoft/AIforEarthDatasets#noaa-global-forecast-system-gfs",
"azure document": [
"https://github.com/microsoft/AIforEarthDatasets#noaa-global-forecast-system-gfs",
"https://microsoft.github.io/AIforEarthDataSets/data/noaa-gfs.html",
],
"aws document": "https://registry.opendata.aws/noaa-gfs-bdp-pds",
"NCAR Research Data Archive (RDA)": "https://rda.ucar.edu/datasets/d084001/",
"NCEI": "https://www.ncei.noaa.gov/products/weather-climate-models/global-forecast",
}
self.PRODUCTS = {
"pgrb2.0p25": "common fields, 0.25 degree resolution",
Expand All @@ -44,34 +41,41 @@ def template(self):
"pgrb2b.0p50": "uncommon fields, 0.50 degree resolution",
"pgrb2b.1p00": "uncommon fields, 1.00 degree resolution",
"pgrb2full.0p50": "combined grids of 0.50 resolution",
"sfluxgrb": "surface flux fields, T1534 Semi-Lagrangian grid",
"goesimpgrb2.0p25": ", 0.50 degree resolution",
}
self.SOURCES = {
"aws": f"https://noaa-gfs-bdp-pds.s3.amazonaws.com/gfs.{self.date:%Y%m%d/%H}/atmos/gfs.t{self.date:%H}z.{self.product}.f{self.fxx:03d}",
"aws-old": f"https://noaa-gfs-bdp-pds.s3.amazonaws.com/gfs.{self.date:%Y%m%d/%H}/gfs.t{self.date:%H}z.{self.product}.f{self.fxx:03d}",
"ftpprd": f"https://ftpprd.ncep.noaa.gov/data/nccf/com/gfs/prod/gfs.{self.date:%Y%m%d/%H}/atmos/gfs.t{self.date:%H}z.{self.product}.f{self.fxx:03d}",
"nomads": f"https://nomads.ncep.noaa.gov/pub/data/nccf/com/gfs/prod/gfs.{self.date:%Y%m%d/%H}/atmos/gfs.t{self.date:%H}z.{self.product}.f{self.fxx:03d}",
"google": f"https://storage.googleapis.com/global-forecast-system/gfs.{self.date:%Y%m%d/%H}/atmos/gfs.t{self.date:%H}z.{self.product}.f{self.fxx:03d}",
"azure": f"https://noaagfs.blob.core.windows.net/gfs/gfs.{self.date:%Y%m%d/%H}/atmos/gfs.t{self.date:%H}z.{self.product}.f{self.fxx:03d}",
}
self.IDX_SUFFIX = [".idx"]
self.LOCALFILE = f"{self.get_remoteFileName}"

class graphcast:
def template(self):
self.DESCRIPTION = "GraphCast Global Forecast System (EXPERIMENTAL)"
self.DETAILS = {
"aws document": "https://registry.opendata.aws/noaa-nws-graphcastgfs-pds/",
}
self.PRODUCTS = {
"pgrb2.0p25": "common fields, 0.25 degree resolution",
}
if self.date < datetime(2021, 3, 23):
post_root = f"gfs.{self.date:%Y%m%d/%H}/gfs.t{self.date:%H}z.{self.product}.f{self.fxx:03d}"
else:
# GFS update version 16.0
# https://www.emc.ncep.noaa.gov/emc/pages/numerical_forecast_systems/gfs/implementations.php
post_root = f"gfs.{self.date:%Y%m%d/%H}/atmos/gfs.t{self.date:%H}z.{self.product}.f{self.fxx:03d}"

if self.product == "sfluxgrb":
post_root = post_root.replace("sfluxgrb.", "sfluxgrb")

self.SOURCES = {
"aws": f"https://noaa-nws-graphcastgfs-pds.s3.amazonaws.com/graphcastgfs.{self.date:%Y%m%d/%H}/forecasts_13_levels/graphcastgfs.t{self.date:%H}z.{self.product}.f{self.fxx:03d}",
"aws": f"https://noaa-gfs-bdp-pds.s3.amazonaws.com/{post_root}",
"ftpprd": f"https://ftpprd.ncep.noaa.gov/data/nccf/com/gfs/prod/{post_root}",
"nomads": f"https://nomads.ncep.noaa.gov/pub/data/nccf/com/gfs/prod/{post_root}",
"google": f"https://storage.googleapis.com/global-forecast-system/{post_root}",
"azure": f"https://noaagfs.blob.core.windows.net/gfs/{post_root}",
"ncar_rda": f"https://data.rda.ucar.edu/d084001/{self.date:%Y/%Y%m%d}/gfs.0p25.{self.date:%Y%m%d%H}.f{self.fxx:03d}.grib2",
}
self.IDX_SUFFIX = [".idx"]
self.LOCALFILE = f"{self.get_remoteFileName}"


class gfs_wave:
"""
Global Forecast System Wave Products.
Wave products were made available with the GFS v16.0 upgrade on
March 22, 2021.
https://www.emc.ncep.noaa.gov/emc/pages/numerical_forecast_systems/gfs/implementations.php
"""

def template(self):
self.DESCRIPTION = "Global Forecast System - Wave Products"
self.DETAILS = {
Expand All @@ -86,15 +90,58 @@ def template(self):
"gsouth.0p25": "Global South; 0.25 deg resolution",
"wcoast.0p16": "West Coast; 0.16 deg resolution",
}

post_root = f"gfs.{self.date:%Y%m%d/%H}/wave/gridded/gfswave.t{self.date:%H}z.{self.product}.f{self.fxx:03d}.grib2"

self.SOURCES = {
"aws": f"https://noaa-gfs-bdp-pds.s3.amazonaws.com/gfs.{self.date:%Y%m%d/%H}/wave/gridded/gfswave.t{self.date:%H}z.{self.product}.f{self.fxx:03d}.grib2",
"ftpprd": f"https://ftpprd.ncep.noaa.gov/data/nccf/com/gfs/prod/gfs.{self.date:%Y%m%d/%H}/wave/gridded/gfswave.t{self.date:%H}z.{self.product}.f{self.fxx:03d}.grib2",
"nomads": f"https://nomads.ncep.noaa.gov/pub/data/nccf/com/gfs/prod/gfs.{self.date:%Y%m%d/%H}/wave/gridded/gfswave.t{self.date:%H}z.{self.product}.f{self.fxx:03d}.grib2",
"google": f"https://storage.googleapis.com/global-forecast-system/gfs.{self.date:%Y%m%d/%H}/wave/gridded/gfswave.t{self.date:%H}z.{self.product}.f{self.fxx:03d}.grib2",
"azure": f"https://noaahrrr.blob.core.windows.net/gfs/gfs.{self.date:%Y%m%d/%H}/wave/gridded/gfswave.t{self.date:%H}z.{self.product}.f{self.fxx:03d}.grib2",
"aws": f"https://noaa-gfs-bdp-pds.s3.amazonaws.com/{post_root}",
"ftpprd": f"https://ftpprd.ncep.noaa.gov/data/nccf/com/gfs/prod/{post_root}",
"nomads": f"https://nomads.ncep.noaa.gov/pub/data/nccf/com/gfs/prod/{post_root}",
"google": f"https://storage.googleapis.com/global-forecast-system/{post_root}",
"azure": f"https://noaahrrr.blob.core.windows.net/gfs/{post_root}",
}
self.LOCALFILE = f"{self.get_remoteFileName}"


class gdas:
"""Global Data Assimilation System."""

def template(self):
self.DESCRIPTION = "Global Data Assimilation System"
self.DETAILS = {
"nomads product description": "https://www.nco.ncep.noaa.gov/pmb/products/gfs/#GDAS",
"google cloud platform": "https://console.cloud.google.com/marketplace/product/noaa-public/gfs?q=search&referrer=search&project=python-232920",
"azure document": "https://github.com/microsoft/AIforEarthDatasets#noaa-global-forecast-system-gfs",
"aws document": "https://registry.opendata.aws/noaa-gfs-bdp-pds",
}
self.PRODUCTS = {
"pgrb2.0p25": "common fields, 0.25 degree resolution",
"pgrb2.1p00": "common fields, 1.00 degree resolution",
"sfluxgrb": "surface flux fields, T1534 Semi-Lagrangian grid",
}

if self.date < datetime(2021, 3, 23):
post_root = f"gdas.{self.date:%Y%m%d/%H}/gdas.t{self.date:%H}z.{self.product}.f{self.fxx:03d}"
else:
# GFS update version 16.0
# https://www.emc.ncep.noaa.gov/emc/pages/numerical_forecast_systems/gfs/implementations.php
post_root = f"gdas.{self.date:%Y%m%d/%H}/atmos/gdas.t{self.date:%H}z.{self.product}.f{self.fxx:03d}"

if self.product == "sfluxgrb":
post_root = post_root.replace("sfluxgrb.", "sfluxgrb")

self.SOURCES = {
"aws": f"https://noaa-gfs-bdp-pds.s3.amazonaws.com/{post_root}",
"aws-old": f"https://noaa-gfs-bdp-pds.s3.amazonaws.com/{post_root}",
"ftpprd": f"https://ftpprd.ncep.noaa.gov/data/nccf/com/gfs/prod/{post_root}",
"nomads": f"https://nomads.ncep.noaa.gov/pub/data/nccf/com/gfs/prod/{post_root}",
"google": f"https://storage.googleapis.com/global-forecast-system/{post_root}",
"azure": f"https://noaagfs.blob.core.windows.net/gfs/{post_root}",
}
self.IDX_SUFFIX = [".idx"]
self.LOCALFILE = f"{self.get_remoteFileName}"


class gdas_wave:
def template(self):
self.DESCRIPTION = "Global Data Assimilation System - Wave Products"
Expand All @@ -110,11 +157,32 @@ def template(self):
"gsouth.0p25": "Global South; 0.25 deg resolution",
"wcoast.0p16": "West Coast; 0.16 deg resolution",
}

post_root = f"gdas.{self.date:%Y%m%d/%H}/wave/gridded/gdaswave.t{self.date:%H}z.{self.product}.f{self.fxx:03d}.grib2"

self.SOURCES = {
"aws": f"https://noaa-gfs-bdp-pds.s3.amazonaws.com/gdas.{self.date:%Y%m%d/%H}/wave/gridded/gdaswave.t{self.date:%H}z.{self.product}.f{self.fxx:03d}.grib2",
"ftpprd": f"https://ftpprd.ncep.noaa.gov/data/nccf/com/gfs/prod/gdas.{self.date:%Y%m%d/%H}/wave/gridded/gdaswave.t{self.date:%H}z.{self.product}.f{self.fxx:03d}.grib2",
"nomads": f"https://nomads.ncep.noaa.gov/pub/data/nccf/com/gfs/prod/gdas.{self.date:%Y%m%d/%H}/wave/gridded/gdaswave.t{self.date:%H}z.{self.product}.f{self.fxx:03d}.grib2",
"google": f"https://storage.googleapis.com/global-forecast-system/gdas.{self.date:%Y%m%d/%H}/wave/gridded/gdaswave.t{self.date:%H}z.{self.product}.f{self.fxx:03d}.grib2",
"azure": f"https://noaahrrr.blob.core.windows.net/gfs/gdas.{self.date:%Y%m%d/%H}/wave/gridded/gdaswave.t{self.date:%H}z.{self.product}.f{self.fxx:03d}.grib2",
"aws": f"https://noaa-gfs-bdp-pds.s3.amazonaws.com/{post_root}",
"ftpprd": f"https://ftpprd.ncep.noaa.gov/data/nccf/com/gfs/prod/{post_root}",
"nomads": f"https://nomads.ncep.noaa.gov/pub/data/nccf/com/gfs/prod/{post_root}",
"google": f"https://storage.googleapis.com/global-forecast-system/{post_root}",
"azure": f"https://noaahrrr.blob.core.windows.net/gfs/{post_root}",
}
self.LOCALFILE = f"{self.get_remoteFileName}"


class graphcast:
"""GraphCast Global Forecast System."""

def template(self):
self.DESCRIPTION = "GraphCast Global Forecast System (EXPERIMENTAL)"
self.DETAILS = {
"aws document": "https://registry.opendata.aws/noaa-nws-graphcastgfs-pds/",
}
self.PRODUCTS = {
"pgrb2.0p25": "common fields, 0.25 degree resolution",
}
self.SOURCES = {
"aws": f"https://noaa-nws-graphcastgfs-pds.s3.amazonaws.com/graphcastgfs.{self.date:%Y%m%d/%H}/forecasts_13_levels/graphcastgfs.t{self.date:%H}z.{self.product}.f{self.fxx:03d}",
}
self.IDX_SUFFIX = [".idx"]
self.LOCALFILE = f"{self.get_remoteFileName}"
Loading

0 comments on commit 047ea49

Please sign in to comment.