Skip to content

Commit

Permalink
Merge pull request #74 from eco4cast/prod
Browse files Browse the repository at this point in the history 
Prod to main
  • Loading branch information
@jzwart
jzwart authored Jun 13, 2024
2 parents 494d89e + 9bb8ce4 commit 7791959
Show file tree
Hide file tree
Showing 37 changed files with 1,729 additions and 1,183 deletions.
225 changes: 220 additions & 5 deletions .github/workflows/catalog.yaml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,143 @@ on:
name: catalog

jobs:
catalog:
metadata_catalog:
runs-on: ubuntu-latest
env:
GITHUB_PAT: ${{ secrets.GITHUB_TOKEN }}
OSN_KEY: ${{ secrets.OSN_KEY }}
OSN_SECRET: ${{ secrets.OSN_SECRET }}
container: eco4cast/rocker-neon4cast:latest
steps:
- run: git config --system --add safe.directory '*'

- uses: actions/checkout@v4
with:
ref: prod
fetch-depth: 0
set-safe-directory: '*'

- name: install validator
run: |
pip install stac-validator
- name: Render metadata
shell: Rscript {0}
run: source("catalog/model_metadata.R")

- name: Render catalog
shell: Rscript {0}
run: source("catalog/catalog.R")

- name: Commit and Push
run: |
git pull
git config user.name github-actions
git config user.email [email protected]
git add catalog/* .
git commit -a -m "update catalog" || echo "nothing to commit"
git push https://${GITHUB_PAT}:${GITHUB_PAT}@github.com/${GITHUB_REPOSITORY}
forecasts:
needs: metadata_catalog
if: success() || failure()
runs-on: ubuntu-latest
env:
GITHUB_PAT: ${{ secrets.GITHUB_TOKEN }}
OSN_KEY: ${{ secrets.OSN_KEY }}
OSN_SECRET: ${{ secrets.OSN_SECRET }}
container: eco4cast/rocker-neon4cast:latest
steps:
- run: git config --system --add safe.directory '*'

- uses: actions/checkout@v3
with:
ref: prod
fetch-depth: 0
set-safe-directory: '*'

- name: install validator
run: |
pip install stac-validator
- name: Render
shell: Rscript {0}
run: source('catalog/forecasts/forecast_models.R')

- name: Commit and Push
run: |
git pull
git config user.name github-actions
git config user.email [email protected]
git add catalog/* .
git commit -a -m "update catalog" || echo "nothing to commit"
git push https://${GITHUB_PAT}:${GITHUB_PAT}@github.com/${GITHUB_REPOSITORY}
scores:
needs: forecasts
if: success() || failure()
runs-on: ubuntu-latest
env:
GITHUB_PAT: ${{ secrets.GITHUB_TOKEN }}
OSN_KEY: ${{ secrets.OSN_KEY }}
OSN_SECRET: ${{ secrets.OSN_SECRET }}
container: eco4cast/rocker-neon4cast:latest
steps:
- run: git config --system --add safe.directory '*'

- uses: actions/checkout@v3
with:
ref: prod
fetch-depth: 0
set-safe-directory: '*'

- name: install validator
run: |
pip install stac-validator
- name: Render
shell: Rscript {0}
run: source('catalog/scores/scores_models.R')

- name: Commit and Push
run: |
git pull
git config user.name github-actions
git config user.email [email protected]
git add catalog/* .
git commit -a -m "update catalog" || echo "nothing to commit"
git push https://${GITHUB_PAT}:${GITHUB_PAT}@github.com/${GITHUB_REPOSITORY}
inventory:
needs: scores
if: success() || failure()
runs-on: ubuntu-latest
env:
GITHUB_PAT: ${{ secrets.GITHUB_TOKEN }}
OSN_KEY: ${{ secrets.OSN_KEY }}
OSN_SECRET: ${{ secrets.OSN_SECRET }}
container: eco4cast/rocker-neon4cast:latest
steps:
- run: git config --system --add safe.directory '*'

- uses: actions/checkout@v3
with:
ref: prod
fetch-depth: 0
set-safe-directory: '*'

- name: install validator
run: |
pip install stac-validator
- name: Render
shell: Rscript {0}
run: source('catalog/inventory/create_inventory_page.R')

- name: Commit and Push
run: |
git pull
git config user.name github-actions
git config user.email [email protected]
git add catalog/* .
git commit -a -m "update catalog" || echo "nothing to commit"
git push https://${GITHUB_PAT}:${GITHUB_PAT}@github.com/${GITHUB_REPOSITORY}
summaries:
needs: inventory
if: success() || failure()
runs-on: ubuntu-latest
env:
GITHUB_PAT: ${{ secrets.GITHUB_TOKEN }}
Expand All @@ -18,23 +154,102 @@ jobs:

- uses: actions/checkout@v3
with:
ref: prod
fetch-depth: 0
set-safe-directory: '*'
ref: prod
fetch-depth: 0
set-safe-directory: '*'

- name: install validator
run: |
pip install stac-validator
- name: Render
shell: Rscript {0}
run: source('catalog/summaries/summaries_models.R')

- name: Commit and Push
run: |
git pull
git config user.name github-actions
git config user.email [email protected]
git add catalog/* .
git commit -a -m "update catalog" || echo "nothing to commit"
git push https://${GITHUB_PAT}:${GITHUB_PAT}@github.com/${GITHUB_REPOSITORY}
noaa:
needs: summaries
if: success() || failure()
runs-on: ubuntu-latest
env:
GITHUB_PAT: ${{ secrets.GITHUB_TOKEN }}
OSN_KEY: ${{ secrets.OSN_KEY }}
OSN_SECRET: ${{ secrets.OSN_SECRET }}
container: eco4cast/rocker-neon4cast:latest
steps:
- run: git config --system --add safe.directory '*'

- uses: actions/checkout@v3
with:
ref: prod
fetch-depth: 0
set-safe-directory: '*'

- name: install validator
run: |
pip install stac-validator
- name: Render
shell: Rscript {0}
run: source("catalog/update_stac.R")
run: source('catalog/noaa_forecasts/noaa_forecasts.R')

- name: Commit and Push
run: |
git pull
git config user.name github-actions
git config user.email [email protected]
git add catalog/* .
git commit -a -m "update catalog" || echo "nothing to commit"
git push https://${GITHUB_PAT}:${GITHUB_PAT}@github.com/${GITHUB_REPOSITORY}
targets_sites:
needs: noaa
if: success() || failure()
runs-on: ubuntu-latest
env:
GITHUB_PAT: ${{ secrets.GITHUB_TOKEN }}
OSN_KEY: ${{ secrets.OSN_KEY }}
OSN_SECRET: ${{ secrets.OSN_SECRET }}
container: eco4cast/rocker-neon4cast:latest
steps:
- run: git config --system --add safe.directory '*'

- uses: actions/checkout@v3
with:
ref: prod
fetch-depth: 0
set-safe-directory: '*'

- name: install validator
run: |
pip install stac-validator
- name: Render targets
shell: Rscript {0}
run: source('catalog/targets/create_targets_page.R')

- name: Render sites
shell: Rscript {0}
run: source('catalog/sites/build_sites_page.R')

- name: Commit and Push
run: |
git pull
git config user.name github-actions
git config user.email [email protected]
git add catalog/* .
git commit -a -m "update catalog" || echo "nothing to commit"
git push https://${GITHUB_PAT}:${GITHUB_PAT}@github.com/${GITHUB_REPOSITORY}
Healthcheck:
needs: [metadata_catalog, forecasts, scores, inventory, summaries, noaa, targets_sites]
runs-on: ubuntu-latest
env:
GITHUB_PAT: ${{ secrets.GITHUB_TOKEN }}
container: eco4cast/rocker-neon4cast:latest
steps:
- name: Call Healthcheck
run: |
curl -m 10 --retry 5 https://hc-ping.com/22fcd257-7930-455c-948d-0f913743b0c3
56 changes: 34 additions & 22 deletions catalog/forecasts/aquatics/Daily_Chlorophyll_a/collection.json
View file
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
{
"id": "Daily_Chlorophyll_a",
"description": "This page includes all models for the Daily_Chlorophyll_a variable.",
"description": "All models for the Daily_Chlorophyll_a variable. The variable description is as follows: daily mean Chlorophyll-a (ug/L)",
"stac_version": "1.0.0",
"license": "CC0-1.0",
"stac_extensions": [
Expand All @@ -9,16 +9,23 @@
"https://stac-extensions.github.io/table/v1.2.0/schema.json"
],
"type": "Collection",
"sci:doi": "10.1002/fee.2616",
"sci:publications": {},
"links": [
{
"rel": "item",
"type": "application/json",
"href": "../../models/model_items/climatology.json"
"href": "./models/persistenceRW.json"
},
{
"rel": "item",
"type": "application/json",
"href": "../../models/model_items/persistenceRW.json"
"href": "./models/USGSHABs1.json"
},
{
"rel": "item",
"type": "application/json",
"href": "./models/climatology.json"
},
{
"rel": "parent",
Expand Down Expand Up @@ -47,7 +54,7 @@
},
{
"rel": "describedby",
"href": "https://projects.ecoforecast.org/usgsrc4cast-docs/",
"href": "https://projects.ecoforecast.org/usgsrc4cast-ci/",
"title": "EFI-USGS River Chlorophyll Forecast Challenge Dashboard",
"type": "text/html"
}
Expand All @@ -56,23 +63,23 @@
"extent": {
"spatial": {
"bbox": [
["Inf", "Inf", "-Inf", "-Inf"]
[-122.6692, 39.6328, -74.7781, 45.5175]
]
},
"temporal": {
"interval": [
[
"2024-02-07T00:00:00Z",
"2024-03-17T00:00:00Z"
"2024-07-17T00:00:00Z"
]
]
}
},
"table:columns": [
{
"name": "reference_datetime",
"type": "timestamp[us, tz=UTC]",
"description": "datetime that the forecast was initiated (horizon = 0)"
"name": "13 columns",
"type": null,
"description": {}
},
{
"name": "datetime",
Expand All @@ -85,25 +92,35 @@
"description": "For forecasts that are not on a spatial grid, use of a site dimension that maps to a more detailed geometry (points, polygons, etc.) is allowable. In general this would be documented in the external metadata (e.g., alook-up table that provides lon and lat)"
},
{
"name": "family",
"type": "string",
"description": "For ensembles: “ensemble.” Default value if unspecified for probability distributions: Name of the statistical distribution associated with the reported statistics. The “sample” distribution is synonymous with “ensemble.”For summary statistics: “summary.”"
"name": "prediction",
"type": "double",
"description": "predicted value for variable"
},
{
"name": "parameter",
"type": "string",
"description": "ensemble member or distribution parameter"
},
{
"name": "prediction",
"type": "double",
"description": "predicted value for variable"
"name": "family",
"type": "string",
"description": "For ensembles: “ensemble.” Default value if unspecified for probability distributions: Name of the statistical distribution associated with the reported statistics. The “sample” distribution is synonymous with “ensemble.”For summary statistics: “summary.”"
},
{
"name": "reference_datetime",
"type": "timestamp[us, tz=UTC]",
"description": "datetime that the forecast was initiated (horizon = 0)"
},
{
"name": "pub_datetime",
"type": "timestamp[us, tz=UTC]",
"description": "datetime that forecast was submitted"
},
{
"name": "date",
"type": "date32[day]",
"description": "date of the forecasted value"
},
{
"name": "project_id",
"type": "string",
Expand All @@ -123,11 +140,6 @@
"name": "model_id",
"type": "string",
"description": "unique model identifier"
},
{
"name": "reference_date",
"type": "string",
"description": "date that the forecast was initiated"
}
],
"assets": {
Expand All @@ -141,12 +153,12 @@
"description": "Use `arrow` for remote access to the database. This R code will return results for forecasts of the variable by the specific model .\n\n### R\n\n```{r}\n# Use code below\n\nall_results <- arrow::open_dataset(\"s3://anonymous@bio230014-bucket01/challenges/forecasts/parquet/project_id=usgsrc4cast/duration=P1D/variable=chla?endpoint_override=sdsc.osn.xsede.org\")\ndf <- all_results |> dplyr::collect()\n\n```\n \n\nYou can use dplyr operations before calling `dplyr::collect()` to `summarise`, `select` columns, and/or `filter` rows prior to pulling the data into a local `data.frame`. Reducing the data that is pulled locally will speed up the data download speed and reduce your memory usage.\n\n\n"
},
"thumbnail": {
"href": "pending",
"href": "https://d9-wret.s3.us-west-2.amazonaws.com/assets/palladium/production/s3fs-public/thumbnails/image/Back-b.jpg",
"type": "image/JPEG",
"roles": [
"thumbnail"
],
"title": "pending"
"title": "Thumbnail Image"
}
}
}
Loading

0 comments on commit 7791959

Please sign in to comment.