Skip to content

Commit

Permalink
Debug and refine config schema reference
Browse files Browse the repository at this point in the history
  • Loading branch information
@dostuffthatmatters
dostuffthatmatters committed Sep 18, 2024
1 parent 6014f7b commit 87d2003
Show file tree
Hide file tree
Showing 7 changed files with 55 additions and 31 deletions.
8 changes: 6 additions & 2 deletions config/config.template.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8,14 +8,18 @@
"data": {
"ground_pressure": {
"path": "path-to-ground-pressure-data",
"file_regex": "^$(SENSOR_ID)$(DATE).*\\.csv$",
"file_regex": "^ground-pressure-$(SENSOR_ID)-$(YYYY)-$(MM)-$(DD).csv$",
"separator": ",",
"pressure_column": "pressure",
"pressure_column_format": "hPa",
"date_column": "UTCdate_____",
"date_column_format": "%Y-%m-%d",
"time_column": "UTCtime_____",
"time_column_format": "%H:%M:%S"
"time_column_format": "%H:%M:%S",
"datetime_column": null,
"datetime_column_format": null,
"unix_timestamp_column": null,
"unix_timestamp_column_format": null
},
"atmospheric_profiles": "path-to-atmospheric-profiles",
"interferograms": "path-to-ifg-upload-directory",
Expand Down
25 changes: 25 additions & 0 deletions docs/components/config-schema.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -103,6 +103,11 @@ const CONFIG_SCHEMA: any = {
],
"default": null,
"description": "Column name in the ground pressure files that contains the datetime.",
"examples": [
"datetime",
"dt",
"utc-datetime"
],
"title": "Datetime Column"
},
"datetime_column_format": {
Expand Down Expand Up @@ -132,6 +137,11 @@ const CONFIG_SCHEMA: any = {
],
"default": null,
"description": "Column name in the ground pressure files that contains the date.",
"examples": [
"date",
"d",
"utc-date"
],
"title": "Date Column"
},
"date_column_format": {
Expand Down Expand Up @@ -163,6 +173,11 @@ const CONFIG_SCHEMA: any = {
],
"default": null,
"description": "Column name in the ground pressure files that contains the time.",
"examples": [
"time",
"t",
"utc-time"
],
"title": "Time Column"
},
"time_column_format": {
Expand Down Expand Up @@ -194,6 +209,11 @@ const CONFIG_SCHEMA: any = {
],
"default": null,
"description": "Column name in the ground pressure files that contains the unix timestamp.",
"examples": [
"unix-timestamp",
"timestamp",
"ts"
],
"title": "Unix Timestamp Column"
},
"unix_timestamp_column_format": {
Expand All @@ -217,6 +237,11 @@ const CONFIG_SCHEMA: any = {
},
"pressure_column": {
"description": "Column name in the ground pressure files that contains the pressure.",
"examples": [
"pressure",
"p",
"ground_pressure"
],
"title": "Pressure Column",
"type": "string"
},
Expand Down
24 changes: 15 additions & 9 deletions docs/components/schema-renderers.tsx
View file
Original file line number Diff line number Diff line change
Expand Up @@ -67,14 +67,16 @@ function ObjectAttribute(props: { name: string; value: any }) {
const: "allowed value",
enum: "allowed values",
};
if (
props.name in knownAttributes &&
props.value !== undefined &&
props.value !== null
) {
if (props.name in knownAttributes) {
let prettyValue: string;
if (props.value === null || props.value === undefined) {
prettyValue = "null";
} else {
prettyValue = JSON.stringify(props.value, null, 4);
}
return (
<div className={`text-slate-800 dark:text-slate-300`}>
{knownAttributes[props.name]}: {JSON.stringify(props.value, null, 4)}
{knownAttributes[props.name]}: {prettyValue}
</div>
);
} else {
Expand All @@ -89,7 +91,7 @@ function ObjectAttributeList(props: {
const attributes: { name: string; value: any }[] = [];

props.consideredAttributes.forEach((name) => {
if (name in props.object && props.object[name] !== null) {
if (name in props.object) {
attributes.push({ name, value: props.object[name] });
}
});
Expand Down Expand Up @@ -230,9 +232,9 @@ function renderConfigProperty(
propertyObject: any,
required: boolean
) {
if (!required && propertyObject.default === undefined) {
/*if (!required && propertyObject.default === undefined) {
propertyObject.default = null;
}
}*/
if (propertyObject.deprecated === true) {
return null;
}
Expand Down Expand Up @@ -320,6 +322,10 @@ function renderConfigProperty(
type="union"
/>
<ObjectDescription text={propertyObject.description} />
<ObjectAttributeList
consideredAttributes={["default", "examples"]}
object={propertyObject}
/>
<div className={`flex flex-col rounded-lg mt-2`}>
<div className={`flex flex-col gap-y-1`}>
{propertyObject.anyOf.map((option: any, i: number) =>
Expand Down
2 changes: 1 addition & 1 deletion docs/pages/api-reference/configuration.mdx
View file
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,7 @@ import {
"data": {
"ground_pressure": {
"path": "path-to-ground-pressure-data",
"file_regex": "^$(SENSOR_ID)$(DATE).*\\.csv$",
"file_regex": "^ground-pressure-$(SENSOR_ID)-$(YYYY)-$(MM)-$(DD).csv$",
"separator": ",",
"datetime_column": null,
"datetime_column_format": null,
Expand Down
2 changes: 1 addition & 1 deletion docs/pages/guides/configuration.mdx
View file
Original file line number Diff line number Diff line change
Expand Up @@ -25,7 +25,7 @@ Template:
"data": {
"ground_pressure": {
"path": "path-to-ground-pressure-data",
"file_regex": "^$(SENSOR_ID)$(DATE).*\\.csv$",
"file_regex": "^ground-pressure-$(SENSOR_ID)-$(YYYY)-$(MM)-$(DD).csv$",
"separator": ",",
"datetime_column": null,
"datetime_column_format": null,
Expand Down
20 changes: 2 additions & 18 deletions docs/pages/index.mdx
View file
Original file line number Diff line number Diff line change
@@ -1,22 +1,6 @@
# EM27 Retrieval Pipeline

<img
alt="GitHub License"
src="https://img.shields.io/github/license/tum-esm/em27-retrieval-pipeline?style=flat&label=License&labelColor=%230f172a&color=%23fef08a&cacheSeconds=60"
className="inline p-0 m-px mt-4"
/> <img
alt="GitHub Tag"
src="https://img.shields.io/github/v/tag/tum-esm/em27-retrieval-pipeline?sort=semver&style=flat&label=Latest%20Pipeline%20Version&color=%23fef08a&cacheSeconds=60&labelColor=%230f172a"
className="inline p-0 m-px mt-4"
/> <br /> <img
alt="Static Badge"
src="https://img.shields.io/badge/Proffast%201.0%20%7C%202.2%20%7C%202.3%20%7C%202.4-whydoineedthis?style=flat&label=Retrieval%20Algorithms&labelColor=%230f172a&color=%2399f6e4&cacheSeconds=60"
className="inline p-0 m-px "
/> <br /> <img
alt="Static Badge"
src="https://img.shields.io/badge/GGG2014%20%7C%20GGG2020%20-%20whydoineedthis?style=flat&label=Atmospheric%20Profile%20Models&labelColor=%230f172a&color=%2399f6e4&cacheSeconds=60"
className="inline p-0 m-px "
/>
<img alt="GitHub License" src="https://img.shields.io/github/license/tum-esm/em27-retrieval-pipeline?style=flat&label=License&labelColor=%230f172a&color=%23fef08a&cacheSeconds=60" className="inline p-0 m-px mt-4"/> <img alt="GitHub Tag" src="https://img.shields.io/github/v/tag/tum-esm/em27-retrieval-pipeline?sort=semver&style=flat&label=Latest%20Pipeline%20Version&color=%23fef08a&cacheSeconds=60&labelColor=%230f172a" className="inline p-0 m-px mt-4"/> <br/> <img alt="Static Badge" src="https://img.shields.io/badge/Proffast%201.0%20%7C%202.2%20%7C%202.3%20%7C%202.4-whydoineedthis?style=flat&label=Retrieval%20Algorithms&labelColor=%230f172a&color=%2399f6e4&cacheSeconds=60" className="inline p-0 m-px "/> <br/> <img alt="Static Badge" src="https://img.shields.io/badge/GGG2014%20%7C%20GGG2020%20-%20whydoineedthis?style=flat&label=Atmospheric%20Profile%20Models&labelColor=%230f172a&color=%2399f6e4&cacheSeconds=60" className="inline p-0 m-px "/>

We retrieve a lot of EM27/SUN data, produced mainly by [MUCCnet (Dietrich et al., 2021)](https://doi.org/10.5194/amt-14-1111-2021), and have used this pipeline since end of 2021.

Expand All @@ -28,7 +12,7 @@ This codebase provides an automated data pipeline for [Proffast 1 and 2.X](https

**Related Projects:**

⚙️ Many of our projects use much functionality from the [`tum-esm-utils` package](https://github.com/tum-esm/utils).<br/>
⚙️ Many of our projects (including this pipeline) use functionality from the [`tum-esm-utils` package](https://github.com/tum-esm/utils).<br/>
🤖 Our EM27/SUN systems are running autonomously with the help of [Pyra](https://github.com/tum-esm/pyra).

## EM27 Retrieval Pipeline vs. Proffast Pylot
Expand Down
5 changes: 5 additions & 0 deletions src/types/config.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -67,6 +67,7 @@ class GroundPressureConfig(pydantic.BaseModel):
datetime_column: Optional[str] = pydantic.Field(
None,
description="Column name in the ground pressure files that contains the datetime.",
examples=["datetime", "dt", "utc-datetime"],
)
datetime_column_format: Optional[str] = pydantic.Field(
None,
Expand All @@ -78,6 +79,7 @@ class GroundPressureConfig(pydantic.BaseModel):
date_column: Optional[str] = pydantic.Field(
None,
description="Column name in the ground pressure files that contains the date.",
examples=["date", "d", "utc-date"],
)
date_column_format: Optional[str] = pydantic.Field(
None,
Expand All @@ -87,6 +89,7 @@ class GroundPressureConfig(pydantic.BaseModel):
time_column: Optional[str] = pydantic.Field(
None,
description="Column name in the ground pressure files that contains the time.",
examples=["time", "t", "utc-time"],
)
time_column_format: Optional[str] = pydantic.Field(
None,
Expand All @@ -98,6 +101,7 @@ class GroundPressureConfig(pydantic.BaseModel):
unix_timestamp_column: Optional[str] = pydantic.Field(
None,
description="Column name in the ground pressure files that contains the unix timestamp.",
examples=["unix-timestamp", "timestamp", "ts"],
)
unix_timestamp_column_format: Optional[Literal["s", "ms", "us", "ns"]] = pydantic.Field(
None,
Expand All @@ -109,6 +113,7 @@ class GroundPressureConfig(pydantic.BaseModel):
pressure_column: str = pydantic.Field(
...,
description="Column name in the ground pressure files that contains the pressure.",
examples=["pressure", "p", "ground_pressure"],
)
pressure_column_format: Literal["hPa", "Pa"] = pydantic.Field(
...,
Expand Down

0 comments on commit 87d2003

Please sign in to comment.