Skip to content

Commit

Permalink
Add aggregate_temporal and minor fixes
Browse files Browse the repository at this point in the history
  • Loading branch information
@m-mohr
m-mohr committed May 19, 2024
1 parent 96a4b06 commit d792b82
Show file tree
Hide file tree
Showing 6 changed files with 356 additions and 30 deletions.
3 changes: 3 additions & 0 deletions README.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -102,3 +102,6 @@ There are various process graph examples in the following folder: [examples](./e
> For example, the handling of no-data, NaN and infinity values works differently.
> Be aware that in some cases the results may slightly differ when compared to other openEO implementations.
Other known issues:
- Ideally, all comutations - including the datacube management - run on GEE's side.
Currently, some smaller checks still run in the proxy implementation due to the issue that we can't easily abort execution on GEE in case we identify anomalies or other issues.
83 changes: 83 additions & 0 deletions src/processes/aggregate_temporal.js
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,83 @@
import GeeProcess from '../processgraph/process.js';

export default class aggregate_temporal extends GeeProcess {

async validate(node) {
await super.validate(node);

const intervals = node.getArgument('intervals');
let labels = node.getArgument('labels', []);

if (labels.length > 0 && labels.length !== intervals.length) {
throw node.invalidArgument('labels', 'The number of labels must be equal to the number of intervals.');
}

if (labels.length === 0) {
labels = intervals.map(range => range[0]);
}
const uniqueLabels = [...new Set(labels)];
if (labels.length !== uniqueLabels.length) {
throw node.invalidArgument('labels', 'Labels must be unique.');
}
}

executeSync(node) {
// STEP 1: Get parameters
const ee = node.ee;
const dc = node.getDataCubeWithEE('data');
const intervals = node.getArgument('intervals');
let labels = node.getArgument('labels', []);
const reducer = node.getCallback('reducer');
const dimensionName = node.getArgument('dimension', null);
const context = node.getArgument("context", null);

// STEP 2: Validate parameters
const data = node.getData();
if (!(data instanceof ee.ImageCollection)) {
throw node.invalidArgument('data', 'Data must have a temporal dimension and/or more than one timestamp.');
}

const dimension = dimensionName ? dc.dim(dimensionName) : dc.dimT();
if (dimension.getType() !== 'temporal') {
throw node.invalidArgument('dimension', 'The specified dimension must be a temporal dimension.');
}

// STEP 3: Prepare labels and intervals
if (labels.length === 0) {
labels = intervals.map(range => range[0]);
}

let ranges = ee.List([]);
for (let interval of intervals) {
ranges = ranges.add(ee.List([
ee.Date(interval[0] || "0000-01-01"),
ee.Date(interval[1] || "9999-12-31")
]));
}

// STEP 4: Aggregate data
const aggregatedImages = ee.Dictionary
.fromLists(labels, ranges)
.map((label, range) => {
let dates = ee.List(range);
let collection = data.filterDate(dates.get(0), dates.get(1));
const image = ee.Image(reducer.execute({
data: collection,
context,
executionContext: {
type: "reducer",
parameter: "dimension",
dimension
}
}));
return image
.set('label', label)
.set('system:time_start', label);
});

// STEP 5: Update data cube
dimension.setValues(labels);
return dc.setData(ee.ImageCollection(aggregatedImages));
}

}
243 changes: 243 additions & 0 deletions src/processes/aggregate_temporal.json
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,243 @@
{
"id": "aggregate_temporal",
"summary": "Temporal aggregations",
"description": "Computes a temporal aggregation based on an array of temporal intervals.\n\nFor common regular calendar hierarchies such as year, month, week or seasons ``aggregate_temporal_period()`` can be used. Other calendar hierarchies must be transformed into specific intervals by the clients.\n\nFor each interval, all data along the dimension will be passed through the reducer.\n\nThe computed values will be projected to the labels. If no labels are specified, the start of the temporal interval will be used as label for the corresponding values. In case of a conflict (i.e. the user-specified values for the start times of the temporal intervals are not distinct), the user-defined labels must be specified in the parameter `labels` as otherwise a `DistinctDimensionLabelsRequired` exception would be thrown. The number of user-defined labels and the number of intervals need to be equal.\n\nIf the dimension is not set or is set to `null`, the data cube is expected to only have one temporal dimension.",
"categories": [
"cubes",
"aggregate"
],
"parameters": [
{
"name": "data",
"description": "A data cube.",
"schema": {
"type": "object",
"subtype": "raster-cube"
}
},
{
"name": "intervals",
"description": "Left-closed temporal intervals, which are allowed to overlap. Each temporal interval in the array has exactly two elements:\n\n1. The first element is the start of the temporal interval. The specified time instant is **included** in the interval.\n2. The second element is the end of the temporal interval. The specified time instant is **excluded** from the interval.\n\nThe second element must always be greater/later than the first element, except when using time without date. Otherwise, a `TemporalExtentEmpty` exception is thrown.",
"schema": {
"type": "array",
"subtype": "temporal-intervals",
"minItems": 1,
"items": {
"type": "array",
"subtype": "temporal-interval",
"uniqueItems": true,
"minItems": 2,
"maxItems": 2,
"items": {
"anyOf": [
{
"type": "string",
"format": "date-time",
"subtype": "date-time",
"description": "Date and time with a time zone."
},
{
"type": "string",
"format": "date",
"subtype": "date",
"description": "Date only, formatted as `YYYY-MM-DD`. The time zone is UTC. Missing time components are all 0."
},
{
"type": "string",
"subtype": "time",
"pattern": "^\\d{2}:\\d{2}:\\d{2}$",
"description": "Time only, formatted as `HH:MM:SS`. The time zone is UTC."
},
{
"type": "null"
}
]
}
},
"examples": [
[
[
"2015-01-01",
"2016-01-01"
],
[
"2016-01-01",
"2017-01-01"
],
[
"2017-01-01",
"2018-01-01"
]
],
[
[
"06:00:00",
"18:00:00"
],
[
"18:00:00",
"06:00:00"
]
]
]
}
},
{
"name": "reducer",
"description": "A reducer to be applied for the values contained in each interval. A reducer is a single process such as ``mean()`` or a set of processes, which computes a single value for a list of values, see the category 'reducer' for such processes. Intervals may not contain any values, which for most reducers leads to no-data (`null`) values by default.",
"schema": {
"type": "object",
"subtype": "process-graph",
"parameters": [
{
"name": "data",
"description": "A labeled array with elements of any type. If there's no data for the interval, the array is empty.",
"schema": {
"type": "array",
"subtype": "labeled-array",
"items": {
"description": "Any data type."
}
}
},
{
"name": "context",
"description": "Additional data passed by the user.",
"schema": {
"description": "Any data type."
},
"optional": true,
"default": null
}
],
"returns": {
"description": "The value to be set in the new data cube.",
"schema": {
"description": "Any data type."
}
}
}
},
{
"name": "labels",
"description": "Distinct labels for the intervals, which can contain dates and/or times. Is only required to be specified if the values for the start of the temporal intervals are not distinct and thus the default labels would not be unique. The number of labels and the number of groups need to be equal.",
"schema": {
"type": "array",
"uniqueItems": true,
"items": {
"type": [
"number",
"string"
]
}
},
"default": [],
"optional": true
},
{
"name": "dimension",
"description": "The name of the temporal dimension for aggregation. All data along the dimension is passed through the specified reducer. If the dimension is not set or set to `null`, the data cube is expected to only have one temporal dimension. Fails with a `TooManyDimensions` exception if it has more dimensions. Fails with a `DimensionNotAvailable` exception if the specified dimension does not exist or no temporal dimension is available.",
"schema": {
"type": [
"string",
"null"
]
},
"default": null,
"optional": true
},
{
"name": "context",
"description": "Additional data to be passed to the reducer.",
"schema": {
"description": "Any data type."
},
"optional": true,
"default": null
}
],
"returns": {
"description": "A new data cube with the same dimensions. The dimension properties (name, type, labels, reference system and resolution) remain unchanged, except for the resolution and dimension labels of the given temporal dimension.",
"schema": {
"type": "object",
"subtype": "raster-cube"
}
},
"examples": [
{
"arguments": {
"data": {
"from_parameter": "data"
},
"intervals": [
[
"2015-01-01",
"2016-01-01"
],
[
"2016-01-01",
"2017-01-01"
],
[
"2017-01-01",
"2018-01-01"
],
[
"2018-01-01",
"2019-01-01"
],
[
"2019-01-01",
"2020-01-01"
]
],
"labels": [
"2015",
"2016",
"2017",
"2018",
"2019"
],
"reducer": {
"process_graph": {
"mean1": {
"process_id": "mean",
"arguments": {
"data": {
"from_parameter": "data"
}
},
"result": true
}
}
}
}
}
],
"exceptions": {
"TooManyDimensions": {
"message": "The data cube contains multiple temporal dimensions. The parameter `dimension` must be specified."
},
"DimensionNotAvailable": {
"message": "A dimension with the specified name does not exist or no temporal dimension is available."
},
"DistinctDimensionLabelsRequired": {
"message": "The dimension labels have duplicate values. Distinct labels must be specified."
},
"TemporalExtentEmpty": {
"message": "At least one of the intervals is empty. The second instant in time must always be greater/later than the first instant."
}
},
"links": [
{
"href": "https://openeo.org/documentation/1.0/datacubes.html#aggregate",
"rel": "about",
"title": "Aggregation explained in the openEO documentation"
},
{
"href": "https://www.rfc-editor.org/rfc/rfc3339.html",
"rel": "about",
"title": "RFC3339: Details about formatting temporal strings"
}
]
}
Loading

0 comments on commit d792b82

Please sign in to comment.