From 790f9a4869f500e7d45d4ab1ae063e647c0e2549 Mon Sep 17 00:00:00 2001 From: Andreas Schmitz Date: Tue, 29 Oct 2024 15:29:13 +0100 Subject: [PATCH] feat: configure results --- src/ump/api/README.md | 7 +++++-- src/ump/api/job.py | 9 ++++++--- src/ump/api/routes/ensembles.py | 3 ++- src/ump/geoserver/geoserver.py | 4 ++-- 4 files changed, 15 insertions(+), 8 deletions(-) diff --git a/src/ump/api/README.md b/src/ump/api/README.md index fd0c88a..1a8a85f 100644 --- a/src/ump/api/README.md +++ b/src/ump/api/README.md @@ -11,7 +11,7 @@ https://developer.ogc.org/api/processes/index.html#tag/JobList For convenience there is still some documentation below. ## Setup -Also the list of providers delivering the OGC API Processes have to be configured in [providers.yaml](../../../providers.yaml.example) along with the processes that the UMP should provide. The structure looks as following: +Also the list of providers delivering the OGC API Processes have to be configured in [providers.yaml](../../../providers.yaml.example) along with the processes that the UMP should provide. The structure looks as following: ``` modelserver-1: @@ -25,6 +25,7 @@ modelserver-1: processes: process-1: result-storage: "geoserver" + result-path: simulation_geometry process-2 result-storage: "remote" process-3 @@ -32,9 +33,11 @@ modelserver-1: ``` -For each process, it is possible to choose from result-storage options. If the attribute `result-storage` is set to `remote`, no results will be stored in the UMP itself, but provided directly from the model server. In case it is set to `geoserver`, UMP will load the geoserver component and tries to store the result data in a specific Geoserver layer. +For each process, it is possible to choose from result-storage options. If the attribute `result-storage` is set to `remote`, no results will be stored in the UMP itself, but provided directly from the model server. In case it is set to `geoserver`, UMP will load the geoserver component and tries to store the result data in a specific Geoserver layer. You can specify the object path to the feature collection using `result-path`. Use dots to separate a path with several components: `result-path: result.some_obj.some_features`. +## Keycloak +You can secure processes and model servers in keycloak by adding users to special client roles. In order to secure a specific process, create a role named `modelserver_processid`, in order to secure all processes of a model server just create a role named `modelserver`. The ids correspond to the keys used in the providers.yaml. ### GET api/jobs diff --git a/src/ump/api/job.py b/src/ump/api/job.py index 0a91655..ebb55e7 100644 --- a/src/ump/api/job.py +++ b/src/ump/api/job.py @@ -103,7 +103,6 @@ def create( (%(job_id)s, %(remote_job_id)s, %(process_id)s, %(provider_prefix)s, %(provider_url)s, %(status)s, %(progress)s, %(parameters)s, %(message)s, %(created)s, %(started)s, %(finished)s, %(updated)s, %(user_id)s, %(process_title)s, %(name)s, %(process_version)s) """ with DBHandler() as db: - logging.error(self._to_dict()) db.run_query(query, query_params=self._to_dict()) logging.info(" --> Job %s for %s created.", self.job_id, self.process_id) @@ -164,7 +163,6 @@ def _init_from_db(self, job_id, user): job_details = db.run_query(query, query_params={"job_id": job_id}) if len(job_details) > 0: - logging.error(job_details[0]) self._init_from_dict(dict(job_details[0])) return True return False @@ -189,7 +187,7 @@ def _init_from_dict(self, data): self.process_title = data["process_title"] self.name = data["name"] self.process_version = data["process_version"] - + def _to_dict(self): return { "process_id": self.process_id, @@ -336,10 +334,15 @@ async def results(self): async def results_to_geoserver(self): try: + p = providers.PROVIDERS[self.provider_prefix]['processes'][self.process_id] results = await self.results() while "results" in results: results = results["results"] + if 'result-path' in p: + parts = p['result-path'].split('.') + for part in parts: + results = results[part] geoserver = Geoserver() self.set_results_metadata(results) diff --git a/src/ump/api/routes/ensembles.py b/src/ump/api/routes/ensembles.py index f714f41..dfca6f4 100644 --- a/src/ump/api/routes/ensembles.py +++ b/src/ump/api/routes/ensembles.py @@ -40,7 +40,8 @@ def add_job_fields(ensemble: Ensemble): ) ensemble.jobs_metadata = {} for row in result: - ensemble.jobs_metadata[row.status] = row.count + if row.status is not None: + ensemble.jobs_metadata[row.status] = row.count @ensembles.route("/", methods=["GET"]) diff --git a/src/ump/geoserver/geoserver.py b/src/ump/geoserver/geoserver.py index d8a73f5..d7c6297 100644 --- a/src/ump/geoserver/geoserver.py +++ b/src/ump/geoserver/geoserver.py @@ -80,7 +80,7 @@ def publish_layer(self, store_name: str, layer_name: str): try: response = requests.post( f"{config.geoserver_workspaces_url}/{self.workspace}" + - "/datastores/{store_name}/featuretypes", + f"/datastores/{store_name}/featuretypes", auth=(config.geoserver_admin_user, config.geoserver_admin_password), data=f"{layer_name}", headers={"Content-type": "text/xml"}, @@ -143,7 +143,7 @@ def create_store(self, store_name: str): def geojson_to_postgis(self, table_name: str, data: dict): engine = create_engine( f"postgresql://{config.postgres_user}:{config.postgres_password}" + - "@{config.postgres_host}/{config.postgres_db}" + f"@{config.postgres_host}/{config.postgres_db}" ) gdf = gpd.GeoDataFrame.from_features(data["features"], crs = 'EPSG:4326') table = Identifier(table_name)