From a10ff503df9ccddb29ff3c72d2728832641518cf Mon Sep 17 00:00:00 2001 From: dlohmeier Date: Thu, 12 Jan 2023 10:59:53 +0100 Subject: [PATCH 01/84] Added registry class to JSON Decoder upon calling json.loads in method serialize_and_update_data: added json decoder registry for pandapipes to decode jsons, as otherwise the components cannot be detected. Please do not merge, before pandapower has been updated to process this change! --- pandahub/lib/PandaHub.py | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/pandahub/lib/PandaHub.py b/pandahub/lib/PandaHub.py index ec39158..07f07e4 100644 --- a/pandahub/lib/PandaHub.py +++ b/pandahub/lib/PandaHub.py @@ -15,7 +15,7 @@ from pymongo import MongoClient, ReplaceOne, DESCENDING import pandapipes as pps -from pandapipes import from_json_string as from_json_pps +from pandapipes import from_json_string as from_json_pps, FromSerializableRegistryPpipe import pandapower as pp import pandapower.io_utils as io_pp from pandahub.api.internal import settings @@ -551,17 +551,16 @@ def _get_net_from_db_by_id(self, id_, include_results=True, only_tables=None, co return net def deserialize_and_update_data(self, net, meta): + registry = io_pp.FromSerializableRegistry if meta.get("sector", "power") == "power" \ + else FromSerializableRegistryPpipe if version.parse(self.get_project_version()) <= version.parse("0.2.3"): - if meta.get("sector", "power") == "power": - data = dict((k, json.loads(v, cls=io_pp.PPJSONDecoder)) for k, v in meta['data'].items()) - net.update(data) - else: - data = dict((k, from_json_pps(v)) for k, v in meta['data'].items()) - net.update(data) + data = dict((k, json.loads(v, cls=io_pp.PPJSONDecoder, registry_class=registry)) + for k, v in meta['data'].items()) + net.update(data) else: for key, value in meta["data"].items(): if type(value) == str and value.startswith("serialized_"): - value = json.loads(value[11:], cls=io_pp.PPJSONDecoder) + value = json.loads(value[11:], cls=io_pp.PPJSONDecoder, registry_class=registry) net[key] = value def get_subnet_from_db(self, name, bus_filter=None, include_results=True, From 6943d679d0aaef523f038e0d0946f3152e3e816a Mon Sep 17 00:00:00 2001 From: mvogt Date: Fri, 22 Sep 2023 12:11:19 +0200 Subject: [PATCH 02/84] changed write_net_to_db so that results get written by default but can be skipped. --- pandahub/lib/PandaHub.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/pandahub/lib/PandaHub.py b/pandahub/lib/PandaHub.py index 2d85918..661ba2e 100644 --- a/pandahub/lib/PandaHub.py +++ b/pandahub/lib/PandaHub.py @@ -708,7 +708,7 @@ def _element_name_of_collection(self, collection): return collection[4:] # remove "net_" prefix def write_network_to_db(self, net, name, sector="power", overwrite=True, project_id=None, - metadata=None): + metadata=None, skip_results=False, ): if project_id: self.set_active_project_by_id(project_id) self.check_permission("write") @@ -729,7 +729,9 @@ def write_network_to_db(self, net, name, sector="power", overwrite=True, project dtypes = {} version_ = version.parse(self.get_project_version()) for element, element_data in net.items(): - if element.startswith("_") or element.startswith("res"): + if skip_results and element.startswith("res"): + continue + if element.startswith("_"): continue if isinstance(element_data, pd.core.frame.DataFrame): # create type lookup From 705c53eaf7dbc361e854c8c2257e998077d98381 Mon Sep 17 00:00:00 2001 From: mvogt Date: Fri, 22 Sep 2023 12:19:00 +0200 Subject: [PATCH 03/84] fixed some import bugs and a bug concerning realm change. --- pandahub/lib/PandaHub.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/pandahub/lib/PandaHub.py b/pandahub/lib/PandaHub.py index 661ba2e..f119771 100644 --- a/pandahub/lib/PandaHub.py +++ b/pandahub/lib/PandaHub.py @@ -14,6 +14,7 @@ from bson.objectid import ObjectId from pydantic.types import UUID4 from pymongo import MongoClient, ReplaceOne, DESCENDING +from pymongo.errors import ServerSelectionTimeoutError import pandapipes as pps from pandapipes import from_json_string as from_json_pps @@ -98,10 +99,10 @@ def check_connection_status(self): Checks if the database is accessible """ try: - status = self.find({}, collection_name="__connection_test_collection") + status = self.mongo_client.find({}, collection_name="__connection_test_collection") if status == []: return "ok" - except (ServerSelectionTimeoutError, timeout) as e: + except (ServerSelectionTimeoutError) as e: return "connection timeout" # ------------------------- @@ -250,7 +251,7 @@ def change_realm(self, realm): self.has_permission("write") project_collection = self.mongo_client["user_management"].projects project_name = self.active_project["name"] - if self.project_exists(active_project_name, realm): + if self.project_exists(project_name, realm): raise PandaHubError("Can't change realm - project with this name already exists") project_collection.find_one_and_update({"_id": self.active_project["_id"]}, {"$set": {"realm": realm}}) From eaf4ae09ab2702fb78d44440f1d10e0c99b7d26e Mon Sep 17 00:00:00 2001 From: Joschka Thurner Date: Wed, 13 Sep 2023 08:37:34 +0200 Subject: [PATCH 04/84] vectorized element deletion --- pandahub/lib/PandaHub.py | 40 ++++++++++++++++++++++++++++++---------- 1 file changed, 30 insertions(+), 10 deletions(-) diff --git a/pandahub/lib/PandaHub.py b/pandahub/lib/PandaHub.py index f119771..472d0ee 100644 --- a/pandahub/lib/PandaHub.py +++ b/pandahub/lib/PandaHub.py @@ -917,10 +917,19 @@ def get_net_value_from_db(self, net, element, element_index, return document[parameter] def delete_net_element(self, net, element, element_index, variant=None, project_id=None): + return self.delete_net_elements(net, element, [element_index], variant, project_id)[0] + + def delete_net_elements(self, net: Union[int, str], element: str, element_indexes: Union[list[int], int], + variant: Union[int, list[int], None] = None, project_id: Union[str, None] = None) -> list[ + dict]: + + if isinstance(element_indexes, int): + element_indexes = [element_indexes] if variant is not None: variant = int(variant) if project_id: self.set_active_project_by_id(project_id) + self.check_permission("write") db = self._get_project_database() collection = self._collection_name_of_element(element) @@ -930,18 +939,29 @@ def delete_net_element(self, net, element, element_index, variant=None, project_ else: net_id = net - element_filter = {"index": element_index, "net_id": int(net_id), **self.get_variant_filter(variant)} + element_filter = {"index": {"$in": element_indexes}, "net_id": int(net_id), **self.get_variant_filter(variant)} - target = db[collection].find_one(element_filter) - if target is None: - # element does not exist in net - return - if variant and target["var_type"] == "base": - db[collection].update_one({"_id": target["_id"]}, - {"$addToSet": {"not_in_var": variant}}) + deletion_targets = list(db[collection].find(element_filter)) + if not deletion_targets: + return [] + + if variant: + delete_ids_variant, delete_ids = [], [] + for target in deletion_targets: + delete_ids_variant.append(target["_id"]) if target["var_type"] == "base" else delete_ids.append( + target["_id"]) + db[collection].update_many({"_id": {"$in": delete_ids_variant}}, + {"$addToSet": {"not_in_var": variant}}) else: - db[collection].delete_one({"_id": target["_id"]}) - return target + delete_ids = [target["_id"] for target in deletion_targets] + db[collection].delete_many({"_id": {"$in": delete_ids}}) + return deletion_targets + + def delete_multiple_elements(self, net, elements: dict[str, Union[list[int], int]], variant=None, project_id=None): + deleted = {} + for element_type, element_indexes in elements.items(): + deleted[element_type] = self.delete_net_elements(net, element_type, element_indexes, variant, project_id) + return deleted def set_net_value_in_db(self, net, element, element_index, parameter, value, variant=None, project_id=None): From 8389db0f6999ec1531c33d68305dddf168051506 Mon Sep 17 00:00:00 2001 From: Joschka Thurner Date: Wed, 13 Sep 2023 14:47:59 +0200 Subject: [PATCH 05/84] drop delete_multiple_elements --- pandahub/lib/PandaHub.py | 5 ----- 1 file changed, 5 deletions(-) diff --git a/pandahub/lib/PandaHub.py b/pandahub/lib/PandaHub.py index 472d0ee..7fbafc7 100644 --- a/pandahub/lib/PandaHub.py +++ b/pandahub/lib/PandaHub.py @@ -957,11 +957,6 @@ def delete_net_elements(self, net: Union[int, str], element: str, element_indexe db[collection].delete_many({"_id": {"$in": delete_ids}}) return deletion_targets - def delete_multiple_elements(self, net, elements: dict[str, Union[list[int], int]], variant=None, project_id=None): - deleted = {} - for element_type, element_indexes in elements.items(): - deleted[element_type] = self.delete_net_elements(net, element_type, element_indexes, variant, project_id) - return deleted def set_net_value_in_db(self, net, element, element_index, parameter, value, variant=None, project_id=None): From 88af390445615e2127e1096382dbce823b176ad6 Mon Sep 17 00:00:00 2001 From: Joschka Thurner Date: Thu, 14 Sep 2023 08:24:03 +0200 Subject: [PATCH 06/84] only allow index list in delete_net_elements --- pandahub/lib/PandaHub.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/pandahub/lib/PandaHub.py b/pandahub/lib/PandaHub.py index 7fbafc7..fc59e1f 100644 --- a/pandahub/lib/PandaHub.py +++ b/pandahub/lib/PandaHub.py @@ -919,12 +919,13 @@ def get_net_value_from_db(self, net, element, element_index, def delete_net_element(self, net, element, element_index, variant=None, project_id=None): return self.delete_net_elements(net, element, [element_index], variant, project_id)[0] - def delete_net_elements(self, net: Union[int, str], element: str, element_indexes: Union[list[int], int], + def delete_net_elements(self, net: Union[int, str], element: str, element_indexes: list[int], variant: Union[int, list[int], None] = None, project_id: Union[str, None] = None) -> list[ dict]: - if isinstance(element_indexes, int): - element_indexes = [element_indexes] + if not isinstance(element_indexes, list): + raise TypeError("Parameter element_indexes must be a list of ints!") + if variant is not None: variant = int(variant) if project_id: From 7148a52a5daf8e4a37206462d81298bcbb4d78cc Mon Sep 17 00:00:00 2001 From: Joschka Thurner Date: Fri, 15 Sep 2023 09:19:24 +0200 Subject: [PATCH 07/84] rename element to element_type in function arguments --- pandahub/lib/PandaHub.py | 64 ++++++++++++++++++++-------------------- 1 file changed, 32 insertions(+), 32 deletions(-) diff --git a/pandahub/lib/PandaHub.py b/pandahub/lib/PandaHub.py index fc59e1f..5efe23a 100644 --- a/pandahub/lib/PandaHub.py +++ b/pandahub/lib/PandaHub.py @@ -763,7 +763,7 @@ def _write_net_collections_to_db(self, db, collections): for element, element_data in collections.items(): self._write_element_to_db(db, element, element_data) - def _write_element_to_db(self, db, element, element_data): + def _write_element_to_db(self, db, element_type, element_data): existing_collections = set(db.list_collection_names()) def add_index(element): columns = {"bus": ["net_id", "index"], @@ -780,15 +780,15 @@ def add_index(element): db[self._collection_name_of_element(element)].create_index([(c, DESCENDING)]) - collection_name = self._collection_name_of_element(element) + collection_name = self._collection_name_of_element(element_type) if len(element_data) > 0: try: db[collection_name].insert_many(element_data, ordered=False) if collection_name not in existing_collections: - add_index(element) + add_index(element_type) except: traceback.print_exc() - print(f"\nFAILED TO WRITE TABLE '{element}' TO DATABASE! (details above)") + print(f"\nFAILED TO WRITE TABLE '{element_type}' TO DATABASE! (details above)") def delete_net_from_db(self, name): self.check_permission("write") @@ -839,12 +839,12 @@ def _get_net_collections(self, db, with_areas=True): def _get_network_metadata(self, db, net_id): return db["_networks"].find_one({"_id": net_id}) - def _add_element_from_collection(self, net, db, element, net_id, + def _add_element_from_collection(self, net, db, element_type, net_id, filter=None, include_results=True, only_tables=None, geo_mode="string", variants=[], dtypes=None): - if only_tables is not None and not element in only_tables: + if only_tables is not None and not element_type in only_tables: return - if not include_results and element.startswith("res_"): + if not include_results and element_type.startswith("res_"): return variants_filter = self.get_variant_filter(variants) filter_dict = {"net_id": net_id, **variants_filter} @@ -856,15 +856,15 @@ def _add_element_from_collection(self, net, db, element, net_id, filter_dict = {**filter_dict, **filter, **filter_and} else: filter_dict = {**filter_dict, **filter} - data = list(db[self._collection_name_of_element(element)].find(filter_dict)) + data = list(db[self._collection_name_of_element(element_type)].find(filter_dict)) if len(data) == 0: return if dtypes is None: dtypes = db["_networks"].find_one({"_id": net_id}, projection={"dtypes"})['dtypes'] df = pd.DataFrame.from_records(data, index="index") - if element in dtypes: + if element_type in dtypes: dtypes_found_columns = { - column: dtype for column, dtype in dtypes[element].items() if column in df.columns + column: dtype for column, dtype in dtypes[element_type].items() if column in df.columns } df = df.astype(dtypes_found_columns, errors="ignore") df.index.name = None @@ -873,18 +873,18 @@ def _add_element_from_collection(self, net, db, element, net_id, convert_geojsons(df, geo_mode) if "object" in df.columns: df["object"] = df["object"].apply(json_to_object) - if not element in net or net[element].empty: - net[element] = df + if not element_type in net or net[element_type].empty: + net[element_type] = df else: - new_rows = set(df.index) - set(net[element].index) + new_rows = set(df.index) - set(net[element_type].index) if new_rows: - net[element] = pd.concat([net[element], df.loc[list(new_rows)]]) + net[element_type] = pd.concat([net[element_type], df.loc[list(new_rows)]]) # ------------------------- # Net element handling # ------------------------- - def get_net_value_from_db(self, net, element, element_index, + def get_net_value_from_db(self, net, element_type, element_index, parameter, variant=None, project_id=None): if variant is not None: variant = int(variant) @@ -897,8 +897,8 @@ def get_net_value_from_db(self, net, element, element_index, else: net_id = net - collection = self._collection_name_of_element(element) - dtypes = self._datatypes.get(element) + collection = self._collection_name_of_element(element_type) + dtypes = self._datatypes.get(element_type) variant_filter = self.get_variant_filter(variant) documents = list(db[collection].find({"index": element_index, "net_id": net_id, **variant_filter})) @@ -916,10 +916,10 @@ def get_net_value_from_db(self, net, element, element_index, else: return document[parameter] - def delete_net_element(self, net, element, element_index, variant=None, project_id=None): - return self.delete_net_elements(net, element, [element_index], variant, project_id)[0] + def delete_net_element(self, net, element_type, element_index, variant=None, project_id=None): + return self.delete_net_elements(net, element_type, [element_index], variant, project_id)[0] - def delete_net_elements(self, net: Union[int, str], element: str, element_indexes: list[int], + def delete_net_elements(self, net: Union[int, str], element_type: str, element_indexes: list[int], variant: Union[int, list[int], None] = None, project_id: Union[str, None] = None) -> list[ dict]: @@ -933,7 +933,7 @@ def delete_net_elements(self, net: Union[int, str], element: str, element_indexe self.check_permission("write") db = self._get_project_database() - collection = self._collection_name_of_element(element) + collection = self._collection_name_of_element(element_type) if type(net) == str: net_id = self._get_id_from_name(net, db) @@ -959,19 +959,19 @@ def delete_net_elements(self, net: Union[int, str], element: str, element_indexe return deletion_targets - def set_net_value_in_db(self, net, element, element_index, + def set_net_value_in_db(self, net, element_type, element_index, parameter, value, variant=None, project_id=None): - logger.info(f"Setting {parameter} = {value} in {element} with index {element_index} and variant {variant}") + logger.info(f"Setting {parameter} = {value} in {element_type} with index {element_index} and variant {variant}") if variant is not None: variant = int(variant) if project_id: self.set_active_project_by_id(project_id) self.check_permission("write") db = self._get_project_database() - dtypes = self._datatypes.get(element) + dtypes = self._datatypes.get(element_type) if value is not None and dtypes is not None and parameter in dtypes: value = dtypes[parameter](value) - collection = self._collection_name_of_element(element) + collection = self._collection_name_of_element(element_type) if type(net) == str: net_id = self._get_id_from_name(net, db) else: @@ -979,7 +979,7 @@ def set_net_value_in_db(self, net, element, element_index, element_filter = {"index": element_index, "net_id": int(net_id), **self.get_variant_filter(variant)} document = db[collection].find_one({**element_filter}) if not document: - raise UserWarning(f"No element '{element}' to change with index '{element_index}' in this variant") + raise UserWarning(f"No element '{element_type}' to change with index '{element_index}' in this variant") old_value = document.get(parameter, None) if old_value == value: @@ -1010,16 +1010,16 @@ def set_net_value_in_db(self, net, element, element_index, update_dict) return {"document": document, parameter: {"previous": old_value, "current": value}} - def set_object_attribute(self, net, element, element_index, + def set_object_attribute(self, net, element_type, element_index, parameter, value, variant=None, project_id=None): if project_id: self.set_active_project_by_id(project_id) self.check_permission("write") db = self._get_project_database() - dtypes = self._datatypes.get(element) + dtypes = self._datatypes.get(element_type) if dtypes is not None and parameter in dtypes: value = dtypes[parameter](value) - collection = self._collection_name_of_element(element) + collection = self._collection_name_of_element(element_type) if type(net) == str: net_id = self._get_id_from_name(net, db) else: @@ -1045,7 +1045,7 @@ def set_object_attribute(self, net, element, element_index, element_filter = {**element_filter, **self.get_variant_filter(variant)} document = db[collection].find_one({**element_filter}) if not document: - raise UserWarning(f"No element '{element}' to change with index '{element_index}' in this variant") + raise UserWarning(f"No element '{element_type}' to change with index '{element_index}' in this variant") obj = json_to_object(document["object"]) setattr(obj, parameter, value) if document["var_type"] == "base": @@ -1138,8 +1138,8 @@ def _add_missing_defaults(self, db, net_id, element_type, element_data): if "g_us_per_km" not in element_data: element_data["g_us_per_km"] = 0 - def _ensure_dtypes(self, element, data): - dtypes = self._datatypes.get(element) + def _ensure_dtypes(self, element_type, data): + dtypes = self._datatypes.get(element_type) if dtypes is None: return for key, val in data.items(): From 7fcab3301b9060f973acfca161fea8a98aa23dcd Mon Sep 17 00:00:00 2001 From: Joschka Thurner Date: Fri, 15 Sep 2023 09:22:11 +0200 Subject: [PATCH 08/84] move create elements logic into vectorized function --- pandahub/lib/PandaHub.py | 30 ++++++------------------------ 1 file changed, 6 insertions(+), 24 deletions(-) diff --git a/pandahub/lib/PandaHub.py b/pandahub/lib/PandaHub.py index 5efe23a..1e637bd 100644 --- a/pandahub/lib/PandaHub.py +++ b/pandahub/lib/PandaHub.py @@ -1059,31 +1059,13 @@ def set_object_attribute(self, net, element_type, element_index, db[collection].update_one({"_id": document["_id"]}, {"$set": {"object._object": obj}}) - def create_element_in_db(self, net, element, element_index, data, variant=None, project_id=None): - logger.info(f"Creating element {element} with index {element_index} and variant {variant}, data: {data}") - if project_id: - self.set_active_project_by_id(project_id) - self.check_permission("write") - db = self._get_project_database() - if type(net) == str: - net_id = self._get_id_from_name(net, db) - else: - net_id = net + def create_element_in_db(self, net: Union[int, str], element_type: str, element_index: int, data: dict, + variant=None, project_id=None): + return self.create_elements_in_db(net, element_type, [{"index": element_index, **data}], + project_id, variant)[0] - element_data = {**data, **{"index": element_index, "net_id": int(net_id)}} - if not variant: - element_data.update(var_type="base", not_in_var=[]) - else: - element_data.update(var_type="addition", variant=int(variant)) - self._add_missing_defaults(db, net_id, element, element_data) - self._ensure_dtypes(element, element_data) - collection = self._collection_name_of_element(element) - insert_result = db[collection].insert_one(element_data) - element_data["_id"] = insert_result.inserted_id - return element_data - - def create_elements_in_db(self, net: Union[int,str], element_type: str, elements_data: list, project_id=None, - variant=None): + def create_elements_in_db(self, net: Union[int, str], element_type: str, elements_data: list[dict], + project_id: str = None, variant: int = None): if project_id: self.set_active_project_by_id(project_id) self.check_permission("write") From ca7d43ffa3fb381ffed2ed502a538f46564ce0fc Mon Sep 17 00:00:00 2001 From: Joschka Thurner Date: Fri, 15 Sep 2023 09:50:53 +0200 Subject: [PATCH 09/84] rename and document create functions --- pandahub/lib/PandaHub.py | 68 +++++++++++++++++++++++++++++++++++++--- 1 file changed, 63 insertions(+), 5 deletions(-) diff --git a/pandahub/lib/PandaHub.py b/pandahub/lib/PandaHub.py index 1e637bd..5523fef 100644 --- a/pandahub/lib/PandaHub.py +++ b/pandahub/lib/PandaHub.py @@ -1059,13 +1059,55 @@ def set_object_attribute(self, net, element_type, element_index, db[collection].update_one({"_id": document["_id"]}, {"$set": {"object._object": obj}}) - def create_element_in_db(self, net: Union[int, str], element_type: str, element_index: int, data: dict, - variant=None, project_id=None): - return self.create_elements_in_db(net, element_type, [{"index": element_index, **data}], + def create_element(self, net: Union[int, str], element_type: str, element_index: int, element_data: dict, + variant=None, project_id=None) -> dict: + """ + Creates an element in the database. + + Parameters + ---------- + net: str or int + Network to add elements to, either as name or numeric id + element_type: str + Name of the element type (e.g. bus, line) + element_index: int + Index of the element to add + element_data: dict + Field-value dict to create element from + project_id: str or None + ObjectId (as str) of the project in which the network is stored. Defaults to current active project if None + variant: int or None + Variant index if elements should be created in a variant + Returns + ------- + dict + The created element (element_data with added _id field) + """ + return self.create_elements_in_db(net, element_type, [{"index": element_index, **element_data}], project_id, variant)[0] - def create_elements_in_db(self, net: Union[int, str], element_type: str, elements_data: list[dict], - project_id: str = None, variant: int = None): + def create_elements(self, net: Union[int, str], element_type: str, elements_data: list[dict], + project_id: str = None, variant: int = None) -> list[dict]: + """ + Creates multiple elements of the same type in the database. + + Parameters + ---------- + net: str or int + Network to add elements to, either a name or numeric id + element_type: str + Name of the element type (e.g. bus, line) + elements_data: list of dict + Field-value dicts to create elements from - must include a valid "index" field! + project_id: str or None + ObjectId (as str) of the project in which the network is stored. Defaults to current active project if None + variant: int or None + Variant index if elements should be created in a variant + Returns + ------- + list + A list of the created elements (elements_data with added _id fields) + """ if project_id: self.set_active_project_by_id(project_id) self.check_permission("write") @@ -1974,6 +2016,22 @@ def bulk_del_timeseries_from_db(self, filter_document, del_res = db[collection_name].delete_many(match_filter) return del_res + #### for + + def create_element_in_db(self, net: Union[int, str], element_type: str, element_index: int, data: dict, + variant=None, project_id=None): + warnings.warn( + "ph.create_element_in_db was renamed - use ph.create_element instead" + ) + return self.create_element(net, element_type, element_index, data, variant, project_id) + + + def create_elements_in_db(self, net: Union[int, str], element_type: str, elements_data: list[dict], + project_id: str = None, variant: int = None): + warnings.warn( + "ph.create_elements_in_db was renamed - use ph.create_elements instead" + ) + return self.create_elements(net, element_type, elements_data, project_id, variant) if __name__ == '__main__': self = PandaHub() From ffea872755a1b7fc87a3ebc4d94bcb2fae6a4763 Mon Sep 17 00:00:00 2001 From: Joschka Thurner Date: Fri, 15 Sep 2023 10:27:47 +0200 Subject: [PATCH 10/84] rename and document delete functions --- pandahub/lib/PandaHub.py | 63 ++++++++++++++++++++++++++++++++++++---- 1 file changed, 58 insertions(+), 5 deletions(-) diff --git a/pandahub/lib/PandaHub.py b/pandahub/lib/PandaHub.py index 5523fef..a8e35d1 100644 --- a/pandahub/lib/PandaHub.py +++ b/pandahub/lib/PandaHub.py @@ -1,10 +1,10 @@ # -*- coding: utf-8 -*- - import builtins import importlib import json import logging import traceback +import warnings from inspect import signature, _empty from typing import Optional, Union @@ -916,13 +916,52 @@ def get_net_value_from_db(self, net, element_type, element_index, else: return document[parameter] - def delete_net_element(self, net, element_type, element_index, variant=None, project_id=None): - return self.delete_net_elements(net, element_type, [element_index], variant, project_id)[0] + def delete_element(self, net, element_type, element_index, variant=None, project_id=None): + """ + Delete an element from the database. - def delete_net_elements(self, net: Union[int, str], element_type: str, element_indexes: list[int], + Parameters + ---------- + net: str or int + Network to add elements to, either a name or numeric id + element_type: str + Name of the element type (e.g. bus, line) + element_index: int + Index of the element to delete + project_id: str or None + ObjectId (as str) of the project in which the network is stored. Defaults to current active project if None + variant: int or None + Variant index if elements should be created in a variant + Returns + ------- + dict + The deleted element as dict with all fields + """ + return self.delete_elements(net, element_type, [element_index], variant, project_id)[0] + + def delete_elements(self, net: Union[int, str], element_type: str, element_indexes: list[int], variant: Union[int, list[int], None] = None, project_id: Union[str, None] = None) -> list[ dict]: + """ + Delete multiple elements of the same type from the database. + Parameters + ---------- + net: str or int + Network to add elements to, either a name or numeric id + element_type: str + Name of the element type (e.g. bus, line) + element_indexes: list of int + Indexes of the elements to delete + project_id: str or None + ObjectId (as str) of the project in which the network is stored. Defaults to current active project if None + variant: int or None + Variant index if elements should be created in a variant + Returns + ------- + list + A list of deleted elements as dicts with all fields + """ if not isinstance(element_indexes, list): raise TypeError("Parameter element_indexes must be a list of ints!") @@ -1083,7 +1122,7 @@ def create_element(self, net: Union[int, str], element_type: str, element_index: dict The created element (element_data with added _id field) """ - return self.create_elements_in_db(net, element_type, [{"index": element_index, **element_data}], + return self.create_elements(net, element_type, [{"index": element_index, **element_data}], project_id, variant)[0] def create_elements(self, net: Union[int, str], element_type: str, elements_data: list[dict], @@ -2033,6 +2072,20 @@ def create_elements_in_db(self, net: Union[int, str], element_type: str, element ) return self.create_elements(net, element_type, elements_data, project_id, variant) + def delete_net_element(self, net, element_type, element_index, variant=None, project_id=None): + warnings.warn( + "ph.delete_net_element was renamed - use ph.delete_element instead" + ) + return self.delete_element(net, element_type, element_index, variant, project_id) + + def delete_net_elements(self, net: Union[int, str], element_type: str, element_indexes: list[int], + variant: Union[int, list[int], None] = None, project_id: Union[str, None] = None): + warnings.warn( + "ph.delete_net_elementS was renamed - use ph.delete_elementS instead" + ) + return self.delete_elements(net, element_type, element_indexes,variant, project_id) + + if __name__ == '__main__': self = PandaHub() project_name = 'test_project' From 9567dd03cd73cb99e58d1e5b69f5dc92beea23f5 Mon Sep 17 00:00:00 2001 From: Joschka Thurner Date: Thu, 21 Sep 2023 14:09:50 +0200 Subject: [PATCH 11/84] refactor: new function names in routes and client --- pandahub/api/routers/net.py | 63 ++++++++++++++++++------------- pandahub/client/PandaHubClient.py | 30 ++++++++++++--- pandahub/lib/PandaHub.py | 29 ++++---------- pandahub/test/test_networks.py | 2 +- 4 files changed, 71 insertions(+), 53 deletions(-) diff --git a/pandahub/api/routers/net.py b/pandahub/api/routers/net.py index 3bf8d39..449f777 100644 --- a/pandahub/api/routers/net.py +++ b/pandahub/api/routers/net.py @@ -44,13 +44,15 @@ def write_network_to_db(data: WriteNetwork, ph=Depends(pandahub)): # ------------------------- -# Element handling +# Element CRUD # ------------------------- -class GetNetValueModel(BaseModel): +class BaseCRUDModel(BaseModel): project_id: str net_name: str - element: str + element_type: str + +class GetNetValueModel(BaseCRUDModel): element_index: int parameter: str @@ -58,10 +60,7 @@ class GetNetValueModel(BaseModel): def get_net_value_from_db(data: GetNetValueModel, ph=Depends(pandahub)): return ph.get_net_value_from_db(**data.dict()) -class SetNetValueModel(BaseModel): - project_id: str - net_name: str - element: str +class SetNetValueModel(BaseCRUDModel): element_index: int parameter: str value: Any @@ -70,33 +69,45 @@ class SetNetValueModel(BaseModel): def set_net_value_in_db(data: SetNetValueModel, ph=Depends(pandahub)): return ph.set_net_value_in_db(**data.dict()) -class CreateElementModel(BaseModel): - project_id: str - net_name: str - element: str +class CreateElementModel(BaseCRUDModel): element_index: int - data: dict + element_data: dict -@router.post("/create_element_in_db") +@router.post("/create_element") def create_element_in_db(data: CreateElementModel, ph=Depends(pandahub)): - return ph.create_element_in_db(**data.dict()) + return ph.create_element(**data.dict()) -class CreateElementsModel(BaseModel): - project_id: str - net_name: str - element_type: str - elements_data: list +class CreateElementsModel(BaseCRUDModel): + elements_data: list[dict[str,Any]] -@router.post("/create_elements_in_db") +@router.post("/create_elements") def create_elements_in_db(data: CreateElementsModel, ph=Depends(pandahub)): - return ph.create_elements_in_db(**data.dict()) + return ph.create_elements(**data.dict()) -class DeleteElementModel(BaseModel): - project_id: str - net_name: str - element: str +class DeleteElementModel(BaseCRUDModel): element_index: int +@router.post("/delete_element") +def delete_net_element(data: DeleteElementModel, ph=Depends(pandahub)): + return ph.delete_element(**data.dict()) + +class DeleteElementsModel(BaseCRUDModel): + element_indexes: list[int] + +@router.post("/delete_elements") +def delete_net_elements(data: DeleteElementsModel, ph=Depends(pandahub)): + return ph.delete_elements(**data.dict()) + +### deprecated routes + +@router.post("/create_element_in_db") +def create_element_in_db(data: CreateElementModel, ph=Depends(pandahub)): + return ph.create_element(**data.dict()) + +@router.post("/create_elements_in_db") +def create_elements_in_db(data: CreateElementsModel, ph=Depends(pandahub)): + return ph.create_elements(**data.dict()) + @router.post("/delete_net_element") def delete_net_element(data: DeleteElementModel, ph=Depends(pandahub)): - return ph.delete_net_element(**data.dict()) + return ph.delete_element(**data.dict()) diff --git a/pandahub/client/PandaHubClient.py b/pandahub/client/PandaHubClient.py index 0ca11f4..f627cb0 100644 --- a/pandahub/client/PandaHubClient.py +++ b/pandahub/client/PandaHubClient.py @@ -1,3 +1,5 @@ +import warnings + import requests import pandapower as pp import pandas as pd @@ -74,17 +76,35 @@ def get_net_from_db(self, name, include_results=True, only_tables=None): def get_net_value_from_db(self, net_name, element, element_index, parameter): return self._post("/net/get_net_value_from_db", json=locals()).json() - def set_net_value_in_db(self, net_name, element, element_index, parameter, value): + def set_net_value_in_db(self, net_name, element_type, element_index, parameter, value): return self._post("/net/set_net_value_in_db", json=locals()) - def create_element_in_db(self, net_name, element, element_index, data): - return self._post("/net/create_element_in_db", json=locals()) + def create_element(self, net_name, element_type, element_index, element_data): + return self._post("/net/create_element", json=locals()) + + def create_elements(self, net_name, element_type, elements_data): + return self._post("/net/create_elements", json=locals()) + + def delete_element(self, net_name, element_type, element_index): + return self._post("/net/delete_element", json=locals()) + + def delete_elements(self, net_name, element_type, element_index): + return self._post("/net/delete_elements", json=locals()) + + + ### deprecated functions + + def create_element_in_db(self, net_name, element_type, element_index, data): + warnings.warn("ph.create_element_in_db was renamed - use ph.create_element instead") + return self._post("/net/create_element", json=locals()) def create_elements_in_db(self, net_name, element_type, elements_data): - return self._post("/net/create_elements_in_db", json=locals()) + warnings.warn("ph.create_elements_in_db was renamed - use ph.create_elements instead") + return self._post("/net/create_elements", json=locals()) def delete_net_element(self, net_name, element, element_index): - return self._post("/net/delete_net_element", json=locals()) + warnings.warn("ph.delete_net_element was renamed - use ph.delete_element instead") + return self._post("/net/delete_element", json=locals()) ### TIMESERIES diff --git a/pandahub/lib/PandaHub.py b/pandahub/lib/PandaHub.py index a8e35d1..b85a290 100644 --- a/pandahub/lib/PandaHub.py +++ b/pandahub/lib/PandaHub.py @@ -6,7 +6,7 @@ import traceback import warnings from inspect import signature, _empty -from typing import Optional, Union +from typing import Optional, Union, Any import numpy as np import pandas as pd @@ -940,8 +940,8 @@ def delete_element(self, net, element_type, element_index, variant=None, project return self.delete_elements(net, element_type, [element_index], variant, project_id)[0] def delete_elements(self, net: Union[int, str], element_type: str, element_indexes: list[int], - variant: Union[int, list[int], None] = None, project_id: Union[str, None] = None) -> list[ - dict]: + variant: Union[int, list[int], None] = None, project_id: Union[str, None] = None) -> list[ + dict[str, Any]]: """ Delete multiple elements of the same type from the database. @@ -1126,7 +1126,7 @@ def create_element(self, net: Union[int, str], element_type: str, element_index: project_id, variant)[0] def create_elements(self, net: Union[int, str], element_type: str, elements_data: list[dict], - project_id: str = None, variant: int = None) -> list[dict]: + project_id: str = None, variant: int = None) -> list[dict[str, Any]]: """ Creates multiple elements of the same type in the database. @@ -2055,36 +2055,23 @@ def bulk_del_timeseries_from_db(self, filter_document, del_res = db[collection_name].delete_many(match_filter) return del_res - #### for + #### deprecated functions def create_element_in_db(self, net: Union[int, str], element_type: str, element_index: int, data: dict, variant=None, project_id=None): - warnings.warn( - "ph.create_element_in_db was renamed - use ph.create_element instead" - ) + warnings.warn("ph.create_element_in_db was renamed - use ph.create_element instead") return self.create_element(net, element_type, element_index, data, variant, project_id) def create_elements_in_db(self, net: Union[int, str], element_type: str, elements_data: list[dict], project_id: str = None, variant: int = None): - warnings.warn( - "ph.create_elements_in_db was renamed - use ph.create_elements instead" - ) + warnings.warn("ph.create_elements_in_db was renamed - use ph.create_elements instead") return self.create_elements(net, element_type, elements_data, project_id, variant) def delete_net_element(self, net, element_type, element_index, variant=None, project_id=None): - warnings.warn( - "ph.delete_net_element was renamed - use ph.delete_element instead" - ) + warnings.warn("ph.delete_net_element was renamed - use ph.delete_element instead") return self.delete_element(net, element_type, element_index, variant, project_id) - def delete_net_elements(self, net: Union[int, str], element_type: str, element_indexes: list[int], - variant: Union[int, list[int], None] = None, project_id: Union[str, None] = None): - warnings.warn( - "ph.delete_net_elementS was renamed - use ph.delete_elementS instead" - ) - return self.delete_elements(net, element_type, element_indexes,variant, project_id) - if __name__ == '__main__': self = PandaHub() diff --git a/pandahub/test/test_networks.py b/pandahub/test/test_networks.py index da140d8..91884ad 100644 --- a/pandahub/test/test_networks.py +++ b/pandahub/test/test_networks.py @@ -131,7 +131,7 @@ def test_access_and_set_single_values(ph): value = ph.get_net_value_from_db(name, element, index, parameter) assert value == p_mw_new - ph.delete_net_element(name, element, index) + ph.delete_element(name, element, index) with pytest.raises(PandaHubError): ph.get_net_value_from_db(name, element, index, parameter) net = ph.get_net_from_db(name) From c1e72591de280dccbe7fee2c4406b978917cddeb Mon Sep 17 00:00:00 2001 From: Joschka Thurner Date: Sat, 23 Sep 2023 10:04:45 +0200 Subject: [PATCH 12/84] add **kwargs to CUD functions --- pandahub/lib/PandaHub.py | 25 ++++++++++++++----------- 1 file changed, 14 insertions(+), 11 deletions(-) diff --git a/pandahub/lib/PandaHub.py b/pandahub/lib/PandaHub.py index b85a290..33d70be 100644 --- a/pandahub/lib/PandaHub.py +++ b/pandahub/lib/PandaHub.py @@ -6,7 +6,8 @@ import traceback import warnings from inspect import signature, _empty -from typing import Optional, Union, Any +from pymongo.errors import ServerSelectionTimeoutError +from typing import Optional, Union import numpy as np import pandas as pd @@ -916,7 +917,7 @@ def get_net_value_from_db(self, net, element_type, element_index, else: return document[parameter] - def delete_element(self, net, element_type, element_index, variant=None, project_id=None): + def delete_element(self, net, element_type, element_index, variant=None, project_id=None, **kwargs) -> dict: """ Delete an element from the database. @@ -937,11 +938,13 @@ def delete_element(self, net, element_type, element_index, variant=None, project dict The deleted element as dict with all fields """ - return self.delete_elements(net, element_type, [element_index], variant, project_id)[0] + return self.delete_elements( + net, element_type, [element_index], variant, project_id, **kwargs + )[0] def delete_elements(self, net: Union[int, str], element_type: str, element_indexes: list[int], - variant: Union[int, list[int], None] = None, project_id: Union[str, None] = None) -> list[ - dict[str, Any]]: + variant: Union[int, list[int], None] = None, project_id: Union[str, None] = None, **kwargs) -> \ + list[dict]: """ Delete multiple elements of the same type from the database. @@ -999,7 +1002,7 @@ def delete_elements(self, net: Union[int, str], element_type: str, element_index def set_net_value_in_db(self, net, element_type, element_index, - parameter, value, variant=None, project_id=None): + parameter, value, variant=None, project_id=None, **kwargs): logger.info(f"Setting {parameter} = {value} in {element_type} with index {element_index} and variant {variant}") if variant is not None: variant = int(variant) @@ -1099,7 +1102,7 @@ def set_object_attribute(self, net, element_type, element_index, {"$set": {"object._object": obj}}) def create_element(self, net: Union[int, str], element_type: str, element_index: int, element_data: dict, - variant=None, project_id=None) -> dict: + variant=None, project_id=None, **kwargs) -> dict: """ Creates an element in the database. @@ -1123,10 +1126,10 @@ def create_element(self, net: Union[int, str], element_type: str, element_index: The created element (element_data with added _id field) """ return self.create_elements(net, element_type, [{"index": element_index, **element_data}], - project_id, variant)[0] + project_id, variant, **kwargs)[0] def create_elements(self, net: Union[int, str], element_type: str, elements_data: list[dict], - project_id: str = None, variant: int = None) -> list[dict[str, Any]]: + project_id: str = None, variant: int = None, **kwargs) -> list[dict]: """ Creates multiple elements of the same type in the database. @@ -1166,8 +1169,8 @@ def create_elements(self, net: Union[int, str], element_type: str, elements_data self._ensure_dtypes(element_type, elm_data) data.append({**elm_data, **var_data, "net_id": net_id}) collection = self._collection_name_of_element(element_type) - insert_result = db[collection].insert_many(data) - return [z[0] | {"_id": z[1]} for z in zip(data, insert_result.inserted_ids)] + db[collection].insert_many(data, ordered=False) + return data def _add_missing_defaults(self, db, net_id, element_type, element_data): func_str = f"create_{element_type}" From d28ce5c87b29cbd0d2401182ae1ac7a4a1900687 Mon Sep 17 00:00:00 2001 From: Joschka Thurner Date: Mon, 25 Sep 2023 12:10:21 +0200 Subject: [PATCH 13/84] implement get_project_setting_value --- pandahub/lib/PandaHub.py | 31 +++++++++++++++++++++++++++++++ 1 file changed, 31 insertions(+) diff --git a/pandahub/lib/PandaHub.py b/pandahub/lib/PandaHub.py index 33d70be..85fe6ab 100644 --- a/pandahub/lib/PandaHub.py +++ b/pandahub/lib/PandaHub.py @@ -13,6 +13,8 @@ import pandas as pd from bson.errors import InvalidId from bson.objectid import ObjectId +from functools import reduce +from operator import getitem from pydantic.types import UUID4 from pymongo import MongoClient, ReplaceOne, DESCENDING from pymongo.errors import ServerSelectionTimeoutError @@ -379,6 +381,35 @@ def get_project_settings(self, project_id=None): self.check_permission("read") return self.active_project["settings"] + + def get_project_setting_value(self, setting, project_id=None): + """ + Retrieve the value of a setting. + + Parameters + ---------- + setting: str + The setting to retrieve - use dot notation to index into nested settings. + project_id: str or None + The project id to retrieve the setting from. Applies to the current active project if None. + Returns + ------- + Settings value or None + The settings' value if set in the database or None if the setting is not defined. + """ + if project_id: + self.set_active_project_by_id(project_id) + self.check_permission("read") + _id = self.active_project["_id"] + project_collection = self.mongo_client["user_management"]["projects"] + setting_string = f"settings.{setting}" + setting = project_collection.find_one({"_id": _id}, {"_id": 0, setting_string: 1}) + try: + return reduce(getitem, setting_string.split("."), setting) + except KeyError: + return None + + def set_project_settings(self, settings, project_id=None): if project_id: self.set_active_project_by_id(project_id) From 3b971f5b124f45722e51b745ba3b252d6c614452 Mon Sep 17 00:00:00 2001 From: Jannis Kupka Date: Wed, 11 Oct 2023 16:05:51 +0200 Subject: [PATCH 14/84] pass arguments as kwargs to create_elements --- pandahub/lib/PandaHub.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pandahub/lib/PandaHub.py b/pandahub/lib/PandaHub.py index 85fe6ab..b962dda 100644 --- a/pandahub/lib/PandaHub.py +++ b/pandahub/lib/PandaHub.py @@ -1157,7 +1157,7 @@ def create_element(self, net: Union[int, str], element_type: str, element_index: The created element (element_data with added _id field) """ return self.create_elements(net, element_type, [{"index": element_index, **element_data}], - project_id, variant, **kwargs)[0] + project_id=project_id, variant=variant, **kwargs)[0] def create_elements(self, net: Union[int, str], element_type: str, elements_data: list[dict], project_id: str = None, variant: int = None, **kwargs) -> list[dict]: From 6d57c8a098a149c03214561e06fe6a9df6a1c8a8 Mon Sep 17 00:00:00 2001 From: Joschka Thurner Date: Mon, 2 Oct 2023 10:13:51 +0200 Subject: [PATCH 15/84] replace find_one_and_* with *_one --- pandahub/lib/PandaHub.py | 26 ++++++++++++-------------- 1 file changed, 12 insertions(+), 14 deletions(-) diff --git a/pandahub/lib/PandaHub.py b/pandahub/lib/PandaHub.py index b962dda..10e4bf6 100644 --- a/pandahub/lib/PandaHub.py +++ b/pandahub/lib/PandaHub.py @@ -246,7 +246,7 @@ def rename_project(self, project_name): realm = self.active_project["realm"] if self.project_exists(project_name, realm): raise PandaHubError("Can't rename - project with this name already exists") - project_collection.find_one_and_update({"_id": self.active_project["_id"]}, + project_collection.update_one({"_id": self.active_project["_id"]}, {"$set": {"name": project_name}}) self.set_active_project(project_name, realm) @@ -256,7 +256,7 @@ def change_realm(self, realm): project_name = self.active_project["name"] if self.project_exists(project_name, realm): raise PandaHubError("Can't change realm - project with this name already exists") - project_collection.find_one_and_update({"_id": self.active_project["_id"]}, + project_collection.update_one({"_id": self.active_project["_id"]}, {"$set": {"realm": realm}}) self.set_active_project(project_name, realm) @@ -361,11 +361,11 @@ def upgrade_project_to_latest_version(self): dat = f"serialized_{json.dumps(data, cls=io_pp.PPJSONEncoder)}" data[key] = dat - db["_networks"].find_one_and_update({"_id":d["_id"]}, + db["_networks"].update_one({"_id":d["_id"]}, {"$set": {"data": data}}) project_collection = self.mongo_client["user_management"].projects - project_collection.find_one_and_update({"_id": self.active_project["_id"]}, + project_collection.update_one({"_id": self.active_project["_id"]}, {"$set": {"version": __version__}}) logger.info(f"upgraded projekt '{self.active_project['name']}' from version" f" {self.get_project_version()} to version {__version__}") @@ -417,7 +417,7 @@ def set_project_settings(self, settings, project_id=None): _id = self.active_project["_id"] new_settings = {**self.active_project["settings"], **settings} project_collection = self.mongo_client["user_management"]["projects"] - project_collection.find_one_and_update({"_id": _id}, {"$set": {"settings": new_settings}}) + project_collection.update_one({"_id": _id}, {"$set": {"settings": new_settings}}) self.active_project["settings"] = new_settings def set_project_settings_value(self, parameter, value, project_id=None): @@ -427,7 +427,7 @@ def set_project_settings_value(self, parameter, value, project_id=None): _id = self.active_project["_id"] project_collection = self.mongo_client["user_management"]["projects"] setting_string = "settings.{}".format(parameter) - project_collection.find_one_and_update({"_id": _id}, {"$set": {setting_string: value}}) + project_collection.update_one({"_id": _id}, {"$set": {setting_string: value}}) self.active_project["settings"][parameter] = value def get_project_metadata(self, project_id=None): @@ -1101,7 +1101,7 @@ def set_object_attribute(self, net, element_type, element_index, js = list(db[collection].find({"index": element_index, "net_id": net_id}))[0] obj = json_to_object(js["object"]) setattr(obj, parameter, value) - db[collection].find_one_and_update({"index": element_index, "net_id": net_id}, + db[collection].update_one({"index": element_index, "net_id": net_id}, {"$set": {"object._object": obj.to_json()}}) element_filter = {"index": element_index, "net_id": int(net_id)} @@ -1110,7 +1110,7 @@ def set_object_attribute(self, net, element_type, element_index, document = db[collection].find_one({**element_filter, **self.base_variant_filter}) obj = json_to_object(document["object"]) setattr(obj, parameter, value) - db[collection].find_one_and_update( + db[collection].update_one( {**element_filter, **self.base_variant_filter}, {"$set": {"object._object": obj.to_json()}} ) else: @@ -1469,7 +1469,7 @@ def write_timeseries_to_db(self, ts_format=ts_format, compress_ts_data=compress_ts_data, **kwargs) - db[collection_name].find_one_and_replace( + db[collection_name].replace_one( {"_id": document["_id"]}, document, upsert=True @@ -1586,10 +1586,8 @@ def update_timeseries_in_db(self, new_ts_content, document_id, collection_name=" self.check_permission("write") db = self._get_project_database() ts_update = {"timeseries_data": {"$each": convert_timeseries_to_subdocuments(new_ts_content)}} - db[collection_name].find_one_and_update({"_id": document_id}, - {"$push": ts_update}, - upsert=False - ) + db[collection_name].update_one({"_id": document_id}, + {"$push": ts_update},) # logger.info("document updated in database") def bulk_update_timeseries_in_db(self, new_ts_content, document_ids, project_id=None, collection_name="timeseries", @@ -1808,7 +1806,7 @@ def add_metadata(self, filter_document, add_meta, global_database=False, raise PandaHubError meta_copy = {**meta_before.iloc[0].to_dict(), **add_meta} # write new metadata to mongo db - db[collection_name].find_one_and_replace({"_id": meta_before.index[0]}, + db[collection_name].replace_one({"_id": meta_before.index[0]}, meta_copy, upsert=True) return meta_copy From 5223afd6effb20b191527a07615db062fb17d52d Mon Sep 17 00:00:00 2001 From: Joschka Thurner Date: Fri, 13 Oct 2023 11:27:44 +0200 Subject: [PATCH 16/84] fix create/delete function signatures --- pandahub/api/routers/net.py | 25 ++++++++++---- pandahub/client/PandaHubClient.py | 17 +++++----- pandahub/lib/PandaHub.py | 56 ++++++++++++++++++++++++------- 3 files changed, 72 insertions(+), 26 deletions(-) diff --git a/pandahub/api/routers/net.py b/pandahub/api/routers/net.py index 449f777..a7b8f66 100644 --- a/pandahub/api/routers/net.py +++ b/pandahub/api/routers/net.py @@ -49,7 +49,7 @@ def write_network_to_db(data: WriteNetwork, ph=Depends(pandahub)): class BaseCRUDModel(BaseModel): project_id: str - net_name: str + net: str element_type: str class GetNetValueModel(BaseCRUDModel): @@ -100,14 +100,27 @@ def delete_net_elements(data: DeleteElementsModel, ph=Depends(pandahub)): ### deprecated routes +class CreateElementModelDeprecated(BaseModel): + project_id: str + net_name: str + element: str + element_index: int + data: dict + @router.post("/create_element_in_db") -def create_element_in_db(data: CreateElementModel, ph=Depends(pandahub)): - return ph.create_element(**data.dict()) +def create_element_in_db(data: CreateElementModelDeprecated, ph=Depends(pandahub)): + return ph.create_element_in_db(**data.dict()) @router.post("/create_elements_in_db") def create_elements_in_db(data: CreateElementsModel, ph=Depends(pandahub)): - return ph.create_elements(**data.dict()) + return ph.create_elements_in_db(**data.dict()) + +class DeleteElementModelDeprecated(BaseModel): + project_id: str + net_name: str + element: str + element_index: int @router.post("/delete_net_element") -def delete_net_element(data: DeleteElementModel, ph=Depends(pandahub)): - return ph.delete_element(**data.dict()) +def delete_net_element(data: DeleteElementModelDeprecated, ph=Depends(pandahub)): + return ph.delete_net_element(**data.dict()) diff --git a/pandahub/client/PandaHubClient.py b/pandahub/client/PandaHubClient.py index f627cb0..cb61ee2 100644 --- a/pandahub/client/PandaHubClient.py +++ b/pandahub/client/PandaHubClient.py @@ -79,31 +79,32 @@ def get_net_value_from_db(self, net_name, element, element_index, parameter): def set_net_value_in_db(self, net_name, element_type, element_index, parameter, value): return self._post("/net/set_net_value_in_db", json=locals()) - def create_element(self, net_name, element_type, element_index, element_data): + def create_element(self, net, element_type, element_index, element_data): return self._post("/net/create_element", json=locals()) - def create_elements(self, net_name, element_type, elements_data): + def create_elements(self, net, element_type, elements_data): return self._post("/net/create_elements", json=locals()) - def delete_element(self, net_name, element_type, element_index): + def delete_element(self, net, element_type, element_index): return self._post("/net/delete_element", json=locals()) - def delete_elements(self, net_name, element_type, element_index): + def delete_elements(self, net, element_type, element_index): return self._post("/net/delete_elements", json=locals()) ### deprecated functions - def create_element_in_db(self, net_name, element_type, element_index, data): - warnings.warn("ph.create_element_in_db was renamed - use ph.create_element instead") + def create_element_in_db(self, net_name, element, element_index, data): + warnings.warn("create_element_in_db was renamed - use create_element instead!") return self._post("/net/create_element", json=locals()) def create_elements_in_db(self, net_name, element_type, elements_data): - warnings.warn("ph.create_elements_in_db was renamed - use ph.create_elements instead") + warnings.warn("ph.create_elements_in_db was renamed - use ph.create_elements instead! " + "Watch out for changed order of project_id and variant args") return self._post("/net/create_elements", json=locals()) def delete_net_element(self, net_name, element, element_index): - warnings.warn("ph.delete_net_element was renamed - use ph.delete_element instead") + warnings.warn("ph.delete_net_element was renamed - use ph.delete_element instead!") return self._post("/net/delete_element", json=locals()) diff --git a/pandahub/lib/PandaHub.py b/pandahub/lib/PandaHub.py index 10e4bf6..ae6ecf1 100644 --- a/pandahub/lib/PandaHub.py +++ b/pandahub/lib/PandaHub.py @@ -970,7 +970,12 @@ def delete_element(self, net, element_type, element_index, variant=None, project The deleted element as dict with all fields """ return self.delete_elements( - net, element_type, [element_index], variant, project_id, **kwargs + net=net, + element_type=element_type, + element_indexes=[element_index], + variant=variant, + project_id=project_id, + **kwargs, )[0] def delete_elements(self, net: Union[int, str], element_type: str, element_indexes: list[int], @@ -1156,11 +1161,17 @@ def create_element(self, net: Union[int, str], element_type: str, element_index: dict The created element (element_data with added _id field) """ - return self.create_elements(net, element_type, [{"index": element_index, **element_data}], - project_id=project_id, variant=variant, **kwargs)[0] + return self.create_elements( + net=net, + element_type=element_type, + elements_data=[{"index": element_index, **element_data}], + variant=variant, + project_id=project_id, + **kwargs, + )[0] def create_elements(self, net: Union[int, str], element_type: str, elements_data: list[dict], - project_id: str = None, variant: int = None, **kwargs) -> list[dict]: + variant: int = None, project_id: str = None, **kwargs) -> list[dict]: """ Creates multiple elements of the same type in the database. @@ -2089,20 +2100,41 @@ def bulk_del_timeseries_from_db(self, filter_document, #### deprecated functions - def create_element_in_db(self, net: Union[int, str], element_type: str, element_index: int, data: dict, + def create_element_in_db(self, net: Union[int, str], element: str, element_index: int, data: dict, variant=None, project_id=None): - warnings.warn("ph.create_element_in_db was renamed - use ph.create_element instead") - return self.create_element(net, element_type, element_index, data, variant, project_id) + warnings.warn("ph.create_element_in_db was renamed - use ph.create_element instead!") + return self.create_element( + net=net, + element_type=element, + element_index=element_index, + element_data=data, + variant=variant, + project_id=project_id, + ) def create_elements_in_db(self, net: Union[int, str], element_type: str, elements_data: list[dict], project_id: str = None, variant: int = None): - warnings.warn("ph.create_elements_in_db was renamed - use ph.create_elements instead") - return self.create_elements(net, element_type, elements_data, project_id, variant) + warnings.warn("ph.create_elements_in_db was renamed - use ph.create_elements instead! " + "Watch out for changed order of project_id and variant args") + return self.create_elements( + net=net, + element_type=element_type, + elements_data=elements_data, + variant=variant, + project_id=project_id, + ) - def delete_net_element(self, net, element_type, element_index, variant=None, project_id=None): - warnings.warn("ph.delete_net_element was renamed - use ph.delete_element instead") - return self.delete_element(net, element_type, element_index, variant, project_id) + + def delete_net_element(self, net, element, element_index, variant=None, project_id=None): + warnings.warn("ph.delete_net_element was renamed - use ph.delete_element instead!") + return self.delete_element( + net=net, + element_type=element, + element_index=element_index, + variant=variant, + project_id=project_id, + ) if __name__ == '__main__': From 2661b415348c2b078b999b55c976476b3d86f354 Mon Sep 17 00:00:00 2001 From: Joschka Thurner Date: Fri, 13 Oct 2023 11:43:03 +0200 Subject: [PATCH 17/84] raise instead of warn on deprecated routes/client functions --- pandahub/api/routers/net.py | 25 ++++++------------------- pandahub/client/PandaHubClient.py | 15 ++++++--------- 2 files changed, 12 insertions(+), 28 deletions(-) diff --git a/pandahub/api/routers/net.py b/pandahub/api/routers/net.py index a7b8f66..fbe8152 100644 --- a/pandahub/api/routers/net.py +++ b/pandahub/api/routers/net.py @@ -99,28 +99,15 @@ def delete_net_elements(data: DeleteElementsModel, ph=Depends(pandahub)): return ph.delete_elements(**data.dict()) ### deprecated routes - -class CreateElementModelDeprecated(BaseModel): - project_id: str - net_name: str - element: str - element_index: int - data: dict - @router.post("/create_element_in_db") -def create_element_in_db(data: CreateElementModelDeprecated, ph=Depends(pandahub)): - return ph.create_element_in_db(**data.dict()) +def create_element_in_db(*args, **kwargs): + raise RuntimeError("create_element_in_db was deprecated - use create_element instead!") @router.post("/create_elements_in_db") -def create_elements_in_db(data: CreateElementsModel, ph=Depends(pandahub)): - return ph.create_elements_in_db(**data.dict()) +def create_elements_in_db(*args, **kwargs): + raise RuntimeError("create_elements_in_db was deprecated - use create_elements instead!") -class DeleteElementModelDeprecated(BaseModel): - project_id: str - net_name: str - element: str - element_index: int @router.post("/delete_net_element") -def delete_net_element(data: DeleteElementModelDeprecated, ph=Depends(pandahub)): - return ph.delete_net_element(**data.dict()) +def delete_net_element(*args, **kwargs): + raise RuntimeError("delete_net_element was deprecated - use delete_element instead!") diff --git a/pandahub/client/PandaHubClient.py b/pandahub/client/PandaHubClient.py index cb61ee2..579f6e4 100644 --- a/pandahub/client/PandaHubClient.py +++ b/pandahub/client/PandaHubClient.py @@ -94,18 +94,15 @@ def delete_elements(self, net, element_type, element_index): ### deprecated functions - def create_element_in_db(self, net_name, element, element_index, data): - warnings.warn("create_element_in_db was renamed - use create_element instead!") - return self._post("/net/create_element", json=locals()) + def create_element_in_db(self, *args, **kwargs): + raise RuntimeError("create_element_in_db was deprecated - use create_element instead!") - def create_elements_in_db(self, net_name, element_type, elements_data): - warnings.warn("ph.create_elements_in_db was renamed - use ph.create_elements instead! " + def create_elements_in_db(self, *args, **kwargs): + raise RuntimeError("ph.create_elements_in_db was deprecated - use ph.create_elements instead! " "Watch out for changed order of project_id and variant args") - return self._post("/net/create_elements", json=locals()) - def delete_net_element(self, net_name, element, element_index): - warnings.warn("ph.delete_net_element was renamed - use ph.delete_element instead!") - return self._post("/net/delete_element", json=locals()) + def delete_net_element(self, *args, **kwargs): + raise RuntimeError("ph.delete_net_element was deprecated - use ph.delete_element instead!") ### TIMESERIES From 99b18537052fc8a562a85e686a3fd41a40134a34 Mon Sep 17 00:00:00 2001 From: dlohmeier Date: Thu, 12 Oct 2023 14:44:34 +0200 Subject: [PATCH 18/84] added collection_name to GetTimeseriesModel for API calls --- pandahub/api/routers/timeseries.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/pandahub/api/routers/timeseries.py b/pandahub/api/routers/timeseries.py index ab226bc..4e46112 100644 --- a/pandahub/api/routers/timeseries.py +++ b/pandahub/api/routers/timeseries.py @@ -23,6 +23,7 @@ class GetTimeSeriesModel(BaseModel): project_id: Optional[str] = None timestamp_range: Optional[tuple] = None exclude_timestamp_range: Optional[tuple] = None + collection_name: Optional[str] = "timeseries" @router.post("/get_timeseries_from_db") @@ -39,6 +40,7 @@ class MultiGetTimeSeriesModel(BaseModel): project_id: Optional[str] = None timestamp_range: Optional[tuple] = None exclude_timestamp_range: Optional[tuple] = None + collection_name: Optional[str] = "timeseries" @router.post("/multi_get_timeseries_from_db") From 7fdab750ca7ddfee6109394f339bf0d142cc8e6c Mon Sep 17 00:00:00 2001 From: Leon Hillmann Date: Thu, 14 Sep 2023 16:21:43 +0200 Subject: [PATCH 19/84] Feat: Add custom filter parameter to `get_subnet_from_db_by_id` Add the `additional_filters` argument to `get_subnet_from_db_by_id` for the user to provide additional filters when retrieving elements from the database. --- pandahub/lib/PandaHub.py | 51 ++++++++++++++++++++++++++-------------- 1 file changed, 34 insertions(+), 17 deletions(-) diff --git a/pandahub/lib/PandaHub.py b/pandahub/lib/PandaHub.py index ae6ecf1..7515d6f 100644 --- a/pandahub/lib/PandaHub.py +++ b/pandahub/lib/PandaHub.py @@ -7,7 +7,7 @@ import warnings from inspect import signature, _empty from pymongo.errors import ServerSelectionTimeoutError -from typing import Optional, Union +from typing import Optional, Union, Callable import numpy as np import pandas as pd @@ -619,9 +619,17 @@ def get_subnet_from_db(self, name, bus_filter=None, include_results=True, return self.get_subnet_from_db_by_id(_id, bus_filter=bus_filter, include_results=include_results, add_edge_branches=add_edge_branches, geo_mode=geo_mode, variants=variants) - def get_subnet_from_db_by_id(self, net_id, bus_filter=None, include_results=True, - add_edge_branches=True, geo_mode="string", variants=[], - ignore_elements=[]): + def get_subnet_from_db_by_id( + self, + net_id, + bus_filter=None, + include_results=True, + add_edge_branches=True, + geo_mode="string", + variants=[], + ignore_elements=[], + additional_filters: dict[str, Callable[[list[int]], dict]] = {} + ) -> pp.pandapowerNet: db = self._get_project_database() meta = self._get_network_metadata(db, net_id) dtypes = db["_networks"].find_one({"_id": net_id}, projection={"dtypes"}) @@ -673,7 +681,7 @@ def get_subnet_from_db_by_id(self, net_id, bus_filter=None, include_results=True set(net.trafo3w.lv_bus.values) | set(net.switch.bus) | set(net.switch.element) branch_buses_outside = [int(b) for b in branch_buses - set(buses)] self._add_element_from_collection(net, db, "bus", net_id, geo_mode=geo_mode, variants=variants, - filter={"index": {"$in": branch_buses_outside}}, + element_filter={"index": {"$in": branch_buses_outside}}, dtypes=dtypes) buses = net.bus.index.tolist() @@ -701,14 +709,22 @@ def get_subnet_from_db_by_id(self, net_id, bus_filter=None, include_results=True # add node elements node_elements = ["load", "sgen", "gen", "ext_grid", "shunt", "xward", "ward", "motor", "storage"] branch_elements = ["trafo", "line", "trafo3w", "switch", "impedance"] - all_elements = node_elements + branch_elements + ["bus"] + all_elements = node_elements + branch_elements + ["bus"] + list(additional_filters.keys()) all_elements = list(set(all_elements) - set(ignore_elements)) + # Add elements for which the user has provided a filter function + for element, filter_func in additional_filters.items(): + element_filter = filter_func(buses) + self._add_element_from_collection(net, db, element, net_id, + element_filter=element_filter, geo_mode=geo_mode, + include_results=include_results, + variants=variants, dtypes=dtypes) + # add all node elements that are connected to buses within the network for element in node_elements: - filter = {"bus": {"$in": buses}} + element_filter = {"bus": {"$in": buses}} self._add_element_from_collection(net, db, element, net_id, - filter=filter, geo_mode=geo_mode, + element_filter=element_filter, geo_mode=geo_mode, include_results=include_results, variants=variants, dtypes=dtypes) @@ -722,13 +738,13 @@ def get_subnet_from_db_by_id(self, net_id, bus_filter=None, include_results=True # for tables that share an index with an element (e.g. load->res_load) load only relevant entries for element in all_elements: if table_name.startswith(element + "_") or table_name.startswith("net_res_" + element): - filter = {"index": {"$in": net[element].index.tolist()}} + element_filter = {"index": {"$in": net[element].index.tolist()}} break else: # all other tables (e.g. std_types) are loaded without filter - filter = None + element_filter = None self._add_element_from_collection(net, db, table_name, net_id, - filter=filter, geo_mode=geo_mode, + element_filter=element_filter, geo_mode=geo_mode, include_results=include_results, variants=variants, dtypes=dtypes) self.deserialize_and_update_data(net, meta) @@ -872,7 +888,7 @@ def _get_network_metadata(self, db, net_id): return db["_networks"].find_one({"_id": net_id}) def _add_element_from_collection(self, net, db, element_type, net_id, - filter=None, include_results=True, + element_filter=None, include_results=True, only_tables=None, geo_mode="string", variants=[], dtypes=None): if only_tables is not None and not element_type in only_tables: return @@ -880,14 +896,15 @@ def _add_element_from_collection(self, net, db, element_type, net_id, return variants_filter = self.get_variant_filter(variants) filter_dict = {"net_id": net_id, **variants_filter} - if filter is not None: - if "$or" in filter_dict.keys() and "$or" in filter.keys(): + if element_filter is not None: + if "$or" in filter_dict.keys() and "$or" in element_filter.keys(): # if 'or' is in both filters create 'and' with # both to avoid override during filter merge - filter_and = {"$and": [{"$or": filter_dict.pop("$or")}, {"$or": filter.pop("$or")}]} - filter_dict = {**filter_dict, **filter, **filter_and} + filter_and = {"$and": [{"$or": filter_dict.pop("$or")}, {"$or": element_filter.pop("$or")}]} + filter_dict = {**filter_dict, **element_filter, **filter_and} else: - filter_dict = {**filter_dict, **filter} + filter_dict = {**filter_dict, **element_filter} + data = list(db[self._collection_name_of_element(element_type)].find(filter_dict)) if len(data) == 0: return From 7a36cf0df9f283f6c2cb6b50995215d4f9065759 Mon Sep 17 00:00:00 2001 From: Joschka Thurner Date: Wed, 11 Oct 2023 12:32:08 +0200 Subject: [PATCH 20/84] make ignore_elements overwrite additional_filters --- pandahub/lib/PandaHub.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/pandahub/lib/PandaHub.py b/pandahub/lib/PandaHub.py index 7515d6f..654577f 100644 --- a/pandahub/lib/PandaHub.py +++ b/pandahub/lib/PandaHub.py @@ -714,6 +714,8 @@ def get_subnet_from_db_by_id( # Add elements for which the user has provided a filter function for element, filter_func in additional_filters.items(): + if element in ignore_elements: + continue element_filter = filter_func(buses) self._add_element_from_collection(net, db, element, net_id, element_filter=element_filter, geo_mode=geo_mode, From 7a12d47d0ebf9fb703bb15e084a4bfe385ff738e Mon Sep 17 00:00:00 2001 From: Joschka Thurner Date: Wed, 11 Oct 2023 13:36:02 +0200 Subject: [PATCH 21/84] net as argument for additional filters --- pandahub/lib/PandaHub.py | 21 ++++++++++++++------- 1 file changed, 14 insertions(+), 7 deletions(-) diff --git a/pandahub/lib/PandaHub.py b/pandahub/lib/PandaHub.py index 654577f..b39cda6 100644 --- a/pandahub/lib/PandaHub.py +++ b/pandahub/lib/PandaHub.py @@ -6,8 +6,8 @@ import traceback import warnings from inspect import signature, _empty -from pymongo.errors import ServerSelectionTimeoutError -from typing import Optional, Union, Callable +from collections.abc import Callable +from typing import Optional, Union import numpy as np import pandas as pd @@ -609,15 +609,22 @@ def deserialize_and_update_data(self, net, meta): value = json.loads(value[11:], cls=io_pp.PPJSONDecoder) net[key] = value - def get_subnet_from_db(self, name, bus_filter=None, include_results=True, - add_edge_branches=True, geo_mode="string", variants=[]): + def get_subnet_from_db(self, + name, + bus_filter=None, + include_results=True, + add_edge_branches=True, + geo_mode="string", + variants=[], + additional_filters: dict[str, Callable[[pp.auxiliary.pandapowerNet], dict]] = {}): self.check_permission("read") db = self._get_project_database() _id = self._get_id_from_name(name, db) if _id is None: return None return self.get_subnet_from_db_by_id(_id, bus_filter=bus_filter, include_results=include_results, - add_edge_branches=add_edge_branches, geo_mode=geo_mode, variants=variants) + add_edge_branches=add_edge_branches, geo_mode=geo_mode, variants=variants, + additional_filters=additional_filters) def get_subnet_from_db_by_id( self, @@ -628,7 +635,7 @@ def get_subnet_from_db_by_id( geo_mode="string", variants=[], ignore_elements=[], - additional_filters: dict[str, Callable[[list[int]], dict]] = {} + additional_filters: dict[str, Callable[[pp.auxiliary.pandapowerNet], dict]] = {} ) -> pp.pandapowerNet: db = self._get_project_database() meta = self._get_network_metadata(db, net_id) @@ -716,7 +723,7 @@ def get_subnet_from_db_by_id( for element, filter_func in additional_filters.items(): if element in ignore_elements: continue - element_filter = filter_func(buses) + element_filter = filter_func(net) self._add_element_from_collection(net, db, element, net_id, element_filter=element_filter, geo_mode=geo_mode, include_results=include_results, From 120f033b304fae31ff0c82fa35463212acc4220f Mon Sep 17 00:00:00 2001 From: Joschka Thurner Date: Tue, 17 Oct 2023 11:20:39 +0200 Subject: [PATCH 22/84] revert renaming of function argument --- pandahub/lib/PandaHub.py | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/pandahub/lib/PandaHub.py b/pandahub/lib/PandaHub.py index b39cda6..c83dc5b 100644 --- a/pandahub/lib/PandaHub.py +++ b/pandahub/lib/PandaHub.py @@ -688,7 +688,7 @@ def get_subnet_from_db_by_id( set(net.trafo3w.lv_bus.values) | set(net.switch.bus) | set(net.switch.element) branch_buses_outside = [int(b) for b in branch_buses - set(buses)] self._add_element_from_collection(net, db, "bus", net_id, geo_mode=geo_mode, variants=variants, - element_filter={"index": {"$in": branch_buses_outside}}, + filter={"index": {"$in": branch_buses_outside}}, dtypes=dtypes) buses = net.bus.index.tolist() @@ -725,7 +725,7 @@ def get_subnet_from_db_by_id( continue element_filter = filter_func(net) self._add_element_from_collection(net, db, element, net_id, - element_filter=element_filter, geo_mode=geo_mode, + filter=element_filter, geo_mode=geo_mode, include_results=include_results, variants=variants, dtypes=dtypes) @@ -733,7 +733,7 @@ def get_subnet_from_db_by_id( for element in node_elements: element_filter = {"bus": {"$in": buses}} self._add_element_from_collection(net, db, element, net_id, - element_filter=element_filter, geo_mode=geo_mode, + filter=element_filter, geo_mode=geo_mode, include_results=include_results, variants=variants, dtypes=dtypes) @@ -753,7 +753,7 @@ def get_subnet_from_db_by_id( # all other tables (e.g. std_types) are loaded without filter element_filter = None self._add_element_from_collection(net, db, table_name, net_id, - element_filter=element_filter, geo_mode=geo_mode, + filter=element_filter, geo_mode=geo_mode, include_results=include_results, variants=variants, dtypes=dtypes) self.deserialize_and_update_data(net, meta) @@ -897,7 +897,7 @@ def _get_network_metadata(self, db, net_id): return db["_networks"].find_one({"_id": net_id}) def _add_element_from_collection(self, net, db, element_type, net_id, - element_filter=None, include_results=True, + filter=None, include_results=True, only_tables=None, geo_mode="string", variants=[], dtypes=None): if only_tables is not None and not element_type in only_tables: return @@ -905,14 +905,14 @@ def _add_element_from_collection(self, net, db, element_type, net_id, return variants_filter = self.get_variant_filter(variants) filter_dict = {"net_id": net_id, **variants_filter} - if element_filter is not None: - if "$or" in filter_dict.keys() and "$or" in element_filter.keys(): + if filter is not None: + if "$or" in filter_dict.keys() and "$or" in filter.keys(): # if 'or' is in both filters create 'and' with # both to avoid override during filter merge - filter_and = {"$and": [{"$or": filter_dict.pop("$or")}, {"$or": element_filter.pop("$or")}]} - filter_dict = {**filter_dict, **element_filter, **filter_and} + filter_and = {"$and": [{"$or": filter_dict.pop("$or")}, {"$or": filter.pop("$or")}]} + filter_dict = {**filter_dict, **filter, **filter_and} else: - filter_dict = {**filter_dict, **element_filter} + filter_dict = {**filter_dict, **filter} data = list(db[self._collection_name_of_element(element_type)].find(filter_dict)) if len(data) == 0: From eb1d1971f07a372d4674cb8cbe81053d7406c31f Mon Sep 17 00:00:00 2001 From: Joschka Thurner Date: Mon, 30 Oct 2023 21:25:03 +0100 Subject: [PATCH 23/84] clean up mongodb index creation --- pandahub/api/internal/settings.py | 1 + pandahub/lib/PandaHub.py | 93 +++++++++++++++++++------------ pandahub/lib/mongodb_indexes.py | 61 ++++++++++++++++++++ 3 files changed, 120 insertions(+), 35 deletions(-) create mode 100644 pandahub/lib/mongodb_indexes.py diff --git a/pandahub/api/internal/settings.py b/pandahub/api/internal/settings.py index 07ed322..0d7491a 100644 --- a/pandahub/api/internal/settings.py +++ b/pandahub/api/internal/settings.py @@ -49,3 +49,4 @@ def get_secret(key, default=None): DATATYPES_MODULE = os.getenv("DATATYPES_MODULE") or "pandahub.lib.datatypes" COSMOSDB_COMPAT = settings_bool("COSMOSDB_COMPAT", default=False) +CREATE_INDEXES_WITH_PROJECT=settings_bool("CREATE_INDEXES_WITH_PROJECT", default=True) diff --git a/pandahub/lib/PandaHub.py b/pandahub/lib/PandaHub.py index c83dc5b..e9d4d0e 100644 --- a/pandahub/lib/PandaHub.py +++ b/pandahub/lib/PandaHub.py @@ -16,17 +16,25 @@ from functools import reduce from operator import getitem from pydantic.types import UUID4 -from pymongo import MongoClient, ReplaceOne, DESCENDING +from pymongo import MongoClient, ReplaceOne from pymongo.errors import ServerSelectionTimeoutError import pandapipes as pps from pandapipes import from_json_string as from_json_pps import pandapower as pp import pandapower.io_utils as io_pp -from pandahub.api.internal import settings -from pandahub.lib.database_toolbox import create_timeseries_document, convert_timeseries_to_subdocuments, \ - convert_element_to_dict, json_to_object, serialize_object_data, get_dtypes -from pandahub.lib.database_toolbox import decompress_timeseries_data, convert_geojsons +import pandahub.api.internal.settings as SETTINGS +from pandahub.lib.database_toolbox import ( + create_timeseries_document, + convert_timeseries_to_subdocuments, + convert_element_to_dict, + json_to_object, + serialize_object_data, + get_dtypes, + decompress_timeseries_data, + convert_geojsons, +) +from pandahub.lib.mongodb_indexes import mongodb_indexes logger = logging.getLogger(__name__) from pandahub import __version__ @@ -55,14 +63,14 @@ class PandaHub: "user_management": ["owner"] } - _datatypes = getattr(importlib.import_module(settings.DATATYPES_MODULE), "datatypes") + _datatypes = getattr(importlib.import_module(SETTINGS.DATATYPES_MODULE), "datatypes") # ------------------------- # Initialization # ------------------------- - def __init__(self, connection_url=settings.MONGODB_URL, connection_user = settings.MONGODB_USER, - connection_password=settings.MONGODB_PASSWORD, check_server_available=False, user_id=None): + def __init__(self, connection_url=SETTINGS.MONGODB_URL, connection_user = SETTINGS.MONGODB_USER, + connection_password=SETTINGS.MONGODB_PASSWORD, check_server_available=False, user_id=None): mongo_client_args = {"host": connection_url, "uuidRepresentation": "standard", "connect":False} if connection_user: @@ -78,6 +86,7 @@ def __init__(self, connection_url=settings.MONGODB_URL, connection_user = settin {"var_type": np.nan}, ] } + self.mongodb_indexes = mongodb_indexes if check_server_available: self.server_is_available() @@ -185,6 +194,8 @@ def create_project(self, name, settings=None, realm=None, metadata=None, project if self.user_id is not None: project_data["users"] = {self.user_id: "owner"} self.mongo_client["user_management"]["projects"].insert_one(project_data) + if SETTINGS.CREATE_INDEXES_WITH_PROJECT: + self._create_mongodb_indexes(project_data["_id"]) if activate: self.set_active_project(name, realm) return project_data @@ -315,11 +326,11 @@ def _get_project_database(self): return self.mongo_client[str(self.active_project["_id"])] def _get_global_database(self): - if self.mongo_client_global_db is None and settings.MONGODB_GLOBAL_DATABASE_URL is not None: - mongo_client_args = {"host": settings.MONGODB_GLOBAL_DATABASE_URL, "uuidRepresentation": "standard"} - if settings.MONGODB_GLOBAL_DATABASE_USER: - mongo_client_args |= {"username": settings.MONGODB_GLOBAL_DATABASE_USER, - "password": settings.MONGODB_GLOBAL_DATABASE_PASSWORD} + if self.mongo_client_global_db is None and SETTINGS.MONGODB_GLOBAL_DATABASE_URL is not None: + mongo_client_args = {"host": SETTINGS.MONGODB_GLOBAL_DATABASE_URL, "uuidRepresentation": "standard"} + if SETTINGS.MONGODB_GLOBAL_DATABASE_USER: + mongo_client_args |= {"username": SETTINGS.MONGODB_GLOBAL_DATABASE_USER, + "password": SETTINGS.MONGODB_GLOBAL_DATABASE_PASSWORD} self.mongo_client_global_db = MongoClient(**mongo_client_args) if self.mongo_client_global_db is None: return self.mongo_client["global_data"] @@ -822,30 +833,12 @@ def _write_net_collections_to_db(self, db, collections): def _write_element_to_db(self, db, element_type, element_data): existing_collections = set(db.list_collection_names()) - def add_index(element): - columns = {"bus": ["net_id", "index"], - "line": ["net_id", "index", "from_bus", "to_bus"], - "trafo": ["net_id", "index", "hv_bus", "lv_bus"], - "switch": ["net_id", "index", "bus", "element", "et"], - "substation": ["net_id", "index"], - "area": ["net_id", "index", "name"]}.get(element, []) - if element in ["load", "sgen", "gen", "ext_grid", "shunt", "xward", "ward", "motor", - "storage"]: - columns = ["net_id", "bus"] - for c in columns: - logger.info(f"creating index on '{c}' in collection '{element}'") - db[self._collection_name_of_element(element)].create_index([(c, DESCENDING)]) - - collection_name = self._collection_name_of_element(element_type) if len(element_data) > 0: - try: - db[collection_name].insert_many(element_data, ordered=False) - if collection_name not in existing_collections: - add_index(element_type) - except: - traceback.print_exc() - print(f"\nFAILED TO WRITE TABLE '{element_type}' TO DATABASE! (details above)") + if collection_name not in existing_collections: + self._create_mongodb_indexes(collection=collection_name) + db[collection_name].insert_many(element_data, ordered=False) + # print(f"\nFAILED TO WRITE TABLE '{element_type}' TO DATABASE! (details above)") def delete_net_from_db(self, name): self.check_permission("write") @@ -1280,6 +1273,36 @@ def _ensure_dtypes(self, element_type, data): if not val is None and key in dtypes and not dtypes[key] == object: data[key] = dtypes[key](val) + def _create_mongodb_indexes(self, project_id: Optional[str]=None, collection: Optional["str"]=None): + """ + Create indexes on mongodb collections. Indexes are defined in pandahub.lib.mongodb_indexes + + Parameters + ---------- + project_id: str or None + Project to create indexes in - if None, current active project is used. + collection: str or None + Single collection to create index for - if None, alle defined Indexes are created. + + Returns + ------- + None + """ + if project_id: + project_db = self.mongo_client[str(project_id)] + else: + project_db = self._get_project_database() + if collection: + if collection in self.mongodb_indexes: + indexes_to_set = {collection: self.mongodb_indexes[collection]} + else: + return + else: + indexes_to_set = self.mongodb_indexes + for collection, indexes in indexes_to_set.items(): + logger.info(f"creating indexes in {collection} collection") + project_db[collection].create_indexes(indexes) + # ------------------------- # Variants # ------------------------- diff --git a/pandahub/lib/mongodb_indexes.py b/pandahub/lib/mongodb_indexes.py new file mode 100644 index 0000000..32f5e37 --- /dev/null +++ b/pandahub/lib/mongodb_indexes.py @@ -0,0 +1,61 @@ +from pymongo import DESCENDING, GEOSPHERE, IndexModel + +mongodb_indexes = { + "net_bus": [ + IndexModel([("net_id", DESCENDING), ("index", DESCENDING), ("variant", DESCENDING)], unique=True), + IndexModel([("geo", GEOSPHERE)]), + ], + "net_line": [ + IndexModel([("net_id", DESCENDING), ("index", DESCENDING), ("variant", DESCENDING)], unique=True), + IndexModel([("from_bus", DESCENDING)]), + IndexModel([("to_bus", DESCENDING)]), + IndexModel([("geo", GEOSPHERE)]), + ], + "net_trafo":[ + IndexModel([("net_id", DESCENDING), ("index", DESCENDING), ("variant", DESCENDING)], unique=True), + IndexModel([("hv_bus", DESCENDING)]), + IndexModel([("lv_bus", DESCENDING)]), + ], + "net_switch": [ + IndexModel([("net_id", DESCENDING), ("index", DESCENDING), ("variant", DESCENDING)], unique=True), + IndexModel([("bus", DESCENDING)]), + IndexModel([("element", DESCENDING)]), + IndexModel([("et", DESCENDING)]), + ], + "net_load": [ + IndexModel([("net_id", DESCENDING), ("index", DESCENDING), ("variant", DESCENDING)], unique=True), + IndexModel([("bus", DESCENDING)]), + ], + "net_sgen": [ + IndexModel([("net_id", DESCENDING), ("index", DESCENDING), ("variant", DESCENDING)], unique=True), + IndexModel([("bus", DESCENDING)]), + ], + "net_gen": [ + IndexModel([("net_id", DESCENDING), ("index", DESCENDING), ("variant", DESCENDING)], unique=True), + IndexModel([("bus", DESCENDING)]), + ], + "net_ext_grid": [ + IndexModel([("net_id", DESCENDING), ("index", DESCENDING), ("variant", DESCENDING)], unique=True), + IndexModel([("bus", DESCENDING)]), + ], + "net_shunt": [ + IndexModel([("net_id", DESCENDING), ("index", DESCENDING), ("variant", DESCENDING)], unique=True), + IndexModel([("bus", DESCENDING)]), + ], + "net_xward": [ + IndexModel([("net_id", DESCENDING), ("index", DESCENDING), ("variant", DESCENDING)], unique=True), + IndexModel([("bus", DESCENDING)]), + ], + "net_ward": [ + IndexModel([("net_id", DESCENDING), ("index", DESCENDING), ("variant", DESCENDING)], unique=True), + IndexModel([("bus", DESCENDING)]), + ], + "net_motor": [ + IndexModel([("net_id", DESCENDING), ("index", DESCENDING), ("variant", DESCENDING)], unique=True), + IndexModel([("bus", DESCENDING)]), + ], + "net_storage": [ + IndexModel([("net_id", DESCENDING), ("index", DESCENDING), ("variant", DESCENDING)], unique=True), + IndexModel([("bus", DESCENDING)]), + ], +} From d6d0ab1e9db90490f4e19c7068ece65633d69da5 Mon Sep 17 00:00:00 2001 From: Joschka Thurner Date: Tue, 31 Oct 2023 17:07:00 +0100 Subject: [PATCH 24/84] add pandapipes indexes Co-authored-by: Daniel Lohmeier --- pandahub/lib/mongodb_indexes.py | 44 +++++++++++++++++++++++++++++++++ 1 file changed, 44 insertions(+) diff --git a/pandahub/lib/mongodb_indexes.py b/pandahub/lib/mongodb_indexes.py index 32f5e37..4d0775f 100644 --- a/pandahub/lib/mongodb_indexes.py +++ b/pandahub/lib/mongodb_indexes.py @@ -58,4 +58,48 @@ IndexModel([("net_id", DESCENDING), ("index", DESCENDING), ("variant", DESCENDING)], unique=True), IndexModel([("bus", DESCENDING)]), ], + "net_water_tank": [ + IndexModel([("net_id", DESCENDING), ("index", DESCENDING), ("variant", DESCENDING)], unique=True), + IndexModel([("junction", DESCENDING)]), + ], + "net_flow_control": [ + IndexModel([("net_id", DESCENDING), ("index", DESCENDING), ("variant", DESCENDING)], unique=True), + IndexModel([("from_junction", DESCENDING)]), + IndexModel([("to_junction", DESCENDING)]), + ], + "net_press_control": [ + IndexModel([("net_id", DESCENDING), ("index", DESCENDING), ("variant", DESCENDING)], unique=True), + IndexModel([("from_junction", DESCENDING)]), + IndexModel([("to_junction", DESCENDING)]), + ], + "net_compressor": [ + IndexModel([("net_id", DESCENDING), ("index", DESCENDING), ("variant", DESCENDING)], unique=True), + IndexModel([("from_junction", DESCENDING)]), + IndexModel([("to_junction", DESCENDING)]), + ], + "net_pump": [ + IndexModel([("net_id", DESCENDING), ("index", DESCENDING), ("variant", DESCENDING)], unique=True), + IndexModel([("from_junction", DESCENDING)]), + IndexModel([("to_junction", DESCENDING)]), + ], + "net_circ_pump_mass": [ + IndexModel([("net_id", DESCENDING), ("index", DESCENDING), ("variant", DESCENDING)], unique=True), + IndexModel([("return_junction", DESCENDING)]), + IndexModel([("flow_junction", DESCENDING)]), + ], + "net_circ_pump_pressure": [ + IndexModel([("net_id", DESCENDING), ("index", DESCENDING), ("variant", DESCENDING)], unique=True), + IndexModel([("return_junction", DESCENDING)]), + IndexModel([("flow_junction", DESCENDING)]), + ], + "net_heat_exchanger": [ + IndexModel([("net_id", DESCENDING), ("index", DESCENDING), ("variant", DESCENDING)], unique=True), + IndexModel([("from_junction", DESCENDING)]), + IndexModel([("to_junction", DESCENDING)]), + ], + "net_heat_consumer": [ + IndexModel([("net_id", DESCENDING), ("index", DESCENDING), ("variant", DESCENDING)], unique=True), + IndexModel([("from_junction", DESCENDING)]), + IndexModel([("to_junction", DESCENDING)]), + ], } From 143fd8f799ca1c90c58d09a14ffe984d5e98fdcb Mon Sep 17 00:00:00 2001 From: Joschka Thurner Date: Tue, 31 Oct 2023 17:10:05 +0100 Subject: [PATCH 25/84] do not create indexes on project creation by default --- pandahub/api/internal/settings.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pandahub/api/internal/settings.py b/pandahub/api/internal/settings.py index 0d7491a..f721ed0 100644 --- a/pandahub/api/internal/settings.py +++ b/pandahub/api/internal/settings.py @@ -49,4 +49,4 @@ def get_secret(key, default=None): DATATYPES_MODULE = os.getenv("DATATYPES_MODULE") or "pandahub.lib.datatypes" COSMOSDB_COMPAT = settings_bool("COSMOSDB_COMPAT", default=False) -CREATE_INDEXES_WITH_PROJECT=settings_bool("CREATE_INDEXES_WITH_PROJECT", default=True) +CREATE_INDEXES_WITH_PROJECT=settings_bool("CREATE_INDEXES_WITH_PROJECT", default=False) From ada506f88c5f011ae6fad422bd4f23963a444f94 Mon Sep 17 00:00:00 2001 From: Joschka Thurner Date: Tue, 31 Oct 2023 17:13:02 +0100 Subject: [PATCH 26/84] add back area/substation indexes --- pandahub/lib/mongodb_indexes.py | 28 ++++++++++++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/pandahub/lib/mongodb_indexes.py b/pandahub/lib/mongodb_indexes.py index 4d0775f..6eaa18d 100644 --- a/pandahub/lib/mongodb_indexes.py +++ b/pandahub/lib/mongodb_indexes.py @@ -102,4 +102,32 @@ IndexModel([("from_junction", DESCENDING)]), IndexModel([("to_junction", DESCENDING)]), ], + "net_area": [ + IndexModel( + [("net_id", DESCENDING), ("index", DESCENDING), ("variant", DESCENDING)], + unique=True, + ), + IndexModel([("index", DESCENDING)]), + IndexModel([("buses", DESCENDING)]), + IndexModel([("lines", DESCENDING)]), + IndexModel([("connection_points", DESCENDING)]), + IndexModel([("feeders", DESCENDING)]), + IndexModel([("trafos", DESCENDING)]), + IndexModel([("substations", DESCENDING)]), + IndexModel([("type", DESCENDING)]), + IndexModel([("substation_buses", DESCENDING)]), + IndexModel([("level", DESCENDING)]), + IndexModel([("geo", GEOSPHERE)]), + IndexModel([("variant", DESCENDING)]), + ], + "net_substation": [ + IndexModel( + [("net_id", DESCENDING), ("index", DESCENDING), ("variant", DESCENDING)], + unique=True, + ), + IndexModel([("index", DESCENDING)]), + IndexModel([("type", DESCENDING)]), + IndexModel([("level", DESCENDING)]), + IndexModel([("geo", GEOSPHERE)]), + ], } From 8edd222df75a2d0f8612216eca5cb553f5468876 Mon Sep 17 00:00:00 2001 From: Joschka Thurner Date: Wed, 1 Nov 2023 14:50:27 +0100 Subject: [PATCH 27/84] add missing pandapipes indexes --- pandahub/lib/mongodb_indexes.py | 29 +++++++++++++++++++++++++++++ 1 file changed, 29 insertions(+) diff --git a/pandahub/lib/mongodb_indexes.py b/pandahub/lib/mongodb_indexes.py index 6eaa18d..951ef9b 100644 --- a/pandahub/lib/mongodb_indexes.py +++ b/pandahub/lib/mongodb_indexes.py @@ -1,6 +1,7 @@ from pymongo import DESCENDING, GEOSPHERE, IndexModel mongodb_indexes = { + # pandapower "net_bus": [ IndexModel([("net_id", DESCENDING), ("index", DESCENDING), ("variant", DESCENDING)], unique=True), IndexModel([("geo", GEOSPHERE)]), @@ -37,6 +38,7 @@ "net_ext_grid": [ IndexModel([("net_id", DESCENDING), ("index", DESCENDING), ("variant", DESCENDING)], unique=True), IndexModel([("bus", DESCENDING)]), + IndexModel([("junction", DESCENDING)]), ], "net_shunt": [ IndexModel([("net_id", DESCENDING), ("index", DESCENDING), ("variant", DESCENDING)], unique=True), @@ -58,6 +60,31 @@ IndexModel([("net_id", DESCENDING), ("index", DESCENDING), ("variant", DESCENDING)], unique=True), IndexModel([("bus", DESCENDING)]), ], + + # pandapipes + "net_junction": [ + IndexModel([("net_id", DESCENDING), ("index", DESCENDING), ("variant", DESCENDING)], unique=True), + IndexModel([("geo", GEOSPHERE)]), + ], + "net_pipe": [ + IndexModel([("net_id", DESCENDING), ("index", DESCENDING), ("variant", DESCENDING)], unique=True), + IndexModel([("from_junction", DESCENDING)]), + IndexModel([("to_junction", DESCENDING)]), + IndexModel([("geo", GEOSPHERE)]), + ], + "net_valve": [ + IndexModel([("net_id", DESCENDING), ("index", DESCENDING), ("variant", DESCENDING)], unique=True), + IndexModel([("from_junction", DESCENDING)]), + IndexModel([("to_junction", DESCENDING)]), + ], + "net_sink": [ + IndexModel([("net_id", DESCENDING), ("index", DESCENDING), ("variant", DESCENDING)], unique=True), + IndexModel([("junction", DESCENDING)]), + ], + "net_source": [ + IndexModel([("net_id", DESCENDING), ("index", DESCENDING), ("variant", DESCENDING)], unique=True), + IndexModel([("junction", DESCENDING)]), + ], "net_water_tank": [ IndexModel([("net_id", DESCENDING), ("index", DESCENDING), ("variant", DESCENDING)], unique=True), IndexModel([("junction", DESCENDING)]), @@ -102,6 +129,8 @@ IndexModel([("from_junction", DESCENDING)]), IndexModel([("to_junction", DESCENDING)]), ], + + # others "net_area": [ IndexModel( [("net_id", DESCENDING), ("index", DESCENDING), ("variant", DESCENDING)], From a3dce730d96b9686b7cb593819d7ca374288fcd4 Mon Sep 17 00:00:00 2001 From: Joschka Thurner Date: Thu, 23 Nov 2023 11:31:24 +0100 Subject: [PATCH 28/84] bump pandahub version to 0.2.9 --- pandahub/__init__.py | 2 +- setup.py | 7 +++---- 2 files changed, 4 insertions(+), 5 deletions(-) diff --git a/pandahub/__init__.py b/pandahub/__init__.py index 194a810..8815810 100644 --- a/pandahub/__init__.py +++ b/pandahub/__init__.py @@ -1,4 +1,4 @@ -__version__ = "0.2.8" +__version__ = "0.2.9" from pandahub.lib.PandaHub import PandaHub, PandaHubError from pandahub.client.PandaHubClient import PandaHubClient diff --git a/setup.py b/setup.py index 264167c..1ee1f8d 100644 --- a/setup.py +++ b/setup.py @@ -22,10 +22,9 @@ 'Intended Audience :: Developers', 'Natural Language :: English', 'Programming Language :: Python :: 3', - 'Programming Language :: Python :: 3.6', - 'Programming Language :: Python :: 3.7', - 'Programming Language :: Python :: 3.8', 'Programming Language :: Python :: 3.9', + 'Programming Language :: Python :: 3.10', + 'Programming Language :: Python :: 3.11', ], description="Data hub for pandapower and pandapipes networks based on MongoDB", install_requires=requirements, @@ -37,7 +36,7 @@ name='pandahub', packages=find_packages(), url='https://github.com/e2nIEE/pandahub', - version='0.2.8', + version='0.2.9', include_package_data=True, long_description_content_type='text/markdown', zip_safe=False, From 03fc0140c37c44e2ba8706431808005302b84d91 Mon Sep 17 00:00:00 2001 From: Leon Thurner Date: Sun, 26 Nov 2023 19:37:01 +0100 Subject: [PATCH 29/84] add performance test file --- pandahub/test/performance_test.py | 69 +++++++++++++++++++++++++++++++ 1 file changed, 69 insertions(+) create mode 100644 pandahub/test/performance_test.py diff --git a/pandahub/test/performance_test.py b/pandahub/test/performance_test.py new file mode 100644 index 0000000..05b2639 --- /dev/null +++ b/pandahub/test/performance_test.py @@ -0,0 +1,69 @@ +# -*- coding: utf-8 -*- +""" +Created on Sun Nov 26 12:25:55 2023 + +@author: LeonThurner +""" + +import pandapower as pp +from pandahub import PandaHub +import time +from line_profiler import LineProfiler + +def get_test_net(n_buses): + net = pp.create_empty_network() + pp.create_buses(net, n_buses, vn_kv=0.4) + pp.create_loads(net, net.bus.index[::50], p_mw=0, q_mvar=0) + pp.create_sgens(net, net.bus.index[::50], p_mw=0, q_mvar=0) + pp.create_gens(net, net.bus.index[::1000], p_mw=0, q_mvar=0) + pp.create_lines(net, net.bus.index[::2], net.bus.index[1::2], 1.0, "NAYY 4x50 SE") + pp.create_transformers_from_parameters(net, net.bus.index[::1000], + net.bus.index[1::1000], + vn_hv_kv=20., + vn_lv_kv=0.4, + sn_mva=0.4, + vkr_percent=5., + vk_percent=0.5, + pfe_kw=0, + i0_percent=0, + ) + line_buses = list(net.line.from_bus.values) + list(net.line.to_bus.values) + line_indices = list(net.line.index) + list(net.line.index) + pp.create_switches(net, line_buses, line_indices, "l") + return net + +def write_test_net_to_mongodb(net, project_name): + ph = PandaHub() + if ph.project_exists(project_name): + ph.set_active_project(project_name) + ph.delete_project(True) + ph.create_project(project_name) + ph.write_network_to_db(net, "net") + +def load_test_subnet_from_mongodb(project_name): + ph = PandaHub() + ph.set_active_project(project_name) + buses = list(range(1000)) + t0 = time.time() + subnet = ph.get_subnet_from_db("net", + bus_filter={"index": {"$in": buses}}) + t1 = time.time() + print("LOADED NET IN %.2fs"%(t1-t0)) + return subnet + +def profile_load_test(project_name): + lp = LineProfiler() + lp_wrapper = lp(load_test_subnet_from_mongodb) + lp.add_function(PandaHub.get_subnet_from_db_by_id) + lp.add_function(PandaHub._add_element_from_collection) + lp_wrapper(project_name) + lp.print_stats() + +if __name__ == '__main__': + n_buses = 3e6 #3 Million buses + project_name = "test_%u"%n_buses + net = get_test_net(n_buses) + write_test_net_to_mongodb(net, project_name) + + profile_load_test(project_name) + # subnet = load_test_subnet_from_mongodb() From 677a4dd9587fa984bcdbc144b5deabf50ecf420d Mon Sep 17 00:00:00 2001 From: Leon Thurner Date: Sun, 26 Nov 2023 19:57:11 +0100 Subject: [PATCH 30/84] replace inefficient cout_documents call with find_one solution --- pandahub/lib/PandaHub.py | 2 +- pandahub/test/performance_test.py | 7 +++---- 2 files changed, 4 insertions(+), 5 deletions(-) diff --git a/pandahub/lib/PandaHub.py b/pandahub/lib/PandaHub.py index e9d4d0e..519c1fe 100644 --- a/pandahub/lib/PandaHub.py +++ b/pandahub/lib/PandaHub.py @@ -654,7 +654,7 @@ def get_subnet_from_db_by_id( net = pp.create_empty_network() - if db[self._collection_name_of_element("bus")].count_documents({}) == 0: + if db[self._collection_name_of_element("bus")].find_one() is None: net["empty"] = True # Add buses with filter diff --git a/pandahub/test/performance_test.py b/pandahub/test/performance_test.py index 05b2639..0cb6bdd 100644 --- a/pandahub/test/performance_test.py +++ b/pandahub/test/performance_test.py @@ -38,14 +38,14 @@ def write_test_net_to_mongodb(net, project_name): ph.set_active_project(project_name) ph.delete_project(True) ph.create_project(project_name) - ph.write_network_to_db(net, "net") + ph.write_network_to_db(net, "test_net") def load_test_subnet_from_mongodb(project_name): ph = PandaHub() ph.set_active_project(project_name) buses = list(range(1000)) t0 = time.time() - subnet = ph.get_subnet_from_db("net", + subnet = ph.get_subnet_from_db("test_net", bus_filter={"index": {"$in": buses}}) t1 = time.time() print("LOADED NET IN %.2fs"%(t1-t0)) @@ -64,6 +64,5 @@ def profile_load_test(project_name): project_name = "test_%u"%n_buses net = get_test_net(n_buses) write_test_net_to_mongodb(net, project_name) - profile_load_test(project_name) - # subnet = load_test_subnet_from_mongodb() + # subnet = load_test_subnet_from_mongodb(project_name) From 96d6556bfd707c0451044fbdfd2991606502846e Mon Sep 17 00:00:00 2001 From: Joschka Thurner Date: Mon, 27 Nov 2023 09:01:44 +0100 Subject: [PATCH 31/84] bump pandahub version to 0.2.10 --- pandahub/__init__.py | 2 +- setup.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/pandahub/__init__.py b/pandahub/__init__.py index 8815810..16cb9c0 100644 --- a/pandahub/__init__.py +++ b/pandahub/__init__.py @@ -1,4 +1,4 @@ -__version__ = "0.2.9" +__version__ = "0.2.10" from pandahub.lib.PandaHub import PandaHub, PandaHubError from pandahub.client.PandaHubClient import PandaHubClient diff --git a/setup.py b/setup.py index 1ee1f8d..0832d2d 100644 --- a/setup.py +++ b/setup.py @@ -36,7 +36,7 @@ name='pandahub', packages=find_packages(), url='https://github.com/e2nIEE/pandahub', - version='0.2.9', + version='0.2.10', include_package_data=True, long_description_content_type='text/markdown', zip_safe=False, From a45982da5c48fe0538a29c854e7d7a66963c95ba Mon Sep 17 00:00:00 2001 From: Joschka Thurner Date: Mon, 27 Nov 2023 09:09:57 +0100 Subject: [PATCH 32/84] Update fastapi-stack (#32) * drop cosmosdb-compat * update fastapi-users and fastapi * replace id with _id when dealing with raw user object * db init: switch to lifespan events * fix pydantic BaseModel.dict method deprecation * bump dependencies * add userdb migration function for >= 0.3.0 --- pandahub/api/dependencies.py | 4 +-- pandahub/api/internal/db.py | 53 +++++++++++++----------------- pandahub/api/internal/models.py | 24 -------------- pandahub/api/internal/schemas.py | 14 ++++++++ pandahub/api/internal/settings.py | 5 ++- pandahub/api/internal/toolbox.py | 4 +-- pandahub/api/internal/users.py | 42 +++++++++++------------ pandahub/api/main.py | 19 +++++++++-- pandahub/api/routers/auth.py | 19 ++++------- pandahub/api/routers/net.py | 18 +++++----- pandahub/api/routers/projects.py | 20 +++++------ pandahub/api/routers/timeseries.py | 8 ++--- pandahub/api/routers/users.py | 3 +- pandahub/api/routers/variants.py | 2 -- pandahub/lib/PandaHub.py | 18 +++++----- pandahub/lib/database_toolbox.py | 45 +++++++++++++++++++++++++ requirements.txt | 8 ++--- 17 files changed, 168 insertions(+), 138 deletions(-) delete mode 100644 pandahub/api/internal/models.py create mode 100644 pandahub/api/internal/schemas.py diff --git a/pandahub/api/dependencies.py b/pandahub/api/dependencies.py index 3e7cec3..d21cadf 100644 --- a/pandahub/api/dependencies.py +++ b/pandahub/api/dependencies.py @@ -1,11 +1,11 @@ from fastapi import Depends from pandahub import PandaHub -from .internal.models import UserDB +from .internal.db import User from .internal.users import fastapi_users current_active_user = fastapi_users.current_user(active=True) -def pandahub(user: UserDB = Depends(current_active_user)): +def pandahub(user: User = Depends(current_active_user)): return PandaHub(user_id=str(user.id)) diff --git a/pandahub/api/internal/db.py b/pandahub/api/internal/db.py index 236f40d..83be7c8 100644 --- a/pandahub/api/internal/db.py +++ b/pandahub/api/internal/db.py @@ -1,49 +1,40 @@ import asyncio +import uuid import motor.motor_asyncio -from fastapi_users.db import MongoDBUserDatabase -from fastapi_users_db_mongodb.access_token import MongoDBAccessTokenDatabase +from beanie import Document +from fastapi_users.db import BeanieBaseUser, BeanieUserDatabase +from fastapi_users_db_beanie.access_token import ( + BeanieAccessTokenDatabase, + BeanieBaseAccessToken, +) from pandahub.api.internal import settings -from pandahub.api.internal.models import AccessToken, UserDB +from pydantic import Field mongo_client_args = {"host": settings.MONGODB_URL, "uuidRepresentation": "standard", "connect": False} if settings.MONGODB_USER: mongo_client_args |= {"username": settings.MONGODB_USER, "password": settings.MONGODB_PASSWORD} client = motor.motor_asyncio.AsyncIOMotorClient(**mongo_client_args) - client.get_io_loop = asyncio.get_event_loop - db = client["user_management"] -collection = db["users"] -access_tokens_collection = db["access_tokens"] +class User(BeanieBaseUser, Document): + # pass + # id: Optional[UUID4] = Field(alias="id") + id: uuid.UUID = Field(default_factory=uuid.uuid4) + class Settings(BeanieBaseUser.Settings): + name = "users" -async def get_user_db(): - if settings.COSMOSDB_COMPAT: - yield MongoDBUserDatabaseCosmos(UserDB, collection) - else: - yield MongoDBUserDatabase(UserDB, collection) +class AccessToken(BeanieBaseAccessToken, Document): + user_id: uuid.UUID = Field(default_factory=uuid.uuid4) + class Settings(BeanieBaseAccessToken.Settings): + name = "access_tokens" + +async def get_user_db(): + yield BeanieUserDatabase(User) async def get_access_token_db(): - yield MongoDBAccessTokenDatabase(AccessToken, access_tokens_collection) - -class MongoDBUserDatabaseCosmos(MongoDBUserDatabase): - from typing import Optional - from fastapi_users.models import UD - async def get_by_email(self, email: str) -> Optional[UD]: - await self._initialize() - - user = await self.collection.find_one( - {"email": email} - ) - return self.user_db_model(**user) if user else None - - async def _initialize(self): - if not self.initialized: - if "email_1" not in await self.collection.index_information(): - await self.collection.create_index("id", unique=True) - await self.collection.create_index("email", unique=True) - self.initialized = True + yield BeanieAccessTokenDatabase(AccessToken) diff --git a/pandahub/api/internal/models.py b/pandahub/api/internal/models.py deleted file mode 100644 index 7f9dfec..0000000 --- a/pandahub/api/internal/models.py +++ /dev/null @@ -1,24 +0,0 @@ -from fastapi_users import models -from fastapi_users.authentication.strategy.db import BaseAccessToken - -from pandahub.api.internal.settings import REGISTRATION_ADMIN_APPROVAL - - -class User(models.BaseUser): - is_active: bool = not REGISTRATION_ADMIN_APPROVAL - - -class UserCreate(models.BaseUserCreate): - pass - - -class UserUpdate(models.BaseUserUpdate): - pass - - -class UserDB(User, models.BaseUserDB): - pass - - -class AccessToken(BaseAccessToken): - pass diff --git a/pandahub/api/internal/schemas.py b/pandahub/api/internal/schemas.py new file mode 100644 index 0000000..63b4f57 --- /dev/null +++ b/pandahub/api/internal/schemas.py @@ -0,0 +1,14 @@ +import uuid + +from fastapi_users import schemas +from pandahub.api.internal.settings import REGISTRATION_ADMIN_APPROVAL + + +class UserRead(schemas.BaseUser[uuid.UUID]): + pass + +class UserCreate(schemas.BaseUserCreate): + is_active: bool = not REGISTRATION_ADMIN_APPROVAL + +class UserUpdate(schemas.BaseUserUpdate): + pass diff --git a/pandahub/api/internal/settings.py b/pandahub/api/internal/settings.py index f721ed0..d79fde7 100644 --- a/pandahub/api/internal/settings.py +++ b/pandahub/api/internal/settings.py @@ -37,8 +37,8 @@ def get_secret(key, default=None): MAIL_PASSWORD = os.getenv("MAIL_PASSWORD") or "" MAIL_PORT = os.getenv("MAIL_PORT") or 587 MAIL_SMTP_SERVER = os.getenv("MAIL_SMTP_SERVER") or "" -MAIL_TLS = os.getenv("MAIL_TLS") or True -MAIL_SSL = os.getenv("MAIL_SSL") or False +MAIL_STARTTLS = os.getenv("MAIL_STARTTLS") or True +MAIL_SSL_TLS = os.getenv("MAIL_SSL_TLS") or False PASSWORD_RESET_URL = os.getenv("PASSWORD_RESET_URL") or "" EMAIL_VERIFY_URL = os.getenv("EMAIL_VERIFY_URL") or "" @@ -48,5 +48,4 @@ def get_secret(key, default=None): REGISTRATION_ADMIN_APPROVAL = settings_bool("REGISTRATION_ADMIN_APPROVAL", default=False) DATATYPES_MODULE = os.getenv("DATATYPES_MODULE") or "pandahub.lib.datatypes" -COSMOSDB_COMPAT = settings_bool("COSMOSDB_COMPAT", default=False) CREATE_INDEXES_WITH_PROJECT=settings_bool("CREATE_INDEXES_WITH_PROJECT", default=False) diff --git a/pandahub/api/internal/toolbox.py b/pandahub/api/internal/toolbox.py index 969cdf7..ca0bd41 100644 --- a/pandahub/api/internal/toolbox.py +++ b/pandahub/api/internal/toolbox.py @@ -8,8 +8,8 @@ MAIL_FROM=settings.MAIL_USERNAME, MAIL_PORT=settings.MAIL_PORT, MAIL_SERVER=settings.MAIL_SMTP_SERVER, - MAIL_TLS=settings.MAIL_TLS, - MAIL_SSL=settings.MAIL_SSL + MAIL_SSL_TLS=settings.MAIL_SSL_TLS, + MAIL_STARTTLS=settings.MAIL_STARTTLS, ) fast_mail = FastMail(email_conf) diff --git a/pandahub/api/internal/users.py b/pandahub/api/internal/users.py index b2f0230..230f250 100644 --- a/pandahub/api/internal/users.py +++ b/pandahub/api/internal/users.py @@ -1,38 +1,43 @@ +import uuid + from typing import Optional from fastapi import Depends, Request -from fastapi_users import BaseUserManager, FastAPIUsers +from fastapi_users import BaseUserManager, FastAPIUsers, UUIDIDMixin from fastapi_users.authentication import AuthenticationBackend, BearerTransport -from fastapi_users.authentication.strategy.db import AccessTokenDatabase, DatabaseStrategy -from fastapi_users.db import MongoDBUserDatabase +from fastapi_users.authentication.strategy.db import ( + AccessTokenDatabase, + DatabaseStrategy, +) +from fastapi_users.db import BeanieUserDatabase from ..internal import settings -from ..internal.db import get_user_db, get_access_token_db -from ..internal.models import User, UserCreate, UserDB, UserUpdate, AccessToken +from ..internal.db import get_user_db, get_access_token_db, User, AccessToken from ..internal.toolbox import send_password_reset_mail, send_verification_email -class UserManager(BaseUserManager[UserCreate, UserDB]): - user_db_model = UserDB +class UserManager(UUIDIDMixin, BaseUserManager[User, uuid.UUID]): if settings.SECRET is None: - raise UserWarning("You must specify a SECRET in the environment variables or .env file") + raise UserWarning( + "You must specify a SECRET in the environment variables or .env file" + ) reset_password_token_secret = settings.SECRET verification_token_secret = settings.SECRET - async def on_after_register(self, user: UserDB, request: Optional[Request] = None): - if (settings.EMAIL_VERIFICATION_REQUIRED): + async def on_after_register(self, user: User, request: Optional[Request] = None): + if settings.EMAIL_VERIFICATION_REQUIRED: await self.request_verify(user) print(f"User {user.id} has registered.") async def on_after_forgot_password( - self, user: UserDB, token: str, request: Optional[Request] = None + self, user: User, token: str, request: Optional[Request] = None ): await send_password_reset_mail(user, token) # print(f"User {user.id} has forgot their password. Reset token: {token}") async def on_after_request_verify( - self, user: UserDB, token: str, request: Optional[Request] = None + self, user: User, token: str, request: Optional[Request] = None ): await send_verification_email(user, token) @@ -40,7 +45,7 @@ async def on_after_request_verify( # print(f"Verification requested for user {user.id}. Verification token: {token}") -async def get_user_manager(user_db: MongoDBUserDatabase = Depends(get_user_db)): +async def get_user_manager(user_db:BeanieUserDatabase = Depends(get_user_db)): yield UserManager(user_db) @@ -49,7 +54,7 @@ async def get_user_manager(user_db: MongoDBUserDatabase = Depends(get_user_db)): def get_database_strategy( access_token_db: AccessTokenDatabase[AccessToken] = Depends(get_access_token_db), -) -> DatabaseStrategy[UserCreate, UserDB, AccessToken]: +) -> DatabaseStrategy: return DatabaseStrategy(access_token_db) @@ -58,11 +63,4 @@ def get_database_strategy( transport=bearer_transport, get_strategy=get_database_strategy, ) -fastapi_users = FastAPIUsers( - get_user_manager, - [auth_backend], - User, - UserCreate, - UserUpdate, - UserDB, -) +fastapi_users = FastAPIUsers(get_user_manager, [auth_backend]) diff --git a/pandahub/api/main.py b/pandahub/api/main.py index 8fd49b8..0d15938 100644 --- a/pandahub/api/main.py +++ b/pandahub/api/main.py @@ -1,3 +1,5 @@ +from contextlib import asynccontextmanager + import uvicorn from fastapi import FastAPI, Request from fastapi.middleware.cors import CORSMiddleware @@ -5,8 +7,21 @@ from pandahub.lib.PandaHub import PandaHubError from pandahub.api.routers import net, projects, timeseries, users, auth, variants +from pandahub.api.internal.db import User, db, AccessToken +from beanie import init_beanie + +@asynccontextmanager +async def lifespan(app: FastAPI): + await init_beanie( + database=db, + document_models=[ + User, + AccessToken + ], + ) + yield -app = FastAPI() +app = FastAPI(lifespan=lifespan) origins = [ "http://localhost:8080", @@ -23,7 +38,7 @@ app.include_router(net.router) app.include_router(projects.router) app.include_router(timeseries.router) -app.include_router(users.router) +app.include_router(User.router) app.include_router(auth.router) app.include_router(variants.router) diff --git a/pandahub/api/routers/auth.py b/pandahub/api/routers/auth.py index 093ba7f..d9f0aa4 100644 --- a/pandahub/api/routers/auth.py +++ b/pandahub/api/routers/auth.py @@ -1,27 +1,20 @@ from fastapi import APIRouter from pandahub.api.internal.users import auth_backend, fastapi_users - +from pandahub.api.internal.schemas import UserCreate, UserRead from pandahub.api.internal.settings import REGISTRATION_ENABLED -router = APIRouter( - prefix="/auth", - tags=["auth"] -) +router = APIRouter(prefix="/auth", tags=["auth"]) -router.include_router( - fastapi_users.get_auth_router(auth_backend) -) +router.include_router(fastapi_users.get_auth_router(auth_backend)) -router.include_router( - fastapi_users.get_reset_password_router() -) +router.include_router(fastapi_users.get_reset_password_router()) if REGISTRATION_ENABLED: router.include_router( - fastapi_users.get_register_router() + fastapi_users.get_register_router(UserRead, UserCreate), ) router.include_router( - fastapi_users.get_verify_router() + fastapi_users.get_verify_router(UserRead), ) diff --git a/pandahub/api/routers/net.py b/pandahub/api/routers/net.py index fbe8152..011c5e9 100644 --- a/pandahub/api/routers/net.py +++ b/pandahub/api/routers/net.py @@ -25,7 +25,7 @@ class GetNetFromDB(BaseModel): @router.post("/get_net_from_db") def get_net_from_db(data: GetNetFromDB, ph=Depends(pandahub)): - net = ph.get_net_from_db(**data.dict()) + net = ph.get_net_from_db(**data.model_dump()) return pp.to_json(net) @@ -38,7 +38,7 @@ class WriteNetwork(BaseModel): @router.post("/write_network_to_db") def write_network_to_db(data: WriteNetwork, ph=Depends(pandahub)): - params = data.dict() + params = data.model_dump() params["net"] = pp.from_json_string(params["net"]) ph.write_network_to_db(**params) @@ -58,16 +58,16 @@ class GetNetValueModel(BaseCRUDModel): @router.post("/get_net_value_from_db") def get_net_value_from_db(data: GetNetValueModel, ph=Depends(pandahub)): - return ph.get_net_value_from_db(**data.dict()) + return ph.get_net_value_from_db(**data.model_dump()) class SetNetValueModel(BaseCRUDModel): element_index: int parameter: str - value: Any + value: Any = None @router.post("/set_net_value_in_db") def set_net_value_in_db(data: SetNetValueModel, ph=Depends(pandahub)): - return ph.set_net_value_in_db(**data.dict()) + return ph.set_net_value_in_db(**data.model_dump()) class CreateElementModel(BaseCRUDModel): element_index: int @@ -75,28 +75,28 @@ class CreateElementModel(BaseCRUDModel): @router.post("/create_element") def create_element_in_db(data: CreateElementModel, ph=Depends(pandahub)): - return ph.create_element(**data.dict()) + return ph.create_element(**data.model_dump()) class CreateElementsModel(BaseCRUDModel): elements_data: list[dict[str,Any]] @router.post("/create_elements") def create_elements_in_db(data: CreateElementsModel, ph=Depends(pandahub)): - return ph.create_elements(**data.dict()) + return ph.create_elements(**data.model_dump()) class DeleteElementModel(BaseCRUDModel): element_index: int @router.post("/delete_element") def delete_net_element(data: DeleteElementModel, ph=Depends(pandahub)): - return ph.delete_element(**data.dict()) + return ph.delete_element(**data.model_dump()) class DeleteElementsModel(BaseCRUDModel): element_indexes: list[int] @router.post("/delete_elements") def delete_net_elements(data: DeleteElementsModel, ph=Depends(pandahub)): - return ph.delete_elements(**data.dict()) + return ph.delete_elements(**data.model_dump()) ### deprecated routes @router.post("/create_element_in_db") diff --git a/pandahub/api/routers/projects.py b/pandahub/api/routers/projects.py index aadc16d..3bf67b3 100644 --- a/pandahub/api/routers/projects.py +++ b/pandahub/api/routers/projects.py @@ -21,7 +21,7 @@ class CreateProject(BaseModel): @router.post("/create_project") def create_project(data: CreateProject, ph=Depends(pandahub)): - ph.create_project(**data.dict(), realm=ph.user_id) + ph.create_project(**data.model_dump(), realm=ph.user_id) return {"message": f"Project {data.name} created !"} class DeleteProject(BaseModel): @@ -30,7 +30,7 @@ class DeleteProject(BaseModel): @router.post("/delete_project") def delete_project(data: DeleteProject, ph=Depends(pandahub)): - ph.delete_project(**data.dict()) + ph.delete_project(**data.model_dump()) return True @router.post("/get_projects") @@ -43,7 +43,7 @@ class Project(BaseModel): @router.post("/project_exists") def project_exists(data: Project, ph=Depends(pandahub)): - return ph.project_exists(**data.dict(), realm=ph.user_id) + return ph.project_exists(**data.model_dump(), realm=ph.user_id) class SetActiveProjectModel(BaseModel): @@ -51,7 +51,7 @@ class SetActiveProjectModel(BaseModel): @router.post("/set_active_project") def set_active_project(data: SetActiveProjectModel, ph=Depends(pandahub)): - ph.set_active_project(**data.dict()) + ph.set_active_project(**data.model_dump()) return str(ph.active_project["_id"]) @@ -64,7 +64,7 @@ class GetProjectSettingsModel(BaseModel): @router.post("/get_project_settings") def get_project_settings(data: GetProjectSettingsModel, ph=Depends(pandahub)): - settings = ph.get_project_settings(**data.dict()) + settings = ph.get_project_settings(**data.model_dump()) return settings class SetProjectSettingsModel(BaseModel): @@ -73,16 +73,16 @@ class SetProjectSettingsModel(BaseModel): @router.post("/set_project_settings") def set_project_settings(data: SetProjectSettingsModel, ph=Depends(pandahub)): - ph.set_project_settings(**data.dict()) + ph.set_project_settings(**data.model_dump()) class SetProjectSettingsValueModel(BaseModel): project_id: str parameter: str - value: Any + value: Any = None @router.post("/set_project_settings_value") def set_project_settings_value(data: SetProjectSettingsValueModel, ph=Depends(pandahub)): - ph.set_project_settings_value(**data.dict()) + ph.set_project_settings_value(**data.model_dump()) # ------------------------- # Metadata @@ -93,7 +93,7 @@ class GetProjectMetadataModel(BaseModel): @router.post("/get_project_metadata") def get_project_metadata(data: GetProjectMetadataModel, ph=Depends(pandahub)): - metadata = ph.get_project_metadata(**data.dict()) + metadata = ph.get_project_metadata(**data.model_dump()) return metadata class SetProjectMetadataModel(BaseModel): @@ -102,4 +102,4 @@ class SetProjectMetadataModel(BaseModel): @router.post("/set_project_metadata") def set_project_metadata(data: SetProjectMetadataModel, ph=Depends(pandahub)): - return ph.set_project_metadata(**data.dict()) + return ph.set_project_metadata(**data.model_dump()) diff --git a/pandahub/api/routers/timeseries.py b/pandahub/api/routers/timeseries.py index 4e46112..8d453fd 100644 --- a/pandahub/api/routers/timeseries.py +++ b/pandahub/api/routers/timeseries.py @@ -3,7 +3,7 @@ import pandas as pd from fastapi import APIRouter, Depends from pydantic import BaseModel -from pydantic.typing import Optional +from typing import Optional from pandahub.api.dependencies import pandahub @@ -30,7 +30,7 @@ class GetTimeSeriesModel(BaseModel): def get_timeseries_from_db(data: GetTimeSeriesModel, ph=Depends(pandahub)): if data.timestamp_range is not None: data.timestamp_range = [pd.Timestamp(t) for t in data.timestamp_range] - ts = ph.get_timeseries_from_db(**data.dict()) + ts = ph.get_timeseries_from_db(**data.model_dump()) return ts.to_json(date_format="iso") @@ -47,7 +47,7 @@ class MultiGetTimeSeriesModel(BaseModel): def multi_get_timeseries_from_db(data: MultiGetTimeSeriesModel, ph=Depends(pandahub)): if data.timestamp_range is not None: data.timestamp_range = [pd.Timestamp(t) for t in data.timestamp_range] - ts = ph.multi_get_timeseries_from_db(**data.dict(), include_metadata=True) + ts = ph.multi_get_timeseries_from_db(**data.model_dump(), include_metadata=True) for i, data in enumerate(ts): ts[i]["timeseries_data"] = data["timeseries_data"].to_json(date_format="iso") return ts @@ -85,5 +85,5 @@ class WriteTimeSeriesModel(BaseModel): def write_timeseries_to_db(data: WriteTimeSeriesModel, ph=Depends(pandahub)): data.timeseries = pd.Series(json.loads(data.timeseries)) data.timeseries.index = pd.to_datetime(data.timeseries.index) - ph.write_timeseries_to_db(**data.dict()) + ph.write_timeseries_to_db(**data.model_dump()) return True diff --git a/pandahub/api/routers/users.py b/pandahub/api/routers/users.py index 1893cce..0a5bf53 100644 --- a/pandahub/api/routers/users.py +++ b/pandahub/api/routers/users.py @@ -1,6 +1,7 @@ from fastapi import APIRouter from pandahub.api.internal import settings +from pandahub.api.internal.schemas import UserRead, UserUpdate from pandahub.api.internal.users import fastapi_users router = APIRouter( @@ -9,6 +10,6 @@ ) router.include_router( - fastapi_users.get_users_router( + fastapi_users.get_users_router(UserRead, UserUpdate, requires_verification=settings.EMAIL_VERIFICATION_REQUIRED), ) diff --git a/pandahub/api/routers/variants.py b/pandahub/api/routers/variants.py index 91321f4..220fbb0 100644 --- a/pandahub/api/routers/variants.py +++ b/pandahub/api/routers/variants.py @@ -1,9 +1,7 @@ import json -import pandas as pd from fastapi import APIRouter, Depends from pydantic import BaseModel -from pydantic.typing import Optional from pandahub.api.dependencies import pandahub diff --git a/pandahub/lib/PandaHub.py b/pandahub/lib/PandaHub.py index 519c1fe..b89ff85 100644 --- a/pandahub/lib/PandaHub.py +++ b/pandahub/lib/PandaHub.py @@ -15,7 +15,7 @@ from bson.objectid import ObjectId from functools import reduce from operator import getitem -from pydantic.types import UUID4 +from uuid import UUID from pymongo import MongoClient, ReplaceOne from pymongo.errors import ServerSelectionTimeoutError @@ -161,14 +161,14 @@ def get_user_by_email(self, email): user = user_mgmnt_db["users"].find_one({"email": email}) if user is None: return None - if str(user["id"]) != self.user_id: + if str(user["_id"]) != self.user_id: self.check_permission("user_management") return user def _get_user(self): user_mgmnt_db = self.mongo_client["user_management"] user = user_mgmnt_db["users"].find_one( - {"id": UUID4(self.user_id)}, projection={"_id": 0, "hashed_password": 0} + {"_id": UUID(self.user_id)}, projection= {"hashed_password": 0} ) return user @@ -507,13 +507,13 @@ def get_project_users(self): self.check_permission("user_management") project_users = self.active_project["users"] users = self.mongo_client["user_management"]["users"].find( - {"id": {"$in": [UUID4(user_id) for user_id in project_users.keys()]}} + {"_id": {"$in": [UUID(user_id) for user_id in project_users.keys()]}} ) enriched_users = [] for user in users: enriched_users.append({ "email": user["email"], - "role": project_users[str(user["id"])] + "role": project_users[str(user["_id"])] }) return enriched_users @@ -522,7 +522,7 @@ def add_user_to_project(self, email, role): user = self.get_user_by_email(email) if user is None: return - user_id = user["id"] + user_id = user["_id"] self.mongo_client["user_management"]["projects"].update_one( {"_id": self.active_project["_id"]}, {"$set": {f"users.{user_id}": role}} @@ -534,7 +534,7 @@ def change_project_user_role(self, email, new_role): user = self.get_user_by_email(email) if user is None: return - user_id = user["id"] + user_id = user["_id"] self.mongo_client["user_management"]["projects"].update_one( {"_id": self.active_project["_id"]}, {"$set": {f"users.{user_id}": new_role}} @@ -546,9 +546,9 @@ def remove_user_from_project(self, email): return # check permission only if the user tries to remove a different user to # allow leaving a project with just 'read' permission - if str(user["id"]) != self.user_id: + if str(user["_id"]) != self.user_id: self.check_permission("user_management") - user_id = user["id"] + user_id = user["_id"] self.mongo_client["user_management"]["projects"].update_one( {"_id": self.active_project["_id"]}, {"$unset": {f"users.{user_id}": ""}} diff --git a/pandahub/lib/database_toolbox.py b/pandahub/lib/database_toolbox.py index 6cc9a86..1741318 100644 --- a/pandahub/lib/database_toolbox.py +++ b/pandahub/lib/database_toolbox.py @@ -1,4 +1,5 @@ # -*- coding: utf-8 -*- +from typing import Optional import numpy as np import pandas as pd @@ -379,3 +380,47 @@ def object_to_json(obj): "_class": obj.__class__.__name__, "_object": obj.to_json() } + +def migrate_userdb_to_beanie(ph): + """Migrate existing users to beanie backend used by pandahub >= 0.3.0. + + Will raise an exception if the user database is inconsistent, and return silently if no users need to be migrated. + See pandahub v0.3.0 release notes for details! + + Parameters + ---------- + ph: pandahub.PandaHub + PandaHub instance with connected mongodb database to apply migrations to. + Returns + ------- + None + """ + from datetime import datetime + from pymongo.errors import OperationFailure + userdb_backup = ph.mongo_client["user_management"][datetime.now().strftime("users_fa9_%Y-%m-%d_%H-%M")] + userdb = ph.mongo_client["user_management"]["users"] + old_users = list(userdb.find({"_id": {"$type": "objectId"}})) + new_users = list(userdb.find({"_id": {"$not": {"$type": "objectId"}}})) + if old_users and new_users: + old_users = [user.get("email") for user in old_users] + new_users = [user.get("email") for user in new_users] + raise RuntimeError("Inconsistent user database - you need to resolve conflicts manually! " + "See pandahub v0.3.0 release notes for details." + f"pandahub < 0.3.0 users: {old_users}" + f"pandahub >= 0.3.0 users: {new_users}" + ) + elif not old_users: + return + userdb_backup.insert_many(old_users) + try: + userdb.drop_index("id_1") + except OperationFailure as e: + if e.code == 27: + pass + else: + raise e + + migration = [{'$addFields': {'_id': '$id'}}, + {'$unset': 'id'}, + {'$out': 'users'}] + userdb.aggregate(migration) diff --git a/requirements.txt b/requirements.txt index eac539b..29e29f5 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,7 +1,7 @@ -uvicorn>=0.17.4 -fastapi~=0.75.0 -fastapi-users[mongodb]~=9.0 -fastapi-mail>=1.0.4 +uvicorn>=0.24.0 +fastapi-users[beanie]>=12.0 +fastapi>=0.104.0 +fastapi-mail>=1.4.1 pandapower>=2.10.1 pandapipes>=0.7.0 pymongo From a22558425061504a0461d35877dab9bd9ff40a0d Mon Sep 17 00:00:00 2001 From: Joschka Thurner Date: Mon, 27 Nov 2023 09:13:14 +0100 Subject: [PATCH 33/84] bump pandahub version to 0.3.0 --- pandahub/__init__.py | 2 +- setup.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/pandahub/__init__.py b/pandahub/__init__.py index 16cb9c0..c5b5cbb 100644 --- a/pandahub/__init__.py +++ b/pandahub/__init__.py @@ -1,4 +1,4 @@ -__version__ = "0.2.10" +__version__ = "0.3.0" from pandahub.lib.PandaHub import PandaHub, PandaHubError from pandahub.client.PandaHubClient import PandaHubClient diff --git a/setup.py b/setup.py index 0832d2d..9206d41 100644 --- a/setup.py +++ b/setup.py @@ -36,7 +36,7 @@ name='pandahub', packages=find_packages(), url='https://github.com/e2nIEE/pandahub', - version='0.2.10', + version='0.3.0', include_package_data=True, long_description_content_type='text/markdown', zip_safe=False, From 8db0b10f6cb71940ee4f116b91ef925c8300f580 Mon Sep 17 00:00:00 2001 From: mvogt Date: Mon, 27 Nov 2023 16:33:57 +0100 Subject: [PATCH 34/84] added some more error handling. --- pandahub/lib/PandaHub.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/pandahub/lib/PandaHub.py b/pandahub/lib/PandaHub.py index f119771..13b1159 100644 --- a/pandahub/lib/PandaHub.py +++ b/pandahub/lib/PandaHub.py @@ -234,8 +234,10 @@ def set_active_project_by_id(self, project_id): try: project_id = ObjectId(project_id) except InvalidId: - pass + raise PandaHubError('Project ID wrong.', 400) self.active_project = self._get_project_document({"_id": project_id}) + if self.active_project == None: + raise PandaHubError('Project not found!', 404) def rename_project(self, project_name): self.has_permission("write") From 1ac62faa1b126e3b4a3c125d69157b5923e7152d Mon Sep 17 00:00:00 2001 From: Joschka Thurner Date: Tue, 28 Nov 2023 08:18:42 +0100 Subject: [PATCH 35/84] allow other types than ObjectId as project IDs --- pandahub/lib/PandaHub.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pandahub/lib/PandaHub.py b/pandahub/lib/PandaHub.py index 064fafd..4ec11e8 100644 --- a/pandahub/lib/PandaHub.py +++ b/pandahub/lib/PandaHub.py @@ -248,9 +248,9 @@ def set_active_project_by_id(self, project_id): try: project_id = ObjectId(project_id) except InvalidId: - raise PandaHubError('Project ID wrong.', 400) + pass self.active_project = self._get_project_document({"_id": project_id}) - if self.active_project == None: + if self.active_project is None: raise PandaHubError('Project not found!', 404) def rename_project(self, project_name): From 84c3263d4251338a4c140704f930aec1ba0bca25 Mon Sep 17 00:00:00 2001 From: Joschka Thurner Date: Fri, 1 Dec 2023 12:07:43 +0100 Subject: [PATCH 36/84] create indexes with project by default --- pandahub/api/internal/settings.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pandahub/api/internal/settings.py b/pandahub/api/internal/settings.py index d79fde7..0fdb121 100644 --- a/pandahub/api/internal/settings.py +++ b/pandahub/api/internal/settings.py @@ -48,4 +48,4 @@ def get_secret(key, default=None): REGISTRATION_ADMIN_APPROVAL = settings_bool("REGISTRATION_ADMIN_APPROVAL", default=False) DATATYPES_MODULE = os.getenv("DATATYPES_MODULE") or "pandahub.lib.datatypes" -CREATE_INDEXES_WITH_PROJECT=settings_bool("CREATE_INDEXES_WITH_PROJECT", default=False) +CREATE_INDEXES_WITH_PROJECT = settings_bool("CREATE_INDEXES_WITH_PROJECT", default=True) From 9400085cf359f99a8cb8a65cb289ddb32fe8b36b Mon Sep 17 00:00:00 2001 From: Joschka Thurner Date: Thu, 14 Dec 2023 10:27:12 +0100 Subject: [PATCH 37/84] fix: filter asymmetric loads/sgens when getting subnets --- pandahub/api/internal/db.py | 2 -- pandahub/lib/PandaHub.py | 4 ++-- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/pandahub/api/internal/db.py b/pandahub/api/internal/db.py index 83be7c8..b87fc48 100644 --- a/pandahub/api/internal/db.py +++ b/pandahub/api/internal/db.py @@ -21,8 +21,6 @@ db = client["user_management"] class User(BeanieBaseUser, Document): - # pass - # id: Optional[UUID4] = Field(alias="id") id: uuid.UUID = Field(default_factory=uuid.uuid4) class Settings(BeanieBaseUser.Settings): name = "users" diff --git a/pandahub/lib/PandaHub.py b/pandahub/lib/PandaHub.py index 4ec11e8..cd72966 100644 --- a/pandahub/lib/PandaHub.py +++ b/pandahub/lib/PandaHub.py @@ -727,7 +727,8 @@ def get_subnet_from_db_by_id( geo_mode=geo_mode, variants=variants, dtypes=dtypes) # add node elements - node_elements = ["load", "sgen", "gen", "ext_grid", "shunt", "xward", "ward", "motor", "storage"] + node_elements = ["load", "asymmetric_load", "sgen", "asymmetric_sgen", "gen", "ext_grid", "shunt", "xward", + "ward", "motor", "storage"] branch_elements = ["trafo", "line", "trafo3w", "switch", "impedance"] all_elements = node_elements + branch_elements + ["bus"] + list(additional_filters.keys()) all_elements = list(set(all_elements) - set(ignore_elements)) @@ -1057,7 +1058,6 @@ def delete_elements(self, net: Union[int, str], element_type: str, element_index db[collection].delete_many({"_id": {"$in": delete_ids}}) return deletion_targets - def set_net_value_in_db(self, net, element_type, element_index, parameter, value, variant=None, project_id=None, **kwargs): logger.info(f"Setting {parameter} = {value} in {element_type} with index {element_index} and variant {variant}") From 9e6a654d8b78b892b02c17b3cba02e57ce89e64a Mon Sep 17 00:00:00 2001 From: Joschka Thurner Date: Thu, 14 Dec 2023 10:35:12 +0100 Subject: [PATCH 38/84] bump pandahub version to 0.3.1 --- pandahub/__init__.py | 2 +- setup.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/pandahub/__init__.py b/pandahub/__init__.py index c5b5cbb..bd2fba2 100644 --- a/pandahub/__init__.py +++ b/pandahub/__init__.py @@ -1,4 +1,4 @@ -__version__ = "0.3.0" +__version__ = "0.3.1" from pandahub.lib.PandaHub import PandaHub, PandaHubError from pandahub.client.PandaHubClient import PandaHubClient diff --git a/setup.py b/setup.py index 9206d41..7e146ca 100644 --- a/setup.py +++ b/setup.py @@ -36,7 +36,7 @@ name='pandahub', packages=find_packages(), url='https://github.com/e2nIEE/pandahub', - version='0.3.0', + version='0.3.1', include_package_data=True, long_description_content_type='text/markdown', zip_safe=False, From 61fc61e3172d7da770990ab80461a4be188bb8a9 Mon Sep 17 00:00:00 2001 From: Leon Thurner Date: Mon, 1 Jan 2024 16:30:59 +0100 Subject: [PATCH 39/84] add support for timeseries collections --- pandahub/api/routers/timeseries.py | 11 +- pandahub/lib/PandaHub.py | 1587 ++++++++++++++++++++-------- requirements.txt | 1 + 3 files changed, 1182 insertions(+), 417 deletions(-) diff --git a/pandahub/api/routers/timeseries.py b/pandahub/api/routers/timeseries.py index 8d453fd..b1238f1 100644 --- a/pandahub/api/routers/timeseries.py +++ b/pandahub/api/routers/timeseries.py @@ -7,16 +7,14 @@ from pandahub.api.dependencies import pandahub -router = APIRouter( - prefix="/timeseries", - tags=["timeseries"] -) +router = APIRouter(prefix="/timeseries", tags=["timeseries"]) # ------------------------------- # ROUTES # ------------------------------- + class GetTimeSeriesModel(BaseModel): filter_document: Optional[dict] = {} global_database: Optional[bool] = False @@ -57,13 +55,16 @@ class GetTimeseriesMetadataModel(BaseModel): project_id: str filter_document: Optional[dict] = {} global_database: Optional[bool] = False + collection_name: Optional[str] = "timeseries" + @router.post("/get_timeseries_metadata") def get_timeseries_metadata(data: GetTimeseriesMetadataModel, ph=Depends(pandahub)): ph.set_active_project_by_id(data.project_id) ts = ph.get_timeseries_metadata( filter_document=data.filter_document, - global_database=data.global_database + global_database=data.global_database, + collection_name=data.collection_name, ) ts = json.loads(ts.to_json(orient="index")) return ts diff --git a/pandahub/lib/PandaHub.py b/pandahub/lib/PandaHub.py index cd72966..c72a0ad 100644 --- a/pandahub/lib/PandaHub.py +++ b/pandahub/lib/PandaHub.py @@ -41,11 +41,15 @@ from packaging import version +import pymongoarrow.monkey + +pymongoarrow.monkey.patch_all() # ------------------------- # Exceptions # ------------------------- + class PandaHubError(Exception): def __init__(self, message, status_code=400): self.status_code = status_code @@ -56,25 +60,40 @@ def __init__(self, message, status_code=400): # PandaHub # ------------------------- + class PandaHub: permissions = { "read": ["owner", "developer", "guest"], "write": ["owner", "developer"], - "user_management": ["owner"] + "user_management": ["owner"], } - _datatypes = getattr(importlib.import_module(SETTINGS.DATATYPES_MODULE), "datatypes") + _datatypes = getattr( + importlib.import_module(SETTINGS.DATATYPES_MODULE), "datatypes" + ) # ------------------------- # Initialization # ------------------------- - def __init__(self, connection_url=SETTINGS.MONGODB_URL, connection_user = SETTINGS.MONGODB_USER, - connection_password=SETTINGS.MONGODB_PASSWORD, check_server_available=False, user_id=None): - - mongo_client_args = {"host": connection_url, "uuidRepresentation": "standard", "connect":False} + def __init__( + self, + connection_url=SETTINGS.MONGODB_URL, + connection_user=SETTINGS.MONGODB_USER, + connection_password=SETTINGS.MONGODB_PASSWORD, + check_server_available=False, + user_id=None, + ): + mongo_client_args = { + "host": connection_url, + "uuidRepresentation": "standard", + "connect": False, + } if connection_user: - mongo_client_args |= {"username": connection_user, "password": connection_password} + mongo_client_args |= { + "username": connection_user, + "password": connection_password, + } self.mongo_client = MongoClient(**mongo_client_args) self.mongo_client_global_db = None self.active_project = None @@ -100,10 +119,14 @@ def server_is_available(self): """ try: self.mongo_client.server_info() - logger.debug("connected to mongoDB server %s" % self.get_masked_mongodb_url()) + logger.debug( + "connected to mongoDB server %s" % self.get_masked_mongodb_url() + ) return True except ServerSelectionTimeoutError: - logger.error("could not connect to mongoDB server %s" % self.get_masked_mongodb_url()) + logger.error( + "could not connect to mongoDB server %s" % self.get_masked_mongodb_url() + ) return False def check_connection_status(self): @@ -111,10 +134,12 @@ def check_connection_status(self): Checks if the database is accessible """ try: - status = self.mongo_client.find({}, collection_name="__connection_test_collection") + status = self.mongo_client.find( + {}, collection_name="__connection_test_collection" + ) if status == []: return "ok" - except (ServerSelectionTimeoutError) as e: + except ServerSelectionTimeoutError as e: return "connection timeout" # ------------------------- @@ -125,7 +150,9 @@ def check_permission(self, permission): if self.active_project is None: raise PandaHubError("No project is activated") if not self.has_permission(permission): - raise PandaHubError("You don't have {} rights on this project".format(permission), 403) + raise PandaHubError( + "You don't have {} rights on this project".format(permission), 403 + ) def has_permission(self, permission): if not "users" in self.active_project: @@ -168,7 +195,7 @@ def get_user_by_email(self, email): def _get_user(self): user_mgmnt_db = self.mongo_client["user_management"] user = user_mgmnt_db["users"].find_one( - {"_id": UUID(self.user_id)}, projection= {"hashed_password": 0} + {"_id": UUID(self.user_id)}, projection={"hashed_password": 0} ) return user @@ -176,19 +203,28 @@ def _get_user(self): # Project handling # ------------------------- - def create_project(self, name, settings=None, realm=None, metadata=None, project_id=None, - activate=True): + def create_project( + self, + name, + settings=None, + realm=None, + metadata=None, + project_id=None, + activate=True, + ): if self.project_exists(name, realm): raise PandaHubError("Project already exists") if settings is None: settings = {} if metadata is None: metadata = {} - project_data = {"name": name, - "realm": realm, - "settings": settings, - "metadata": metadata, - "version": __version__} + project_data = { + "name": name, + "realm": realm, + "settings": settings, + "metadata": metadata, + "version": __version__, + } if project_id: project_data["_id"] = project_id if self.user_id is not None: @@ -207,7 +243,8 @@ def delete_project(self, i_know_this_action_is_final=False, project_id=None): self.check_permission("write") if not i_know_this_action_is_final: raise PandaHubError( - "Calling this function will delete the whole project and all the nets stored within. It can not be reversed. Add 'i_know_this_action_is_final=True' to confirm.") + "Calling this function will delete the whole project and all the nets stored within. It can not be reversed. Add 'i_know_this_action_is_final=True' to confirm." + ) self.mongo_client.drop_database(str(project_id)) self.mongo_client.user_management.projects.delete_one({"_id": project_id}) self.active_project = None @@ -223,19 +260,28 @@ def get_projects(self): filter_dict = {"users": {"$exists": False}} db = self.mongo_client["user_management"] projects = list(db["projects"].find(filter_dict)) - return [{ - "id": str(p["_id"]), - "name": p["name"], - "realm": str(p["realm"]), - "settings": p["settings"], - "locked": p.get("locked"), - "locked_by": p.get("locked_by"), - "permissions": self.get_permissions_by_role(p.get("users").get(self.user_id)) if self.user_id else None - } for p in projects] + return [ + { + "id": str(p["_id"]), + "name": p["name"], + "realm": str(p["realm"]), + "settings": p["settings"], + "locked": p.get("locked"), + "locked_by": p.get("locked_by"), + "permissions": self.get_permissions_by_role( + p.get("users").get(self.user_id) + ) + if self.user_id + else None, + } + for p in projects + ] def set_active_project(self, project_name, realm=None): projects = self.get_projects() - active_projects = [project for project in projects if project["name"] == project_name] + active_projects = [ + project for project in projects if project["name"] == project_name + ] if len(active_projects) == 0: raise PandaHubError("Project not found!", 404) elif len(active_projects) > 1: @@ -251,7 +297,7 @@ def set_active_project_by_id(self, project_id): pass self.active_project = self._get_project_document({"_id": project_id}) if self.active_project is None: - raise PandaHubError('Project not found!', 404) + raise PandaHubError("Project not found!", 404) def rename_project(self, project_name): self.has_permission("write") @@ -259,8 +305,9 @@ def rename_project(self, project_name): realm = self.active_project["realm"] if self.project_exists(project_name, realm): raise PandaHubError("Can't rename - project with this name already exists") - project_collection.update_one({"_id": self.active_project["_id"]}, - {"$set": {"name": project_name}}) + project_collection.update_one( + {"_id": self.active_project["_id"]}, {"$set": {"name": project_name}} + ) self.set_active_project(project_name, realm) def change_realm(self, realm): @@ -268,16 +315,21 @@ def change_realm(self, realm): project_collection = self.mongo_client["user_management"].projects project_name = self.active_project["name"] if self.project_exists(project_name, realm): - raise PandaHubError("Can't change realm - project with this name already exists") - project_collection.update_one({"_id": self.active_project["_id"]}, - {"$set": {"realm": realm}}) + raise PandaHubError( + "Can't change realm - project with this name already exists" + ) + project_collection.update_one( + {"_id": self.active_project["_id"]}, {"$set": {"realm": realm}} + ) self.set_active_project(project_name, realm) def lock_project(self): db = self.mongo_client["user_management"]["projects"] result = db.update_one( - {"_id": self.active_project["_id"], }, - {"$set": {"locked": True, "locked_by": self.user_id}} + { + "_id": self.active_project["_id"], + }, + {"$set": {"locked": True, "locked_by": self.user_id}}, ) return result.acknowledged and result.modified_count > 0 @@ -285,7 +337,7 @@ def unlock_project(self): db = self.mongo_client["user_management"]["projects"] return db.update_one( {"_id": self.active_project["_id"], "locked_by": self.user_id}, - {"$set": {"locked": False, "locked_by": None}} + {"$set": {"locked": False, "locked_by": None}}, ) def force_unlock_project(self, project_id): @@ -294,8 +346,15 @@ def force_unlock_project(self, project_id): user = self._get_user() if project is None: return None - if "users" not in project or self.user_id in project["users"].keys() or user["is_superuser"]: - return db.update_one({"_id": ObjectId(project_id)}, {"$set": {"locked": False, "locked_by": None}}) + if ( + "users" not in project + or self.user_id in project["users"].keys() + or user["is_superuser"] + ): + return db.update_one( + {"_id": ObjectId(project_id)}, + {"$set": {"locked": False, "locked_by": None}}, + ) else: raise PandaHubError("You don't have rights to access this project", 403) @@ -311,7 +370,8 @@ def _get_project_document(self, filter_dict: dict) -> Optional[dict]: return None if len(projects) > 1: raise PandaHubError( - "Duplicate Project detected. This should never happen if you create projects through the API. Remove duplicate projects manually in the database.") + "Duplicate Project detected. This should never happen if you create projects through the API. Remove duplicate projects manually in the database." + ) project_doc = projects[0] if "users" not in project_doc: return project_doc # project is not user protected @@ -328,11 +388,19 @@ def _get_project_database(self): return self.mongo_client[str(self.active_project["_id"])] def _get_global_database(self): - if self.mongo_client_global_db is None and SETTINGS.MONGODB_GLOBAL_DATABASE_URL is not None: - mongo_client_args = {"host": SETTINGS.MONGODB_GLOBAL_DATABASE_URL, "uuidRepresentation": "standard"} + if ( + self.mongo_client_global_db is None + and SETTINGS.MONGODB_GLOBAL_DATABASE_URL is not None + ): + mongo_client_args = { + "host": SETTINGS.MONGODB_GLOBAL_DATABASE_URL, + "uuidRepresentation": "standard", + } if SETTINGS.MONGODB_GLOBAL_DATABASE_USER: - mongo_client_args |= {"username": SETTINGS.MONGODB_GLOBAL_DATABASE_USER, - "password": SETTINGS.MONGODB_GLOBAL_DATABASE_PASSWORD} + mongo_client_args |= { + "username": SETTINGS.MONGODB_GLOBAL_DATABASE_USER, + "password": SETTINGS.MONGODB_GLOBAL_DATABASE_PASSWORD, + } self.mongo_client_global_db = MongoClient(**mongo_client_args) if self.mongo_client_global_db is None: return self.mongo_client["global_data"] @@ -350,9 +418,13 @@ def upgrade_project_to_latest_version(self): if version.parse(self.get_project_version()) < version.parse("0.2.3"): db = self._get_project_database() all_collection_names = db.list_collection_names() - old_net_collections = [name for name in all_collection_names if - not name.startswith("_") and - not name == "timeseries" and not name.startswith("net_")] + old_net_collections = [ + name + for name in all_collection_names + if not name.startswith("_") + and not name == "timeseries" + and not name.startswith("net_") + ] for element in old_net_collections: db[element].rename(self._collection_name_of_element(element)) @@ -360,12 +432,17 @@ def upgrade_project_to_latest_version(self): if version.parse(self.get_project_version()) < version.parse("0.2.3"): db = self._get_project_database() # for all networks - for d in list(db["_networks"].find({}, projection={"sector":1, "data":1})): + for d in list( + db["_networks"].find({}, projection={"sector": 1, "data": 1}) + ): # load old format if d.get("sector", "power") == "power": - data = dict((k, json.loads(v, cls=io_pp.PPJSONDecoder)) for k, v in d['data'].items()) + data = dict( + (k, json.loads(v, cls=io_pp.PPJSONDecoder)) + for k, v in d["data"].items() + ) else: - data = dict((k, from_json_pps(v)) for k, v in d['data'].items()) + data = dict((k, from_json_pps(v)) for k, v in d["data"].items()) # save new format for key, dat in data.items(): try: @@ -374,14 +451,16 @@ def upgrade_project_to_latest_version(self): dat = f"serialized_{json.dumps(data, cls=io_pp.PPJSONEncoder)}" data[key] = dat - db["_networks"].update_one({"_id":d["_id"]}, - {"$set": {"data": data}}) + db["_networks"].update_one({"_id": d["_id"]}, {"$set": {"data": data}}) project_collection = self.mongo_client["user_management"].projects - project_collection.update_one({"_id": self.active_project["_id"]}, - {"$set": {"version": __version__}}) - logger.info(f"upgraded projekt '{self.active_project['name']}' from version" - f" {self.get_project_version()} to version {__version__}") + project_collection.update_one( + {"_id": self.active_project["_id"]}, {"$set": {"version": __version__}} + ) + logger.info( + f"upgraded projekt '{self.active_project['name']}' from version" + f" {self.get_project_version()} to version {__version__}" + ) self.active_project["version"] = __version__ # ------------------------- @@ -394,7 +473,6 @@ def get_project_settings(self, project_id=None): self.check_permission("read") return self.active_project["settings"] - def get_project_setting_value(self, setting, project_id=None): """ Retrieve the value of a setting. @@ -416,13 +494,14 @@ def get_project_setting_value(self, setting, project_id=None): _id = self.active_project["_id"] project_collection = self.mongo_client["user_management"]["projects"] setting_string = f"settings.{setting}" - setting = project_collection.find_one({"_id": _id}, {"_id": 0, setting_string: 1}) + setting = project_collection.find_one( + {"_id": _id}, {"_id": 0, setting_string: 1} + ) try: return reduce(getitem, setting_string.split("."), setting) except KeyError: return None - def set_project_settings(self, settings, project_id=None): if project_id: self.set_active_project_by_id(project_id) @@ -430,7 +509,9 @@ def set_project_settings(self, settings, project_id=None): _id = self.active_project["_id"] new_settings = {**self.active_project["settings"], **settings} project_collection = self.mongo_client["user_management"]["projects"] - project_collection.update_one({"_id": _id}, {"$set": {"settings": new_settings}}) + project_collection.update_one( + {"_id": _id}, {"$set": {"settings": new_settings}} + ) self.active_project["settings"] = new_settings def set_project_settings_value(self, parameter, value, project_id=None): @@ -493,11 +574,13 @@ def replace_empty(updated, data): replace_empty(update_metadata, new_metadata) self.mongo_client.user_management.projects.update_one( - {"_id": project_data['_id']}, + {"_id": project_data["_id"]}, [ - {"$unset": "metadata"}, # deletion needed because set won't delete not existing fields - {"$set": {"metadata": update_metadata}} - ] + { + "$unset": "metadata" + }, # deletion needed because set won't delete not existing fields + {"$set": {"metadata": update_metadata}}, + ], ) self.active_project["metadata"] = update_metadata @@ -513,10 +596,9 @@ def get_project_users(self): ) enriched_users = [] for user in users: - enriched_users.append({ - "email": user["email"], - "role": project_users[str(user["_id"])] - }) + enriched_users.append( + {"email": user["email"], "role": project_users[str(user["_id"])]} + ) return enriched_users def add_user_to_project(self, email, role): @@ -526,8 +608,7 @@ def add_user_to_project(self, email, role): return user_id = user["_id"] self.mongo_client["user_management"]["projects"].update_one( - {"_id": self.active_project["_id"]}, - {"$set": {f"users.{user_id}": role}} + {"_id": self.active_project["_id"]}, {"$set": {f"users.{user_id}": role}} ) return user @@ -539,7 +620,7 @@ def change_project_user_role(self, email, new_role): user_id = user["_id"] self.mongo_client["user_management"]["projects"].update_one( {"_id": self.active_project["_id"]}, - {"$set": {f"users.{user_id}": new_role}} + {"$set": {f"users.{user_id}": new_role}}, ) def remove_user_from_project(self, email): @@ -552,8 +633,7 @@ def remove_user_from_project(self, email): self.check_permission("user_management") user_id = user["_id"] self.mongo_client["user_management"]["projects"].update_one( - {"_id": self.active_project["_id"]}, - {"$unset": {f"users.{user_id}": ""}} + {"_id": self.active_project["_id"]}, {"$unset": {f"users.{user_id}": ""}} ) # ------------------------- @@ -563,13 +643,19 @@ def remove_user_from_project(self, email): def get_all_nets_metadata_from_db(self, project_id=None): if project_id: self.set_active_project_by_id(project_id) - self.check_permission('read') + self.check_permission("read") db = self._get_project_database() - return list(db['_networks'].find()) + return list(db["_networks"].find()) - - def get_net_from_db(self, name, include_results=True, only_tables=None, project_id=None, - geo_mode="string", variants=[]): + def get_net_from_db( + self, + name, + include_results=True, + only_tables=None, + project_id=None, + geo_mode="string", + variants=[], + ): if project_id: self.set_active_project_by_id(project_id) self.check_permission("read") @@ -577,16 +663,38 @@ def get_net_from_db(self, name, include_results=True, only_tables=None, project_ _id = self._get_id_from_name(name, db) if _id is None: return None - return self.get_net_from_db_by_id(_id, include_results, only_tables, geo_mode=geo_mode, variants=variants) + return self.get_net_from_db_by_id( + _id, include_results, only_tables, geo_mode=geo_mode, variants=variants + ) - def get_net_from_db_by_id(self, id, include_results=True, only_tables=None, convert=True, - geo_mode="string", variants=[]): + def get_net_from_db_by_id( + self, + id, + include_results=True, + only_tables=None, + convert=True, + geo_mode="string", + variants=[], + ): self.check_permission("read") - return self._get_net_from_db_by_id(id, include_results, only_tables, convert=convert, - geo_mode=geo_mode, variants=variants) + return self._get_net_from_db_by_id( + id, + include_results, + only_tables, + convert=convert, + geo_mode=geo_mode, + variants=variants, + ) - def _get_net_from_db_by_id(self, id_, include_results=True, only_tables=None, convert=True, - geo_mode="string", variants=[]): + def _get_net_from_db_by_id( + self, + id_, + include_results=True, + only_tables=None, + convert=True, + geo_mode="string", + variants=[], + ): db = self._get_project_database() meta = self._get_network_metadata(db, id_) @@ -597,9 +705,16 @@ def _get_net_from_db_by_id(self, id_, include_results=True, only_tables=None, co collection_names = self._get_net_collections(db) for collection_name in collection_names: el = self._element_name_of_collection(collection_name) - self._add_element_from_collection(net, db, el, id_, include_results=include_results, - only_tables=only_tables, geo_mode=geo_mode, - variants=variants) + self._add_element_from_collection( + net, + db, + el, + id_, + include_results=include_results, + only_tables=only_tables, + geo_mode=geo_mode, + variants=variants, + ) # add data that is not stored in dataframes self.deserialize_and_update_data(net, meta) @@ -611,10 +726,13 @@ def _get_net_from_db_by_id(self, id_, include_results=True, only_tables=None, co def deserialize_and_update_data(self, net, meta): if version.parse(self.get_project_version()) <= version.parse("0.2.3"): if meta.get("sector", "power") == "power": - data = dict((k, json.loads(v, cls=io_pp.PPJSONDecoder)) for k, v in meta['data'].items()) + data = dict( + (k, json.loads(v, cls=io_pp.PPJSONDecoder)) + for k, v in meta["data"].items() + ) net.update(data) else: - data = dict((k, from_json_pps(v)) for k, v in meta['data'].items()) + data = dict((k, from_json_pps(v)) for k, v in meta["data"].items()) net.update(data) else: for key, value in meta["data"].items(): @@ -622,22 +740,32 @@ def deserialize_and_update_data(self, net, meta): value = json.loads(value[11:], cls=io_pp.PPJSONDecoder) net[key] = value - def get_subnet_from_db(self, - name, - bus_filter=None, - include_results=True, - add_edge_branches=True, - geo_mode="string", - variants=[], - additional_filters: dict[str, Callable[[pp.auxiliary.pandapowerNet], dict]] = {}): + def get_subnet_from_db( + self, + name, + bus_filter=None, + include_results=True, + add_edge_branches=True, + geo_mode="string", + variants=[], + additional_filters: dict[ + str, Callable[[pp.auxiliary.pandapowerNet], dict] + ] = {}, + ): self.check_permission("read") db = self._get_project_database() _id = self._get_id_from_name(name, db) if _id is None: return None - return self.get_subnet_from_db_by_id(_id, bus_filter=bus_filter, include_results=include_results, - add_edge_branches=add_edge_branches, geo_mode=geo_mode, variants=variants, - additional_filters=additional_filters) + return self.get_subnet_from_db_by_id( + _id, + bus_filter=bus_filter, + include_results=include_results, + add_edge_branches=add_edge_branches, + geo_mode=geo_mode, + variants=variants, + additional_filters=additional_filters, + ) def get_subnet_from_db_by_id( self, @@ -648,7 +776,9 @@ def get_subnet_from_db_by_id( geo_mode="string", variants=[], ignore_elements=[], - additional_filters: dict[str, Callable[[pp.auxiliary.pandapowerNet], dict]] = {} + additional_filters: dict[ + str, Callable[[pp.auxiliary.pandapowerNet], dict] + ] = {}, ) -> pp.pandapowerNet: db = self._get_project_database() meta = self._get_network_metadata(db, net_id) @@ -661,76 +791,149 @@ def get_subnet_from_db_by_id( # Add buses with filter if bus_filter is not None: - self._add_element_from_collection(net, db, "bus", net_id, bus_filter, geo_mode=geo_mode, - variants=variants, dtypes=dtypes) + self._add_element_from_collection( + net, + db, + "bus", + net_id, + bus_filter, + geo_mode=geo_mode, + variants=variants, + dtypes=dtypes, + ) buses = net.bus.index.tolist() branch_operator = "$or" if add_edge_branches else "$and" # Add branch elements connected to at least one bus - self._add_element_from_collection(net, db, "line", net_id, - {branch_operator: [ - {"from_bus": {"$in": buses}}, - {"to_bus": {"$in": buses}}]}, geo_mode=geo_mode, - variants=variants, dtypes=dtypes) - self._add_element_from_collection(net, db, "trafo", net_id, - {branch_operator: [ - {"hv_bus": {"$in": buses}}, - {"lv_bus": {"$in": buses}}]}, geo_mode=geo_mode, - variants=variants, dtypes=dtypes) - self._add_element_from_collection(net, db, "trafo3w", net_id, - {branch_operator: [ - {"hv_bus": {"$in": buses}}, - {"mv_bus": {"$in": buses}}, - {"lv_bus": {"$in": buses}}]}, geo_mode=geo_mode, - variants=variants, dtypes=dtypes) - - self._add_element_from_collection(net, db, "switch", net_id, - {"$and": [ - {"et": "b"}, - {branch_operator: [ - {"bus": {"$in": buses}}, - {"element": {"$in": buses}} - ]} - ] - }, geo_mode=geo_mode, variants=variants, dtypes=dtypes) + self._add_element_from_collection( + net, + db, + "line", + net_id, + { + branch_operator: [ + {"from_bus": {"$in": buses}}, + {"to_bus": {"$in": buses}}, + ] + }, + geo_mode=geo_mode, + variants=variants, + dtypes=dtypes, + ) + self._add_element_from_collection( + net, + db, + "trafo", + net_id, + {branch_operator: [{"hv_bus": {"$in": buses}}, {"lv_bus": {"$in": buses}}]}, + geo_mode=geo_mode, + variants=variants, + dtypes=dtypes, + ) + self._add_element_from_collection( + net, + db, + "trafo3w", + net_id, + { + branch_operator: [ + {"hv_bus": {"$in": buses}}, + {"mv_bus": {"$in": buses}}, + {"lv_bus": {"$in": buses}}, + ] + }, + geo_mode=geo_mode, + variants=variants, + dtypes=dtypes, + ) + + self._add_element_from_collection( + net, + db, + "switch", + net_id, + { + "$and": [ + {"et": "b"}, + { + branch_operator: [ + {"bus": {"$in": buses}}, + {"element": {"$in": buses}}, + ] + }, + ] + }, + geo_mode=geo_mode, + variants=variants, + dtypes=dtypes, + ) if add_edge_branches: # Add buses on the other side of the branches - branch_buses = set(net.trafo.hv_bus.values) | set(net.trafo.lv_bus.values) | \ - set(net.line.from_bus) | set(net.line.to_bus) | \ - set(net.trafo3w.hv_bus.values) | set(net.trafo3w.mv_bus.values) | \ - set(net.trafo3w.lv_bus.values) | set(net.switch.bus) | set(net.switch.element) + branch_buses = ( + set(net.trafo.hv_bus.values) + | set(net.trafo.lv_bus.values) + | set(net.line.from_bus) + | set(net.line.to_bus) + | set(net.trafo3w.hv_bus.values) + | set(net.trafo3w.mv_bus.values) + | set(net.trafo3w.lv_bus.values) + | set(net.switch.bus) + | set(net.switch.element) + ) branch_buses_outside = [int(b) for b in branch_buses - set(buses)] - self._add_element_from_collection(net, db, "bus", net_id, geo_mode=geo_mode, variants=variants, - filter={"index": {"$in": branch_buses_outside}}, - dtypes=dtypes) + self._add_element_from_collection( + net, + db, + "bus", + net_id, + geo_mode=geo_mode, + variants=variants, + filter={"index": {"$in": branch_buses_outside}}, + dtypes=dtypes, + ) buses = net.bus.index.tolist() - switch_filter = {"$or": [ - {"$and": [ - {"et": "t"}, - {"element": {"$in": net.trafo.index.tolist()}} - ] - }, - {"$and": [ - {"et": "l"}, - {"element": {"$in": net.line.index.tolist()}} - ] - }, - {"$and": [ - {"et": "t3"}, - {"element": {"$in": net.trafo3w.index.tolist()}} + switch_filter = { + "$or": [ + {"$and": [{"et": "t"}, {"element": {"$in": net.trafo.index.tolist()}}]}, + {"$and": [{"et": "l"}, {"element": {"$in": net.line.index.tolist()}}]}, + { + "$and": [ + {"et": "t3"}, + {"element": {"$in": net.trafo3w.index.tolist()}}, + ] + }, ] - } - ] } - self._add_element_from_collection(net, db, "switch", net_id, switch_filter, - geo_mode=geo_mode, variants=variants, dtypes=dtypes) + self._add_element_from_collection( + net, + db, + "switch", + net_id, + switch_filter, + geo_mode=geo_mode, + variants=variants, + dtypes=dtypes, + ) # add node elements - node_elements = ["load", "asymmetric_load", "sgen", "asymmetric_sgen", "gen", "ext_grid", "shunt", "xward", - "ward", "motor", "storage"] + node_elements = [ + "load", + "asymmetric_load", + "sgen", + "asymmetric_sgen", + "gen", + "ext_grid", + "shunt", + "xward", + "ward", + "motor", + "storage", + ] branch_elements = ["trafo", "line", "trafo3w", "switch", "impedance"] - all_elements = node_elements + branch_elements + ["bus"] + list(additional_filters.keys()) + all_elements = ( + node_elements + branch_elements + ["bus"] + list(additional_filters.keys()) + ) all_elements = list(set(all_elements) - set(ignore_elements)) # Add elements for which the user has provided a filter function @@ -738,18 +941,32 @@ def get_subnet_from_db_by_id( if element in ignore_elements: continue element_filter = filter_func(net) - self._add_element_from_collection(net, db, element, net_id, - filter=element_filter, geo_mode=geo_mode, - include_results=include_results, - variants=variants, dtypes=dtypes) + self._add_element_from_collection( + net, + db, + element, + net_id, + filter=element_filter, + geo_mode=geo_mode, + include_results=include_results, + variants=variants, + dtypes=dtypes, + ) # add all node elements that are connected to buses within the network for element in node_elements: element_filter = {"bus": {"$in": buses}} - self._add_element_from_collection(net, db, element, net_id, - filter=element_filter, geo_mode=geo_mode, - include_results=include_results, - variants=variants, dtypes=dtypes) + self._add_element_from_collection( + net, + db, + element, + net_id, + filter=element_filter, + geo_mode=geo_mode, + include_results=include_results, + variants=variants, + dtypes=dtypes, + ) # add all other collections collection_names = self._get_net_collections(db) @@ -760,16 +977,25 @@ def get_subnet_from_db_by_id( continue # for tables that share an index with an element (e.g. load->res_load) load only relevant entries for element in all_elements: - if table_name.startswith(element + "_") or table_name.startswith("net_res_" + element): + if table_name.startswith(element + "_") or table_name.startswith( + "net_res_" + element + ): element_filter = {"index": {"$in": net[element].index.tolist()}} break else: # all other tables (e.g. std_types) are loaded without filter element_filter = None - self._add_element_from_collection(net, db, table_name, net_id, - filter=element_filter, geo_mode=geo_mode, - include_results=include_results, - variants=variants, dtypes=dtypes) + self._add_element_from_collection( + net, + db, + table_name, + net_id, + filter=element_filter, + geo_mode=geo_mode, + include_results=include_results, + variants=variants, + dtypes=dtypes, + ) self.deserialize_and_update_data(net, meta) return net @@ -779,15 +1005,23 @@ def _collection_name_of_element(self, element): def _element_name_of_collection(self, collection): return collection[4:] # remove "net_" prefix - def write_network_to_db(self, net, name, sector="power", overwrite=True, project_id=None, - metadata=None, skip_results=False, ): + def write_network_to_db( + self, + net, + name, + sector="power", + overwrite=True, + project_id=None, + metadata=None, + skip_results=False, + ): if project_id: self.set_active_project_by_id(project_id) self.check_permission("write") db = self._get_project_database() -# if not isinstance(net, pp.pandapowerNet) and not isinstance(net, pps.pandapipesNet): -# raise PandaHubError("net must be a pandapower or pandapipes object") + # if not isinstance(net, pp.pandapowerNet) and not isinstance(net, pps.pandapipesNet): + # raise PandaHubError("net must be a pandapower or pandapipes object") if self._network_with_name_exists(name, db): if overwrite: @@ -811,7 +1045,9 @@ def write_network_to_db(self, net, name, sector="power", overwrite=True, project if element_data.empty: continue # convert pandapower dataframe object to dict and save to db - element_data = convert_element_to_dict(element_data.copy(deep=True), _id, self._datatypes.get(element)) + element_data = convert_element_to_dict( + element_data.copy(deep=True), _id, self._datatypes.get(element) + ) self._write_element_to_db(db, element, element_data) else: @@ -820,11 +1056,13 @@ def write_network_to_db(self, net, name, sector="power", overwrite=True, project data[element] = element_data # write network metadata - net_dict = {"_id": _id, - "name": name, - "sector": sector, - "dtypes": dtypes, - "data": data} + net_dict = { + "_id": _id, + "name": name, + "sector": sector, + "dtypes": dtypes, + "data": data, + } if metadata is not None: net_dict.update(metadata) @@ -851,7 +1089,7 @@ def delete_net_from_db(self, name): raise PandaHubError("Network does not exist", 404) collection_names = self._get_net_collections(db) # TODO for collection_name in collection_names: - db[collection_name].delete_many({'net_id': _id}) + db[collection_name].delete_many({"net_id": _id}) db["_networks"].delete_one({"_id": _id}) def network_with_name_exists(self, name): @@ -883,18 +1121,27 @@ def _network_with_name_exists(self, name, db): def _get_net_collections(self, db, with_areas=True): if with_areas: - collection_filter = {'name': {'$regex': '^net_'}} + collection_filter = {"name": {"$regex": "^net_"}} else: - collection_filter = {'name': {'$regex': '^net_.*(? dict: + def delete_element( + self, net, element_type, element_index, variant=None, project_id=None, **kwargs + ) -> dict: """ Delete an element from the database. @@ -1000,9 +1267,15 @@ def delete_element(self, net, element_type, element_index, variant=None, project **kwargs, )[0] - def delete_elements(self, net: Union[int, str], element_type: str, element_indexes: list[int], - variant: Union[int, list[int], None] = None, project_id: Union[str, None] = None, **kwargs) -> \ - list[dict]: + def delete_elements( + self, + net: Union[int, str], + element_type: str, + element_indexes: list[int], + variant: Union[int, list[int], None] = None, + project_id: Union[str, None] = None, + **kwargs, + ) -> list[dict]: """ Delete multiple elements of the same type from the database. @@ -1040,7 +1313,11 @@ def delete_elements(self, net: Union[int, str], element_type: str, element_index else: net_id = net - element_filter = {"index": {"$in": element_indexes}, "net_id": int(net_id), **self.get_variant_filter(variant)} + element_filter = { + "index": {"$in": element_indexes}, + "net_id": int(net_id), + **self.get_variant_filter(variant), + } deletion_targets = list(db[collection].find(element_filter)) if not deletion_targets: @@ -1049,18 +1326,32 @@ def delete_elements(self, net: Union[int, str], element_type: str, element_index if variant: delete_ids_variant, delete_ids = [], [] for target in deletion_targets: - delete_ids_variant.append(target["_id"]) if target["var_type"] == "base" else delete_ids.append( - target["_id"]) - db[collection].update_many({"_id": {"$in": delete_ids_variant}}, - {"$addToSet": {"not_in_var": variant}}) + delete_ids_variant.append(target["_id"]) if target[ + "var_type" + ] == "base" else delete_ids.append(target["_id"]) + db[collection].update_many( + {"_id": {"$in": delete_ids_variant}}, + {"$addToSet": {"not_in_var": variant}}, + ) else: delete_ids = [target["_id"] for target in deletion_targets] db[collection].delete_many({"_id": {"$in": delete_ids}}) return deletion_targets - def set_net_value_in_db(self, net, element_type, element_index, - parameter, value, variant=None, project_id=None, **kwargs): - logger.info(f"Setting {parameter} = {value} in {element_type} with index {element_index} and variant {variant}") + def set_net_value_in_db( + self, + net, + element_type, + element_index, + parameter, + value, + variant=None, + project_id=None, + **kwargs, + ): + logger.info( + f"Setting {parameter} = {value} in {element_type} with index {element_index} and variant {variant}" + ) if variant is not None: variant = int(variant) if project_id: @@ -1075,14 +1366,22 @@ def set_net_value_in_db(self, net, element_type, element_index, net_id = self._get_id_from_name(net, db) else: net_id = net - element_filter = {"index": element_index, "net_id": int(net_id), **self.get_variant_filter(variant)} + element_filter = { + "index": element_index, + "net_id": int(net_id), + **self.get_variant_filter(variant), + } document = db[collection].find_one({**element_filter}) if not document: - raise UserWarning(f"No element '{element_type}' to change with index '{element_index}' in this variant") + raise UserWarning( + f"No element '{element_type}' to change with index '{element_index}' in this variant" + ) old_value = document.get(parameter, None) if old_value == value: - logger.warning(f'Value "{value}" for "{parameter}" identical to database element - no change applied') + logger.warning( + f'Value "{value}" for "{parameter}" identical to database element - no change applied' + ) return None if "." in parameter: key, subkey = parameter.split(".") @@ -1091,26 +1390,41 @@ def set_net_value_in_db(self, net, element_type, element_index, document[parameter] = value if variant is None: - db[collection].update_one({**element_filter, **self.base_variant_filter}, - {"$set": {parameter: value}}) + db[collection].update_one( + {**element_filter, **self.base_variant_filter}, + {"$set": {parameter: value}}, + ) else: if document["var_type"] == "base": base_variant_id = document.pop("_id") - db[collection].update_one({"_id": base_variant_id}, - {"$addToSet": {"not_in_var": variant}}) - document.update(var_type="change", variant=variant, changed_fields=[parameter]) + db[collection].update_one( + {"_id": base_variant_id}, {"$addToSet": {"not_in_var": variant}} + ) + document.update( + var_type="change", variant=variant, changed_fields=[parameter] + ) insert_result = db[collection].insert_one(document) document["_id"] = insert_result.inserted_id else: update_dict = {"$set": {parameter: value}, "$unset": {"not_in_var": ""}} if document["var_type"] == "change": update_dict["$addToSet"] = {"changed_fields": parameter} - db[collection].update_one({"_id": document["_id"]}, - update_dict) - return {"document": document, parameter: {"previous": old_value, "current": value}} + db[collection].update_one({"_id": document["_id"]}, update_dict) + return { + "document": document, + parameter: {"previous": old_value, "current": value}, + } - def set_object_attribute(self, net, element_type, element_index, - parameter, value, variant=None, project_id=None): + def set_object_attribute( + self, + net, + element_type, + element_index, + parameter, + value, + variant=None, + project_id=None, + ): if project_id: self.set_active_project_by_id(project_id) self.check_permission("write") @@ -1127,39 +1441,56 @@ def set_object_attribute(self, net, element_type, element_index, js = list(db[collection].find({"index": element_index, "net_id": net_id}))[0] obj = json_to_object(js["object"]) setattr(obj, parameter, value) - db[collection].update_one({"index": element_index, "net_id": net_id}, - {"$set": {"object._object": obj.to_json()}}) + db[collection].update_one( + {"index": element_index, "net_id": net_id}, + {"$set": {"object._object": obj.to_json()}}, + ) element_filter = {"index": element_index, "net_id": int(net_id)} if variant is None: - document = db[collection].find_one({**element_filter, **self.base_variant_filter}) + document = db[collection].find_one( + {**element_filter, **self.base_variant_filter} + ) obj = json_to_object(document["object"]) setattr(obj, parameter, value) db[collection].update_one( - {**element_filter, **self.base_variant_filter}, {"$set": {"object._object": obj.to_json()}} + {**element_filter, **self.base_variant_filter}, + {"$set": {"object._object": obj.to_json()}}, ) else: variant = int(variant) element_filter = {**element_filter, **self.get_variant_filter(variant)} document = db[collection].find_one({**element_filter}) if not document: - raise UserWarning(f"No element '{element_type}' to change with index '{element_index}' in this variant") + raise UserWarning( + f"No element '{element_type}' to change with index '{element_index}' in this variant" + ) obj = json_to_object(document["object"]) setattr(obj, parameter, value) if document["var_type"] == "base": base_variant_id = document.pop("_id") - db[collection].update_one({"_id": base_variant_id}, - {"$addToSet": {"not_in_var": variant}}) + db[collection].update_one( + {"_id": base_variant_id}, {"$addToSet": {"not_in_var": variant}} + ) document["object"]["_object"] = obj document["var_type"] = "change" db[collection].insert_one(document) else: - db[collection].update_one({"_id": document["_id"]}, - {"$set": {"object._object": obj}}) + db[collection].update_one( + {"_id": document["_id"]}, {"$set": {"object._object": obj}} + ) - def create_element(self, net: Union[int, str], element_type: str, element_index: int, element_data: dict, - variant=None, project_id=None, **kwargs) -> dict: + def create_element( + self, + net: Union[int, str], + element_type: str, + element_index: int, + element_data: dict, + variant=None, + project_id=None, + **kwargs, + ) -> dict: """ Creates an element in the database. @@ -1191,8 +1522,15 @@ def create_element(self, net: Union[int, str], element_type: str, element_index: **kwargs, )[0] - def create_elements(self, net: Union[int, str], element_type: str, elements_data: list[dict], - variant: int = None, project_id: str = None, **kwargs) -> list[dict]: + def create_elements( + self, + net: Union[int, str], + element_type: str, + elements_data: list[dict], + variant: int = None, + project_id: str = None, + **kwargs, + ) -> list[dict]: """ Creates multiple elements of the same type in the database. @@ -1275,7 +1613,9 @@ def _ensure_dtypes(self, element_type, data): if not val is None and key in dtypes and not dtypes[key] == object: data[key] = dtypes[key](val) - def _create_mongodb_indexes(self, project_id: Optional[str]=None, collection: Optional["str"]=None): + def _create_mongodb_indexes( + self, project_id: Optional[str] = None, collection: Optional["str"] = None + ): """ Create indexes on mongodb collections. Indexes are defined in pandahub.lib.mongodb_indexes @@ -1312,13 +1652,16 @@ def _create_mongodb_indexes(self, project_id: Optional[str]=None, collection: Op def create_variant(self, data): db = self._get_project_database() net_id = int(data["net_id"]) - max_index = list(db["variant"].find({"net_id": net_id}, - projection={"_id": 0, "index": 1}).sort("index", -1).limit(1)) + max_index = list( + db["variant"] + .find({"net_id": net_id}, projection={"_id": 0, "index": 1}) + .sort("index", -1) + .limit(1) + ) if not max_index: index = 1 for coll in self._get_net_collections(db): - update = {"$set": {"var_type": "base", - "not_in_var": []}} + update = {"$set": {"var_type": "base", "not_in_var": []}} db[coll].update_many({}, update) else: @@ -1339,11 +1682,18 @@ def delete_variant(self, net_id, index): collection_names = self._get_net_collections(db) for coll in collection_names: # remove references to deleted objects - db[coll].update_many({"net_id": net_id, "var_type": "base", "not_in_var": index}, - {"$pull": {"not_in_var": index}}) + db[coll].update_many( + {"net_id": net_id, "var_type": "base", "not_in_var": index}, + {"$pull": {"not_in_var": index}}, + ) # remove changes and additions - db[coll].delete_many({"net_id": net_id, "var_type": {"$in": ["change", "addition"]}, - "variant": index}) + db[coll].delete_many( + { + "net_id": net_id, + "var_type": {"$in": ["change", "addition"]}, + "variant": index, + } + ) # delete variant db["variant"].delete_one({"net_id": net_id, "index": index}) @@ -1367,15 +1717,28 @@ def get_variant_filter(self, variants): """ if type(variants) is list and variants: if len(variants) > 1: - variants = [int(var) for var in variants] # make sure variants are of type int - return {"$or": [{"var_type": "base", "not_in_var": {"$nin": variants}}, - {"var_type": {"$in": ["change", "addition"]}, "variant": {"$in": variants}}]} + variants = [ + int(var) for var in variants + ] # make sure variants are of type int + return { + "$or": [ + {"var_type": "base", "not_in_var": {"$nin": variants}}, + { + "var_type": {"$in": ["change", "addition"]}, + "variant": {"$in": variants}, + }, + ] + } else: variants = variants[0] if variants: variants = int(variants) - return {"$or": [{"var_type": "base", "not_in_var": {"$ne": variants}}, - {"var_type": {"$in": ["change", "addition"]}, "variant": variants}]} + return { + "$or": [ + {"var_type": "base", "not_in_var": {"$ne": variants}}, + {"var_type": {"$in": ["change", "addition"]}, "variant": variants}, + ] + } else: return self.base_variant_filter @@ -1383,7 +1746,9 @@ def get_variant_filter(self, variants): # Bulk operations # ------------------------- - def bulk_write_to_db(self, data, collection_name="tasks", global_database=True, project_id=None): + def bulk_write_to_db( + self, data, collection_name="tasks", global_database=True, project_id=None + ): """ Writes any number of documents to the database at once. Checks, if any document with the same _id already exists in the database. Already existing @@ -1411,12 +1776,20 @@ def bulk_write_to_db(self, data, collection_name="tasks", global_database=True, else: self.check_permission("write") db = self._get_project_database() - operations = [ReplaceOne(replacement=d, filter={"_id": d["_id"]}, - upsert=True) - for d in data] + operations = [ + ReplaceOne(replacement=d, filter={"_id": d["_id"]}, upsert=True) + for d in data + ] db[collection_name].bulk_write(operations) - def bulk_update_in_db(self, data, document_ids, collection_name="tasks", global_database=False, project_id=None): + def bulk_update_in_db( + self, + data, + document_ids, + collection_name="tasks", + global_database=False, + project_id=None, + ): """ Updates any number of documents in the database at once, according to their document_ids. @@ -1449,11 +1822,13 @@ def bulk_update_in_db(self, data, document_ids, collection_name="tasks", global_ operations["UpdateOne"] = [] i = 0 for d in data: - operations["UpdateOne"].append({ - "filter": {"_id": document_ids[i]}, - "update": {"$push": d}, - "upsert": False - }) + operations["UpdateOne"].append( + { + "filter": {"_id": document_ids[i]}, + "update": {"$push": d}, + "upsert": False, + } + ) i += 1 db[collection_name].bulk_write(operations) @@ -1462,15 +1837,17 @@ def bulk_update_in_db(self, data, document_ids, collection_name="tasks", global_ # Timeseries # ------------------------- - def write_timeseries_to_db(self, - timeseries, - data_type, - ts_format="timestamp_value", - compress_ts_data=False, - global_database=False, - collection_name="timeseries", - project_id=None, - **kwargs): + def write_timeseries_to_db( + self, + timeseries, + data_type, + ts_format="timestamp_value", + compress_ts_data=False, + global_database=False, + collection_name="timeseries", + project_id=None, + **kwargs, + ): """ This function can be used to write a timeseries to a MongoDB database. The timeseries must be provided as a pandas Series with the timestamps as @@ -1526,29 +1903,46 @@ def write_timeseries_to_db(self, else: self.check_permission("write") db = self._get_project_database() - document = create_timeseries_document(timeseries=timeseries, - data_type=data_type, - ts_format=ts_format, - compress_ts_data=compress_ts_data, - **kwargs) - db[collection_name].replace_one( - {"_id": document["_id"]}, - document, - upsert=True + if self.collection_is_timeseries(collection_name, project_id, global_database): + metadata = kwargs + if data_type is not None: + metadata["data_type"] = data_type + if isinstance(timeseries, pd.Series): + documents = [ + {"metadata": metadata, "timestamp": idx, "value": value} + for idx, value in timeseries.items() + ] + elif isinstance(timeseries, pd.DataFrame): + documents = [ + {"metadata": metadata, "timestamp": idx, **row.to_dict()} + for idx, row in timeseries.iterrows() + ] + return db.measurements.insert_many(documents) + document = create_timeseries_document( + timeseries=timeseries, + data_type=data_type, + ts_format=ts_format, + compress_ts_data=compress_ts_data, + **kwargs, ) + db[collection_name].replace_one({"_id": document["_id"]}, document, upsert=True) logger.debug("document with _id {document['_id']} added to database") if kwargs.get("return_id"): return document["_id"] return None - def bulk_write_timeseries_to_db(self, timeseries, data_type, - meta_frame=None, - ts_format="timestamp_value", - compress_ts_data=False, - global_database=False, - collection_name="timeseries", - project_id=None, - **kwargs): + def bulk_write_timeseries_to_db( + self, + timeseries, + data_type, + meta_frame=None, + ts_format="timestamp_value", + compress_ts_data=False, + global_database=False, + collection_name="timeseries", + project_id=None, + **kwargs, + ): """ This function can be used to write a pandas DataFrame, containing multiple timeseries of the same element_type and data_type at once to a MongoDB @@ -1595,26 +1989,36 @@ def bulk_write_timeseries_to_db(self, timeseries, data_type, documents = [] if project_id: self.set_active_project_by_id(project_id) + if self.collection_is_timeseries(collection_name, project_id, global_database): + raise NotImplementedError("Not implemented yet for timeseries collections") for col in timeseries.columns: if meta_frame is not None: args = {**kwargs, **meta_frame.loc[col]} else: args = kwargs - doc = create_timeseries_document(timeseries[col], - data_type, - ts_format=ts_format, - compress_ts_data=compress_ts_data, - element_index=col, - **args) + doc = create_timeseries_document( + timeseries[col], + data_type, + ts_format=ts_format, + compress_ts_data=compress_ts_data, + element_index=col, + **args, + ) documents.append(doc) - self.bulk_write_to_db(documents, collection_name=collection_name, - global_database=global_database) + self.bulk_write_to_db( + documents, collection_name=collection_name, global_database=global_database + ) logger.debug(f"{len(documents)} documents added to database") return [d["_id"] for d in documents] - def update_timeseries_in_db(self, new_ts_content, document_id, collection_name="timeseries", - global_database=False, project_id=None): - + def update_timeseries_in_db( + self, + new_ts_content, + document_id, + collection_name="timeseries", + global_database=False, + project_id=None, + ): """ This function can be used to append a timeseries to an existing timseries in the MongoDB database. @@ -1647,14 +2051,25 @@ def update_timeseries_in_db(self, new_ts_content, document_id, collection_name=" else: self.check_permission("write") db = self._get_project_database() - ts_update = {"timeseries_data": {"$each": convert_timeseries_to_subdocuments(new_ts_content)}} - db[collection_name].update_one({"_id": document_id}, - {"$push": ts_update},) + ts_update = { + "timeseries_data": { + "$each": convert_timeseries_to_subdocuments(new_ts_content) + } + } + db[collection_name].update_one( + {"_id": document_id}, + {"$push": ts_update}, + ) # logger.info("document updated in database") - def bulk_update_timeseries_in_db(self, new_ts_content, document_ids, project_id=None, collection_name="timeseries", - global_database=False): - + def bulk_update_timeseries_in_db( + self, + new_ts_content, + document_ids, + project_id=None, + collection_name="timeseries", + global_database=False, + ): """ This function can be used to append a pandas DataFrame, containing multiple timeseries of the same element_type and data_type at once, to an already @@ -1684,23 +2099,38 @@ def bulk_update_timeseries_in_db(self, new_ts_content, document_ids, project_id= """ if project_id: self.set_active_project_by_id(project_id) - + if self.collection_is_timeseries(collection_name, project_id, global_database): + raise NotImplementedError("Not implemented yet for timeseries collections") documents = [] for i in range(len(new_ts_content.columns)): col = new_ts_content.columns[i] document = {} - document["timeseries_data"] = {"$each": convert_timeseries_to_subdocuments(new_ts_content[col])} + document["timeseries_data"] = { + "$each": convert_timeseries_to_subdocuments(new_ts_content[col]) + } documents.append(document) - self.bulk_update_in_db(documents, document_ids, project_id=project_id, - collection_name="timeseries", global_database=global_database) + self.bulk_update_in_db( + documents, + document_ids, + project_id=project_id, + collection_name="timeseries", + global_database=global_database, + ) # logger.debug(f"{len(documents)} documents added to database") - def get_timeseries_from_db(self, filter_document={}, timestamp_range=None, - ts_format="timestamp_value", - compressed_ts_data=False, - global_database=False, collection_name="timeseries", - include_metadata=False, project_id=None, **kwargs): + def get_timeseries_from_db( + self, + filter_document={}, + timestamp_range=None, + ts_format="timestamp_value", + compressed_ts_data=False, + global_database=False, + collection_name="timeseries", + include_metadata=False, + project_id=None, + **kwargs, + ): """ This function can be used to retrieve a single timeseries from a MongoDB database that matches the provided metadata filter_document. @@ -1751,28 +2181,67 @@ def get_timeseries_from_db(self, filter_document={}, timestamp_range=None, else: self.check_permission("read") db = self._get_project_database() + if self.collection_is_timeseries(collection_name, project_id, global_database): + meta_filter = { + "metadata." + key: value for key, value in filter_document.items() + } + pipeline = [] + pipeline.append({"$match": meta_filter}) + pipeline.append({"$project": {"_id": 0, "metadata": 0}}) + timeseries = db[collection_name].aggregate_pandas_all(pipeline) + timeseries.set_index("timestamp", inplace=True) + if include_metadata: + raise NotImplementedError( + "Not implemented yet for timeseries collections" + ) + return timeseries filter_document = {**filter_document, **kwargs} pipeline = [{"$match": filter_document}] if not compressed_ts_data: if ts_format == "timestamp_value": if timestamp_range: - pipeline.append({"$project": {"timeseries_data": {"$filter": {"input": "$timeseries_data", - "as": "timeseries_data", - "cond": {"$and": [{"$gte": [ - "$$timeseries_data.timestamp", - timestamp_range[0]]}, - {"$lt": [ - "$$timeseries_data.timestamp", - timestamp_range[ - 1]]}]}}}}}) - pipeline.append({"$addFields": {"timestamps": "$timeseries_data.timestamp", - "values": "$timeseries_data.value"}}) + pipeline.append( + { + "$project": { + "timeseries_data": { + "$filter": { + "input": "$timeseries_data", + "as": "timeseries_data", + "cond": { + "$and": [ + { + "$gte": [ + "$$timeseries_data.timestamp", + timestamp_range[0], + ] + }, + { + "$lt": [ + "$$timeseries_data.timestamp", + timestamp_range[1], + ] + }, + ] + }, + } + } + } + } + ) + pipeline.append( + { + "$addFields": { + "timestamps": "$timeseries_data.timestamp", + "values": "$timeseries_data.value", + } + } + ) if include_metadata: pipeline.append({"$project": {"timeseries_data": 0}}) else: - pipeline.append({"$project": {"timestamps": 1, - "values": 1, - "_id": 0}}) + pipeline.append( + {"$project": {"timestamps": 1, "values": 1, "_id": 0}} + ) elif ts_format == "array": if not include_metadata: pipeline.append({"$project": {"timeseries_data": 1}}) @@ -1787,12 +2256,14 @@ def get_timeseries_from_db(self, filter_document={}, timestamp_range=None, else: data = data[0] if compressed_ts_data: - timeseries_data = decompress_timeseries_data(data["timeseries_data"], ts_format) + timeseries_data = decompress_timeseries_data( + data["timeseries_data"], ts_format + ) else: if ts_format == "timestamp_value": - timeseries_data = pd.Series(data["values"], - index=data["timestamps"], - dtype="float64") + timeseries_data = pd.Series( + data["values"], index=data["timestamps"], dtype="float64" + ) elif ts_format == "array": timeseries_data = data["timeseries_data"] if include_metadata: @@ -1803,7 +2274,13 @@ def get_timeseries_from_db(self, filter_document={}, timestamp_range=None, else: return timeseries_data - def get_timeseries_metadata(self, filter_document, collection_name="timeseries", global_database=False): + def get_timeseries_metadata( + self, + filter_document, + collection_name="timeseries", + global_database=False, + project_id=None, + ): """ Returns a DataFrame, containing all metadata matching the provided filter. A filter document has to be provided in form of a dictionary, containing @@ -1827,33 +2304,65 @@ def get_timeseries_metadata(self, filter_document, collection_name="timeseries", DataFrame, containing all metadata matching the provided filter. """ + if project_id: + self.set_active_project_by_id(project_id) if global_database: db = self._get_global_database() else: self.check_permission("read") db = self._get_project_database() - match_filter = [] - pipeline = [] - for key in filter_document: - if key == "timestamp_range": - continue - filter_value = filter_document[key] - if type(filter_value) == list: - match_filter.append({key: {"$in": filter_value}}) + if self.collection_is_timeseries(collection_name, project_id, global_database): + pipeline = [] + if len(filter_document) > 0: + document_filter = { + "metadata." + key: value for key, value in filter_document.items() + } + pipeline.append({"$match": document_filter}) else: - match_filter.append({key: filter_value}) - if match_filter: - pipeline.append({"$match": {"$and": match_filter}}) - projection = {"$project": {"timeseries_data": 0}} - pipeline.append(projection) + document_filter = {} + document = db[collection_name].find_one( + document_filter, projection={"timestamp": 0, "metadata": 0, "_id": 0} + ) + value_fields = ["$%s" % field for field in document.keys()] + pipeline.append( + { + "$group": { + "_id": "$metadata._id", + "max_value": {"$max": {"$max": value_fields}}, + "min_value": {"$min": {"$min": value_fields}}, + "first_timestamp": {"$min": "$timestamp"}, + "last_timestamp": {"$max": "$timestamp"}, + } + } + ) + else: + match_filter = [] + pipeline = [] + for key in filter_document: + if key == "timestamp_range": + continue + filter_value = filter_document[key] + if type(filter_value) == list: + match_filter.append({key: {"$in": filter_value}}) + else: + match_filter.append({key: filter_value}) + if match_filter: + pipeline.append({"$match": {"$and": match_filter}}) + projection = {"$project": {"timeseries_data": 0}} + pipeline.append(projection) metadata = list(db[collection_name].aggregate(pipeline)) df_metadata = pd.DataFrame(metadata) if len(df_metadata): df_metadata.set_index("_id", inplace=True) return df_metadata - def add_metadata(self, filter_document, add_meta, global_database=False, - collection_name="timeseries"): + def add_metadata( + self, + filter_document, + add_meta, + global_database=False, + collection_name="timeseries", + ): if global_database: db = self._get_global_database() else: @@ -1861,25 +2370,36 @@ def add_metadata(self, filter_document, add_meta, global_database=False, db = self._get_project_database() # get metada before change - meta_before = self.get_timeseries_metadata(filter_document, global_database=global_database, - collection_name=collection_name) + meta_before = self.get_timeseries_metadata( + filter_document, + global_database=global_database, + collection_name=collection_name, + ) # add the new information to the metadata dict of the existing timeseries - if len(meta_before) > 1: # TODO is this the desired behaviour? Needs to specified + if ( + len(meta_before) > 1 + ): # TODO is this the desired behaviour? Needs to specified raise PandaHubError meta_copy = {**meta_before.iloc[0].to_dict(), **add_meta} # write new metadata to mongo db - db[collection_name].replace_one({"_id": meta_before.index[0]}, - meta_copy, upsert=True) + db[collection_name].replace_one( + {"_id": meta_before.index[0]}, meta_copy, upsert=True + ) return meta_copy - def multi_get_timeseries_from_db(self, filter_document={}, - timestamp_range=None, - exclude_timestamp_range=None, - include_metadata=False, - ts_format="timestamp_value", - compressed_ts_data=False, - global_database=False, collection_name="timeseries", - project_id=None, **kwargs): + def multi_get_timeseries_from_db( + self, + filter_document={}, + timestamp_range=None, + exclude_timestamp_range=None, + include_metadata=False, + ts_format="timestamp_value", + compressed_ts_data=False, + global_database=False, + collection_name="timeseries", + project_id=None, + **kwargs, + ): if project_id: self.set_active_project_by_id(project_id) if global_database: @@ -1887,6 +2407,74 @@ def multi_get_timeseries_from_db(self, filter_document={}, else: self.check_permission("read") db = self._get_project_database() + if self.collection_is_timeseries(collection_name, project_id, global_database): + pipeline = [] + if timestamp_range is not None: + pipeline.append( + { + "$match": { + "timestamp": { + "$gte": timestamp_range[0], + "$lt": timestamp_range[1], + } + } + } + ) + if exclude_timestamp_range is not None: + pipeline.append( + { + "$match": { + "timestamp": { + "$gte": exclude_timestamp_range[0], + "$lt": exclude_timestamp_range[1], + } + } + } + ) + if filter_document is not None: + document_filter = { + "metadata." + key: value for key, value in filter_document.items() + } + pipeline.append({"$match": document_filter}) + + pipeline.append({"$addFields": {"_id": "$metadata._id"}}) + pipeline.append({"$project": {"metadata": 0}}) + + if include_metadata: + document = db[collection_name].find_one( + document_filter, + projection={"timestamp": 0, "metadata": 0, "_id": 0}, + ) + meta_pipeline = [] + meta_pipeline.append({"$match": document_filter}) + value_fields = ["$%s" % field for field in document.keys()] + meta_pipeline.append( + { + "$group": { + "_id": "$metadata._id", + "max_value": {"$max": {"$max": value_fields}}, + "min_value": {"$min": {"$min": value_fields}}, + "first_timestamp": {"$min": "$timestamp"}, + "last_timestamp": {"$max": "$timestamp"}, + "name": {"$first": "$metadata.name"}, + "data_type": {"$first": "$metadata.data_type"}, + } + } + ) + meta_data = { + d["_id"]: d for d in db[collection_name].aggregate(meta_pipeline) + } + timeseries = [] + ts_all = db.measurements.aggregate_pandas_all(pipeline) + for _id, ts in ts_all.groupby("_id"): + ts.set_index("timestamp", inplace=True) + for col in set(ts.columns) - {"timestamp", "_id"}: + timeseries_dict = {"timeseries_data": ts[col]} + if include_metadata: + timeseries_dict.update(meta_data[_id]) + timeseries.append(timeseries_dict) + return timeseries + filter_document = {**filter_document, **kwargs} match_filter = [] for key in filter_document: @@ -1901,20 +2489,56 @@ def multi_get_timeseries_from_db(self, filter_document={}, else: pipeline = [] if timestamp_range: - projection = {"timeseries_data": {"$filter": {"input": "$timeseries_data", - "as": "timeseries_data", - "cond": {"$and": [{"$gte": ["$$timeseries_data.timestamp", - timestamp_range[0]]}, - {"$lt": ["$$timeseries_data.timestamp", - timestamp_range[1]]}]}}}} + projection = { + "timeseries_data": { + "$filter": { + "input": "$timeseries_data", + "as": "timeseries_data", + "cond": { + "$and": [ + { + "$gte": [ + "$$timeseries_data.timestamp", + timestamp_range[0], + ] + }, + { + "$lt": [ + "$$timeseries_data.timestamp", + timestamp_range[1], + ] + }, + ] + }, + } + } + } pipeline.append({"$project": projection}) if exclude_timestamp_range: - projection = {"timeseries_data": {"$filter": {"input": "$timeseries_data", - "as": "timeseries_data", - "cond": {"$or": [{"$lt": ["$$timeseries_data.timestamp", - timestamp_range[0]]}, - {"$gte": ["$$timeseries_data.timestamp", - timestamp_range[1]]}]}}}} + projection = { + "timeseries_data": { + "$filter": { + "input": "$timeseries_data", + "as": "timeseries_data", + "cond": { + "$or": [ + { + "$lt": [ + "$$timeseries_data.timestamp", + timestamp_range[0], + ] + }, + { + "$gte": [ + "$$timeseries_data.timestamp", + timestamp_range[1], + ] + }, + ] + }, + } + } + } pipeline.append({"$project": projection}) if not include_metadata: pipeline.append({"$project": {"timeseries_data": 1}}) @@ -1937,7 +2561,9 @@ def multi_get_timeseries_from_db(self, filter_document={}, timeseries.append(ts) if exclude_timestamp_range is not None or timestamp_range is not None: # TODO: Second query to get the metadata, since metadata is not returned if a projection on the subfield is used - metadata = db[collection_name].find_one({"_id": ts["_id"]}, projection={"timeseries_data": 0}) + metadata = db[collection_name].find_one( + {"_id": ts["_id"]}, projection={"timeseries_data": 0} + ) ts.update(metadata) else: if ts_format == "timestamp_value": @@ -1951,11 +2577,18 @@ def multi_get_timeseries_from_db(self, filter_document={}, return pd.DataFrame(np.array(timeseries).T, index=timeseries_data.index) return pd.DataFrame(np.array(timeseries).T) - def bulk_get_timeseries_from_db(self, filter_document={}, timestamp_range=None, - exclude_timestamp_range=None, - additional_columns=None, pivot_by_column=None, - global_database=False, collection_name="timeseries", - **kwargs): + def bulk_get_timeseries_from_db( + self, + filter_document={}, + timestamp_range=None, + exclude_timestamp_range=None, + additional_columns=None, + pivot_by_column=None, + global_database=False, + collection_name="timeseries", + project_id=None, + **kwargs, + ): """ This function can be used to retrieve multiple timeseries at once from a MongoDB database. The timeseries will be filtered by their metadata. @@ -2006,6 +2639,30 @@ def bulk_get_timeseries_from_db(self, filter_document={}, timestamp_range=None, self.check_permission("read") db = self._get_project_database() + if self.collection_is_timeseries(collection_name, project_id, global_database): + document_filter = { + "metadata." + key: value for key, value in filter_document.items() + } + if timestamp_range is not None and exclude_timestamp_range is not None: + raise NotImplementedError( + "timestamp_range and exclude_timestamp_range cannot be used at the same time with timeseries collections" + ) + if timestamp_range is not None: + document_filter["timestamp"] = { + "$gte": timestamp_range[0], + "$lte": timestamp_range[1], + } + if exclude_timestamp_range is not None: + document_filter["timestamp"] = { + "$lte": exclude_timestamp_range[0], + "$gte": exclude_timestamp_range[1], + } + timeseries = { + d["timestamp"]: d["value"] + for d in db[collection_name].find(document_filter) + } + return pd.Series(timeseries) + filter_document = {**filter_document, **kwargs} match_filter = [] for key in filter_document: @@ -2023,26 +2680,64 @@ def bulk_get_timeseries_from_db(self, filter_document={}, timestamp_range=None, if pivot_by_column: custom_projection[pivot_by_column] = 1 if timestamp_range: - projection = {"timeseries_data": {"$filter": {"input": "$timeseries_data", - "as": "timeseries_data", - "cond": {"$and": [{"$gte": ["$$timeseries_data.timestamp", - timestamp_range[0]]}, - {"$lt": ["$$timeseries_data.timestamp", - timestamp_range[1]]}]}}}} + projection = { + "timeseries_data": { + "$filter": { + "input": "$timeseries_data", + "as": "timeseries_data", + "cond": { + "$and": [ + { + "$gte": [ + "$$timeseries_data.timestamp", + timestamp_range[0], + ] + }, + { + "$lt": [ + "$$timeseries_data.timestamp", + timestamp_range[1], + ] + }, + ] + }, + } + } + } projection = {**projection, **custom_projection} pipeline.append({"$project": projection}) if exclude_timestamp_range: - projection = {"timeseries_data": {"$filter": {"input": "$timeseries_data", - "as": "timeseries_data", - "cond": {"$or": [{"$lt": ["$$timeseries_data.timestamp", - timestamp_range[0]]}, - {"$gte": ["$$timeseries_data.timestamp", - timestamp_range[1]]}]}}}} + projection = { + "timeseries_data": { + "$filter": { + "input": "$timeseries_data", + "as": "timeseries_data", + "cond": { + "$or": [ + { + "$lt": [ + "$$timeseries_data.timestamp", + timestamp_range[0], + ] + }, + { + "$gte": [ + "$$timeseries_data.timestamp", + timestamp_range[1], + ] + }, + ] + }, + } + } + } projection = {**projection, **custom_projection} pipeline.append({"$project": projection}) pipeline.append({"$unwind": "$timeseries_data"}) - projection = {"value": "$timeseries_data.value", - "timestamp": "$timeseries_data.timestamp"} + projection = { + "value": "$timeseries_data.value", + "timestamp": "$timeseries_data.timestamp", + } projection = {**projection, **custom_projection} pipeline.append({"$project": projection}) timeseries = pd.DataFrame(db[collection_name].aggregate(pipeline)) @@ -2057,9 +2752,15 @@ def bulk_get_timeseries_from_db(self, filter_document={}, timestamp_range=None, timeseries = timeseries.pivot(columns=pivot_by_column, values="value") return timeseries - def delete_timeseries_from_db(self, element_type, data_type, netname=None, - element_index=None, collection_name="timeseries", - **kwargs): + def delete_timeseries_from_db( + self, + element_type, + data_type, + netname=None, + element_index=None, + collection_name="timeseries", + **kwargs, + ): """ This function can be used to delete a single timeseries that matches the provided metadata from a MongoDB database. The element_type and data_type @@ -2097,8 +2798,7 @@ def delete_timeseries_from_db(self, element_type, data_type, netname=None, self.check_permission("write") db = self._get_project_database() - filter_document = {"element_type": element_type, - "data_type": data_type} + filter_document = {"element_type": element_type, "data_type": data_type} if netname is not None: filter_document["netname"] = netname if element_index is not None: @@ -2107,8 +2807,9 @@ def delete_timeseries_from_db(self, element_type, data_type, netname=None, del_res = db[collection_name].delete_one(filter_document) return del_res - def bulk_del_timeseries_from_db(self, filter_document, - collection_name="timeseries"): + def bulk_del_timeseries_from_db( + self, filter_document, collection_name="timeseries" + ): """ This function can be used to delete multiple timeseries at once from a MongoDB database. The timeseries will be filtered by their metadata. @@ -2136,6 +2837,12 @@ def bulk_del_timeseries_from_db(self, filter_document, """ self.check_permission("write") db = self._get_project_database() + if self.collection_is_timeseries(collection_name): + meta_filter = { + "metadata." + key: value for key, value in filter_document.items() + } + return db[collection_name].delete_many(meta_filter) + db = self._get_project_database() match_filter = {} for key in filter_document: if key == "timestamp_range": @@ -2149,11 +2856,57 @@ def bulk_del_timeseries_from_db(self, filter_document, del_res = db[collection_name].delete_many(match_filter) return del_res + def create_timeseries_collection(self, collection_name, overwrite=False): + db = self._get_project_database() + collection_exists = collection_name in db.list_collection_names() + if collection_exists: + if overwrite: + db.drop_collection(collection_name) + else: + print("Collection exists, skipping") + return + db.create_collection( + collection_name, + timeseries={ + "timeField": "timestamp", + "metaField": "metadata", + "granularity": "minutes", + }, + ) + db[collection_name].create_index({"metadata._id": 1}) + + def collection_is_timeseries( + self, + collection_name, + project_id=None, + global_database=False, + ): + db = self._get_project_or_global_db(project_id, global_database) + collections = list(db.list_collections(filter={"name": collection_name})) + return len(collections) == 1 and collections[0]["type"] == "timeseries" + + def _get_project_or_global_db(self, project_id=None, global_database=False): + if project_id: + self.set_active_project_by_id(project_id) + if global_database: + return self._get_global_database() + else: + return self._get_project_database() + #### deprecated functions - def create_element_in_db(self, net: Union[int, str], element: str, element_index: int, data: dict, - variant=None, project_id=None): - warnings.warn("ph.create_element_in_db was renamed - use ph.create_element instead!") + def create_element_in_db( + self, + net: Union[int, str], + element: str, + element_index: int, + data: dict, + variant=None, + project_id=None, + ): + warnings.warn( + "ph.create_element_in_db was renamed - use ph.create_element instead!" + ) return self.create_element( net=net, element_type=element, @@ -2163,11 +2916,18 @@ def create_element_in_db(self, net: Union[int, str], element: str, element_index project_id=project_id, ) - - def create_elements_in_db(self, net: Union[int, str], element_type: str, elements_data: list[dict], - project_id: str = None, variant: int = None): - warnings.warn("ph.create_elements_in_db was renamed - use ph.create_elements instead! " - "Watch out for changed order of project_id and variant args") + def create_elements_in_db( + self, + net: Union[int, str], + element_type: str, + elements_data: list[dict], + project_id: str = None, + variant: int = None, + ): + warnings.warn( + "ph.create_elements_in_db was renamed - use ph.create_elements instead! " + "Watch out for changed order of project_id and variant args" + ) return self.create_elements( net=net, element_type=element_type, @@ -2176,9 +2936,12 @@ def create_elements_in_db(self, net: Union[int, str], element_type: str, element project_id=project_id, ) - - def delete_net_element(self, net, element, element_index, variant=None, project_id=None): - warnings.warn("ph.delete_net_element was renamed - use ph.delete_element instead!") + def delete_net_element( + self, net, element, element_index, variant=None, project_id=None + ): + warnings.warn( + "ph.delete_net_element was renamed - use ph.delete_element instead!" + ) return self.delete_element( net=net, element_type=element, @@ -2188,9 +2951,9 @@ def delete_net_element(self, net, element, element_index, variant=None, project_ ) -if __name__ == '__main__': +if __name__ == "__main__": self = PandaHub() - project_name = 'test_project' + project_name = "test_project" self.set_active_project(project_name) ts = self.multi_get_timeseries_from_db(global_database=True) # r = self.create_account(email, password) diff --git a/requirements.txt b/requirements.txt index 29e29f5..2e6e645 100644 --- a/requirements.txt +++ b/requirements.txt @@ -9,3 +9,4 @@ pydantic simplejson requests python-dotenv +pymongoarrow \ No newline at end of file From 20c4d8edeb77b89b5846f685d5b4deb63b305238 Mon Sep 17 00:00:00 2001 From: Leon Thurner Date: Tue, 2 Jan 2024 23:26:36 +0100 Subject: [PATCH 40/84] fix and improve timeseries collection support --- pandahub/lib/PandaHub.py | 89 ++++++++++++++++++++++++++-------------- 1 file changed, 58 insertions(+), 31 deletions(-) diff --git a/pandahub/lib/PandaHub.py b/pandahub/lib/PandaHub.py index c72a0ad..492399b 100644 --- a/pandahub/lib/PandaHub.py +++ b/pandahub/lib/PandaHub.py @@ -1747,7 +1747,7 @@ def get_variant_filter(self, variants): # ------------------------- def bulk_write_to_db( - self, data, collection_name="tasks", global_database=True, project_id=None + self, data, collection_name="tasks", global_database=False, project_id=None ): """ Writes any number of documents to the database at once. Checks, if any @@ -1776,8 +1776,21 @@ def bulk_write_to_db( else: self.check_permission("write") db = self._get_project_database() + if self.collection_is_timeseries( + collection_name=collection_name, + project_id=project_id, + global_database=global_database, + ): + raise NotImplementedError( + "Bulk write is not fully supported for timeseries collections in MongoDB" + ) + operations = [ - ReplaceOne(replacement=d, filter={"_id": d["_id"]}, upsert=True) + ReplaceOne( + replacement=d, + filter={"_id": d["_id"]}, + upsert=True, + ) for d in data ] db[collection_name].bulk_write(operations) @@ -1917,7 +1930,7 @@ def write_timeseries_to_db( {"metadata": metadata, "timestamp": idx, **row.to_dict()} for idx, row in timeseries.iterrows() ] - return db.measurements.insert_many(documents) + return db[collection_name].insert_many(documents) document = create_timeseries_document( timeseries=timeseries, data_type=data_type, @@ -2321,20 +2334,23 @@ def get_timeseries_metadata( else: document_filter = {} document = db[collection_name].find_one( - document_filter, projection={"timestamp": 0, "metadata": 0, "_id": 0} + document_filter, projection={"timestamp": 0, "_id": 0} ) value_fields = ["$%s" % field for field in document.keys()] - pipeline.append( - { - "$group": { - "_id": "$metadata._id", - "max_value": {"$max": {"$max": value_fields}}, - "min_value": {"$min": {"$min": value_fields}}, - "first_timestamp": {"$min": "$timestamp"}, - "last_timestamp": {"$max": "$timestamp"}, - } - } - ) + group_dict = { + "_id": "$metadata._id", + "max_value": {"$max": {"$max": value_fields}}, + "min_value": {"$min": {"$min": value_fields}}, + "first_timestamp": {"$min": "$timestamp"}, + "last_timestamp": {"$max": "$timestamp"}, + } + metadata_fields = { + metadata_field: {"$first": "$metadata.%s" % metadata_field} + for metadata_field in document["metadata"].keys() + if metadata_field != "_id" + } + group_dict.update(metadata_fields) + pipeline.append({"$group": group_dict}) else: match_filter = [] pipeline = [] @@ -2448,30 +2464,41 @@ def multi_get_timeseries_from_db( meta_pipeline = [] meta_pipeline.append({"$match": document_filter}) value_fields = ["$%s" % field for field in document.keys()] - meta_pipeline.append( - { - "$group": { - "_id": "$metadata._id", - "max_value": {"$max": {"$max": value_fields}}, - "min_value": {"$min": {"$min": value_fields}}, - "first_timestamp": {"$min": "$timestamp"}, - "last_timestamp": {"$max": "$timestamp"}, - "name": {"$first": "$metadata.name"}, - "data_type": {"$first": "$metadata.data_type"}, - } - } - ) + group_dict = { + "_id": "$metadata._id", + "max_value": {"$max": {"$max": value_fields}}, + "min_value": {"$min": {"$min": value_fields}}, + "first_timestamp": {"$min": "$timestamp"}, + "last_timestamp": {"$max": "$timestamp"}, + } + document = db[collection_name].find_one(document_filter) + metadata_fields = { + metadata_field: {"$first": "$metadata.%s" % metadata_field} + for metadata_field in document["metadata"].keys() + if metadata_field != "_id" + } + group_dict.update(metadata_fields) + meta_pipeline.append({"$group": group_dict}) meta_data = { d["_id"]: d for d in db[collection_name].aggregate(meta_pipeline) } timeseries = [] - ts_all = db.measurements.aggregate_pandas_all(pipeline) + ts_all = db[collection_name].aggregate_pandas_all(pipeline) + if len(ts_all) == 0: + return timeseries for _id, ts in ts_all.groupby("_id"): ts.set_index("timestamp", inplace=True) - for col in set(ts.columns) - {"timestamp", "_id"}: + value_columns = list(set(ts.columns) - {"timestamp", "_id"}) + value_columns.sort() + for col in value_columns: timeseries_dict = {"timeseries_data": ts[col]} if include_metadata: timeseries_dict.update(meta_data[_id]) + if len(value_columns) > 1: + timeseries_dict["name"] = "%s, %s" % ( + timeseries_dict["name"], + col, + ) timeseries.append(timeseries_dict) return timeseries @@ -2863,7 +2890,7 @@ def create_timeseries_collection(self, collection_name, overwrite=False): if overwrite: db.drop_collection(collection_name) else: - print("Collection exists, skipping") + logger.info("Collection already exists, skipping") return db.create_collection( collection_name, From 08622a2adb675a23fc51e4518d2efb4f92c7a47b Mon Sep 17 00:00:00 2001 From: Leon Thurner Date: Sun, 7 Jan 2024 14:40:49 +0100 Subject: [PATCH 41/84] fix base variant filter - nan is not compatible with mongodb --- pandahub/lib/PandaHub.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pandahub/lib/PandaHub.py b/pandahub/lib/PandaHub.py index 492399b..959b190 100644 --- a/pandahub/lib/PandaHub.py +++ b/pandahub/lib/PandaHub.py @@ -102,7 +102,7 @@ def __init__( "$or": [ {"var_type": {"$exists": False}}, {"var_type": "base"}, - {"var_type": np.nan}, + {"var_type": None}, ] } self.mongodb_indexes = mongodb_indexes From 37af204f3e564aeec3b05d0b61cb35758061ff4a Mon Sep 17 00:00:00 2001 From: Leon Thurner Date: Thu, 11 Jan 2024 11:57:47 +0100 Subject: [PATCH 42/84] check for nan and None in base variant filter --- pandahub/lib/PandaHub.py | 1 + 1 file changed, 1 insertion(+) diff --git a/pandahub/lib/PandaHub.py b/pandahub/lib/PandaHub.py index 959b190..a0923e9 100644 --- a/pandahub/lib/PandaHub.py +++ b/pandahub/lib/PandaHub.py @@ -103,6 +103,7 @@ def __init__( {"var_type": {"$exists": False}}, {"var_type": "base"}, {"var_type": None}, + {"var_type": np.nan}, ] } self.mongodb_indexes = mongodb_indexes From 5fd35154434a7384a19c90df458dc8535b78d1c6 Mon Sep 17 00:00:00 2001 From: Joschka Thurner Date: Mon, 22 Jan 2024 12:07:15 +0100 Subject: [PATCH 43/84] fix: activate created project by id --- pandahub/lib/PandaHub.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pandahub/lib/PandaHub.py b/pandahub/lib/PandaHub.py index a0923e9..97b0cf5 100644 --- a/pandahub/lib/PandaHub.py +++ b/pandahub/lib/PandaHub.py @@ -234,7 +234,7 @@ def create_project( if SETTINGS.CREATE_INDEXES_WITH_PROJECT: self._create_mongodb_indexes(project_data["_id"]) if activate: - self.set_active_project(name, realm) + self.set_active_project_by_id(project_data["_id"]) return project_data def delete_project(self, i_know_this_action_is_final=False, project_id=None): From d2866268860f87e73f25d22424000537dfd4842f Mon Sep 17 00:00:00 2001 From: Joschka Thurner Date: Tue, 23 Jan 2024 13:20:17 +0100 Subject: [PATCH 44/84] bump pandahub version to 0.3.2 --- pandahub/__init__.py | 2 +- setup.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/pandahub/__init__.py b/pandahub/__init__.py index bd2fba2..85e371f 100644 --- a/pandahub/__init__.py +++ b/pandahub/__init__.py @@ -1,4 +1,4 @@ -__version__ = "0.3.1" +__version__ = "0.3.2" from pandahub.lib.PandaHub import PandaHub, PandaHubError from pandahub.client.PandaHubClient import PandaHubClient diff --git a/setup.py b/setup.py index 7e146ca..7e0a8c7 100644 --- a/setup.py +++ b/setup.py @@ -36,7 +36,7 @@ name='pandahub', packages=find_packages(), url='https://github.com/e2nIEE/pandahub', - version='0.3.1', + version='0.3.2', include_package_data=True, long_description_content_type='text/markdown', zip_safe=False, From d6ceeac4843570a6a0564579d3aa4aa3106a5eb1 Mon Sep 17 00:00:00 2001 From: Leon Thurner Date: Thu, 25 Jan 2024 16:11:37 +0100 Subject: [PATCH 45/84] add node elements before running additional filters, so that additional filters that reference a node element (load, sgen) are considered --- pandahub/lib/PandaHub.py | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/pandahub/lib/PandaHub.py b/pandahub/lib/PandaHub.py index 97b0cf5..78f2e26 100644 --- a/pandahub/lib/PandaHub.py +++ b/pandahub/lib/PandaHub.py @@ -937,11 +937,9 @@ def get_subnet_from_db_by_id( ) all_elements = list(set(all_elements) - set(ignore_elements)) - # Add elements for which the user has provided a filter function - for element, filter_func in additional_filters.items(): - if element in ignore_elements: - continue - element_filter = filter_func(net) + # add all node elements that are connected to buses within the network + for element in node_elements: + element_filter = {"bus": {"$in": buses}} self._add_element_from_collection( net, db, @@ -954,9 +952,11 @@ def get_subnet_from_db_by_id( dtypes=dtypes, ) - # add all node elements that are connected to buses within the network - for element in node_elements: - element_filter = {"bus": {"$in": buses}} + # Add elements for which the user has provided a filter function + for element, filter_func in additional_filters.items(): + if element in ignore_elements: + continue + element_filter = filter_func(net) self._add_element_from_collection( net, db, From d37b35ef232a994257fe3fdcd8d5032f341826af Mon Sep 17 00:00:00 2001 From: Jan Ulffers Date: Tue, 30 Jan 2024 18:47:06 +0100 Subject: [PATCH 46/84] pandas 2.0 compatibility for timeseries compression --- pandahub/lib/PandaHub.py | 9 +++++---- pandahub/lib/database_toolbox.py | 8 ++++---- 2 files changed, 9 insertions(+), 8 deletions(-) diff --git a/pandahub/lib/PandaHub.py b/pandahub/lib/PandaHub.py index 78f2e26..3a352b8 100644 --- a/pandahub/lib/PandaHub.py +++ b/pandahub/lib/PandaHub.py @@ -2261,7 +2261,8 @@ def get_timeseries_from_db( pipeline.append({"$project": {"timeseries_data": 1}}) else: if not include_metadata: - pipeline.append({"$project": {"timeseries_data": 1}}) + pipeline.append({"$project": {"timeseries_data": 1, + "num_timestamps": 1}}) data = list(db[collection_name].aggregate(pipeline)) if len(data) == 0: raise PandaHubError("no documents matching the provided filter found", 404) @@ -2270,9 +2271,9 @@ def get_timeseries_from_db( else: data = data[0] if compressed_ts_data: - timeseries_data = decompress_timeseries_data( - data["timeseries_data"], ts_format - ) + timeseries_data = decompress_timeseries_data(data["timeseries_data"], + ts_format, + num_timestamps=data["num_timestamps"]) else: if ts_format == "timestamp_value": timeseries_data = pd.Series( diff --git a/pandahub/lib/database_toolbox.py b/pandahub/lib/database_toolbox.py index 1741318..4f5826f 100644 --- a/pandahub/lib/database_toolbox.py +++ b/pandahub/lib/database_toolbox.py @@ -114,7 +114,7 @@ def convert_timeseries_to_subdocuments(timeseries): def compress_timeseries_data(timeseries_data, ts_format): import blosc if ts_format == "timestamp_value": - timeseries_data = np.array([timeseries_data.index.astype(int), + timeseries_data = np.array([timeseries_data.index.astype("int64"), timeseries_data.values]) return blosc.compress(timeseries_data.tobytes(), shuffle=blosc.SHUFFLE, @@ -125,11 +125,11 @@ def compress_timeseries_data(timeseries_data, ts_format): cname="zlib") -def decompress_timeseries_data(timeseries_data, ts_format): +def decompress_timeseries_data(timeseries_data, ts_format, num_timestamps): import blosc if ts_format == "timestamp_value": - data = np.frombuffer(blosc.decompress(timeseries_data), - dtype=np.float64).reshape((35040,2), + data = np.frombuffer(blosc.decompress(timeseries_data), + dtype=np.float64).reshape((num_timestamps, 2), order="F") return pd.Series(data[:,1], index=pd.to_datetime(data[:,0])) elif ts_format == "array": From 64bb42f7d96ce1e62101e6a0b4cb2c5ebcf6be2b Mon Sep 17 00:00:00 2001 From: dlohmeier Date: Mon, 5 Feb 2024 16:49:48 +0100 Subject: [PATCH 47/84] allow for additional arguments upon project creation that are handed to the project document --- pandahub/lib/PandaHub.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/pandahub/lib/PandaHub.py b/pandahub/lib/PandaHub.py index 78f2e26..2cf41d7 100644 --- a/pandahub/lib/PandaHub.py +++ b/pandahub/lib/PandaHub.py @@ -212,6 +212,7 @@ def create_project( metadata=None, project_id=None, activate=True, + additional_project_data=None, ): if self.project_exists(name, realm): raise PandaHubError("Project already exists") @@ -219,12 +220,15 @@ def create_project( settings = {} if metadata is None: metadata = {} + if additional_project_data is None: + additional_project_data = {} project_data = { "name": name, "realm": realm, "settings": settings, "metadata": metadata, "version": __version__, + **additional_project_data, } if project_id: project_data["_id"] = project_id From 28be19c5105c154b9566fd14f9ce7095549073e4 Mon Sep 17 00:00:00 2001 From: Joschka Thurner Date: Fri, 9 Feb 2024 11:28:11 +0100 Subject: [PATCH 48/84] make write_network_to_db return net metadata --- pandahub/lib/PandaHub.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/pandahub/lib/PandaHub.py b/pandahub/lib/PandaHub.py index 78f2e26..34ec719 100644 --- a/pandahub/lib/PandaHub.py +++ b/pandahub/lib/PandaHub.py @@ -3,6 +3,7 @@ import importlib import json import logging +import pymongo import traceback import warnings from inspect import signature, _empty @@ -385,7 +386,7 @@ def _get_project_document(self, filter_dict: dict) -> Optional[dict]: else: return project_doc - def _get_project_database(self): + def _get_project_database(self) -> pymongo.mongo_client: return self.mongo_client[str(self.active_project["_id"])] def _get_global_database(self): @@ -1068,6 +1069,7 @@ def write_network_to_db( if metadata is not None: net_dict.update(metadata) db["_networks"].insert_one(net_dict) + return net_dict def _write_net_collections_to_db(self, db, collections): for element, element_data in collections.items(): @@ -1725,6 +1727,7 @@ def get_variant_filter(self, variants): "$or": [ {"var_type": "base", "not_in_var": {"$nin": variants}}, { + # redundant? "var_type": {"$in": ["change", "addition"]}, "variant": {"$in": variants}, }, @@ -1737,6 +1740,7 @@ def get_variant_filter(self, variants): return { "$or": [ {"var_type": "base", "not_in_var": {"$ne": variants}}, + # var type redundant {"var_type": {"$in": ["change", "addition"]}, "variant": variants}, ] } From cfd145341030d6881c91ca61916ba609b8f2d4aa Mon Sep 17 00:00:00 2001 From: Joschka Thurner Date: Wed, 10 Apr 2024 09:56:50 +0200 Subject: [PATCH 49/84] allow passing index to create_variant --- pandahub/lib/PandaHub.py | 42 +++++++++++++++++++--------------------- 1 file changed, 20 insertions(+), 22 deletions(-) diff --git a/pandahub/lib/PandaHub.py b/pandahub/lib/PandaHub.py index 54d3ccd..ed8b1a3 100644 --- a/pandahub/lib/PandaHub.py +++ b/pandahub/lib/PandaHub.py @@ -390,10 +390,10 @@ def _get_project_document(self, filter_dict: dict) -> Optional[dict]: else: return project_doc - def _get_project_database(self) -> pymongo.mongo_client: + def _get_project_database(self) -> MongoClient: return self.mongo_client[str(self.active_project["_id"])] - def _get_global_database(self): + def _get_global_database(self) -> MongoClient: if ( self.mongo_client_global_db is None and SETTINGS.MONGODB_GLOBAL_DATABASE_URL is not None @@ -1652,23 +1652,23 @@ def _create_mongodb_indexes( # Variants # ------------------------- - def create_variant(self, data): + def create_variant(self, data, index: Optional[int] = None): db = self._get_project_database() net_id = int(data["net_id"]) - max_index = list( - db["variant"] - .find({"net_id": net_id}, projection={"_id": 0, "index": 1}) - .sort("index", -1) - .limit(1) - ) - if not max_index: - index = 1 - for coll in self._get_net_collections(db): - update = {"$set": {"var_type": "base", "not_in_var": []}} - db[coll].update_many({}, update) - - else: - index = int(max_index[0]["index"]) + 1 + if index is None: + max_index = list( + db["variant"] + .find({"net_id": net_id}, projection={"_id": 0, "index": 1}) + .sort("index", -1) + .limit(1) + ) + if not max_index: + index = 1 + for coll in self._get_net_collections(db): + update = {"$set": {"var_type": "base", "not_in_var": []}} + db[coll].update_many({}, update) + else: + index = int(max_index[0]["index"]) + 1 data["index"] = index @@ -1704,7 +1704,7 @@ def update_variant(self, net_id, index, data): db = self._get_project_database() db["variant"].update_one({"net_id": net_id, "index": index}, {"$set": data}) - def get_variant_filter(self, variants): + def get_variant_filter(self, variants: Optional[Union[list[int], int]]) -> dict: """ Creates a mongodb query filter to retrieve pandapower elements for the given variant(s). @@ -1718,11 +1718,9 @@ def get_variant_filter(self, variants): dict mongodb query filter for the given variant(s) """ - if type(variants) is list and variants: + if isinstance(variants, list): if len(variants) > 1: - variants = [ - int(var) for var in variants - ] # make sure variants are of type int + variants = [int(var) for var in variants] # make sure variants are of type int return { "$or": [ {"var_type": "base", "not_in_var": {"$nin": variants}}, From 2908596f6a3a1951a46f8617514d747954bd89e3 Mon Sep 17 00:00:00 2001 From: Joschka Thurner Date: Thu, 11 Apr 2024 09:26:10 +0200 Subject: [PATCH 50/84] alleviate race condition creating to non-unique net-ids --- pandahub/lib/PandaHub.py | 20 +++++++++++--------- 1 file changed, 11 insertions(+), 9 deletions(-) diff --git a/pandahub/lib/PandaHub.py b/pandahub/lib/PandaHub.py index ed8b1a3..0fdbb6e 100644 --- a/pandahub/lib/PandaHub.py +++ b/pandahub/lib/PandaHub.py @@ -1016,6 +1016,7 @@ def write_network_to_db( project_id=None, metadata=None, skip_results=False, + net_id=None, ): if project_id: self.set_active_project_by_id(project_id) @@ -1030,8 +1031,11 @@ def write_network_to_db( self.delete_net_from_db(name) else: raise PandaHubError("Network name already exists") - max_id_network = db["_networks"].find_one(sort=[("_id", -1)]) - _id = 0 if max_id_network is None else max_id_network["_id"] + 1 + + if net_id is None: + max_id_network = db["_networks"].find_one(sort=[("_id", -1)]) + net_id = 0 if max_id_network is None else max_id_network["_id"] + 1 + db["_networks"].insert_one({"_id": net_id}) data = {} dtypes = {} @@ -1048,7 +1052,7 @@ def write_network_to_db( continue # convert pandapower dataframe object to dict and save to db element_data = convert_element_to_dict( - element_data.copy(deep=True), _id, self._datatypes.get(element) + element_data.copy(deep=True), net_id, self._datatypes.get(element) ) self._write_element_to_db(db, element, element_data) @@ -1058,18 +1062,16 @@ def write_network_to_db( data[element] = element_data # write network metadata - net_dict = { - "_id": _id, + network_data = { "name": name, "sector": sector, "dtypes": dtypes, "data": data, } - if metadata is not None: - net_dict.update(metadata) - db["_networks"].insert_one(net_dict) - return net_dict + network_data.update(metadata) + db["_networks"].update_one({"_id": net_id}, {"$set": network_data}) + return network_data | {"_id": net_id} def _write_net_collections_to_db(self, db, collections): for element, element_data in collections.items(): From 3866947f6e7bf9194310a8fb428c72508f18a8b9 Mon Sep 17 00:00:00 2001 From: Jan Ulffers Date: Thu, 11 Apr 2024 14:19:19 +0200 Subject: [PATCH 51/84] changed default value of variants from [] to None in PandaHub method get_net_from_db to ensure compatibility with commit cfd1453 --- pandahub/lib/PandaHub.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pandahub/lib/PandaHub.py b/pandahub/lib/PandaHub.py index 2f72110..b9be34c 100644 --- a/pandahub/lib/PandaHub.py +++ b/pandahub/lib/PandaHub.py @@ -660,7 +660,7 @@ def get_net_from_db( only_tables=None, project_id=None, geo_mode="string", - variants=[], + variants=None, ): if project_id: self.set_active_project_by_id(project_id) @@ -2275,7 +2275,7 @@ def get_timeseries_from_db( else: data = data[0] if compressed_ts_data: - timeseries_data = decompress_timeseries_data(data["timeseries_data"], + timeseries_data = decompress_timeseries_data(data["timeseries_data"], ts_format, num_timestamps=data["num_timestamps"]) else: From 010faa5363df5c5d6eb6a08dbc1a987ed8b395ea Mon Sep 17 00:00:00 2001 From: Joschka Thurner Date: Mon, 15 Apr 2024 13:25:49 +0200 Subject: [PATCH 52/84] make variants more robust (#46) * start deprecation of filtering for multiple variants * make variant creation more robust * add "base" as default value for var_type --- pandahub/lib/PandaHub.py | 64 ++++++++++++++++++++++++---------------- 1 file changed, 38 insertions(+), 26 deletions(-) diff --git a/pandahub/lib/PandaHub.py b/pandahub/lib/PandaHub.py index b9be34c..5518a51 100644 --- a/pandahub/lib/PandaHub.py +++ b/pandahub/lib/PandaHub.py @@ -101,7 +101,6 @@ def __init__( self.user_id = user_id self.base_variant_filter = { "$or": [ - {"var_type": {"$exists": False}}, {"var_type": "base"}, {"var_type": None}, {"var_type": np.nan}, @@ -680,7 +679,7 @@ def get_net_from_db_by_id( only_tables=None, convert=True, geo_mode="string", - variants=[], + variants=None, ): self.check_permission("read") return self._get_net_from_db_by_id( @@ -699,7 +698,7 @@ def _get_net_from_db_by_id( only_tables=None, convert=True, geo_mode="string", - variants=[], + variants=None, ): db = self._get_project_database() meta = self._get_network_metadata(db, id_) @@ -749,7 +748,7 @@ def get_subnet_from_db( include_results=True, add_edge_branches=True, geo_mode="string", - variants=[], + variants=None, additional_filters: dict[ str, Callable[[pp.auxiliary.pandapowerNet], dict] ] = {}, @@ -776,7 +775,7 @@ def get_subnet_from_db_by_id( include_results=True, add_edge_branches=True, geo_mode="string", - variants=[], + variants=None, ignore_elements=[], additional_filters: dict[ str, Callable[[pp.auxiliary.pandapowerNet], dict] @@ -1050,10 +1049,12 @@ def write_network_to_db( dtypes[element] = get_dtypes(element_data, self._datatypes.get(element)) if element_data.empty: continue - # convert pandapower dataframe object to dict and save to db - element_data = convert_element_to_dict( - element_data.copy(deep=True), net_id, self._datatypes.get(element) - ) + element_data = element_data.copy(deep=True) + if "var_type" in element_data: + element_data["var_type"] = element_data["var_type"].fillna("base") + else: + element_data["var_type"] = "base" + element_data = convert_element_to_dict(element_data, net_id, self._datatypes.get(element)) self._write_element_to_db(db, element, element_data) else: @@ -1144,7 +1145,7 @@ def _add_element_from_collection( include_results=True, only_tables=None, geo_mode="string", - variants=[], + variants=None, dtypes=None, ): if only_tables is not None and not element_type in only_tables: @@ -1664,22 +1665,21 @@ def create_variant(self, data, index: Optional[int] = None): .sort("index", -1) .limit(1) ) - if not max_index: - index = 1 - for coll in self._get_net_collections(db): - update = {"$set": {"var_type": "base", "not_in_var": []}} - db[coll].update_many({}, update) - else: - index = int(max_index[0]["index"]) + 1 + index = int(max_index[0]["index"]) + 1 if max_index else 1 data["index"] = index - if data.get("default_name") is not None and data.get("name") is None: data["name"] = data.pop("default_name") + " " + str(index) - db["variant"].insert_one(data) del data["_id"] + if index == 1: + for coll in self._get_net_collections(db): + update = {"$set": {"var_type": "base", "not_in_var": []}} + db[coll].update_many( + {"$or": [{"var_type": None}, {"var_type": np.nan}]}, + {"$set": {"var_type": "base"}} + ) return data def delete_variant(self, net_id, index): @@ -1706,7 +1706,7 @@ def update_variant(self, net_id, index, data): db = self._get_project_database() db["variant"].update_one({"net_id": net_id, "index": index}, {"$set": data}) - def get_variant_filter(self, variants: Optional[Union[list[int], int]]) -> dict: + def get_variant_filter(self, variants: Optional[int]) -> dict: """ Creates a mongodb query filter to retrieve pandapower elements for the given variant(s). @@ -1721,26 +1721,38 @@ def get_variant_filter(self, variants: Optional[Union[list[int], int]]) -> dict: mongodb query filter for the given variant(s) """ if isinstance(variants, list): - if len(variants) > 1: - variants = [int(var) for var in variants] # make sure variants are of type int + warnings.warn( + f"Passing variants as list is deprecated, use None or int instead (variants: {variants})", + DeprecationWarning, + stacklevel=2, + ) + if len(variants) == 0: + variants = None + elif len(variants) == 1: + variants = variants[0] + elif len(variants) > 1: + warnings.warn( + f"Passing multiple variants is deprecated, use None or int instead (variants: {variants})", + DeprecationWarning, + stacklevel=2, + ) + variants = [ + int(var) for var in variants + ] # make sure variants are of type int return { "$or": [ {"var_type": "base", "not_in_var": {"$nin": variants}}, { - # redundant? "var_type": {"$in": ["change", "addition"]}, "variant": {"$in": variants}, }, ] } - else: - variants = variants[0] if variants: variants = int(variants) return { "$or": [ {"var_type": "base", "not_in_var": {"$ne": variants}}, - # var type redundant {"var_type": {"$in": ["change", "addition"]}, "variant": variants}, ] } From 34288a8712af168507d21737c352b53b56d87dab Mon Sep 17 00:00:00 2001 From: Joschka Thurner Date: Mon, 15 Apr 2024 13:54:30 +0200 Subject: [PATCH 53/84] add back default not_in_var field on variant init --- pandahub/lib/PandaHub.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/pandahub/lib/PandaHub.py b/pandahub/lib/PandaHub.py index 5518a51..d3569f3 100644 --- a/pandahub/lib/PandaHub.py +++ b/pandahub/lib/PandaHub.py @@ -1675,10 +1675,9 @@ def create_variant(self, data, index: Optional[int] = None): if index == 1: for coll in self._get_net_collections(db): - update = {"$set": {"var_type": "base", "not_in_var": []}} db[coll].update_many( {"$or": [{"var_type": None}, {"var_type": np.nan}]}, - {"$set": {"var_type": "base"}} + {"$set": {"var_type": "base", "not_in_var": []}} ) return data From 9787cef006a81df54a51601ee0a4457568613274 Mon Sep 17 00:00:00 2001 From: Leon Thurner Date: Wed, 24 Apr 2024 23:23:57 +0200 Subject: [PATCH 54/84] remove verbose logging in set_net_value_in_db --- pandahub/lib/PandaHub.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/pandahub/lib/PandaHub.py b/pandahub/lib/PandaHub.py index d3569f3..30fdaab 100644 --- a/pandahub/lib/PandaHub.py +++ b/pandahub/lib/PandaHub.py @@ -1355,9 +1355,6 @@ def set_net_value_in_db( project_id=None, **kwargs, ): - logger.info( - f"Setting {parameter} = {value} in {element_type} with index {element_index} and variant {variant}" - ) if variant is not None: variant = int(variant) if project_id: From 9a46f0067cfdb67442178075876ad2b505866920 Mon Sep 17 00:00:00 2001 From: Joschka Thurner Date: Sat, 20 Apr 2024 00:45:43 +0200 Subject: [PATCH 55/84] add some types --- pandahub/lib/PandaHub.py | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/pandahub/lib/PandaHub.py b/pandahub/lib/PandaHub.py index 30fdaab..a6b3e0e 100644 --- a/pandahub/lib/PandaHub.py +++ b/pandahub/lib/PandaHub.py @@ -8,7 +8,7 @@ import warnings from inspect import signature, _empty from collections.abc import Callable -from typing import Optional, Union +from typing import Optional, Union, TypeVar import numpy as np import pandas as pd @@ -61,6 +61,8 @@ def __init__(self, message, status_code=400): # PandaHub # ------------------------- +ProjectID = TypeVar("ProjectID", str, int, ObjectId) +SettingsValue = TypeVar("SettingsValue", str, int, float, list, dict) class PandaHub: permissions = { @@ -282,7 +284,7 @@ def get_projects(self): for p in projects ] - def set_active_project(self, project_name, realm=None): + def set_active_project(self, project_name:str, realm=None): projects = self.get_projects() active_projects = [ project for project in projects if project["name"] == project_name @@ -295,7 +297,7 @@ def set_active_project(self, project_name, realm=None): project_id = active_projects[0]["id"] self.set_active_project_by_id(project_id) - def set_active_project_by_id(self, project_id): + def set_active_project_by_id(self, project_id:ProjectID): try: project_id = ObjectId(project_id) except InvalidId: @@ -304,7 +306,7 @@ def set_active_project_by_id(self, project_id): if self.active_project is None: raise PandaHubError("Project not found!", 404) - def rename_project(self, project_name): + def rename_project(self, project_name:str): self.has_permission("write") project_collection = self.mongo_client["user_management"].projects realm = self.active_project["realm"] @@ -363,7 +365,7 @@ def force_unlock_project(self, project_id): else: raise PandaHubError("You don't have rights to access this project", 403) - def project_exists(self, project_name=None, realm=None): + def project_exists(self, project_name:Optional[str]=None, realm=None): project_collection = self.mongo_client["user_management"].projects project = project_collection.find_one({"name": project_name, "realm": realm}) return project is not None From 436e0c7c49a6d0e2591b22be010120968bf67385 Mon Sep 17 00:00:00 2001 From: Joschka Thurner Date: Tue, 21 May 2024 21:43:54 +0200 Subject: [PATCH 56/84] add network id getter, make project db getter public --- pandahub/api/routers/variants.py | 7 +++---- pandahub/lib/PandaHub.py | 33 +++++++++++++++++++++++++++++++- 2 files changed, 35 insertions(+), 5 deletions(-) diff --git a/pandahub/api/routers/variants.py b/pandahub/api/routers/variants.py index 220fbb0..2fff2fb 100644 --- a/pandahub/api/routers/variants.py +++ b/pandahub/api/routers/variants.py @@ -21,11 +21,10 @@ class GetVariantsModel(BaseModel): @router.post("/get_variants") def get_variants(data: GetVariantsModel, ph=Depends(pandahub)): - project_id = data.project_id - ph.set_active_project_by_id(project_id) - db = ph._get_project_database() + ph.set_active_project_by_id(data.project_id) + variants_collection = ph.get_project_database("variant") - variants = db["variant"].find({"net_id": data.net_id}, projection={"_id": 0}) + variants = variants_collection.find({"net_id": data.net_id}, projection={"_id": 0}) response = {} for var in variants: response[var.pop("index")] = var diff --git a/pandahub/lib/PandaHub.py b/pandahub/lib/PandaHub.py index a6b3e0e..19df90a 100644 --- a/pandahub/lib/PandaHub.py +++ b/pandahub/lib/PandaHub.py @@ -392,7 +392,25 @@ def _get_project_document(self, filter_dict: dict) -> Optional[dict]: return project_doc def _get_project_database(self) -> MongoClient: - return self.mongo_client[str(self.active_project["_id"])] + return self.get_project_database() + + def get_project_database(self, collection: Optional[str] = None) -> MongoClient: + """ + Get a MongoClient instance connected to the database for the current active project, optionally set to the given collection. + + Parameters + ---------- + collection + Name of document collection + + Returns + ------- + MongoClient + """ + project_db = self.mongo_client[str(self.active_project["_id"])] + if collection is None: + return project_db + return project_db[collection] def _get_global_database(self) -> MongoClient: if ( @@ -414,6 +432,19 @@ def _get_global_database(self) -> MongoClient: else: return self.mongo_client_global_db["global_data"] + def get_network_ids(self) -> list[int]: + """ + Retrieve the id's of all networks in the active project. + + Returns + ------- + list + network ids + """ + if not self.active_project: + raise PandaHubError("No project activated!") + return self.get_project_database("_network").find({}, {"_id:": 1}).distinct("_id") + def get_project_version(self): return self.active_project.get("version", "0.2.2") From 619c862abb27910af442d9c84d4c5179c10b0bab Mon Sep 17 00:00:00 2001 From: Joschka Thurner Date: Tue, 4 Jun 2024 12:09:12 +0200 Subject: [PATCH 57/84] fix user created as active when admin approval is required --- pandahub/api/internal/db.py | 2 ++ pandahub/api/internal/schemas.py | 2 ++ 2 files changed, 4 insertions(+) diff --git a/pandahub/api/internal/db.py b/pandahub/api/internal/db.py index b87fc48..a066e54 100644 --- a/pandahub/api/internal/db.py +++ b/pandahub/api/internal/db.py @@ -9,6 +9,7 @@ BeanieBaseAccessToken, ) +from pandahub.api.internal.settings import REGISTRATION_ADMIN_APPROVAL from pandahub.api.internal import settings from pydantic import Field @@ -22,6 +23,7 @@ class User(BeanieBaseUser, Document): id: uuid.UUID = Field(default_factory=uuid.uuid4) + is_active: bool = not REGISTRATION_ADMIN_APPROVAL class Settings(BeanieBaseUser.Settings): name = "users" diff --git a/pandahub/api/internal/schemas.py b/pandahub/api/internal/schemas.py index 63b4f57..60bc00d 100644 --- a/pandahub/api/internal/schemas.py +++ b/pandahub/api/internal/schemas.py @@ -7,8 +7,10 @@ class UserRead(schemas.BaseUser[uuid.UUID]): pass + class UserCreate(schemas.BaseUserCreate): is_active: bool = not REGISTRATION_ADMIN_APPROVAL + class UserUpdate(schemas.BaseUserUpdate): pass From e816bb67c038ffbf165103eae721dd95dcc3743b Mon Sep 17 00:00:00 2001 From: Joschka Thurner Date: Wed, 5 Jun 2024 12:40:13 +0200 Subject: [PATCH 58/84] add more mongodb indexes for variants --- pandahub/lib/mongodb_indexes.py | 34 ++++++++++++++++++++++++++++++++- 1 file changed, 33 insertions(+), 1 deletion(-) diff --git a/pandahub/lib/mongodb_indexes.py b/pandahub/lib/mongodb_indexes.py index 951ef9b..72241e8 100644 --- a/pandahub/lib/mongodb_indexes.py +++ b/pandahub/lib/mongodb_indexes.py @@ -1,44 +1,57 @@ from pymongo import DESCENDING, GEOSPHERE, IndexModel +VARIANT_INDEXES = [ + IndexModel([("variant", DESCENDING)]), + IndexModel([("var_type", DESCENDING)]), + IndexModel([("not_in_var", DESCENDING)]), +] mongodb_indexes = { # pandapower "net_bus": [ IndexModel([("net_id", DESCENDING), ("index", DESCENDING), ("variant", DESCENDING)], unique=True), IndexModel([("geo", GEOSPHERE)]), + *VARIANT_INDEXES, ], "net_line": [ IndexModel([("net_id", DESCENDING), ("index", DESCENDING), ("variant", DESCENDING)], unique=True), IndexModel([("from_bus", DESCENDING)]), IndexModel([("to_bus", DESCENDING)]), IndexModel([("geo", GEOSPHERE)]), + *VARIANT_INDEXES, ], "net_trafo":[ IndexModel([("net_id", DESCENDING), ("index", DESCENDING), ("variant", DESCENDING)], unique=True), IndexModel([("hv_bus", DESCENDING)]), IndexModel([("lv_bus", DESCENDING)]), + *VARIANT_INDEXES, ], "net_switch": [ IndexModel([("net_id", DESCENDING), ("index", DESCENDING), ("variant", DESCENDING)], unique=True), IndexModel([("bus", DESCENDING)]), IndexModel([("element", DESCENDING)]), IndexModel([("et", DESCENDING)]), + *VARIANT_INDEXES, ], "net_load": [ IndexModel([("net_id", DESCENDING), ("index", DESCENDING), ("variant", DESCENDING)], unique=True), IndexModel([("bus", DESCENDING)]), + *VARIANT_INDEXES, ], "net_sgen": [ IndexModel([("net_id", DESCENDING), ("index", DESCENDING), ("variant", DESCENDING)], unique=True), IndexModel([("bus", DESCENDING)]), + *VARIANT_INDEXES, ], "net_gen": [ IndexModel([("net_id", DESCENDING), ("index", DESCENDING), ("variant", DESCENDING)], unique=True), IndexModel([("bus", DESCENDING)]), + *VARIANT_INDEXES, ], "net_ext_grid": [ IndexModel([("net_id", DESCENDING), ("index", DESCENDING), ("variant", DESCENDING)], unique=True), IndexModel([("bus", DESCENDING)]), IndexModel([("junction", DESCENDING)]), + *VARIANT_INDEXES, ], "net_shunt": [ IndexModel([("net_id", DESCENDING), ("index", DESCENDING), ("variant", DESCENDING)], unique=True), @@ -47,87 +60,105 @@ "net_xward": [ IndexModel([("net_id", DESCENDING), ("index", DESCENDING), ("variant", DESCENDING)], unique=True), IndexModel([("bus", DESCENDING)]), + *VARIANT_INDEXES, ], "net_ward": [ IndexModel([("net_id", DESCENDING), ("index", DESCENDING), ("variant", DESCENDING)], unique=True), IndexModel([("bus", DESCENDING)]), + *VARIANT_INDEXES, ], "net_motor": [ IndexModel([("net_id", DESCENDING), ("index", DESCENDING), ("variant", DESCENDING)], unique=True), IndexModel([("bus", DESCENDING)]), + *VARIANT_INDEXES, ], "net_storage": [ IndexModel([("net_id", DESCENDING), ("index", DESCENDING), ("variant", DESCENDING)], unique=True), IndexModel([("bus", DESCENDING)]), + *VARIANT_INDEXES, ], # pandapipes "net_junction": [ IndexModel([("net_id", DESCENDING), ("index", DESCENDING), ("variant", DESCENDING)], unique=True), IndexModel([("geo", GEOSPHERE)]), + *VARIANT_INDEXES, ], "net_pipe": [ IndexModel([("net_id", DESCENDING), ("index", DESCENDING), ("variant", DESCENDING)], unique=True), IndexModel([("from_junction", DESCENDING)]), IndexModel([("to_junction", DESCENDING)]), IndexModel([("geo", GEOSPHERE)]), + *VARIANT_INDEXES, ], "net_valve": [ IndexModel([("net_id", DESCENDING), ("index", DESCENDING), ("variant", DESCENDING)], unique=True), IndexModel([("from_junction", DESCENDING)]), IndexModel([("to_junction", DESCENDING)]), + *VARIANT_INDEXES, ], "net_sink": [ IndexModel([("net_id", DESCENDING), ("index", DESCENDING), ("variant", DESCENDING)], unique=True), IndexModel([("junction", DESCENDING)]), + *VARIANT_INDEXES, ], "net_source": [ IndexModel([("net_id", DESCENDING), ("index", DESCENDING), ("variant", DESCENDING)], unique=True), IndexModel([("junction", DESCENDING)]), + *VARIANT_INDEXES, ], "net_water_tank": [ IndexModel([("net_id", DESCENDING), ("index", DESCENDING), ("variant", DESCENDING)], unique=True), IndexModel([("junction", DESCENDING)]), + *VARIANT_INDEXES, ], "net_flow_control": [ IndexModel([("net_id", DESCENDING), ("index", DESCENDING), ("variant", DESCENDING)], unique=True), IndexModel([("from_junction", DESCENDING)]), IndexModel([("to_junction", DESCENDING)]), + *VARIANT_INDEXES, ], "net_press_control": [ IndexModel([("net_id", DESCENDING), ("index", DESCENDING), ("variant", DESCENDING)], unique=True), IndexModel([("from_junction", DESCENDING)]), IndexModel([("to_junction", DESCENDING)]), + *VARIANT_INDEXES, ], "net_compressor": [ IndexModel([("net_id", DESCENDING), ("index", DESCENDING), ("variant", DESCENDING)], unique=True), IndexModel([("from_junction", DESCENDING)]), IndexModel([("to_junction", DESCENDING)]), + *VARIANT_INDEXES, ], "net_pump": [ IndexModel([("net_id", DESCENDING), ("index", DESCENDING), ("variant", DESCENDING)], unique=True), IndexModel([("from_junction", DESCENDING)]), IndexModel([("to_junction", DESCENDING)]), + *VARIANT_INDEXES, ], "net_circ_pump_mass": [ IndexModel([("net_id", DESCENDING), ("index", DESCENDING), ("variant", DESCENDING)], unique=True), IndexModel([("return_junction", DESCENDING)]), IndexModel([("flow_junction", DESCENDING)]), + *VARIANT_INDEXES, ], "net_circ_pump_pressure": [ IndexModel([("net_id", DESCENDING), ("index", DESCENDING), ("variant", DESCENDING)], unique=True), IndexModel([("return_junction", DESCENDING)]), IndexModel([("flow_junction", DESCENDING)]), + *VARIANT_INDEXES, ], "net_heat_exchanger": [ IndexModel([("net_id", DESCENDING), ("index", DESCENDING), ("variant", DESCENDING)], unique=True), IndexModel([("from_junction", DESCENDING)]), IndexModel([("to_junction", DESCENDING)]), + *VARIANT_INDEXES, ], "net_heat_consumer": [ IndexModel([("net_id", DESCENDING), ("index", DESCENDING), ("variant", DESCENDING)], unique=True), IndexModel([("from_junction", DESCENDING)]), IndexModel([("to_junction", DESCENDING)]), + *VARIANT_INDEXES, ], # others @@ -147,7 +178,7 @@ IndexModel([("substation_buses", DESCENDING)]), IndexModel([("level", DESCENDING)]), IndexModel([("geo", GEOSPHERE)]), - IndexModel([("variant", DESCENDING)]), + *VARIANT_INDEXES, ], "net_substation": [ IndexModel( @@ -158,5 +189,6 @@ IndexModel([("type", DESCENDING)]), IndexModel([("level", DESCENDING)]), IndexModel([("geo", GEOSPHERE)]), + *VARIANT_INDEXES, ], } From 66ab37b701ff9ca86110af5110fa867c6b261cbb Mon Sep 17 00:00:00 2001 From: Leon Thurner Date: Tue, 11 Jun 2024 09:09:47 +0200 Subject: [PATCH 59/84] allow finer settings of add_edge_branches with a list --- pandahub/lib/PandaHub.py | 20 ++++++++++++++------ 1 file changed, 14 insertions(+), 6 deletions(-) diff --git a/pandahub/lib/PandaHub.py b/pandahub/lib/PandaHub.py index 19df90a..912d0ab 100644 --- a/pandahub/lib/PandaHub.py +++ b/pandahub/lib/PandaHub.py @@ -837,7 +837,14 @@ def get_subnet_from_db_by_id( ) buses = net.bus.index.tolist() - branch_operator = "$or" if add_edge_branches else "$and" + if isinstance(add_edge_branches, bool): + if add_edge_branches: + add_edge_branches = ["line", "trafo", "switch"] + else: + add_edge_branches = [] + elif not isinstance(add_edge_branches, list): + raise ValueError("add_edge_branches must be a list or a boolean") + line_operator = "$or" if "line" in add_edge_branches else "$and" # Add branch elements connected to at least one bus self._add_element_from_collection( net, @@ -845,7 +852,7 @@ def get_subnet_from_db_by_id( "line", net_id, { - branch_operator: [ + line_operator: [ {"from_bus": {"$in": buses}}, {"to_bus": {"$in": buses}}, ] @@ -854,12 +861,13 @@ def get_subnet_from_db_by_id( variants=variants, dtypes=dtypes, ) + trafo_operator = "$or" if "trafo" in add_edge_branches else "$and" self._add_element_from_collection( net, db, "trafo", net_id, - {branch_operator: [{"hv_bus": {"$in": buses}}, {"lv_bus": {"$in": buses}}]}, + {trafo_operator: [{"hv_bus": {"$in": buses}}, {"lv_bus": {"$in": buses}}]}, geo_mode=geo_mode, variants=variants, dtypes=dtypes, @@ -870,7 +878,7 @@ def get_subnet_from_db_by_id( "trafo3w", net_id, { - branch_operator: [ + trafo_operator: [ {"hv_bus": {"$in": buses}}, {"mv_bus": {"$in": buses}}, {"lv_bus": {"$in": buses}}, @@ -880,7 +888,7 @@ def get_subnet_from_db_by_id( variants=variants, dtypes=dtypes, ) - + switch_operator = "$or" if "switch" in add_edge_branches else "$and" self._add_element_from_collection( net, db, @@ -890,7 +898,7 @@ def get_subnet_from_db_by_id( "$and": [ {"et": "b"}, { - branch_operator: [ + switch_operator: [ {"bus": {"$in": buses}}, {"element": {"$in": buses}}, ] From 514682bc3b96f14c1403d3ecb2a8ae4743828149 Mon Sep 17 00:00:00 2001 From: Leon Thurner Date: Tue, 11 Jun 2024 09:10:11 +0200 Subject: [PATCH 60/84] set index on substation field in bus collection for get_net queries --- pandahub/lib/mongodb_indexes.py | 1 + 1 file changed, 1 insertion(+) diff --git a/pandahub/lib/mongodb_indexes.py b/pandahub/lib/mongodb_indexes.py index 72241e8..27f8413 100644 --- a/pandahub/lib/mongodb_indexes.py +++ b/pandahub/lib/mongodb_indexes.py @@ -10,6 +10,7 @@ "net_bus": [ IndexModel([("net_id", DESCENDING), ("index", DESCENDING), ("variant", DESCENDING)], unique=True), IndexModel([("geo", GEOSPHERE)]), + IndexModel([("substation", DESCENDING)]), *VARIANT_INDEXES, ], "net_line": [ From 4dc69929e52e883291ea4bc91c1a21f3088260e1 Mon Sep 17 00:00:00 2001 From: Joschka Thurner Date: Thu, 20 Jun 2024 09:11:27 +0200 Subject: [PATCH 61/84] add mongodb_indexes and datatypes as arguments to PandaHub() (#48) --- pandahub/api/internal/settings.py | 1 - pandahub/lib/PandaHub.py | 51 +++++++++++++++++++------------ pandahub/lib/database_toolbox.py | 8 ++--- pandahub/lib/datatypes.py | 2 +- pandahub/lib/mongodb_indexes.py | 2 +- 5 files changed, 37 insertions(+), 27 deletions(-) diff --git a/pandahub/api/internal/settings.py b/pandahub/api/internal/settings.py index 0fdb121..aaad9da 100644 --- a/pandahub/api/internal/settings.py +++ b/pandahub/api/internal/settings.py @@ -47,5 +47,4 @@ def get_secret(key, default=None): REGISTRATION_ENABLED = settings_bool("REGISTRATION_ENABLED", default=True) REGISTRATION_ADMIN_APPROVAL = settings_bool("REGISTRATION_ADMIN_APPROVAL", default=False) -DATATYPES_MODULE = os.getenv("DATATYPES_MODULE") or "pandahub.lib.datatypes" CREATE_INDEXES_WITH_PROJECT = settings_bool("CREATE_INDEXES_WITH_PROJECT", default=True) diff --git a/pandahub/lib/PandaHub.py b/pandahub/lib/PandaHub.py index 912d0ab..18766fd 100644 --- a/pandahub/lib/PandaHub.py +++ b/pandahub/lib/PandaHub.py @@ -1,10 +1,7 @@ # -*- coding: utf-8 -*- import builtins -import importlib import json import logging -import pymongo -import traceback import warnings from inspect import signature, _empty from collections.abc import Callable @@ -24,7 +21,8 @@ from pandapipes import from_json_string as from_json_pps, FromSerializableRegistryPpipe import pandapower as pp import pandapower.io_utils as io_pp -import pandahub.api.internal.settings as SETTINGS +from pandahub.api.internal.settings import MONGODB_URL, MONGODB_USER, MONGODB_PASSWORD, MONGODB_GLOBAL_DATABASE_URL, \ + MONGODB_GLOBAL_DATABASE_USER, MONGODB_GLOBAL_DATABASE_PASSWORD, CREATE_INDEXES_WITH_PROJECT from pandahub.lib.database_toolbox import ( create_timeseries_document, convert_timeseries_to_subdocuments, @@ -35,10 +33,11 @@ decompress_timeseries_data, convert_geojsons, ) -from pandahub.lib.mongodb_indexes import mongodb_indexes +from pandahub.lib.mongodb_indexes import MONGODB_INDEXES logger = logging.getLogger(__name__) from pandahub import __version__ +from pandahub.lib.datatypes import DATATYPES from packaging import version @@ -71,21 +70,19 @@ class PandaHub: "user_management": ["owner"], } - _datatypes = getattr( - importlib.import_module(SETTINGS.DATATYPES_MODULE), "datatypes" - ) - # ------------------------- # Initialization # ------------------------- def __init__( self, - connection_url=SETTINGS.MONGODB_URL, - connection_user=SETTINGS.MONGODB_USER, - connection_password=SETTINGS.MONGODB_PASSWORD, + connection_url=MONGODB_URL, + connection_user=MONGODB_USER, + connection_password=MONGODB_PASSWORD, check_server_available=False, user_id=None, + datatypes=DATATYPES, + mongodb_indexes=MONGODB_INDEXES, ): mongo_client_args = { "host": connection_url, @@ -97,6 +94,8 @@ def __init__( "username": connection_user, "password": connection_password, } + self._datatypes = datatypes + self.mongodb_indexes = mongodb_indexes self.mongo_client = MongoClient(**mongo_client_args) self.mongo_client_global_db = None self.active_project = None @@ -108,7 +107,6 @@ def __init__( {"var_type": np.nan}, ] } - self.mongodb_indexes = mongodb_indexes if check_server_available: self.server_is_available() @@ -237,7 +235,7 @@ def create_project( if self.user_id is not None: project_data["users"] = {self.user_id: "owner"} self.mongo_client["user_management"]["projects"].insert_one(project_data) - if SETTINGS.CREATE_INDEXES_WITH_PROJECT: + if CREATE_INDEXES_WITH_PROJECT: self._create_mongodb_indexes(project_data["_id"]) if activate: self.set_active_project_by_id(project_data["_id"]) @@ -275,6 +273,7 @@ def get_projects(self): "settings": p["settings"], "locked": p.get("locked"), "locked_by": p.get("locked_by"), + "locked_reason": p.get("locked_reason"), "permissions": self.get_permissions_by_role( p.get("users").get(self.user_id) ) @@ -340,6 +339,20 @@ def lock_project(self): ) return result.acknowledged and result.modified_count > 0 + + # def lock_project(self): + # db = self.mongo_client["user_management"]["projects"] + # result = db.update_one( + # + # result = db.find_one_and_update( + # { + # "_id": self.active_project["_id"], + # "_id": self.active_project["_id"], "locked": False + # }, + # {"$set": {"locked": True, "locked_by": self.user_id}}, + # ) + # return result.acknowledged and result.modified_count > 0 + def unlock_project(self): db = self.mongo_client["user_management"]["projects"] return db.update_one( @@ -415,16 +428,16 @@ def get_project_database(self, collection: Optional[str] = None) -> MongoClient: def _get_global_database(self) -> MongoClient: if ( self.mongo_client_global_db is None - and SETTINGS.MONGODB_GLOBAL_DATABASE_URL is not None + and MONGODB_GLOBAL_DATABASE_URL is not None ): mongo_client_args = { - "host": SETTINGS.MONGODB_GLOBAL_DATABASE_URL, + "host": MONGODB_GLOBAL_DATABASE_URL, "uuidRepresentation": "standard", } - if SETTINGS.MONGODB_GLOBAL_DATABASE_USER: + if MONGODB_GLOBAL_DATABASE_USER: mongo_client_args |= { - "username": SETTINGS.MONGODB_GLOBAL_DATABASE_USER, - "password": SETTINGS.MONGODB_GLOBAL_DATABASE_PASSWORD, + "username": MONGODB_GLOBAL_DATABASE_USER, + "password": MONGODB_GLOBAL_DATABASE_PASSWORD, } self.mongo_client_global_db = MongoClient(**mongo_client_args) if self.mongo_client_global_db is None: diff --git a/pandahub/lib/database_toolbox.py b/pandahub/lib/database_toolbox.py index 4f5826f..c7ffced 100644 --- a/pandahub/lib/database_toolbox.py +++ b/pandahub/lib/database_toolbox.py @@ -3,7 +3,7 @@ import numpy as np import pandas as pd -from pandahub.api.internal import settings +from pandahub.lib.datatypes import DATATYPES import base64 import hashlib import logging @@ -128,7 +128,7 @@ def compress_timeseries_data(timeseries_data, ts_format): def decompress_timeseries_data(timeseries_data, ts_format, num_timestamps): import blosc if ts_format == "timestamp_value": - data = np.frombuffer(blosc.decompress(timeseries_data), + data = np.frombuffer(blosc.decompress(timeseries_data), dtype=np.float64).reshape((num_timestamps, 2), order="F") return pd.Series(data[:,1], index=pd.to_datetime(data[:,0])) @@ -233,10 +233,8 @@ def convert_element_to_dict(element_data, net_id, default_dtypes=None): load_geojsons(element_data) return element_data.to_dict(orient="records") -def convert_dataframes_to_dicts(net, net_id, version_, datatypes=None): - if datatypes is None: - datatypes = getattr(importlib.import_module(settings.DATATYPES_MODULE), "datatypes") +def convert_dataframes_to_dicts(net, net_id, version_, datatypes=DATATYPES): dataframes = {} other_parameters = {} types = {} diff --git a/pandahub/lib/datatypes.py b/pandahub/lib/datatypes.py index bfe10b9..896a609 100644 --- a/pandahub/lib/datatypes.py +++ b/pandahub/lib/datatypes.py @@ -1,5 +1,5 @@ -datatypes = { +DATATYPES = { "bus": { "name": str, "vn_kv": float, diff --git a/pandahub/lib/mongodb_indexes.py b/pandahub/lib/mongodb_indexes.py index 27f8413..ed84334 100644 --- a/pandahub/lib/mongodb_indexes.py +++ b/pandahub/lib/mongodb_indexes.py @@ -5,7 +5,7 @@ IndexModel([("var_type", DESCENDING)]), IndexModel([("not_in_var", DESCENDING)]), ] -mongodb_indexes = { +MONGODB_INDEXES = { # pandapower "net_bus": [ IndexModel([("net_id", DESCENDING), ("index", DESCENDING), ("variant", DESCENDING)], unique=True), From 48cf113ae4155cff98f815c33e354ad356f5e366 Mon Sep 17 00:00:00 2001 From: Joschka Thurner Date: Thu, 20 Jun 2024 09:16:31 +0200 Subject: [PATCH 62/84] bump pandahub version to 0.3.3 --- pandahub/__init__.py | 2 +- setup.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/pandahub/__init__.py b/pandahub/__init__.py index 85e371f..5b774fd 100644 --- a/pandahub/__init__.py +++ b/pandahub/__init__.py @@ -1,4 +1,4 @@ -__version__ = "0.3.2" +__version__ = "0.3.3" from pandahub.lib.PandaHub import PandaHub, PandaHubError from pandahub.client.PandaHubClient import PandaHubClient diff --git a/setup.py b/setup.py index 7e0a8c7..003b4ae 100644 --- a/setup.py +++ b/setup.py @@ -36,7 +36,7 @@ name='pandahub', packages=find_packages(), url='https://github.com/e2nIEE/pandahub', - version='0.3.2', + version='0.3.3', include_package_data=True, long_description_content_type='text/markdown', zip_safe=False, From 459c645c191031fc339a1820afe8705e3b29cc5f Mon Sep 17 00:00:00 2001 From: mvogt Date: Wed, 26 Jun 2024 10:50:16 +0200 Subject: [PATCH 63/84] update of scripts in release.yml --- .github/workflows/release.yml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index b1ae3c1..9ba41b3 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -26,10 +26,10 @@ jobs: # Steps represent a sequence of tasks that will be executed as part of the job steps: # Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 # Sets up python3 - - uses: actions/setup-python@v4 + - uses: actions/setup-python@v5 with: python-version: '3.10' # Installs and upgrades pip, installs other dependencies and installs the package from setup.py @@ -80,7 +80,7 @@ jobs: os: [ ubuntu-latest, windows-latest ] steps: - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v4 + uses: actions/setup-python@v5 with: python-version: ${{ matrix.python-version }} - name: Install dependencies @@ -94,10 +94,10 @@ jobs: - name: Install pandahub from PyPI if: ${{ inputs.upload_server == 'pypi'}} run: | - pip install pandahub + python -m pip install pandahub - name: List all installed packages run: | - pip list + python -m pip list - name: Test with pytest run: | - pytest --pyargs pandahub.test + python -m pytest --pyargs pandahub.test From c794df869bb61e39fcc2c4ba8dfc75e874ac5c46 Mon Sep 17 00:00:00 2001 From: mvogt Date: Wed, 26 Jun 2024 10:53:37 +0200 Subject: [PATCH 64/84] added checks to upload codecoverage reports only from develop and master. --- .github/workflows/github_test_action.yml | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/.github/workflows/github_test_action.yml b/.github/workflows/github_test_action.yml index 7e06179..f1e86f9 100644 --- a/.github/workflows/github_test_action.yml +++ b/.github/workflows/github_test_action.yml @@ -19,9 +19,9 @@ jobs: - 27017:27017 steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v4 - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v4.7.0 + uses: actions/setup-python@v5 with: python-version: ${{ matrix.python-version }} - name: Install dependencies @@ -36,9 +36,12 @@ jobs: pip list - name: Test with pytest and Codecov run: | - pytest --cov=./ --cov-report=xml + python -m pip install pytest-cov + python -m pytest -n=auto --cov=./ --cov-report=xml - name: Upload coverage to Codecov - uses: codecov/codecov-action@v2 + if: ${{ github.ref == 'refs/heads/develop' || github.ref != 'refs/heads/master' }} + uses: codecov/codecov-action@v4 with: - fail_ci_if_error: true + file: ./coverage.xml token: ${{ secrets.CODECOV_TOKEN }} + verbose: true From c2c395a37eb50a2b61aa6991e8057ef7845f1ed4 Mon Sep 17 00:00:00 2001 From: mvogt Date: Wed, 26 Jun 2024 11:00:16 +0200 Subject: [PATCH 65/84] removed requirements_dev.txt to be installed since the versions in there are too old. --- .github/workflows/github_test_action.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/github_test_action.yml b/.github/workflows/github_test_action.yml index f1e86f9..a64698e 100644 --- a/.github/workflows/github_test_action.yml +++ b/.github/workflows/github_test_action.yml @@ -29,7 +29,6 @@ jobs: python -m pip install --upgrade pip python -m pip install -U pytest pip install -r requirements.txt - pip install -r requirements_dev.txt pip install .["all"] - name: List of installed packages run: | From 33978a7383c28a0d6271354ab85d9c035d64094b Mon Sep 17 00:00:00 2001 From: mvogt Date: Wed, 26 Jun 2024 11:05:48 +0200 Subject: [PATCH 66/84] updated test requirements to be able to check also the jupyter notebooks. --- .github/workflows/github_test_action.yml | 4 ++-- setup.py | 3 ++- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/.github/workflows/github_test_action.yml b/.github/workflows/github_test_action.yml index a64698e..9b75ba3 100644 --- a/.github/workflows/github_test_action.yml +++ b/.github/workflows/github_test_action.yml @@ -35,8 +35,8 @@ jobs: pip list - name: Test with pytest and Codecov run: | - python -m pip install pytest-cov - python -m pytest -n=auto --cov=./ --cov-report=xml + python -m pip install pytest-cov pytest-xdist nbmake + python -m pytest -n=auto --nbmake --cov=./ --cov-report=xml - name: Upload coverage to Codecov if: ${{ github.ref == 'refs/heads/develop' || github.ref != 'refs/heads/master' }} uses: codecov/codecov-action@v4 diff --git a/setup.py b/setup.py index 003b4ae..7c0185c 100644 --- a/setup.py +++ b/setup.py @@ -11,7 +11,7 @@ long_description = '\n\n'.join((readme, changelog)) -test_requirements = ['pytest>=3', ] +test_requirements = ['pytest>=3', 'pytest-xdist', 'nbmake'] setup( author="Jan Ulffers, Leon Thurner, Jannis Kupka, Mike Vogt, Joschka Thurner, Alexander Scheidler", @@ -28,6 +28,7 @@ ], description="Data hub for pandapower and pandapipes networks based on MongoDB", install_requires=requirements, + test_requirements=test_requirements, long_description=readme, entry_points = { 'console_scripts': ['pandahub-login=pandahub.client.user_management:login'], From bea079d98d69c87b30d7c76a766958b56b641011 Mon Sep 17 00:00:00 2001 From: mvogt Date: Wed, 26 Jun 2024 11:22:19 +0200 Subject: [PATCH 67/84] added simbench and line-profiler for testing and fixed pandapower version. --- requirements.txt | 4 ++-- setup.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/requirements.txt b/requirements.txt index 2e6e645..fb3efa5 100644 --- a/requirements.txt +++ b/requirements.txt @@ -2,11 +2,11 @@ uvicorn>=0.24.0 fastapi-users[beanie]>=12.0 fastapi>=0.104.0 fastapi-mail>=1.4.1 -pandapower>=2.10.1 +pandapower~=2.14 pandapipes>=0.7.0 pymongo pydantic simplejson requests python-dotenv -pymongoarrow \ No newline at end of file +pymongoarrow diff --git a/setup.py b/setup.py index 7c0185c..a111ef9 100644 --- a/setup.py +++ b/setup.py @@ -11,7 +11,7 @@ long_description = '\n\n'.join((readme, changelog)) -test_requirements = ['pytest>=3', 'pytest-xdist', 'nbmake'] +test_requirements = ['pytest>=3', 'pytest-xdist', 'nbmake', 'simbench', 'line_profiler'] setup( author="Jan Ulffers, Leon Thurner, Jannis Kupka, Mike Vogt, Joschka Thurner, Alexander Scheidler", From b5c2bc84c592257777aeba03123540ed1f466112 Mon Sep 17 00:00:00 2001 From: mvogt Date: Wed, 26 Jun 2024 11:40:58 +0200 Subject: [PATCH 68/84] changed project to use a pyproject.toml --- CHANGELOG.md | 2 +- CHANGELOG.rst | 23 -------------- pandahub/__init__.py | 6 ++-- pyproject.toml | 75 ++++++++++++++++++++++++++++++++++++++++++++ setup.py | 45 ++------------------------ 5 files changed, 81 insertions(+), 70 deletions(-) delete mode 100644 CHANGELOG.rst create mode 100644 pyproject.toml diff --git a/CHANGELOG.md b/CHANGELOG.md index 3846ca9..dea032e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,8 +1,8 @@ # Change Log ## [0.2.4] - - BREAKING drops index argument from create_variant() function + - BREAKING drops index argument from create_variant() function ## [0.2.3]- 2022-08-04 diff --git a/CHANGELOG.rst b/CHANGELOG.rst deleted file mode 100644 index 3f49273..0000000 --- a/CHANGELOG.rst +++ /dev/null @@ -1,23 +0,0 @@ -Changelog -============= - -[XX.XX.XX] - 2023-XX-XX -------------------------------- -- [ADDED] -- [IMPROVED] -- [CHANGED] -- [REMOVED] - -[0.23.0] - 2022-08-04 -------------------------------- -- [ADDED] version property in project data -- [ADDED] method to migrate projects to latest version -- [ADDED] option to disable registration -- [ADDED] option to use a separate mongodb instance as global database -- [ADDED] geo mode to handle geojson columns -- [ADDED] tutorials -- [IMPROVED] collections for element tables now start with :code:`net_` -- [IMPROVED] project IDs now can be any name -- [IMPROVED] compatibility with python < 3.9 -- [IMPROVED] project settings API -- [IMPROVED] timeseries handling diff --git a/pandahub/__init__.py b/pandahub/__init__.py index 5b774fd..b60b201 100644 --- a/pandahub/__init__.py +++ b/pandahub/__init__.py @@ -1,6 +1,6 @@ -__version__ = "0.3.3" +import importlib.metadata + +__version__ = importlib.metadata.version("pandahub") from pandahub.lib.PandaHub import PandaHub, PandaHubError from pandahub.client.PandaHubClient import PandaHubClient - - diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000..d34153d --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,75 @@ +[build-system] +requires = ["build", "wheel", "setuptools"] +build-backend = "setuptools.build_meta" + +[project] +name = "pandahub" +version = "0.3.3" # File format version '__format_version__' is tracked in __init__.py +authors=[ + { name = "Jan Ulffers", email = "jan.ulffers@iee.fraunhofer.de" }, + { name = "Leon Thurner", email = "leon.thurner@retoflow.de" }, + { name = "Jannis Kupka", email = "jannis.kupka@retoflow.de" }, + { name = "Mike Vogt", email = "@iee.fraunhofer.de" }, + { name = "Joschka Thurner", email = "joschka.thurner@retoflow.de" }, + { name = "Alexander Scheidler", email = "alexander.scheidler@iee.fraunhofer.de" }, +] +description = "Data hub for pandapower and pandapipes networks based on MongoDB" +readme = "README.md" +license = { file = "LICENSE" } +requires-python = ">=3.8" +classifiers = [ + "Development Status :: 5 - Production/Stable", + "Environment :: Console", + "Intended Audience :: Developers", + "Intended Audience :: Education", + "Intended Audience :: Science/Research", + "License :: OSI Approved :: BSD License", + "Natural Language :: English", + "Operating System :: OS Independent", + "Programming Language :: Python", + "Programming Language :: Python :: 3", + # Add the specific Python versions supported here, e.g.: + "Programming Language :: Python :: 3.8", + "Programming Language :: Python :: 3.9", + "Programming Language :: Python :: 3.10", + "Programming Language :: Python :: 3.11", + "Programming Language :: Python :: 3.12" +] +dependencies = [ + "uvicorn>=0.24.0", + "fastapi-users[beanie]>=12.0", + "fastapi>=0.104.0", + "fastapi-mail>=1.4.1", + "pandapower~=2.14", + "pandapipes>=0.7.0", + "pymongo", + "pydantic", + "simplejson", + "requests", + "python-dotenv", + "pymongoarrow" +] +keywords = [ + "network", "analysis", "optimization", "automation", "grid", "energy", "engineering", "simulation", +] + +[project.urls] +Homepage = "https://github.com/e2nIEE/pandahub" +Documentation = "https://pandapipes.readthedocs.io" +Source = "https://github.com/e2nIEE/pandahub" +Repository = "https://github.com/e2nIEE/pandahub.git" +Issues = "https://github.com/e2nIEE/pandahub/issues" +Download = "https://pypi.org/project/pandahub/#files" +Changelog = "https://github.com/e2nIEE/pandahub/blob/develop/CHANGELOG.md" + +[project.optional-dependencies] +docs = ["numpydoc", "sphinx", "sphinx_rtd_theme", "sphinxcontrib.bibtex", "sphinx-pyproject"] +test = ["pytest", "pytest-xdist", "nbmake", "simbench", "line_profiler"] +all = [ + "numpydoc", "sphinx", "sphinx_rtd_theme", "sphinxcontrib.bibtex", "sphinx-pyproject", + "pytest", "pytest-xdist", "nbmake", "simbench", "line_profiler", +] + +[tool.setuptools.packages.find] +where = ["pandahub"] +include = ["*"] diff --git a/setup.py b/setup.py index a111ef9..6068493 100644 --- a/setup.py +++ b/setup.py @@ -1,44 +1,3 @@ -from setuptools import setup, find_packages +from setuptools import setup -with open('README.md') as readme_file: - readme = readme_file.read() - -with open('CHANGELOG.md') as change_file: - changelog = change_file.read() - -with open('requirements.txt') as req_file: - requirements = req_file.read() - -long_description = '\n\n'.join((readme, changelog)) - -test_requirements = ['pytest>=3', 'pytest-xdist', 'nbmake', 'simbench', 'line_profiler'] - -setup( - author="Jan Ulffers, Leon Thurner, Jannis Kupka, Mike Vogt, Joschka Thurner, Alexander Scheidler", - author_email='info@pandapower.de', - python_requires='>=3.6', - classifiers=[ - 'Development Status :: 2 - Pre-Alpha', - 'Intended Audience :: Developers', - 'Natural Language :: English', - 'Programming Language :: Python :: 3', - 'Programming Language :: Python :: 3.9', - 'Programming Language :: Python :: 3.10', - 'Programming Language :: Python :: 3.11', - ], - description="Data hub for pandapower and pandapipes networks based on MongoDB", - install_requires=requirements, - test_requirements=test_requirements, - long_description=readme, - entry_points = { - 'console_scripts': ['pandahub-login=pandahub.client.user_management:login'], - }, - keywords='pandahub', - name='pandahub', - packages=find_packages(), - url='https://github.com/e2nIEE/pandahub', - version='0.3.3', - include_package_data=True, - long_description_content_type='text/markdown', - zip_safe=False, -) +setup() From e8232cd15546961fe25d1738c796175b80245039 Mon Sep 17 00:00:00 2001 From: mvogt Date: Wed, 26 Jun 2024 11:43:21 +0200 Subject: [PATCH 69/84] forgot to write an email address... --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index d34153d..6a21428 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -9,7 +9,7 @@ authors=[ { name = "Jan Ulffers", email = "jan.ulffers@iee.fraunhofer.de" }, { name = "Leon Thurner", email = "leon.thurner@retoflow.de" }, { name = "Jannis Kupka", email = "jannis.kupka@retoflow.de" }, - { name = "Mike Vogt", email = "@iee.fraunhofer.de" }, + { name = "Mike Vogt", email = "mike.vogt@iee.fraunhofer.de" }, { name = "Joschka Thurner", email = "joschka.thurner@retoflow.de" }, { name = "Alexander Scheidler", email = "alexander.scheidler@iee.fraunhofer.de" }, ] From a6ccfc7b2424e1a6b5803a8737262c0027881365 Mon Sep 17 00:00:00 2001 From: mvogt Date: Wed, 26 Jun 2024 13:33:44 +0200 Subject: [PATCH 70/84] update of some bugs. --- .github/workflows/github_test_action.yml | 1 - .travis.yml | 2 +- pandahub/lib/database_toolbox.py | 2 +- requirements.txt | 12 ------------ requirements_dev.txt | 16 ---------------- 5 files changed, 2 insertions(+), 31 deletions(-) delete mode 100644 requirements.txt delete mode 100644 requirements_dev.txt diff --git a/.github/workflows/github_test_action.yml b/.github/workflows/github_test_action.yml index 9b75ba3..113d878 100644 --- a/.github/workflows/github_test_action.yml +++ b/.github/workflows/github_test_action.yml @@ -28,7 +28,6 @@ jobs: run: | python -m pip install --upgrade pip python -m pip install -U pytest - pip install -r requirements.txt pip install .["all"] - name: List of installed packages run: | diff --git a/.travis.yml b/.travis.yml index 3362948..c82d8f0 100644 --- a/.travis.yml +++ b/.travis.yml @@ -2,7 +2,7 @@ language: python python: - - 3.8 + - 3.10 # Command to install dependencies, e.g. pip install -r requirements.txt --use-mirrors install: pip install -U tox-travis diff --git a/pandahub/lib/database_toolbox.py b/pandahub/lib/database_toolbox.py index c7ffced..e58178c 100644 --- a/pandahub/lib/database_toolbox.py +++ b/pandahub/lib/database_toolbox.py @@ -244,7 +244,7 @@ def convert_dataframes_to_dicts(net, net_id, version_, datatypes=DATATYPES): if isinstance(data, pd.core.frame.DataFrame): # ------------ # create type lookup - types[key] = get_dtypes(key, data, datatypes.get(key)) + types[key] = get_dtypes(key, data) if data.empty: continue # ------------ diff --git a/requirements.txt b/requirements.txt deleted file mode 100644 index fb3efa5..0000000 --- a/requirements.txt +++ /dev/null @@ -1,12 +0,0 @@ -uvicorn>=0.24.0 -fastapi-users[beanie]>=12.0 -fastapi>=0.104.0 -fastapi-mail>=1.4.1 -pandapower~=2.14 -pandapipes>=0.7.0 -pymongo -pydantic -simplejson -requests -python-dotenv -pymongoarrow diff --git a/requirements_dev.txt b/requirements_dev.txt deleted file mode 100644 index 1824f01..0000000 --- a/requirements_dev.txt +++ /dev/null @@ -1,16 +0,0 @@ -pip==19.2.3 -bump2version==0.5.11 -wheel==0.33.6 -watchdog==0.9.0 -flake8==3.7.8 -tox==3.14.0 -coverage==4.5.4 -Sphinx==1.8.5 -twine==1.14.0 -Click==7.0 -pytest==4.6.5 -pytest-runner==5.1 -simbench -matplotlib -pytest-cov -pytest From be093055b06fc6c43d2d8f23b3efbfa2eaf17dd8 Mon Sep 17 00:00:00 2001 From: mvogt Date: Wed, 26 Jun 2024 13:54:40 +0200 Subject: [PATCH 71/84] changed call of pip, to be more secure --- .github/workflows/github_test_action.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/github_test_action.yml b/.github/workflows/github_test_action.yml index 113d878..2d8b1ae 100644 --- a/.github/workflows/github_test_action.yml +++ b/.github/workflows/github_test_action.yml @@ -28,7 +28,7 @@ jobs: run: | python -m pip install --upgrade pip python -m pip install -U pytest - pip install .["all"] + python -m pip install .["all"] - name: List of installed packages run: | pip list From 70fac8912ca9394ce1b3cf264feb9b44186c2f2f Mon Sep 17 00:00:00 2001 From: mvogt Date: Wed, 26 Jun 2024 13:55:12 +0200 Subject: [PATCH 72/84] set activate to false since when a new project get's created the id is not available. --- pandahub/test/conftest.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pandahub/test/conftest.py b/pandahub/test/conftest.py index 80c1eb4..14821fb 100644 --- a/pandahub/test/conftest.py +++ b/pandahub/test/conftest.py @@ -13,7 +13,7 @@ def ph(): ph.set_active_project(project_name) ph.delete_project(i_know_this_action_is_final=True) - ph.create_project(project_name) + ph.create_project(name=project_name, activate=False) ph.set_active_project(project_name) yield ph @@ -31,7 +31,7 @@ def phc(): # phc.set_active_project(project_name) # phc.delete_project(i_know_this_action_is_final=True) - phc.create_project(project_name) + phc.create_project(name=project_name) phc.set_active_project(project_name) yield phc From e84d1aa49fc541d88491f437a78a21093fd1b108 Mon Sep 17 00:00:00 2001 From: mvogt Date: Wed, 26 Jun 2024 14:01:23 +0200 Subject: [PATCH 73/84] added a fix to use the id if a project get's created without the "_id" field set. --- pandahub/lib/PandaHub.py | 7 +++++-- pandahub/test/conftest.py | 2 +- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/pandahub/lib/PandaHub.py b/pandahub/lib/PandaHub.py index 18766fd..942ce43 100644 --- a/pandahub/lib/PandaHub.py +++ b/pandahub/lib/PandaHub.py @@ -234,11 +234,14 @@ def create_project( project_data["_id"] = project_id if self.user_id is not None: project_data["users"] = {self.user_id: "owner"} - self.mongo_client["user_management"]["projects"].insert_one(project_data) + id = self.mongo_client["user_management"]["projects"].insert_one(project_data) if CREATE_INDEXES_WITH_PROJECT: self._create_mongodb_indexes(project_data["_id"]) if activate: - self.set_active_project_by_id(project_data["_id"]) + if "_id" in project_data.keys(): + id = project_data["_id"] + + self.set_active_project_by_id(id) return project_data def delete_project(self, i_know_this_action_is_final=False, project_id=None): diff --git a/pandahub/test/conftest.py b/pandahub/test/conftest.py index 14821fb..0ae8096 100644 --- a/pandahub/test/conftest.py +++ b/pandahub/test/conftest.py @@ -13,7 +13,7 @@ def ph(): ph.set_active_project(project_name) ph.delete_project(i_know_this_action_is_final=True) - ph.create_project(name=project_name, activate=False) + ph.create_project(name=project_name, activate=True) ph.set_active_project(project_name) yield ph From bc1117c68e9771aa81cded57423168fd0bc3a2e4 Mon Sep 17 00:00:00 2001 From: mvogt Date: Wed, 26 Jun 2024 14:35:31 +0200 Subject: [PATCH 74/84] wrong Id returned... --- pandahub/lib/PandaHub.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pandahub/lib/PandaHub.py b/pandahub/lib/PandaHub.py index 942ce43..9cd4d43 100644 --- a/pandahub/lib/PandaHub.py +++ b/pandahub/lib/PandaHub.py @@ -234,7 +234,7 @@ def create_project( project_data["_id"] = project_id if self.user_id is not None: project_data["users"] = {self.user_id: "owner"} - id = self.mongo_client["user_management"]["projects"].insert_one(project_data) + id = self.mongo_client["user_management"]["projects"].insert_one(project_data).inserted_id if CREATE_INDEXES_WITH_PROJECT: self._create_mongodb_indexes(project_data["_id"]) if activate: From 83cc5210dc9f5b11d43d901d408ee2be90c1203a Mon Sep 17 00:00:00 2001 From: mvogt Date: Wed, 26 Jun 2024 14:43:54 +0200 Subject: [PATCH 75/84] hack to get the tests running again, until issue #49 is solved. --- pandahub/test/conftest.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pandahub/test/conftest.py b/pandahub/test/conftest.py index 0ae8096..14821fb 100644 --- a/pandahub/test/conftest.py +++ b/pandahub/test/conftest.py @@ -13,7 +13,7 @@ def ph(): ph.set_active_project(project_name) ph.delete_project(i_know_this_action_is_final=True) - ph.create_project(name=project_name, activate=True) + ph.create_project(name=project_name, activate=False) ph.set_active_project(project_name) yield ph From bead5f9d170bf7b62f769ed831bc66396a0a548b Mon Sep 17 00:00:00 2001 From: mvogt Date: Wed, 26 Jun 2024 14:56:00 +0200 Subject: [PATCH 76/84] renamed project, to a one existing in the test. --- pandahub/test/test_networks.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pandahub/test/test_networks.py b/pandahub/test/test_networks.py index 91884ad..cac56db 100644 --- a/pandahub/test/test_networks.py +++ b/pandahub/test/test_networks.py @@ -139,7 +139,7 @@ def test_access_and_set_single_values(ph): def test_pandapipes(ph): - ph.set_active_project('Awesome') + ph.set_active_project('pytest') net = nw_pps.gas_versatility() ph.write_network_to_db(net, 'versatility') net2 = ph.get_net_from_db('versatility') @@ -163,7 +163,7 @@ def test_get_set_single_value(ph): from pandahub import PandaHub ph = PandaHub(connection_url="mongodb://localhost:27017") - ph.create_project('Awesome') + ph.create_project('pytest') net = nw_pps.gas_versatility() ph.write_network_to_db(net, 'versatility') net2 = ph.get_net_from_db('versatility') From 8b2464cb6aab122e0f5034455c117f525f11ae77 Mon Sep 17 00:00:00 2001 From: mvogt Date: Wed, 26 Jun 2024 22:53:48 +0200 Subject: [PATCH 77/84] added an env example --- .env.example | 27 +++++++++++++++++++++++++++ 1 file changed, 27 insertions(+) create mode 100644 .env.example diff --git a/.env.example b/.env.example new file mode 100644 index 0000000..1b5cc83 --- /dev/null +++ b/.env.example @@ -0,0 +1,27 @@ +MONGODB_URL="mongodb://localhost:27017" +MONGODB_USER=None +MONGODB_PASSWORD=None + +MONGODB_GLOBAL_DATABASE_URL=None +MONGODB_GLOBAL_DATABASE_USER=None +MONGODB_GLOBAL_DATABASE_PASSWORD=None + +EMAIL_VERIFICATION_REQUIRED=False + +MAIL_USERNAME="dummy@mail.de" +MAIL_PASSWORD="" +MAIL_PORT=587 +MAIL_SMTP_SERVER="" +MAIL_STARTTLS=True +MAIL_SSL_TLS=False + +PASSWORD_RESET_URL="" +EMAIL_VERIFY_URL="" +SECRET="totally secret" + +REGISTRATION_ENABLED=True +REGISTRATION_ADMIN_APPROVAL=False + +CREATE_INDEXES_WITH_PROJECT=True + +DEBUG=False From 956cd6709bd56635192e129619132ae9cbc8d1eb Mon Sep 17 00:00:00 2001 From: mvogt Date: Wed, 26 Jun 2024 22:54:24 +0200 Subject: [PATCH 78/84] added a debug switch, to disable/enable CORS --- pandahub/api/internal/settings.py | 2 ++ pandahub/api/main.py | 18 ++++++++++-------- 2 files changed, 12 insertions(+), 8 deletions(-) diff --git a/pandahub/api/internal/settings.py b/pandahub/api/internal/settings.py index aaad9da..528c577 100644 --- a/pandahub/api/internal/settings.py +++ b/pandahub/api/internal/settings.py @@ -48,3 +48,5 @@ def get_secret(key, default=None): REGISTRATION_ADMIN_APPROVAL = settings_bool("REGISTRATION_ADMIN_APPROVAL", default=False) CREATE_INDEXES_WITH_PROJECT = settings_bool("CREATE_INDEXES_WITH_PROJECT", default=True) + +DEBUG = os.getenv("DEBUG") or False diff --git a/pandahub/api/main.py b/pandahub/api/main.py index 0d15938..debde8f 100644 --- a/pandahub/api/main.py +++ b/pandahub/api/main.py @@ -8,6 +8,7 @@ from pandahub.lib.PandaHub import PandaHubError from pandahub.api.routers import net, projects, timeseries, users, auth, variants from pandahub.api.internal.db import User, db, AccessToken +from pandahub.api.internal.settings import DEBUG from beanie import init_beanie @asynccontextmanager @@ -27,18 +28,19 @@ async def lifespan(app: FastAPI): "http://localhost:8080", ] -app.add_middleware( - CORSMiddleware, - allow_origins=origins, - allow_credentials=True, - allow_methods=["*"], - allow_headers=["*"], -) +if DEBUG: + app.add_middleware( + CORSMiddleware, + allow_origins=origins, + allow_credentials=True, + allow_methods=["*"], + allow_headers=["*"], + ) app.include_router(net.router) app.include_router(projects.router) app.include_router(timeseries.router) -app.include_router(User.router) +app.include_router(users.router) app.include_router(auth.router) app.include_router(variants.router) From 48ef715e60c1d9aeb0009e23b4c9e695c4bbb871 Mon Sep 17 00:00:00 2001 From: mvogt Date: Wed, 26 Jun 2024 22:55:11 +0200 Subject: [PATCH 79/84] changed docker files to be buildable. --- Dockerfile | 17 +++++++++++------ docker-compose.yml | 37 ++++++++++++++++++++++++++++--------- 2 files changed, 39 insertions(+), 15 deletions(-) diff --git a/Dockerfile b/Dockerfile index 3d46090..5a73c21 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,8 +1,13 @@ -FROM python:3.9 as dev +FROM python:3.10 as dev + ENV PYTHONUNBUFFERED 1 -ENV PYTHONPATH /code/ -WORKDIR /code -COPY ./requirements.txt . -RUN pip install -r requirements.txt -RUN pip install watchdog pyyaml +ENV PYTHONPATH /code/pandahub + +COPY ./ /code/pandahub/ +WORKDIR /code/pandahub + +RUN python -m pip install --upgrade pip +RUN python -m pip install .["all"] +RUN python -m pip install watchdog pyyaml + CMD uvicorn --host "0.0.0.0" --port "8002" "pandahub.api.main:app" --reload diff --git a/docker-compose.yml b/docker-compose.yml index 1da5821..d322b5c 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,15 +1,34 @@ -version: '3.7' +version: '3' + +networks: + test: + external: false + services: - pandahub: - image: dev/pandahub - build: . + db: + user: ${CURRENT_UID} + image: mongo:latest + container_name: mongodb + restart: always + networks: + - test + ports: + - "27017:27017" + + pandahub_server: + user: ${CURRENT_UID} + container_name: pandahub + build: + context: . + dockerfile: Dockerfile environment: - SECRET=devonly! - MONGODB_URL=${MONGODB_URL:-mongodb://db:27017} ports: - "8002:8002" - volumes: - - ./pandahub:/code/pandahub - - - +# volumes: +# - ./:/code/pandahub + networks: + - test + depends_on: + - db From be314c0b30ce3624e9a14540a5b40919784c21f1 Mon Sep 17 00:00:00 2001 From: mvogt Date: Thu, 27 Jun 2024 00:31:41 +0200 Subject: [PATCH 80/84] made lot of changes to make the api server better usable. --- .env.example | 3 +++ Dockerfile | 8 +++++++- pandahub/api/internal/settings.py | 11 +++++++---- pandahub/api/main.py | 18 +++++++++++++++--- pandahub/client/PandaHubClient.py | 19 ++++++++++++------- pandahub/lib/PandaHub.py | 3 ++- pandahub/test/conftest.py | 16 +++++++++++++--- pandahub/test/test_networks.py | 2 +- 8 files changed, 60 insertions(+), 20 deletions(-) diff --git a/.env.example b/.env.example index 1b5cc83..3be7447 100644 --- a/.env.example +++ b/.env.example @@ -25,3 +25,6 @@ REGISTRATION_ADMIN_APPROVAL=False CREATE_INDEXES_WITH_PROJECT=True DEBUG=False +PANDAHUB_SERVER_URL="0.0.0.0" +PANDAHUB_SERVER_PORT=8002 +WORKERS=2 diff --git a/Dockerfile b/Dockerfile index 5a73c21..6d15baa 100644 --- a/Dockerfile +++ b/Dockerfile @@ -3,6 +3,11 @@ FROM python:3.10 as dev ENV PYTHONUNBUFFERED 1 ENV PYTHONPATH /code/pandahub +ENV DEBUG False +ENV PANDAHUB_SERVER_URL 0.0.0.0 +ENV PANDAHUB_SERVER_PORT 8002 +ENV WORKERS 2 + COPY ./ /code/pandahub/ WORKDIR /code/pandahub @@ -10,4 +15,5 @@ RUN python -m pip install --upgrade pip RUN python -m pip install .["all"] RUN python -m pip install watchdog pyyaml -CMD uvicorn --host "0.0.0.0" --port "8002" "pandahub.api.main:app" --reload +# CMD uvicorn --host "0.0.0.0" --port "8002" "pandahub.api.main:app" --reload +ENTRYPOINT ["python", "pandahub/api/main.py"] diff --git a/pandahub/api/internal/settings.py b/pandahub/api/internal/settings.py index 528c577..3d8ccca 100644 --- a/pandahub/api/internal/settings.py +++ b/pandahub/api/internal/settings.py @@ -31,14 +31,14 @@ def get_secret(key, default=None): if not MONGODB_GLOBAL_DATABASE_URL: MONGODB_GLOBAL_DATABASE_URL = os.getenv("MONGODB_URL_GLOBAL_DATABASE") or None -EMAIL_VERIFICATION_REQUIRED = settings_bool("EMAIL_VERIFICATION_REQUIRED") +EMAIL_VERIFICATION_REQUIRED = settings_bool("EMAIL_VERIFICATION_REQUIRED", default=False) MAIL_USERNAME = os.getenv("MAIL_USERNAME") or "dummy@mail.de" MAIL_PASSWORD = os.getenv("MAIL_PASSWORD") or "" MAIL_PORT = os.getenv("MAIL_PORT") or 587 MAIL_SMTP_SERVER = os.getenv("MAIL_SMTP_SERVER") or "" -MAIL_STARTTLS = os.getenv("MAIL_STARTTLS") or True -MAIL_SSL_TLS = os.getenv("MAIL_SSL_TLS") or False +MAIL_STARTTLS = settings_bool("MAIL_STARTTLS", default=True) +MAIL_SSL_TLS = settings_bool("MAIL_SSL_TLS", default=False) PASSWORD_RESET_URL = os.getenv("PASSWORD_RESET_URL") or "" EMAIL_VERIFY_URL = os.getenv("EMAIL_VERIFY_URL") or "" @@ -49,4 +49,7 @@ def get_secret(key, default=None): CREATE_INDEXES_WITH_PROJECT = settings_bool("CREATE_INDEXES_WITH_PROJECT", default=True) -DEBUG = os.getenv("DEBUG") or False +DEBUG = settings_bool("DEBUG", default=False) +PANDAHUB_SERVER_URL = os.getenv("PANDAHUB_SERVER_URL", "0.0.0.0") +PANDAHUB_SERVER_PORT = int(os.getenv('PANDAHUB_SERVER_PORT', 8002)) +WORKERS = int(os.getenv('WORKER', 2)) diff --git a/pandahub/api/main.py b/pandahub/api/main.py index debde8f..c13768d 100644 --- a/pandahub/api/main.py +++ b/pandahub/api/main.py @@ -8,7 +8,7 @@ from pandahub.lib.PandaHub import PandaHubError from pandahub.api.routers import net, projects, timeseries, users, auth, variants from pandahub.api.internal.db import User, db, AccessToken -from pandahub.api.internal.settings import DEBUG +from pandahub.api.internal import settings from beanie import init_beanie @asynccontextmanager @@ -28,7 +28,7 @@ async def lifespan(app: FastAPI): "http://localhost:8080", ] -if DEBUG: +if settings.DEBUG: app.add_middleware( CORSMiddleware, allow_origins=origins, @@ -52,6 +52,18 @@ async def pandahub_exception_handler(request: Request, exc: PandaHubError): content=str(exc), ) +@app.get("/") +async def ready(): + if settings.DEBUG: + import os + return os.environ + return "Hello World!" + if __name__ == "__main__": - uvicorn.run("main:app", host="127.0.0.1", port=8002, log_level="info", reload=True) + uvicorn.run("main:app", + host=settings.PANDAHUB_SERVER_URL, + port=settings.PANDAHUB_SERVER_PORT, + log_level="info", + reload=True, + workers=settings.WORKERS) diff --git a/pandahub/client/PandaHubClient.py b/pandahub/client/PandaHubClient.py index 579f6e4..0328784 100644 --- a/pandahub/client/PandaHubClient.py +++ b/pandahub/client/PandaHubClient.py @@ -1,22 +1,27 @@ -import warnings - import requests import pandapower as pp import pandas as pd -import numpy as np from pathlib import Path import os import json from fastapi.encoders import jsonable_encoder + class PandaHubClient: - def __init__(self): - config = os.path.join(Path.home(), "pandahub.config") + def __init__(self, config=None): + d = None + if config is None: + config = os.path.join(Path.home(), "pandahub.config") + elif type(config) == dict: + d = config + try: - with open(config, "r") as f: - d = json.load(f) + if d is None: + with open(config, "r") as f: + d = json.load(f) except FileNotFoundError: raise UserWarning("No pandahub configuration file found - log in first") + self.url = d["url"] self.token = d["token"] self.cert = None diff --git a/pandahub/lib/PandaHub.py b/pandahub/lib/PandaHub.py index 9cd4d43..6a7564b 100644 --- a/pandahub/lib/PandaHub.py +++ b/pandahub/lib/PandaHub.py @@ -709,6 +709,7 @@ def get_net_from_db( project_id=None, geo_mode="string", variants=None, + convert=True, ): if project_id: self.set_active_project_by_id(project_id) @@ -718,7 +719,7 @@ def get_net_from_db( if _id is None: return None return self.get_net_from_db_by_id( - _id, include_results, only_tables, geo_mode=geo_mode, variants=variants + _id, include_results, only_tables, geo_mode=geo_mode, variants=variants, convert=convert ) def get_net_from_db_by_id( diff --git a/pandahub/test/conftest.py b/pandahub/test/conftest.py index 14821fb..adadf9a 100644 --- a/pandahub/test/conftest.py +++ b/pandahub/test/conftest.py @@ -1,11 +1,11 @@ import pytest from pandahub import PandaHub from pandahub import PandaHubClient -from pandahub.client.user_management import _login +from pandahub.api.internal import settings @pytest.fixture(scope="session") def ph(): - ph = PandaHub(connection_url="mongodb://localhost:27017") + ph = PandaHub(connection_url=settings.MONGODB_URL) project_name = "pytest" @@ -23,7 +23,17 @@ def ph(): @pytest.fixture(scope="session") def phc(): - phc = PandaHubClient() + url = settings.PANDAHUB_SERVER_URL + + if url == "0.0.0.0": + url = "127.0.0.1" + + phc = PandaHubClient( + config={ + "url": f"http://{url}:{settings.PANDAHUB_SERVER_PORT}", + "token": settings.SECRET + } + ) project_name = "pandahubclienttest" diff --git a/pandahub/test/test_networks.py b/pandahub/test/test_networks.py index cac56db..9531649 100644 --- a/pandahub/test/test_networks.py +++ b/pandahub/test/test_networks.py @@ -142,7 +142,7 @@ def test_pandapipes(ph): ph.set_active_project('pytest') net = nw_pps.gas_versatility() ph.write_network_to_db(net, 'versatility') - net2 = ph.get_net_from_db('versatility') + net2 = ph.get_net_from_db('versatility', convert=False) pps.pipeflow(net) pps.pipeflow(net2) assert nets_equal(net, net2, check_only_results=True) From 14e564c02843aa6bbe9601f81bf76320953f9d43 Mon Sep 17 00:00:00 2001 From: mvogt Date: Wed, 3 Jul 2024 16:34:29 +0200 Subject: [PATCH 81/84] added blosc for compressed timeseries --- pyproject.toml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 6a21428..04bd0e0 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -47,7 +47,8 @@ dependencies = [ "simplejson", "requests", "python-dotenv", - "pymongoarrow" + "pymongoarrow", + "blosc" ] keywords = [ "network", "analysis", "optimization", "automation", "grid", "energy", "engineering", "simulation", From f3e5c40a7fdc721da1dacadb6dcd92f88dfae8d7 Mon Sep 17 00:00:00 2001 From: mvogt Date: Wed, 3 Jul 2024 16:34:49 +0200 Subject: [PATCH 82/84] moved blosc to general import and fixed a bug --- pandahub/lib/database_toolbox.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/pandahub/lib/database_toolbox.py b/pandahub/lib/database_toolbox.py index e58178c..9da0a20 100644 --- a/pandahub/lib/database_toolbox.py +++ b/pandahub/lib/database_toolbox.py @@ -9,6 +9,7 @@ import logging import json import importlib +import blosc logger = logging.getLogger(__name__) from pandapower.io_utils import PPJSONEncoder from packaging import version @@ -112,7 +113,6 @@ def convert_timeseries_to_subdocuments(timeseries): def compress_timeseries_data(timeseries_data, ts_format): - import blosc if ts_format == "timestamp_value": timeseries_data = np.array([timeseries_data.index.astype("int64"), timeseries_data.values]) @@ -126,7 +126,6 @@ def compress_timeseries_data(timeseries_data, ts_format): def decompress_timeseries_data(timeseries_data, ts_format, num_timestamps): - import blosc if ts_format == "timestamp_value": data = np.frombuffer(blosc.decompress(timeseries_data), dtype=np.float64).reshape((num_timestamps, 2), @@ -244,7 +243,7 @@ def convert_dataframes_to_dicts(net, net_id, version_, datatypes=DATATYPES): if isinstance(data, pd.core.frame.DataFrame): # ------------ # create type lookup - types[key] = get_dtypes(key, data) + types[key] = get_dtypes(data, datatypes.get(key)) if data.empty: continue # ------------ From c401b6cc658e6253ca31df7304b6be12116f8bbd Mon Sep 17 00:00:00 2001 From: mvogt Date: Wed, 3 Jul 2024 16:35:18 +0200 Subject: [PATCH 83/84] changed testing to use values from the env --- pandahub/test/test_networks.py | 3 ++- pandahub/test/test_projects.py | 11 +++++++---- pandahub/test/test_timeseries.py | 3 ++- 3 files changed, 11 insertions(+), 6 deletions(-) diff --git a/pandahub/test/test_networks.py b/pandahub/test/test_networks.py index 9531649..d0fb672 100644 --- a/pandahub/test/test_networks.py +++ b/pandahub/test/test_networks.py @@ -6,6 +6,7 @@ import pandapower.networks as nw_pp from pandahub import PandaHubError from pandapipes.toolbox import nets_equal +from pandahub.api.internal import settings def test_additional_res_tables(ph): @@ -162,7 +163,7 @@ def test_get_set_single_value(ph): if __name__ == '__main__': from pandahub import PandaHub - ph = PandaHub(connection_url="mongodb://localhost:27017") + ph = PandaHub(connection_url=settings.MONGODB_URL) ph.create_project('pytest') net = nw_pps.gas_versatility() ph.write_network_to_db(net, 'versatility') diff --git a/pandahub/test/test_projects.py b/pandahub/test/test_projects.py index 0426fe8..89e5df9 100644 --- a/pandahub/test/test_projects.py +++ b/pandahub/test/test_projects.py @@ -1,11 +1,12 @@ import pandapower.networks as nw import pandahub import pandapower as pp -import pytest +from pandahub.api.internal import settings from pandahub.lib.database_toolbox import convert_dataframes_to_dicts from pymongo import DESCENDING from packaging import version + def test_project_management(ph): project = "pytest2" if not ph.project_exists(project): @@ -74,7 +75,7 @@ def _write_net_collections_to_db(self, db, collections): except: print("FAILED TO WRITE TABLE", key) # we use the implemetation of 0.2.2 to write a net - oldph = PandaHubV0_2_2(connection_url="mongodb://localhost:27017") + oldph = PandaHubV0_2_2(connection_url=settings.MONGODB_URL) if oldph.project_exists("pytest"): oldph.set_active_project("pytest") @@ -83,14 +84,16 @@ def _write_net_collections_to_db(self, db, collections): oldph.create_project("pytest") oldph.set_active_project("pytest") net = nw.simple_four_bus_system() + net.bus.zone = "1" + net.ext_grid.name = "Slack" oldph.write_network_to_db(net, "simple_network") # convert the db to latest version - ph = pandahub.PandaHub(connection_url="mongodb://localhost:27017") + ph = pandahub.PandaHub(connection_url=settings.MONGODB_URL) ph.set_active_project("pytest") ph.upgrade_project_to_latest_version() # and test if everything went fine net2 = ph.get_net_from_db("simple_network") - assert pp.nets_equal(net, net2) + assert pp.nets_equal(net, net2, check_dtype=False) def reset_project(db): diff --git a/pandahub/test/test_timeseries.py b/pandahub/test/test_timeseries.py index 3a5570a..2448073 100644 --- a/pandahub/test/test_timeseries.py +++ b/pandahub/test/test_timeseries.py @@ -6,6 +6,7 @@ import datetime import pandapower.networks as nw import simbench as sb +from pandahub.api.internal import settings code = "1-HV-urban--0--sw" project = "pytest" @@ -258,7 +259,7 @@ def test_bulk_write_with_meta(ph): if __name__ == '__main__': from pandahub import PandaHub - ph = PandaHub(connection_url="mongodb://localhost:27017") + ph = PandaHub(connection_url=settings.MONGODB_URL) project_name = "pytest" From 13262f757ed60e2aee718533d918c6b6a7534df5 Mon Sep 17 00:00:00 2001 From: mvogt Date: Wed, 3 Jul 2024 16:39:36 +0200 Subject: [PATCH 84/84] removed testing of jupyter notebooks. --- .github/workflows/github_test_action.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/github_test_action.yml b/.github/workflows/github_test_action.yml index 2d8b1ae..6e357da 100644 --- a/.github/workflows/github_test_action.yml +++ b/.github/workflows/github_test_action.yml @@ -35,7 +35,7 @@ jobs: - name: Test with pytest and Codecov run: | python -m pip install pytest-cov pytest-xdist nbmake - python -m pytest -n=auto --nbmake --cov=./ --cov-report=xml + python -m pytest -n=auto --cov=./ --cov-report=xml - name: Upload coverage to Codecov if: ${{ github.ref == 'refs/heads/develop' || github.ref != 'refs/heads/master' }} uses: codecov/codecov-action@v4