diff --git a/modules/data_processing/forcings.py b/modules/data_processing/forcings.py index c2499f2..29a9b50 100644 --- a/modules/data_processing/forcings.py +++ b/modules/data_processing/forcings.py @@ -83,7 +83,7 @@ def get_index_chunks(data: xr.DataArray) -> list[tuple[int, int]]: # takes a data array and calculates the start and end index for each chunk # based on the available memory. array_memory_usage = data.nbytes - # free_memory = psutil.virtual_memory().available * 0.8 # 80% of available memory + free_memory = psutil.virtual_memory().available * 0.8 # 80% of available memory # limit the chunk to 20gb, makes things more stable free_memory = min(free_memory, 20 * 1024 * 1024 * 1024) num_chunks = ceil(array_memory_usage / free_memory) diff --git a/modules/map_app/__init__.py b/modules/map_app/__init__.py new file mode 100644 index 0000000..25b4063 --- /dev/null +++ b/modules/map_app/__init__.py @@ -0,0 +1,28 @@ +from flask import Flask +import logging +from map_app.views import main, intra_module_db +from data_sources.source_validation import validate_all + +with open("app.log", "w") as f: + f.write("") + f.write("Starting Application!\n") + +logging.basicConfig( + level=logging.INFO, + format="%(name)-12s: %(levelname)s - %(message)s", + filename="app.log", + filemode="a", +) # Append mode +# Example: Adding a console handler to root logger (optional) +console_handler = logging.StreamHandler() +console_handler.setLevel(logging.INFO) # Or any other level +formatter = logging.Formatter("%(name)-12s: %(levelname)-8s %(message)s") +console_handler.setFormatter(formatter) +logging.getLogger("").addHandler(console_handler) + +validate_all() + +app = Flask(__name__) +app.register_blueprint(main) + +intra_module_db["app"] = app diff --git a/modules/map_app/__main__.py b/modules/map_app/__main__.py index 4a6e718..58cb355 100644 --- a/modules/map_app/__main__.py +++ b/modules/map_app/__main__.py @@ -5,36 +5,8 @@ from threading import Timer from data_processing.file_paths import file_paths -from data_sources.source_validation import validate_all from data_processing.graph_utils import get_graph -from flask import Flask - -from .views import intra_module_db, main - -validate_all() - -with open("app.log", "w") as f: - f.write("") - f.write("Starting Application!\n") - - -app = Flask(__name__) -app.register_blueprint(main) - -intra_module_db["app"] = app - -logging.basicConfig( - level=logging.INFO, - format="%(name)-12s: %(levelname)s - %(message)s", - filename="app.log", - filemode="a", -) # Append mode -# Example: Adding a console handler to root logger (optional) -console_handler = logging.StreamHandler() -console_handler.setLevel(logging.INFO) # Or any other level -formatter = logging.Formatter("%(name)-12s: %(levelname)-8s %(message)s") -console_handler.setFormatter(formatter) -logging.getLogger("").addHandler(console_handler) +from map_app import app, console_handler def open_browser(): @@ -59,8 +31,7 @@ def set_logs_to_warning(): console_handler.setLevel(logging.DEBUG) -if __name__ == "__main__": - +def main(): # call this once to cache the graph Timer(1, get_graph).start() @@ -74,3 +45,6 @@ def set_logs_to_warning(): with open("app.log", "a") as f: f.write("Running in production mode\n") app.run(host="0.0.0.0", port="0") + +if __name__ == "__main__": + main() diff --git a/modules/map_app/views.py b/modules/map_app/views.py index d1798e1..ae3684f 100644 --- a/modules/map_app/views.py +++ b/modules/map_app/views.py @@ -1,6 +1,7 @@ import json import logging from datetime import datetime +from pathlib import Path from data_processing.create_realization import create_realization from data_processing.file_paths import file_paths @@ -43,7 +44,7 @@ def subset_selection(): subset_name = cat_ids[0] run_paths = file_paths(subset_name) subset(cat_ids, output_gpkg_path=run_paths.geopackage_path) - return run_paths.geopackage_path, 200 + return str(run_paths.geopackage_path), 200 @main.route("/subset_to_file", methods=["POST"]) @@ -65,7 +66,9 @@ def subset_to_file(): def get_forcings(): # body: JSON.stringify({'forcing_dir': forcing_dir, 'start_time': start_time, 'end_time': end_time}), data = json.loads(request.data.decode("utf-8")) - cat_id = data.get("forcing_dir").split("/")[-1] + subset_gpkg = Path(data.get("forcing_dir").split("subset to ")[-1]) + output_folder = subset_gpkg.parent.parent.stem + start_time = data.get("start_time") end_time = data.get("end_time") # get the forcings @@ -77,7 +80,7 @@ def get_forcings(): app.debug = False logger.info(f"get_forcings() disabled debug mode at {datetime.now()}") try: - create_forcings(start_time, end_time, cat_id) + create_forcings(start_time, end_time, output_folder) except Exception as e: logger.info(f"get_forcings() failed with error: {str(e)}") return jsonify({"error": str(e)}), 500 @@ -90,13 +93,14 @@ def get_forcings(): def get_realization(): # body: JSON.stringify({'forcing_dir': forcing_dir, 'start_time': start_time, 'end_time': end_time}), data = json.loads(request.data.decode("utf-8")) - cat_id = data.get("forcing_dir").split("/")[-1] + subset_gpkg = Path(data.get("forcing_dir").split("subset to ")[-1]) + output_folder = subset_gpkg.parent.parent.stem start_time = data.get("start_time") end_time = data.get("end_time") # get the forcings start_time = datetime.strptime(start_time, "%Y-%m-%dT%H:%M") end_time = datetime.strptime(end_time, "%Y-%m-%dT%H:%M") - create_realization(cat_id, start_time, end_time) + create_realization(output_folder, start_time, end_time) return "success", 200 diff --git a/pyproject.toml b/pyproject.toml index f573ffb..14aa854 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -57,7 +57,7 @@ Issues = "https://github.com/CIROH-UA/NGIAB_data_preprocess/issues" [project.scripts] cli = "ngiab_data_cli.__main__:main" -map = "map_app.__main__:main" +map_app = "map_app.__main__:main" [build-system] # scm adds files tracked by git to the package