fix forcing memory detection and map_app structure

modules/data_processing/forcings.py

@@ -83,7 +83,7 @@ def get_index_chunks(data: xr.DataArray) -> list[tuple[int, int]]:
     # takes a data array and calculates the start and end index for each chunk
     # based on the available memory.
     array_memory_usage = data.nbytes
-    # free_memory = psutil.virtual_memory().available * 0.8 # 80% of available memory
+    free_memory = psutil.virtual_memory().available * 0.8  # 80% of available memory
     # limit the chunk to 20gb, makes things more stable
     free_memory = min(free_memory, 20 * 1024 * 1024 * 1024)
     num_chunks = ceil(array_memory_usage / free_memory)

modules/map_app/__init__.py

@@ -0,0 +1,28 @@
+from flask import Flask
+import logging
+from map_app.views import main, intra_module_db
+from data_sources.source_validation import validate_all
+
+with open("app.log", "w") as f:
+    f.write("")
+    f.write("Starting Application!\n")
+
+logging.basicConfig(
+    level=logging.INFO,
+    format="%(name)-12s: %(levelname)s - %(message)s",
+    filename="app.log",
+    filemode="a",
+)  # Append mode
+# Example: Adding a console handler to root logger (optional)
+console_handler = logging.StreamHandler()
+console_handler.setLevel(logging.INFO)  # Or any other level
+formatter = logging.Formatter("%(name)-12s: %(levelname)-8s %(message)s")
+console_handler.setFormatter(formatter)
+logging.getLogger("").addHandler(console_handler)
+
+validate_all()
+
+app = Flask(__name__)
+app.register_blueprint(main)
+
+intra_module_db["app"] = app

modules/map_app/__main__.py

@@ -5,36 +5,8 @@
 from threading import Timer
 
 from data_processing.file_paths import file_paths
-from data_sources.source_validation import validate_all
 from data_processing.graph_utils import get_graph
-from flask import Flask
-
-from .views import intra_module_db, main
-
-validate_all()
-
-with open("app.log", "w") as f:
-    f.write("")
-    f.write("Starting Application!\n")
-
-
-app = Flask(__name__)
-app.register_blueprint(main)
-
-intra_module_db["app"] = app
-
-logging.basicConfig(
-    level=logging.INFO,
-    format="%(name)-12s: %(levelname)s - %(message)s",
-    filename="app.log",
-    filemode="a",
-)  # Append mode
-# Example: Adding a console handler to root logger (optional)
-console_handler = logging.StreamHandler()
-console_handler.setLevel(logging.INFO)  # Or any other level
-formatter = logging.Formatter("%(name)-12s: %(levelname)-8s %(message)s")
-console_handler.setFormatter(formatter)
-logging.getLogger("").addHandler(console_handler)
+from map_app import app, console_handler
 
 
 def open_browser():
@@ -59,8 +31,7 @@ def set_logs_to_warning():
     console_handler.setLevel(logging.DEBUG)
 
 
-if __name__ == "__main__":
-
+def main():
     # call this once to cache the graph
     Timer(1, get_graph).start()
 
@@ -74,3 +45,6 @@ def set_logs_to_warning():
         with open("app.log", "a") as f:
             f.write("Running in production mode\n")
         app.run(host="0.0.0.0", port="0")
+
+if __name__ == "__main__":
+    main()

modules/map_app/views.py

@@ -1,6 +1,7 @@
 import json
 import logging
 from datetime import datetime
+from pathlib import Path
 
 from data_processing.create_realization import create_realization
 from data_processing.file_paths import file_paths
@@ -43,7 +44,7 @@ def subset_selection():
     subset_name = cat_ids[0]
     run_paths = file_paths(subset_name)
     subset(cat_ids, output_gpkg_path=run_paths.geopackage_path)
-    return run_paths.geopackage_path, 200
+    return str(run_paths.geopackage_path), 200
 
 
 @main.route("/subset_to_file", methods=["POST"])
@@ -65,7 +66,9 @@ def subset_to_file():
 def get_forcings():
     # body: JSON.stringify({'forcing_dir': forcing_dir, 'start_time': start_time, 'end_time': end_time}),
     data = json.loads(request.data.decode("utf-8"))
-    cat_id = data.get("forcing_dir").split("/")[-1]
+    subset_gpkg = Path(data.get("forcing_dir").split("subset to ")[-1])
+    output_folder = subset_gpkg.parent.parent.stem
+
     start_time = data.get("start_time")
     end_time = data.get("end_time")
     # get the forcings
@@ -77,7 +80,7 @@ def get_forcings():
     app.debug = False
     logger.info(f"get_forcings() disabled debug mode at {datetime.now()}")
     try:
-        create_forcings(start_time, end_time, cat_id)
+        create_forcings(start_time, end_time, output_folder)
     except Exception as e:
         logger.info(f"get_forcings() failed with error: {str(e)}")
         return jsonify({"error": str(e)}), 500
@@ -90,13 +93,14 @@ def get_forcings():
 def get_realization():
     # body: JSON.stringify({'forcing_dir': forcing_dir, 'start_time': start_time, 'end_time': end_time}),
     data = json.loads(request.data.decode("utf-8"))
-    cat_id = data.get("forcing_dir").split("/")[-1]
+    subset_gpkg = Path(data.get("forcing_dir").split("subset to ")[-1])
+    output_folder = subset_gpkg.parent.parent.stem
     start_time = data.get("start_time")
     end_time = data.get("end_time")
     # get the forcings
     start_time = datetime.strptime(start_time, "%Y-%m-%dT%H:%M")
     end_time = datetime.strptime(end_time, "%Y-%m-%dT%H:%M")
-    create_realization(cat_id, start_time, end_time)
+    create_realization(output_folder, start_time, end_time)
     return "success", 200
 
 

pyproject.toml

@@ -57,7 +57,7 @@ Issues = "https://github.com/CIROH-UA/NGIAB_data_preprocess/issues"
 
 [project.scripts]
 cli = "ngiab_data_cli.__main__:main"
-map = "map_app.__main__:main"
+map_app = "map_app.__main__:main"
 
 [build-system]
 # scm adds files tracked by git to the package