Updates to ResultsHandler

.github/workflows/continous_integration.yml

@@ -26,7 +26,7 @@ jobs:
     # Specify the python versions to test
     strategy:
       matrix:
-        python-version: ["3.8", "3.9", "3.10", "3.11"]
+        python-version: ["3.9", "3.10", "3.11"]
 
     # Steps represent a sequence of tasks that will be executed as part of the job
     steps:

flarestack/cluster/submitter.py

@@ -1,6 +1,15 @@
+import copy
 import json
-import os, subprocess, time, logging, shutil, copy, sys
+import logging
 import numpy as np
+import os
+from pathlib import Path
+import shutil
+import subprocess
+import sys
+import time
+from typing import Optional
+
 from flarestack.shared import (
     fs_dir,
     log_dir,
@@ -15,14 +24,14 @@
 from flarestack.core.minimisation import MinimisationHandler
 from flarestack.core.results import ResultsHandler
 
-from pathlib import Path
-
 
 logger = logging.getLogger(__name__)
 
 
 class Submitter(object):
-    submitter_dict = dict()
+    # in other modules, such attribute is called 'subclass'
+    # is this functionally different?
+    submitter_dict: dict[str, object] = dict()
 
     def __init__(
         self,
@@ -302,7 +311,7 @@ class HTCondorSubmitter(Submitter):
 
     # Log path
     log_path = Path(log_dir)
-    override_log_path = None
+    override_log_path: Optional[Path] = None
 
     def __init__(self, *args, **kwargs):
         """

flarestack/core/angular_error_modifier.py

@@ -34,7 +34,7 @@
 
 
 class BaseFloorClass(object):
-    subclasses = {}
+    subclasses: dict[str, object] = {}
 
     def __init__(self, floor_dict):
         self.floor_dict = floor_dict
@@ -149,7 +149,7 @@ def __init__(self, floor_dict):
         if not os.path.isfile(self.pickle_name):
             self.create_pickle()
         else:
-            logger.debug("Loading from".format(self.pickle_name))
+            logger.debug(f"Loading from {self.pickle_name}")
 
         with open(self.pickle_name, "r") as f:
             pickled_data = Pickle.load(f)
@@ -182,6 +182,9 @@ def create_function(self, pickled_array):
         return lambda data, params: np.array([func(params) for _ in data])
 
 
+"""
+# This class has been commented out since its name is redefined below.
+# However, it seems QuantileFloor1D is never explicitly called in the code and is hence likely untested.
 @BaseFloorClass.register_subclass("quantile_floor_1d")
 class QuantileFloor1D(BaseQuantileFloor, BaseStaticFloor):
     def create_pickle(self):
@@ -190,6 +193,7 @@ def create_pickle(self):
     def create_function(self, pickled_array):
         func = interp1d(pickled_array[0], pickled_array[1])
         return lambda data, params: func(data["logE"])
+"""
 
 
 @BaseFloorClass.register_subclass("quantile_floor_1d_e")
@@ -212,7 +216,7 @@ def create_function(self, pickled_array):
 
 
 class BaseAngularErrorModifier(object):
-    subclasses = {}
+    subclasses: dict[str, object] = {}
 
     def __init__(self, pull_dict):
         self.season = pull_dict["season"]
@@ -245,7 +249,7 @@ def create(
         e_pdf_dict,
         floor_name="static_floor",
         aem_name="no_modifier",
-        **kwargs
+        **kwargs,
     ):
         pull_dict = dict()
         pull_dict["season"] = season
@@ -335,7 +339,7 @@ def __init__(self, pull_dict):
         if not os.path.isfile(self.pull_name):
             self.create_pickle()
         else:
-            logger.debug("Loading from".format(self.pull_name))
+            logger.debug(f"Loading from {self.pull_name}")
 
         with open(self.pull_name, "r") as f:
             self.pickled_data = Pickle.load(f)
@@ -466,6 +470,9 @@ def create_dynamic(self, pickled_array):
         return lambda data: func(data["logE"], data["sinDec"])
 
 
+"""
+# NOTE: `mypy` allowed to find some deprecations in the following __main__ block that were never noticed before, because the block is not part of any testing routine. It should be checked whether equivalent code is covered by an unit test, before removing the (now commented) block altogether.
+
 if __name__ == "__main__":
     from flarestack.data.icecube.ps_tracks.ps_v002_p01 import IC86_1_dict
     from flarestack.analyses.angular_error_floor.plot_bias import (
@@ -530,3 +537,4 @@ def med_pull(data):
 # for gamma in np.linspace(1.0, 4.0, 4):
 #     print data_loader(IC86_1_dict["exp_path"])["logE"][:8]
 #     print np.degrees(x.f(data_loader(IC86_1_dict["exp_path"])[:8], [gamma]))
+"""

flarestack/core/energy_pdf.py

@@ -63,7 +63,7 @@ class EnergyPDF(object):
     A base class for Energy PDFs.
     """
 
-    subclasses = {}
+    subclasses: dict[str, object] = {}
 
     def __init__(self, e_pdf_dict):
         """

flarestack/core/injector.py

@@ -64,7 +64,7 @@ def read_injector_dict(inj_dict):
 class BaseInjector:
     """Base Injector Class"""
 
-    subclasses = {}
+    subclasses: dict[str, object] = {}
 
     def __init__(self, season, sources, **kwargs):
         kwargs = read_injector_dict(kwargs)
@@ -224,7 +224,7 @@ class MCInjector(BaseInjector):
     background. This can be either MC background, or scrambled real data.
     """
 
-    subclasses = {}
+    subclasses: dict[str, object] = {}
 
     def __init__(self, season, sources, **kwargs):
         kwargs = read_injector_dict(kwargs)

flarestack/core/llh.py

@@ -80,7 +80,7 @@ def read_llh_dict(llh_dict):
 class LLH(object):
     """Base class LLH."""
 
-    subclasses = {}
+    subclasses: dict[str, object] = {}
 
     def __init__(self, season, sources, llh_dict):
         self.season = season
@@ -900,7 +900,7 @@ def create_kwargs(self, data, pull_corrector, weight_f=None):
 
         return kwargs
 
-    def calculate_test_statistic(self, params, weights, **kwargs):
+    def calculate_test_statistic(self, params, weights, **kwargs) -> float:
         """Calculates the test statistic, given the parameters. Uses numexpr
         for faster calculations.
 
@@ -920,18 +920,18 @@ def calculate_test_statistic(self, params, weights, **kwargs):
         # If n_s if negative, then removes the energy term from the likelihood
 
         for i, n_j in enumerate(all_n_j):
-            SoB_spacetime: list = kwargs["SoB_spacetime_cache"][i]
+            SoB_spacetime_data: list = kwargs["SoB_spacetime_cache"][i]
 
             # Switches off Energy term for negative n_s, which should in theory
             # be a continuous change that does not alter the likelihood for
             # n_s > 0 (as it is not included for n_s=0).
 
-            if len(SoB_spacetime) == 0:
+            if len(SoB_spacetime_data) == 0:
                 x.append(np.array([1.0]))
 
             else:
                 SoB_spacetime = kwargs["pull_corrector"].estimate_spatial(
-                    gamma, SoB_spacetime
+                    gamma, SoB_spacetime_data
                 )
 
                 if n_j < 0:
@@ -1394,13 +1394,18 @@ def find_significant_events(self, coincident_data, source):
     return FlareLLH(season, sources, llh_dict)
 
 
+"""
+# NOTE: `mypy` allowed to find some deprecations in the following __main__ block that were never noticed before, because the block is not part of any testing routine. It should be checked whether equivalent code is covered by an unit test, before removing the (now commented) block altogether.
+
 if __name__ == "__main__":
+
+
     from flarestack.shared import fs_scratch_dir
     from scipy.interpolate import InterpolatedUnivariateSpline
 
     g = EnergyPDF.create({"energy_pdf_name": "PowerLaw", "gamma": 2.2})
 
-    e_range = np.logspace(0, 7, 1e3)
+    e_range = np.logspace(start=0, stop=7, num=1000)
 
     f = InterpolatedUnivariateSpline(e_range, np.log(g.f(e_range)))
 
@@ -1440,3 +1445,4 @@ def find_significant_events(self, coincident_data, source):
 
     for i in np.linspace(0.0, 10.0, 21):
         print(i, f([i], weights))
+"""

flarestack/core/minimisation.py

@@ -93,10 +93,10 @@ class MinimisationHandler(object):
     the injector and the likelihood.
     """
 
-    subclasses = {}
+    subclasses: dict[str, object] = {}
 
     # Each MinimisationHandler must specify which LLH classes are compatible
-    compatible_llh = []
+    compatible_llh: list[str] = []
     compatible_negative_n_s = False
 
     def __init__(self, mh_dict):

flarestack/core/multiprocess_wrapper.py

@@ -40,7 +40,9 @@ def add_injector(self, season, sources):
 
 class MultiProcessor:
     queue = None
-    results = dict()
+
+    # this is not used within the class, maybe it can be safely dropped
+    # results = dict()
 
     def __init__(self, n_cpu, **kwargs):
         self.queue = JoinableQueue()
@@ -149,9 +151,15 @@ def run_multiprocess(n_cpu, mh_dict):
 
     logging.basicConfig(level=logging.INFO)
 
+    # os.cpu_count() may return None, deal safely with that
+    cpu_count = os.cpu_count()
+    available_cpus = cpu_count if cpu_count is not None else 1
+    # 1 <= n_cpu <= 32
+    n_cpu = min(max(1, available_cpus - 1), 32)
+
     parser = argparse.ArgumentParser()
     parser.add_argument("-f", "--file", help="Path for analysis pkl_file")
-    parser.add_argument("-n", "--n_cpu", default=min(max(1, os.cpu_count() - 1), 32))
+    parser.add_argument("-n", "--n_cpu", default=n_cpu)
     cfg = parser.parse_args()
 
     logger.info(f"N CPU available {os.cpu_count()}. Using {cfg.n_cpu}")