add driver.host options to the spark-submit launcher (#115)

* add driver.host options to the spark-submit launcher * fix swift err radius * fix test * ci * ci
FusRoman · Mar 14, 2024 · 904bc3a · 904bc3a
1 parent 1f68b46
commit 904bc3a
Show file tree

Hide file tree

Showing 5 changed files with 12 additions and 6 deletions.
diff --git a/fink_mm/conf/distribute_for_test.conf b/fink_mm/conf/distribute_for_test.conf
@@ -49,6 +49,7 @@ gaia_dist=5
 [STREAM]
 tinterval=30
 manager=local[2]
+driver_host=$HOSTNAME
 principal=
 secret=
 role=

diff --git a/fink_mm/conf/fink_mm.conf b/fink_mm/conf/fink_mm.conf
@@ -49,6 +49,7 @@ gaia_dist=5
 [STREAM]
 tinterval=30
 manager=local[8]
+driver_host=$HOSTNAME
 principal=
 secret=
 role=

diff --git a/fink_mm/observatory/Swift/Swift.py b/fink_mm/observatory/Swift/Swift.py
@@ -51,16 +51,16 @@ def err_to_arcminute(self):
         Example
         -------
         >>> swift_bat.err_to_arcminute()
-        0.05
+        3.0
         >>> swift_xrt.err_to_arcminute()
-        0.0016
+        0.096
         >>> swift_uvot.err_to_arcminute()
-        0.0001
+        0.006
         """
         instrument = self.detect_instruments()
         coords = vp.get_event_position(self.voevent)
 
-        err = 1 / 60 if coords.err == 0.0 else coords.err
+        err = 1 / 60 if coords.err == 0.0 else coords.err * 60
 
         if instrument in ["XRT", "UVOT", "BAT", "FOM"]:
             return err

diff --git a/fink_mm/observatory/observatory.py b/fink_mm/observatory/observatory.py
@@ -331,7 +331,7 @@ def association_proba(
         >>> tr_time = Time("2022-07-30T15:48:54.89").jd
         >>> r = swift_bat.association_proba(225.0206, -69.4968, tr_time+1)
         >>> print(round(r, 13))
-        1.7e-12
+        5.9576e-09
 
         >>> tr_time = Time("2022-07-29T21:13:01").jd
         >>> r = fermi_gbm.association_proba(316.6900, -4.1699, tr_time)

diff --git a/fink_mm/utils/fun_utils.py b/fink_mm/utils/fun_utils.py
@@ -871,13 +871,15 @@ def read_and_build_spark_submit(config, logger):
     >>> spark_str = read_and_build_spark_submit(config, logger)
 
     >>> home_path = os.environ["HOME"]
+    >>> driver_host = os.environ["HOSTNAME"]
     >>> path_bash_profile = os.path.join(home_path, ".bash_profile")
-    >>> test_str = f"if test -f '{path_bash_profile}'; then         source {path_bash_profile}; fi;         `which spark-submit`         --master local[8]         --conf spark.mesos.principal=         --conf spark.mesos.secret=         --conf spark.mesos.role=         --conf spark.executorEnv.HOME=/path/to/user/         --driver-memory 4G         --executor-memory 8G         --conf spark.cores.max=16         --conf spark.executor.cores=8"
+    >>> test_str = f"if test -f '{path_bash_profile}'; then         source {path_bash_profile}; fi;         `which spark-submit`         --master local[8]         --conf spark.driver.host={driver_host}         --conf spark.mesos.principal=         --conf spark.mesos.secret=         --conf spark.mesos.role=         --conf spark.executorEnv.HOME=/path/to/user/         --driver-memory 4G         --executor-memory 8G         --conf spark.cores.max=16         --conf spark.executor.cores=8"
     >>> test_str == spark_str
     True
     """
     try:
         master_manager = config["STREAM"]["manager"]
+        driver_host = config["STREAM"]["driver_host"]
         principal_group = config["STREAM"]["principal"]
         secret = config["STREAM"]["secret"]
         role = config["STREAM"]["role"]
@@ -897,6 +899,7 @@ def read_and_build_spark_submit(config, logger):
         source {}; fi; \
         `which spark-submit` \
         --master {} \
+        --conf spark.driver.host={} \
         --conf spark.mesos.principal={} \
         --conf spark.mesos.secret={} \
         --conf spark.mesos.role={} \
@@ -908,6 +911,7 @@ def read_and_build_spark_submit(config, logger):
         path_bash_profile,
         path_bash_profile,
         master_manager,
+        driver_host,
         principal_group,
         secret,
         role,