restore old test path

FusRoman · Feb 19, 2024 · d2480e6 · d2480e6
1 parent 4375032
commit d2480e6
Show file tree

Hide file tree

Showing 4 changed files with 10 additions and 10 deletions.
diff --git a/fink_mm/conf/fink_mm.conf b/fink_mm/conf/fink_mm.conf
@@ -13,7 +13,7 @@ hdfs_gcn_storage=/user/roman.le-montagner/gcn_storage/raw
 # They can be in local FS (/path/ or files:///path/) or
 # in distributed FS (e.g. hdfs:///path/).
 # Be careful though to have enough disk space!
-online_ztf_data_prefix=fink_mm/test/test_data/ztf_test/online/science
+online_ztf_data_prefix=fink_mm/test/test_data/ztf_test
 
 # Prefix path on disk to save GRB join ZTF data (work for both online and offline).
 online_grb_data_prefix=fink_mm/test/test_output

diff --git a/fink_mm/conftest.py b/fink_mm/conftest.py
@@ -189,7 +189,8 @@ def init_spark(doctest_namespace):
     doctest_namespace["join_data"] = join_data
     doctest_namespace["alert_data"] = alert_data
 
-    ztf_datatest = "fink_mm/test/test_data/ztf_test/online/science"
+    ztf_datatest = "fink_mm/test/test_data/ztf_test"
+
     gcn_datatest = "fink_mm/test/test_data/gcn_test/raw"
     online_data_test = "fink_mm/test/test_data/online"
     offline_data_test = "fink_mm/test/test_data/offline"

diff --git a/fink_mm/utils/fun_utils.py b/fink_mm/utils/fun_utils.py
@@ -1116,7 +1116,7 @@ def read_grb_admin_options(arguments, config, logger, is_test=False):
     >>> logger = init_logging()
 
     >>> read_grb_admin_options(arguments, config, logger, False)
-    ('20221014', '120', 'fink_mm/test/test_data/ztf_test/online/science', 'fink_mm/test/test_data/gcn_test/raw', 'fink_mm/test/test_output', '30', '127.0.0.1', '4', '/home/roman.le-montagner/fink-broker/catalogs_hbase/ztf.jd.json', 7, 'localhost:9092', 'toto', 'tata')
+    ('20221014', '120', 'fink_mm/test/test_data/ztf_test', 'fink_mm/test/test_data/gcn_test/raw', 'fink_mm/test/test_output', '30', '127.0.0.1', '4', '/home/roman.le-montagner/fink-broker/catalogs_hbase/ztf.jd.json', 7, 'localhost:9092', 'toto', 'tata')
 
     >>> res = read_grb_admin_options(arguments, config, logger, True)
 

diff --git a/fink_mm/ztf_join_gcn.py b/fink_mm/ztf_join_gcn.py
@@ -3,6 +3,7 @@
 warnings.filterwarnings("ignore")
 
 import time
+import os
 import subprocess
 from typing import Tuple
 import sys
@@ -184,10 +185,8 @@ def load_dataframe(
     if load_mode == DataMode.STREAMING:
         # connection to the ztf science stream
         ztf_alert = connect_to_raw_database(
-            ztf_path
-            + "/year={}/month={}/day={}".format(night[0:4], night[4:6], night[6:8]),
-            ztf_path
-            + "/year={}/month={}/day={}".format(night[0:4], night[4:6], night[6:8]),
+            os.path.join(ztf_path, f"/online/science/{night}"),
+            os.path.join(ztf_path, f"/online/science/{night}"),
             latestfirst=False,
         )
 
@@ -207,9 +206,9 @@ def load_dataframe(
             spark.read.format("parquet")
             .option("mergeSchema", True)
             .load(
-                ztf_path
-                + "/archive/science/year={}/month={}/day={}".format(
-                    night[0:4], night[4:6], night[6:8]
+                os.path.join(
+                    ztf_path,
+                    f"/archive/science/year={night[0:4]}/month={night[4:6]}/day={night[6:8]}",
                 )
             )
         )