diff --git a/.flake8 b/.flake8 index 2c8f24f4c..6919481c4 100644 --- a/.flake8 +++ b/.flake8 @@ -9,6 +9,10 @@ exclude = old, build, dist + ../fink-mm/test_dir/ + ../fink-mm/prep_ztf_mm.py + ../fink-mm/inspect_data.py + ../fink-mm/generate_test_data.py per-file-ignores = ../fink-mm/fink_mm/ztf_join_gcn.py:W503,E402 ../fink-mm/fink_mm/offline/spark_offline.py:W503,W605 diff --git a/.github/workflows/run_test.yml b/.github/workflows/run_test.yml index a93af667f..8a3061f0e 100644 --- a/.github/workflows/run_test.yml +++ b/.github/workflows/run_test.yml @@ -85,13 +85,13 @@ jobs: source scripts/start_services.sh --kafka-version ${KAFKA_VERSION} --hbase-version ${HBASE_VERSION} cd $FINK_MM - - name: Call raw2science - run: | - fink start raw2science -c ${FINK_MM}/fink_mm/test/test_data/with_hbase/fink_test.conf --night $NIGHT --exit_after 90 +# - name: Call raw2science +# run: | +# fink start raw2science -c ${FINK_MM}/fink_mm/test/test_data/with_hbase/fink_test.conf --night $NIGHT --exit_after 90 - - name: Merge data - run: | - fink start merge -c ${FINK_MM}/fink_mm/test/test_data/with_hbase/fink_test.conf --night $NIGHT +# - name: Merge data +# run: | +# fink start merge -c ${FINK_MM}/fink_mm/test/test_data/with_hbase/fink_test.conf --night $NIGHT - name: Test preamble run: | diff --git a/.gitignore b/.gitignore index 3b4e89a13..03edd2930 100644 --- a/.gitignore +++ b/.gitignore @@ -142,3 +142,4 @@ science2mm_locally.sh clean_it.sh # export_unit_test.sh it_test.sh +test_dir \ No newline at end of file diff --git a/fink_mm/__init__.py b/fink_mm/__init__.py index b2c0f6cf4..c62c9e497 100644 --- a/fink_mm/__init__.py +++ b/fink_mm/__init__.py @@ -12,6 +12,6 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -__version__ = "0.16.13" +__version__ = "0.17.0" __distribution_schema_version__ = "1.3" __observatory_schema_version__ = "1.1" diff --git a/fink_mm/conf/distribute_for_test.conf b/fink_mm/conf/distribute_for_test.conf index d30125b8b..ea36115da 100644 --- a/fink_mm/conf/distribute_for_test.conf +++ b/fink_mm/conf/distribute_for_test.conf @@ -15,8 +15,8 @@ hdfs_gcn_storage=/user/roman.le-montagner/gcn_storage/raw # Be careful though to have enough disk space! online_ztf_data_prefix=fink_mm/test/test_data/distribution_test_data/ -# Prefix path on disk to save GRB join ZTF data (work for both online and offline). -online_grb_data_prefix=fink_mm/test/test_data +# Prefix path on disk to save GRB join ZTF data (is used by online, offline and distribution). +online_grb_data_prefix=fink_mm/test/test_data/distribution_test_data/ # Path where are store the hbase catalog in order to query the hbase database hbase_catalog=/home/roman.le-montagner/fink-broker/catalogs_hbase/ztf.jd.json diff --git a/fink_mm/conf/fink_mm.conf b/fink_mm/conf/fink_mm.conf index 65cd49d40..3455f2e19 100644 --- a/fink_mm/conf/fink_mm.conf +++ b/fink_mm/conf/fink_mm.conf @@ -48,7 +48,7 @@ gaia_dist=5 [STREAM] tinterval=30 -manager=local[2] +manager=local[8] principal= secret= role= diff --git a/fink_mm/conftest.py b/fink_mm/conftest.py index 13bb6ed20..98e0be50f 100644 --- a/fink_mm/conftest.py +++ b/fink_mm/conftest.py @@ -177,11 +177,11 @@ def init_spark(doctest_namespace): doctest_namespace["DataMode"] = DataMode doctest_namespace["sql_func"] = sql_func - grb_data = "fink_mm/test/test_data/gcn_test/raw/year=2019/month=09/day=03" + grb_data = "fink_mm/test/test_data/gcn_test/raw/year=2024/month=01/day=15" gw_data = "fink_mm/test/test_data/S230518h_0_test" join_data = "fink_mm/test/test_data/join_raw_datatest.parquet" alert_data = ( - "fink_mm/test/test_data/ztf_test/online/science/year=2019/month=09/day=03/" + "fink_mm/test/test_data/ztf_test/online/science/year=2024/month=01/day=15/" ) doctest_namespace["grb_data"] = grb_data diff --git a/fink_mm/distribution/apply_filters.py b/fink_mm/distribution/apply_filters.py index 23d05d227..e9a2e305d 100644 --- a/fink_mm/distribution/apply_filters.py +++ b/fink_mm/distribution/apply_filters.py @@ -77,7 +77,7 @@ def apply_filters( df_stream["rb"], df_stream["gcn_loc_error"], df_stream["p_assoc"], - df_stream["rate"], + df_stream["mag_rate"], ), ) .filter("f_gold == True") diff --git a/fink_mm/distribution/distribution.py b/fink_mm/distribution/distribution.py index a85e78c83..55806cfa4 100644 --- a/fink_mm/distribution/distribution.py +++ b/fink_mm/distribution/distribution.py @@ -110,7 +110,7 @@ def grb_distribution( -------- >>> grb_distribution( ... ztfxgcn_test, - ... "20190903", + ... "20240115", ... 30, 40, ... "localhost:9092", ... "toto", "tata" @@ -124,7 +124,8 @@ def grb_distribution( ... topic, ... alert["objectId"], ... alert["fink_class"], - ... alert["rate"] + ... alert["mag_rate"], + ... alert["sigma_rate"] ... ]] >>> len(table) @@ -223,7 +224,7 @@ def launch_distribution(arguments): -------- >>> launch_distribution({ ... "--config" : "fink_mm/conf/distribute_for_test.conf", - ... "--night" : "20190903", + ... "--night" : "20240115", ... "--exit_after" : 30, ... "--verbose" : False ... }) @@ -236,7 +237,8 @@ def launch_distribution(arguments): ... topic, ... alert["objectId"], ... alert["fink_class"], - ... alert["rate"] + ... alert["mag_rate"], + ... alert["sigma_rate"] ... ]] >>> len(table) diff --git a/fink_mm/observatory/LVK/LVK.py b/fink_mm/observatory/LVK/LVK.py index 6f67dff73..a1267c653 100644 --- a/fink_mm/observatory/LVK/LVK.py +++ b/fink_mm/observatory/LVK/LVK.py @@ -21,7 +21,13 @@ from datetime import datetime -def gcn_from_hdfs(client: InsecureClient, root_path: str, triggerId: str, triggerTime: datetime, gcn_status: str) -> pd.DataFrame: +def gcn_from_hdfs( + client: InsecureClient, + root_path: str, + triggerId: str, + triggerTime: datetime, + gcn_status: str, +) -> pd.DataFrame: path_date = os.path.join( root_path, f"year={triggerTime.year:04d}/month={triggerTime.month:02d}/day={triggerTime.day:02d}", @@ -32,7 +38,10 @@ def gcn_from_hdfs(client: InsecureClient, root_path: str, triggerId: str, trigge with client.read(path_to_load) as reader: content = reader.read() pdf = pd.read_parquet(io.BytesIO(content)) - if triggerId in pdf["triggerId"].values and gcn_status in pdf["gcn_status"].values: + if ( + triggerId in pdf["triggerId"].values + and gcn_status in pdf["gcn_status"].values + ): return pdf[ (pdf["triggerId"] == triggerId) & (pdf["gcn_status"] == gcn_status) @@ -82,19 +91,22 @@ def get_skymap(self, **kwargs) -> QTable: >>> np.array(lvk_initial.get_skymap()["UNIQ"]) array([ 1285, 1287, 1296, ..., 162369, 162370, 162371]) """ - if "skymap" in self.voevent["event"]: - skymap_str = self.voevent["event"]["skymap"] - else: - hdfs_adress = kwargs["hdfs_adress"] - hdfs_client = InsecureClient(f"http://{hdfs_adress}:50070") - triggerId = self.get_trigger_id() - gcn_status = kwargs["gcn_status"] - root_path = kwargs["root_path"] - t_obs = Time(self.get_trigger_time()[1], format="jd").to_datetime() - gcn_pdf = gcn_from_hdfs( - hdfs_client, root_path, triggerId, t_obs, gcn_status - ) - skymap_str = json.loads(gcn_pdf["raw_event"].iloc[0])["event"]["skymap"] + + skymap_str = self.voevent["event"]["skymap"] + + # if "skymap" in self.voevent["event"]: + # skymap_str = self.voevent["event"]["skymap"] + # else: + # hdfs_adress = kwargs["hdfs_adress"] + # hdfs_client = InsecureClient(f"http://{hdfs_adress}:50070") + # triggerId = self.get_trigger_id() + # gcn_status = kwargs["gcn_status"] + # root_path = kwargs["root_path"] + # t_obs = Time(self.get_trigger_time()[1], format="jd").to_datetime() + # gcn_pdf = gcn_from_hdfs( + # hdfs_client, root_path, triggerId, t_obs, gcn_status + # ) + # skymap_str = json.loads(gcn_pdf["raw_event"].iloc[0])["event"]["skymap"] # Decode and parse skymap skymap_bytes = b64decode(skymap_str) @@ -136,7 +148,8 @@ def is_observation(self, is_test: bool) -> bool: or self.voevent["superevent_id"][0] == "M" ) return ( - self.voevent["superevent_id"][0] == "S" + self.voevent["superevent_id"][0] + == "S" # comment the significant filter (enable cross-match with 'burst' and subthreshold events) # and self.voevent["event"]["significant"] ) @@ -432,10 +445,13 @@ def association_proba( >>> lvk_initial.association_proba(95.712890625, -10.958863307027668, 0) 0.0054008620296433045 """ - if "hdfs_adress" in kwargs and "gcn_status" in kwargs and "root_path" in kwargs: - skymap = self.get_skymap(**kwargs) - else: - skymap = self.get_skymap() + + skymap = self.get_skymap() + + # if "hdfs_adress" in kwargs and "gcn_status" in kwargs and "root_path" in kwargs: + # skymap = self.get_skymap(**kwargs) + # else: + # skymap = self.get_skymap() max_level = 29 max_nside = ah.level_to_nside(max_level) diff --git a/fink_mm/test/test_data/distribution_test_data/online/year=2019/month=09/day=03/part-00000-c47d82e7-e6e7-40dc-921b-d517971cad63-c000.snappy.parquet b/fink_mm/test/test_data/distribution_test_data/online/year=2019/month=09/day=03/part-00000-c47d82e7-e6e7-40dc-921b-d517971cad63-c000.snappy.parquet deleted file mode 100644 index b35a74fa5..000000000 Binary files a/fink_mm/test/test_data/distribution_test_data/online/year=2019/month=09/day=03/part-00000-c47d82e7-e6e7-40dc-921b-d517971cad63-c000.snappy.parquet and /dev/null differ diff --git a/fink_mm/test/test_data/distribution_test_data/online/year=2019/month=09/day=03/part-00001-c47d82e7-e6e7-40dc-921b-d517971cad63-c000.snappy.parquet b/fink_mm/test/test_data/distribution_test_data/online/year=2019/month=09/day=03/part-00001-c47d82e7-e6e7-40dc-921b-d517971cad63-c000.snappy.parquet deleted file mode 100644 index 50c2060a0..000000000 Binary files a/fink_mm/test/test_data/distribution_test_data/online/year=2019/month=09/day=03/part-00001-c47d82e7-e6e7-40dc-921b-d517971cad63-c000.snappy.parquet and /dev/null differ diff --git a/fink_mm/test/test_data/distribution_test_data/online/year=2019/month=09/day=03/part-00002-c47d82e7-e6e7-40dc-921b-d517971cad63-c000.snappy.parquet b/fink_mm/test/test_data/distribution_test_data/online/year=2019/month=09/day=03/part-00002-c47d82e7-e6e7-40dc-921b-d517971cad63-c000.snappy.parquet deleted file mode 100644 index 50c2060a0..000000000 Binary files a/fink_mm/test/test_data/distribution_test_data/online/year=2019/month=09/day=03/part-00002-c47d82e7-e6e7-40dc-921b-d517971cad63-c000.snappy.parquet and /dev/null differ diff --git a/fink_mm/test/test_data/distribution_test_data/online/year=2024/month=01/day=15/part-00000-16eda71b-a47f-4b6e-8f86-e82999599355.c000.snappy.parquet b/fink_mm/test/test_data/distribution_test_data/online/year=2024/month=01/day=15/part-00000-16eda71b-a47f-4b6e-8f86-e82999599355.c000.snappy.parquet new file mode 100644 index 000000000..c5b898463 Binary files /dev/null and b/fink_mm/test/test_data/distribution_test_data/online/year=2024/month=01/day=15/part-00000-16eda71b-a47f-4b6e-8f86-e82999599355.c000.snappy.parquet differ diff --git a/fink_mm/test/test_data/distribution_test_data/online/year=2024/month=01/day=15/part-00001-f0b82ee7-10a0-4a38-8d78-6ce6264672fb.c000.snappy.parquet b/fink_mm/test/test_data/distribution_test_data/online/year=2024/month=01/day=15/part-00001-f0b82ee7-10a0-4a38-8d78-6ce6264672fb.c000.snappy.parquet new file mode 100644 index 000000000..ac6516507 Binary files /dev/null and b/fink_mm/test/test_data/distribution_test_data/online/year=2024/month=01/day=15/part-00001-f0b82ee7-10a0-4a38-8d78-6ce6264672fb.c000.snappy.parquet differ diff --git a/fink_mm/test/test_data/distribution_test_data/online/year=2024/month=01/day=15/part-00002-50e44c9f-d826-46a9-a8dd-5c5f97d8b917.c000.snappy.parquet b/fink_mm/test/test_data/distribution_test_data/online/year=2024/month=01/day=15/part-00002-50e44c9f-d826-46a9-a8dd-5c5f97d8b917.c000.snappy.parquet new file mode 100644 index 000000000..2955e19e9 Binary files /dev/null and b/fink_mm/test/test_data/distribution_test_data/online/year=2024/month=01/day=15/part-00002-50e44c9f-d826-46a9-a8dd-5c5f97d8b917.c000.snappy.parquet differ diff --git a/fink_mm/test/test_data/distribution_test_data/online/year=2024/month=01/day=15/part-00003-89da5c3b-73a7-4a92-8a61-d84b70c61cfa.c000.snappy.parquet b/fink_mm/test/test_data/distribution_test_data/online/year=2024/month=01/day=15/part-00003-89da5c3b-73a7-4a92-8a61-d84b70c61cfa.c000.snappy.parquet new file mode 100644 index 000000000..821415e45 Binary files /dev/null and b/fink_mm/test/test_data/distribution_test_data/online/year=2024/month=01/day=15/part-00003-89da5c3b-73a7-4a92-8a61-d84b70c61cfa.c000.snappy.parquet differ diff --git a/fink_mm/test/test_data/distribution_test_data/online/year=2024/month=01/day=15/part-00004-457e921b-0180-4752-afc7-128f95668244.c000.snappy.parquet b/fink_mm/test/test_data/distribution_test_data/online/year=2024/month=01/day=15/part-00004-457e921b-0180-4752-afc7-128f95668244.c000.snappy.parquet new file mode 100644 index 000000000..14727af57 Binary files /dev/null and b/fink_mm/test/test_data/distribution_test_data/online/year=2024/month=01/day=15/part-00004-457e921b-0180-4752-afc7-128f95668244.c000.snappy.parquet differ diff --git a/fink_mm/test/test_data/distribution_test_data/online/year=2024/month=01/day=15/part-00005-e4e308da-80cf-4655-9aa4-198e3b0436e9.c000.snappy.parquet b/fink_mm/test/test_data/distribution_test_data/online/year=2024/month=01/day=15/part-00005-e4e308da-80cf-4655-9aa4-198e3b0436e9.c000.snappy.parquet new file mode 100644 index 000000000..c09d113bc Binary files /dev/null and b/fink_mm/test/test_data/distribution_test_data/online/year=2024/month=01/day=15/part-00005-e4e308da-80cf-4655-9aa4-198e3b0436e9.c000.snappy.parquet differ diff --git a/fink_mm/test/test_data/distribution_test_data/online/year=2024/month=01/day=15/part-00006-897eca4d-a526-4a39-8713-0166ee2d93c8.c000.snappy.parquet b/fink_mm/test/test_data/distribution_test_data/online/year=2024/month=01/day=15/part-00006-897eca4d-a526-4a39-8713-0166ee2d93c8.c000.snappy.parquet new file mode 100644 index 000000000..a0917bb20 Binary files /dev/null and b/fink_mm/test/test_data/distribution_test_data/online/year=2024/month=01/day=15/part-00006-897eca4d-a526-4a39-8713-0166ee2d93c8.c000.snappy.parquet differ diff --git a/fink_mm/test/test_data/distribution_test_data/online/year=2024/month=01/day=15/part-00007-bd110783-028f-4996-b172-f98110102202.c000.snappy.parquet b/fink_mm/test/test_data/distribution_test_data/online/year=2024/month=01/day=15/part-00007-bd110783-028f-4996-b172-f98110102202.c000.snappy.parquet new file mode 100644 index 000000000..a3b75c599 Binary files /dev/null and b/fink_mm/test/test_data/distribution_test_data/online/year=2024/month=01/day=15/part-00007-bd110783-028f-4996-b172-f98110102202.c000.snappy.parquet differ diff --git a/fink_mm/test/test_data/distribution_test_data/online/year=2024/month=01/day=15/part-00008-f08f3592-a440-4851-b4bc-a94462b6d341.c000.snappy.parquet b/fink_mm/test/test_data/distribution_test_data/online/year=2024/month=01/day=15/part-00008-f08f3592-a440-4851-b4bc-a94462b6d341.c000.snappy.parquet new file mode 100644 index 000000000..ab88a277b Binary files /dev/null and b/fink_mm/test/test_data/distribution_test_data/online/year=2024/month=01/day=15/part-00008-f08f3592-a440-4851-b4bc-a94462b6d341.c000.snappy.parquet differ diff --git a/fink_mm/test/test_data/distribution_test_data/online/year=2024/month=01/day=15/part-00009-a39412c4-b789-4ddb-b3bf-8944945c847c.c000.snappy.parquet b/fink_mm/test/test_data/distribution_test_data/online/year=2024/month=01/day=15/part-00009-a39412c4-b789-4ddb-b3bf-8944945c847c.c000.snappy.parquet new file mode 100644 index 000000000..bd26fcee7 Binary files /dev/null and b/fink_mm/test/test_data/distribution_test_data/online/year=2024/month=01/day=15/part-00009-a39412c4-b789-4ddb-b3bf-8944945c847c.c000.snappy.parquet differ diff --git a/fink_mm/test/test_data/distribution_test_data/online/year=2024/month=01/day=15/part-00010-aeb604d8-8587-4101-aab2-54429ad3d7d2.c000.snappy.parquet b/fink_mm/test/test_data/distribution_test_data/online/year=2024/month=01/day=15/part-00010-aeb604d8-8587-4101-aab2-54429ad3d7d2.c000.snappy.parquet new file mode 100644 index 000000000..b0f22e75c Binary files /dev/null and b/fink_mm/test/test_data/distribution_test_data/online/year=2024/month=01/day=15/part-00010-aeb604d8-8587-4101-aab2-54429ad3d7d2.c000.snappy.parquet differ diff --git a/fink_mm/test/test_data/distribution_test_data/online/year=2024/month=01/day=15/part-00011-94f9397c-7786-4237-90ce-5ad9d6cee936.c000.snappy.parquet b/fink_mm/test/test_data/distribution_test_data/online/year=2024/month=01/day=15/part-00011-94f9397c-7786-4237-90ce-5ad9d6cee936.c000.snappy.parquet new file mode 100644 index 000000000..05f74384f Binary files /dev/null and b/fink_mm/test/test_data/distribution_test_data/online/year=2024/month=01/day=15/part-00011-94f9397c-7786-4237-90ce-5ad9d6cee936.c000.snappy.parquet differ diff --git a/fink_mm/test/test_data/distribution_test_data/online/year=2024/month=01/day=15/part-00014-4bf2c65f-8eff-465e-8334-81b8e4876b52.c000.snappy.parquet b/fink_mm/test/test_data/distribution_test_data/online/year=2024/month=01/day=15/part-00014-4bf2c65f-8eff-465e-8334-81b8e4876b52.c000.snappy.parquet new file mode 100644 index 000000000..923d3e2d8 Binary files /dev/null and b/fink_mm/test/test_data/distribution_test_data/online/year=2024/month=01/day=15/part-00014-4bf2c65f-8eff-465e-8334-81b8e4876b52.c000.snappy.parquet differ diff --git a/fink_mm/test/test_data/distribution_test_data/online/year=2024/month=01/day=15/part-00017-4b4d7597-5af3-43c6-a4f0-45ad376bccd8.c000.snappy.parquet b/fink_mm/test/test_data/distribution_test_data/online/year=2024/month=01/day=15/part-00017-4b4d7597-5af3-43c6-a4f0-45ad376bccd8.c000.snappy.parquet new file mode 100644 index 000000000..5c54dc4d1 Binary files /dev/null and b/fink_mm/test/test_data/distribution_test_data/online/year=2024/month=01/day=15/part-00017-4b4d7597-5af3-43c6-a4f0-45ad376bccd8.c000.snappy.parquet differ diff --git a/fink_mm/test/test_data/distribution_test_data/online/year=2024/month=01/day=15/part-00018-3166db0a-ee1c-4a1e-aec6-b553b59e143d.c000.snappy.parquet b/fink_mm/test/test_data/distribution_test_data/online/year=2024/month=01/day=15/part-00018-3166db0a-ee1c-4a1e-aec6-b553b59e143d.c000.snappy.parquet new file mode 100644 index 000000000..22e4adba6 Binary files /dev/null and b/fink_mm/test/test_data/distribution_test_data/online/year=2024/month=01/day=15/part-00018-3166db0a-ee1c-4a1e-aec6-b553b59e143d.c000.snappy.parquet differ diff --git a/fink_mm/test/test_data/distribution_test_data/online/year=2024/month=01/day=15/part-00019-5619688b-f40e-444d-bc90-42c680b14d79.c000.snappy.parquet b/fink_mm/test/test_data/distribution_test_data/online/year=2024/month=01/day=15/part-00019-5619688b-f40e-444d-bc90-42c680b14d79.c000.snappy.parquet new file mode 100644 index 000000000..7e7eee4fd Binary files /dev/null and b/fink_mm/test/test_data/distribution_test_data/online/year=2024/month=01/day=15/part-00019-5619688b-f40e-444d-bc90-42c680b14d79.c000.snappy.parquet differ diff --git a/fink_mm/test/test_data/distribution_test_data/online/year=2024/month=01/day=15/part-00020-c3c3ffce-4c8f-40b8-b7c8-b65afd4ce2e9.c000.snappy.parquet b/fink_mm/test/test_data/distribution_test_data/online/year=2024/month=01/day=15/part-00020-c3c3ffce-4c8f-40b8-b7c8-b65afd4ce2e9.c000.snappy.parquet new file mode 100644 index 000000000..3a075f777 Binary files /dev/null and b/fink_mm/test/test_data/distribution_test_data/online/year=2024/month=01/day=15/part-00020-c3c3ffce-4c8f-40b8-b7c8-b65afd4ce2e9.c000.snappy.parquet differ diff --git a/fink_mm/test/test_data/distribution_test_data/online/year=2024/month=01/day=15/part-00021-e60f7ed6-0333-44d1-a876-1e32d3cc2bbf.c000.snappy.parquet b/fink_mm/test/test_data/distribution_test_data/online/year=2024/month=01/day=15/part-00021-e60f7ed6-0333-44d1-a876-1e32d3cc2bbf.c000.snappy.parquet new file mode 100644 index 000000000..dbcc779b0 Binary files /dev/null and b/fink_mm/test/test_data/distribution_test_data/online/year=2024/month=01/day=15/part-00021-e60f7ed6-0333-44d1-a876-1e32d3cc2bbf.c000.snappy.parquet differ diff --git a/fink_mm/test/test_data/distribution_test_data/online/year=2024/month=01/day=15/part-00023-a732675a-9282-47db-b67c-cdc278841d9c.c000.snappy.parquet b/fink_mm/test/test_data/distribution_test_data/online/year=2024/month=01/day=15/part-00023-a732675a-9282-47db-b67c-cdc278841d9c.c000.snappy.parquet new file mode 100644 index 000000000..b53f94469 Binary files /dev/null and b/fink_mm/test/test_data/distribution_test_data/online/year=2024/month=01/day=15/part-00023-a732675a-9282-47db-b67c-cdc278841d9c.c000.snappy.parquet differ diff --git a/fink_mm/test/test_data/distribution_test_data/online/year=2024/month=01/day=15/part-00024-a61c2dfd-fc77-4520-a279-7077f9490759.c000.snappy.parquet b/fink_mm/test/test_data/distribution_test_data/online/year=2024/month=01/day=15/part-00024-a61c2dfd-fc77-4520-a279-7077f9490759.c000.snappy.parquet new file mode 100644 index 000000000..43f681da5 Binary files /dev/null and b/fink_mm/test/test_data/distribution_test_data/online/year=2024/month=01/day=15/part-00024-a61c2dfd-fc77-4520-a279-7077f9490759.c000.snappy.parquet differ diff --git a/fink_mm/test/test_data/distribution_test_data/online/year=2024/month=01/day=15/part-00025-1494bd22-0ad8-40e1-9ac8-353aaf7863b0.c000.snappy.parquet b/fink_mm/test/test_data/distribution_test_data/online/year=2024/month=01/day=15/part-00025-1494bd22-0ad8-40e1-9ac8-353aaf7863b0.c000.snappy.parquet new file mode 100644 index 000000000..bf3341f30 Binary files /dev/null and b/fink_mm/test/test_data/distribution_test_data/online/year=2024/month=01/day=15/part-00025-1494bd22-0ad8-40e1-9ac8-353aaf7863b0.c000.snappy.parquet differ diff --git a/fink_mm/test/test_data/distribution_test_data/online/year=2024/month=01/day=15/part-00027-b693956f-ab21-4971-9265-588d06323ba9.c000.snappy.parquet b/fink_mm/test/test_data/distribution_test_data/online/year=2024/month=01/day=15/part-00027-b693956f-ab21-4971-9265-588d06323ba9.c000.snappy.parquet new file mode 100644 index 000000000..5143031b5 Binary files /dev/null and b/fink_mm/test/test_data/distribution_test_data/online/year=2024/month=01/day=15/part-00027-b693956f-ab21-4971-9265-588d06323ba9.c000.snappy.parquet differ diff --git a/fink_mm/test/test_data/distribution_test_data/online/year=2024/month=01/day=15/part-00028-240b3982-b9c0-4ec5-9854-0d40222f2436.c000.snappy.parquet b/fink_mm/test/test_data/distribution_test_data/online/year=2024/month=01/day=15/part-00028-240b3982-b9c0-4ec5-9854-0d40222f2436.c000.snappy.parquet new file mode 100644 index 000000000..ca2528b15 Binary files /dev/null and b/fink_mm/test/test_data/distribution_test_data/online/year=2024/month=01/day=15/part-00028-240b3982-b9c0-4ec5-9854-0d40222f2436.c000.snappy.parquet differ diff --git a/fink_mm/test/test_data/distribution_test_data/online/year=2024/month=01/day=15/part-00030-e3a62b1d-c182-4a5a-884d-170a6bb6ca10.c000.snappy.parquet b/fink_mm/test/test_data/distribution_test_data/online/year=2024/month=01/day=15/part-00030-e3a62b1d-c182-4a5a-884d-170a6bb6ca10.c000.snappy.parquet new file mode 100644 index 000000000..a94bf871d Binary files /dev/null and b/fink_mm/test/test_data/distribution_test_data/online/year=2024/month=01/day=15/part-00030-e3a62b1d-c182-4a5a-884d-170a6bb6ca10.c000.snappy.parquet differ diff --git a/fink_mm/test/test_data/distribution_test_data/online/year=2024/month=01/day=15/part-00031-f4624b0c-57e3-4495-b652-38329e280fc5.c000.snappy.parquet b/fink_mm/test/test_data/distribution_test_data/online/year=2024/month=01/day=15/part-00031-f4624b0c-57e3-4495-b652-38329e280fc5.c000.snappy.parquet new file mode 100644 index 000000000..886bd82c1 Binary files /dev/null and b/fink_mm/test/test_data/distribution_test_data/online/year=2024/month=01/day=15/part-00031-f4624b0c-57e3-4495-b652-38329e280fc5.c000.snappy.parquet differ diff --git a/fink_mm/test/test_data/distribution_test_data/online/year=2024/month=01/day=15/part-00032-612131bc-6e8a-4cb1-8d58-1f0c81856c0e.c000.snappy.parquet b/fink_mm/test/test_data/distribution_test_data/online/year=2024/month=01/day=15/part-00032-612131bc-6e8a-4cb1-8d58-1f0c81856c0e.c000.snappy.parquet new file mode 100644 index 000000000..53bf8fbe7 Binary files /dev/null and b/fink_mm/test/test_data/distribution_test_data/online/year=2024/month=01/day=15/part-00032-612131bc-6e8a-4cb1-8d58-1f0c81856c0e.c000.snappy.parquet differ diff --git a/fink_mm/test/test_data/distribution_test_data/online/year=2024/month=01/day=15/part-00033-52d352fb-ab06-4db9-84ba-bd78c9c3ca43.c000.snappy.parquet b/fink_mm/test/test_data/distribution_test_data/online/year=2024/month=01/day=15/part-00033-52d352fb-ab06-4db9-84ba-bd78c9c3ca43.c000.snappy.parquet new file mode 100644 index 000000000..976689ddf Binary files /dev/null and b/fink_mm/test/test_data/distribution_test_data/online/year=2024/month=01/day=15/part-00033-52d352fb-ab06-4db9-84ba-bd78c9c3ca43.c000.snappy.parquet differ diff --git a/fink_mm/test/test_data/distribution_test_data/online/year=2024/month=01/day=15/part-00034-c04fa82a-de59-44b2-b1f7-5fb2df5092f7.c000.snappy.parquet b/fink_mm/test/test_data/distribution_test_data/online/year=2024/month=01/day=15/part-00034-c04fa82a-de59-44b2-b1f7-5fb2df5092f7.c000.snappy.parquet new file mode 100644 index 000000000..575d0df68 Binary files /dev/null and b/fink_mm/test/test_data/distribution_test_data/online/year=2024/month=01/day=15/part-00034-c04fa82a-de59-44b2-b1f7-5fb2df5092f7.c000.snappy.parquet differ diff --git a/fink_mm/test/test_data/distribution_test_data/online/year=2024/month=01/day=15/part-00036-dfa55340-c9ea-4bff-932c-ce024831a5e6.c000.snappy.parquet b/fink_mm/test/test_data/distribution_test_data/online/year=2024/month=01/day=15/part-00036-dfa55340-c9ea-4bff-932c-ce024831a5e6.c000.snappy.parquet new file mode 100644 index 000000000..bd0fafd39 Binary files /dev/null and b/fink_mm/test/test_data/distribution_test_data/online/year=2024/month=01/day=15/part-00036-dfa55340-c9ea-4bff-932c-ce024831a5e6.c000.snappy.parquet differ diff --git a/fink_mm/test/test_data/distribution_test_data/online/year=2024/month=01/day=15/part-00037-18245bed-b4e0-421a-8b97-49efc0e79a54.c000.snappy.parquet b/fink_mm/test/test_data/distribution_test_data/online/year=2024/month=01/day=15/part-00037-18245bed-b4e0-421a-8b97-49efc0e79a54.c000.snappy.parquet new file mode 100644 index 000000000..912ea43b5 Binary files /dev/null and b/fink_mm/test/test_data/distribution_test_data/online/year=2024/month=01/day=15/part-00037-18245bed-b4e0-421a-8b97-49efc0e79a54.c000.snappy.parquet differ diff --git a/fink_mm/test/test_data/distribution_test_data/online/year=2024/month=01/day=15/part-00042-aef664c5-758d-45d3-a6e2-c8ef9b54af4b.c000.snappy.parquet b/fink_mm/test/test_data/distribution_test_data/online/year=2024/month=01/day=15/part-00042-aef664c5-758d-45d3-a6e2-c8ef9b54af4b.c000.snappy.parquet new file mode 100644 index 000000000..bf2f1b1b3 Binary files /dev/null and b/fink_mm/test/test_data/distribution_test_data/online/year=2024/month=01/day=15/part-00042-aef664c5-758d-45d3-a6e2-c8ef9b54af4b.c000.snappy.parquet differ diff --git a/fink_mm/test/test_data/distribution_test_data/online/year=2024/month=01/day=15/part-00043-32bb4910-b611-4de2-9e75-3ebc2e9fc984.c000.snappy.parquet b/fink_mm/test/test_data/distribution_test_data/online/year=2024/month=01/day=15/part-00043-32bb4910-b611-4de2-9e75-3ebc2e9fc984.c000.snappy.parquet new file mode 100644 index 000000000..ca15a02d7 Binary files /dev/null and b/fink_mm/test/test_data/distribution_test_data/online/year=2024/month=01/day=15/part-00043-32bb4910-b611-4de2-9e75-3ebc2e9fc984.c000.snappy.parquet differ diff --git a/fink_mm/test/test_data/distribution_test_data/online/year=2024/month=01/day=15/part-00044-fb25faa8-ba77-42a1-90e0-3de971c3b720.c000.snappy.parquet b/fink_mm/test/test_data/distribution_test_data/online/year=2024/month=01/day=15/part-00044-fb25faa8-ba77-42a1-90e0-3de971c3b720.c000.snappy.parquet new file mode 100644 index 000000000..557849429 Binary files /dev/null and b/fink_mm/test/test_data/distribution_test_data/online/year=2024/month=01/day=15/part-00044-fb25faa8-ba77-42a1-90e0-3de971c3b720.c000.snappy.parquet differ diff --git a/fink_mm/test/test_data/distribution_test_data/online/year=2024/month=01/day=15/part-00048-87827af7-ad06-4a57-99a0-a9b530c54114.c000.snappy.parquet b/fink_mm/test/test_data/distribution_test_data/online/year=2024/month=01/day=15/part-00048-87827af7-ad06-4a57-99a0-a9b530c54114.c000.snappy.parquet new file mode 100644 index 000000000..3d897fda8 Binary files /dev/null and b/fink_mm/test/test_data/distribution_test_data/online/year=2024/month=01/day=15/part-00048-87827af7-ad06-4a57-99a0-a9b530c54114.c000.snappy.parquet differ diff --git a/fink_mm/test/test_data/distribution_test_data/online/year=2024/month=01/day=15/part-00050-ea67ad12-8b76-41a7-86c0-b7106b82dc42.c000.snappy.parquet b/fink_mm/test/test_data/distribution_test_data/online/year=2024/month=01/day=15/part-00050-ea67ad12-8b76-41a7-86c0-b7106b82dc42.c000.snappy.parquet new file mode 100644 index 000000000..00ffb50ba Binary files /dev/null and b/fink_mm/test/test_data/distribution_test_data/online/year=2024/month=01/day=15/part-00050-ea67ad12-8b76-41a7-86c0-b7106b82dc42.c000.snappy.parquet differ diff --git a/fink_mm/test/test_data/distribution_test_data/online/year=2024/month=01/day=15/part-00051-a6ab676e-caa6-4cfb-8bea-c7f0910c4ba8.c000.snappy.parquet b/fink_mm/test/test_data/distribution_test_data/online/year=2024/month=01/day=15/part-00051-a6ab676e-caa6-4cfb-8bea-c7f0910c4ba8.c000.snappy.parquet new file mode 100644 index 000000000..ba1a14c71 Binary files /dev/null and b/fink_mm/test/test_data/distribution_test_data/online/year=2024/month=01/day=15/part-00051-a6ab676e-caa6-4cfb-8bea-c7f0910c4ba8.c000.snappy.parquet differ diff --git a/fink_mm/test/test_data/distribution_test_data/online/year=2024/month=01/day=15/part-00052-c761fadc-b5ca-4d38-b924-e2a1e3dcc4c3.c000.snappy.parquet b/fink_mm/test/test_data/distribution_test_data/online/year=2024/month=01/day=15/part-00052-c761fadc-b5ca-4d38-b924-e2a1e3dcc4c3.c000.snappy.parquet new file mode 100644 index 000000000..c472d0209 Binary files /dev/null and b/fink_mm/test/test_data/distribution_test_data/online/year=2024/month=01/day=15/part-00052-c761fadc-b5ca-4d38-b924-e2a1e3dcc4c3.c000.snappy.parquet differ diff --git a/fink_mm/test/test_data/distribution_test_data/online/year=2024/month=01/day=15/part-00053-efbf47f3-cf60-417a-ad50-37a52a040d2a.c000.snappy.parquet b/fink_mm/test/test_data/distribution_test_data/online/year=2024/month=01/day=15/part-00053-efbf47f3-cf60-417a-ad50-37a52a040d2a.c000.snappy.parquet new file mode 100644 index 000000000..5a1a3091d Binary files /dev/null and b/fink_mm/test/test_data/distribution_test_data/online/year=2024/month=01/day=15/part-00053-efbf47f3-cf60-417a-ad50-37a52a040d2a.c000.snappy.parquet differ diff --git a/fink_mm/test/test_data/distribution_test_data/online/year=2024/month=01/day=15/part-00054-72ce4206-4481-4a43-842f-48ef45fdb54f.c000.snappy.parquet b/fink_mm/test/test_data/distribution_test_data/online/year=2024/month=01/day=15/part-00054-72ce4206-4481-4a43-842f-48ef45fdb54f.c000.snappy.parquet new file mode 100644 index 000000000..27e2fbff1 Binary files /dev/null and b/fink_mm/test/test_data/distribution_test_data/online/year=2024/month=01/day=15/part-00054-72ce4206-4481-4a43-842f-48ef45fdb54f.c000.snappy.parquet differ diff --git a/fink_mm/test/test_data/distribution_test_data/online/year=2024/month=01/day=15/part-00057-c7b5fcd7-4a83-40b2-b99a-02f048582e6e.c000.snappy.parquet b/fink_mm/test/test_data/distribution_test_data/online/year=2024/month=01/day=15/part-00057-c7b5fcd7-4a83-40b2-b99a-02f048582e6e.c000.snappy.parquet new file mode 100644 index 000000000..ddf2cda3c Binary files /dev/null and b/fink_mm/test/test_data/distribution_test_data/online/year=2024/month=01/day=15/part-00057-c7b5fcd7-4a83-40b2-b99a-02f048582e6e.c000.snappy.parquet differ diff --git a/fink_mm/test/test_data/distribution_test_data/online/year=2024/month=01/day=15/part-00059-77534333-61a8-4b72-bcfd-01567a038114.c000.snappy.parquet b/fink_mm/test/test_data/distribution_test_data/online/year=2024/month=01/day=15/part-00059-77534333-61a8-4b72-bcfd-01567a038114.c000.snappy.parquet new file mode 100644 index 000000000..bf9439484 Binary files /dev/null and b/fink_mm/test/test_data/distribution_test_data/online/year=2024/month=01/day=15/part-00059-77534333-61a8-4b72-bcfd-01567a038114.c000.snappy.parquet differ diff --git a/fink_mm/test/test_data/distribution_test_data/online/year=2024/month=01/day=15/part-00061-bda2b5b0-a27c-429c-80da-dd77510a4f0d.c000.snappy.parquet b/fink_mm/test/test_data/distribution_test_data/online/year=2024/month=01/day=15/part-00061-bda2b5b0-a27c-429c-80da-dd77510a4f0d.c000.snappy.parquet new file mode 100644 index 000000000..fdccb089e Binary files /dev/null and b/fink_mm/test/test_data/distribution_test_data/online/year=2024/month=01/day=15/part-00061-bda2b5b0-a27c-429c-80da-dd77510a4f0d.c000.snappy.parquet differ diff --git a/fink_mm/test/test_data/distribution_test_data/online/year=2024/month=01/day=15/part-00063-fe110af7-e45f-448c-b43a-ac3c41c63455.c000.snappy.parquet b/fink_mm/test/test_data/distribution_test_data/online/year=2024/month=01/day=15/part-00063-fe110af7-e45f-448c-b43a-ac3c41c63455.c000.snappy.parquet new file mode 100644 index 000000000..46b0fe3b2 Binary files /dev/null and b/fink_mm/test/test_data/distribution_test_data/online/year=2024/month=01/day=15/part-00063-fe110af7-e45f-448c-b43a-ac3c41c63455.c000.snappy.parquet differ diff --git a/fink_mm/test/test_data/distribution_test_data/online/year=2024/month=01/day=15/part-00066-f7b0c76f-551c-40d3-b993-e78a536ac989.c000.snappy.parquet b/fink_mm/test/test_data/distribution_test_data/online/year=2024/month=01/day=15/part-00066-f7b0c76f-551c-40d3-b993-e78a536ac989.c000.snappy.parquet new file mode 100644 index 000000000..52bd50f70 Binary files /dev/null and b/fink_mm/test/test_data/distribution_test_data/online/year=2024/month=01/day=15/part-00066-f7b0c76f-551c-40d3-b993-e78a536ac989.c000.snappy.parquet differ diff --git a/fink_mm/test/test_data/distribution_test_data/online/year=2024/month=01/day=15/part-00067-f1f2f1a2-9c13-4eb5-b7a0-c0bf2c4ad392.c000.snappy.parquet b/fink_mm/test/test_data/distribution_test_data/online/year=2024/month=01/day=15/part-00067-f1f2f1a2-9c13-4eb5-b7a0-c0bf2c4ad392.c000.snappy.parquet new file mode 100644 index 000000000..e096455a2 Binary files /dev/null and b/fink_mm/test/test_data/distribution_test_data/online/year=2024/month=01/day=15/part-00067-f1f2f1a2-9c13-4eb5-b7a0-c0bf2c4ad392.c000.snappy.parquet differ diff --git a/fink_mm/test/test_data/distribution_test_data/online/year=2024/month=01/day=15/part-00068-cc5b9d42-e43c-40e0-925e-f94a4969cbad.c000.snappy.parquet b/fink_mm/test/test_data/distribution_test_data/online/year=2024/month=01/day=15/part-00068-cc5b9d42-e43c-40e0-925e-f94a4969cbad.c000.snappy.parquet new file mode 100644 index 000000000..7c39a91cd Binary files /dev/null and b/fink_mm/test/test_data/distribution_test_data/online/year=2024/month=01/day=15/part-00068-cc5b9d42-e43c-40e0-925e-f94a4969cbad.c000.snappy.parquet differ diff --git a/fink_mm/test/test_data/distribution_test_data/online/year=2024/month=01/day=15/part-00071-d5e9864e-d3af-4d86-a727-1d1cc5f6306e.c000.snappy.parquet b/fink_mm/test/test_data/distribution_test_data/online/year=2024/month=01/day=15/part-00071-d5e9864e-d3af-4d86-a727-1d1cc5f6306e.c000.snappy.parquet new file mode 100644 index 000000000..4badcd83b Binary files /dev/null and b/fink_mm/test/test_data/distribution_test_data/online/year=2024/month=01/day=15/part-00071-d5e9864e-d3af-4d86-a727-1d1cc5f6306e.c000.snappy.parquet differ diff --git a/fink_mm/test/test_data/distribution_test_data/online/year=2024/month=01/day=15/part-00072-5f8feb82-9dd2-4dcf-8b0c-b57044f084ab.c000.snappy.parquet b/fink_mm/test/test_data/distribution_test_data/online/year=2024/month=01/day=15/part-00072-5f8feb82-9dd2-4dcf-8b0c-b57044f084ab.c000.snappy.parquet new file mode 100644 index 000000000..e226c22d3 Binary files /dev/null and b/fink_mm/test/test_data/distribution_test_data/online/year=2024/month=01/day=15/part-00072-5f8feb82-9dd2-4dcf-8b0c-b57044f084ab.c000.snappy.parquet differ diff --git a/fink_mm/test/test_data/distribution_test_data/online/year=2024/month=01/day=15/part-00073-06385970-5174-43a9-aba0-fbe5111bf8a3.c000.snappy.parquet b/fink_mm/test/test_data/distribution_test_data/online/year=2024/month=01/day=15/part-00073-06385970-5174-43a9-aba0-fbe5111bf8a3.c000.snappy.parquet new file mode 100644 index 000000000..9216d3f11 Binary files /dev/null and b/fink_mm/test/test_data/distribution_test_data/online/year=2024/month=01/day=15/part-00073-06385970-5174-43a9-aba0-fbe5111bf8a3.c000.snappy.parquet differ diff --git a/fink_mm/test/test_data/distribution_test_data/online/year=2024/month=01/day=15/part-00074-a21eb37d-fdae-4879-b9fc-74621ef8ef09.c000.snappy.parquet b/fink_mm/test/test_data/distribution_test_data/online/year=2024/month=01/day=15/part-00074-a21eb37d-fdae-4879-b9fc-74621ef8ef09.c000.snappy.parquet new file mode 100644 index 000000000..31b4384cf Binary files /dev/null and b/fink_mm/test/test_data/distribution_test_data/online/year=2024/month=01/day=15/part-00074-a21eb37d-fdae-4879-b9fc-74621ef8ef09.c000.snappy.parquet differ diff --git a/fink_mm/test/test_data/distribution_test_data/online/year=2024/month=01/day=15/part-00077-0d0dd1f3-2e30-4ce0-8fc2-49cfcd740d42.c000.snappy.parquet b/fink_mm/test/test_data/distribution_test_data/online/year=2024/month=01/day=15/part-00077-0d0dd1f3-2e30-4ce0-8fc2-49cfcd740d42.c000.snappy.parquet new file mode 100644 index 000000000..b49a556e0 Binary files /dev/null and b/fink_mm/test/test_data/distribution_test_data/online/year=2024/month=01/day=15/part-00077-0d0dd1f3-2e30-4ce0-8fc2-49cfcd740d42.c000.snappy.parquet differ diff --git a/fink_mm/test/test_data/distribution_test_data/online/year=2024/month=01/day=15/part-00078-7bebba13-8bba-41a5-bbff-3d62c6ed360e.c000.snappy.parquet b/fink_mm/test/test_data/distribution_test_data/online/year=2024/month=01/day=15/part-00078-7bebba13-8bba-41a5-bbff-3d62c6ed360e.c000.snappy.parquet new file mode 100644 index 000000000..1fb48040a Binary files /dev/null and b/fink_mm/test/test_data/distribution_test_data/online/year=2024/month=01/day=15/part-00078-7bebba13-8bba-41a5-bbff-3d62c6ed360e.c000.snappy.parquet differ diff --git a/fink_mm/test/test_data/distribution_test_data/online/year=2024/month=01/day=15/part-00079-7daeceb3-587f-42f7-905d-4f9e5e6c29e6.c000.snappy.parquet b/fink_mm/test/test_data/distribution_test_data/online/year=2024/month=01/day=15/part-00079-7daeceb3-587f-42f7-905d-4f9e5e6c29e6.c000.snappy.parquet new file mode 100644 index 000000000..c9737f464 Binary files /dev/null and b/fink_mm/test/test_data/distribution_test_data/online/year=2024/month=01/day=15/part-00079-7daeceb3-587f-42f7-905d-4f9e5e6c29e6.c000.snappy.parquet differ diff --git a/fink_mm/test/test_data/distribution_test_data/online/year=2024/month=01/day=15/part-00080-e4d48d75-1a8f-4117-bafd-15c851977cf1.c000.snappy.parquet b/fink_mm/test/test_data/distribution_test_data/online/year=2024/month=01/day=15/part-00080-e4d48d75-1a8f-4117-bafd-15c851977cf1.c000.snappy.parquet new file mode 100644 index 000000000..254e70862 Binary files /dev/null and b/fink_mm/test/test_data/distribution_test_data/online/year=2024/month=01/day=15/part-00080-e4d48d75-1a8f-4117-bafd-15c851977cf1.c000.snappy.parquet differ diff --git a/fink_mm/test/test_data/distribution_test_data/online/year=2024/month=01/day=15/part-00082-581e6e03-e7a0-4b67-8002-5a332227f776.c000.snappy.parquet b/fink_mm/test/test_data/distribution_test_data/online/year=2024/month=01/day=15/part-00082-581e6e03-e7a0-4b67-8002-5a332227f776.c000.snappy.parquet new file mode 100644 index 000000000..ca9370d62 Binary files /dev/null and b/fink_mm/test/test_data/distribution_test_data/online/year=2024/month=01/day=15/part-00082-581e6e03-e7a0-4b67-8002-5a332227f776.c000.snappy.parquet differ diff --git a/fink_mm/test/test_data/distribution_test_data/online/year=2024/month=01/day=15/part-00083-01e6ccfd-75d1-4b75-af94-8efde30b9486.c000.snappy.parquet b/fink_mm/test/test_data/distribution_test_data/online/year=2024/month=01/day=15/part-00083-01e6ccfd-75d1-4b75-af94-8efde30b9486.c000.snappy.parquet new file mode 100644 index 000000000..b584ffa7a Binary files /dev/null and b/fink_mm/test/test_data/distribution_test_data/online/year=2024/month=01/day=15/part-00083-01e6ccfd-75d1-4b75-af94-8efde30b9486.c000.snappy.parquet differ diff --git a/fink_mm/test/test_data/distribution_test_data/online/year=2024/month=01/day=15/part-00084-5bf8a3e6-db54-49e5-bbfd-9bf157ec8e49.c000.snappy.parquet b/fink_mm/test/test_data/distribution_test_data/online/year=2024/month=01/day=15/part-00084-5bf8a3e6-db54-49e5-bbfd-9bf157ec8e49.c000.snappy.parquet new file mode 100644 index 000000000..63a781a2a Binary files /dev/null and b/fink_mm/test/test_data/distribution_test_data/online/year=2024/month=01/day=15/part-00084-5bf8a3e6-db54-49e5-bbfd-9bf157ec8e49.c000.snappy.parquet differ diff --git a/fink_mm/test/test_data/distribution_test_data/online/year=2024/month=01/day=15/part-00086-5178ba71-7a49-42c2-9eb8-c94b9777f77e.c000.snappy.parquet b/fink_mm/test/test_data/distribution_test_data/online/year=2024/month=01/day=15/part-00086-5178ba71-7a49-42c2-9eb8-c94b9777f77e.c000.snappy.parquet new file mode 100644 index 000000000..5bc844fd7 Binary files /dev/null and b/fink_mm/test/test_data/distribution_test_data/online/year=2024/month=01/day=15/part-00086-5178ba71-7a49-42c2-9eb8-c94b9777f77e.c000.snappy.parquet differ diff --git a/fink_mm/test/test_data/distribution_test_data/online/year=2024/month=01/day=15/part-00087-8ed3bec0-7681-48cf-9d49-4f2c90ba9544.c000.snappy.parquet b/fink_mm/test/test_data/distribution_test_data/online/year=2024/month=01/day=15/part-00087-8ed3bec0-7681-48cf-9d49-4f2c90ba9544.c000.snappy.parquet new file mode 100644 index 000000000..e9ad7944d Binary files /dev/null and b/fink_mm/test/test_data/distribution_test_data/online/year=2024/month=01/day=15/part-00087-8ed3bec0-7681-48cf-9d49-4f2c90ba9544.c000.snappy.parquet differ diff --git a/fink_mm/test/test_data/distribution_test_data/online/year=2024/month=01/day=15/part-00088-5cb5aaf7-fb77-499b-910d-13f31b10da10.c000.snappy.parquet b/fink_mm/test/test_data/distribution_test_data/online/year=2024/month=01/day=15/part-00088-5cb5aaf7-fb77-499b-910d-13f31b10da10.c000.snappy.parquet new file mode 100644 index 000000000..2c50ccc66 Binary files /dev/null and b/fink_mm/test/test_data/distribution_test_data/online/year=2024/month=01/day=15/part-00088-5cb5aaf7-fb77-499b-910d-13f31b10da10.c000.snappy.parquet differ diff --git a/fink_mm/test/test_data/distribution_test_data/online/year=2024/month=01/day=15/part-00089-ba5b63d6-bbeb-4e53-a2be-756a6479ba4c.c000.snappy.parquet b/fink_mm/test/test_data/distribution_test_data/online/year=2024/month=01/day=15/part-00089-ba5b63d6-bbeb-4e53-a2be-756a6479ba4c.c000.snappy.parquet new file mode 100644 index 000000000..561be677c Binary files /dev/null and b/fink_mm/test/test_data/distribution_test_data/online/year=2024/month=01/day=15/part-00089-ba5b63d6-bbeb-4e53-a2be-756a6479ba4c.c000.snappy.parquet differ diff --git a/fink_mm/test/test_data/distribution_test_data/online/year=2024/month=01/day=15/part-00090-5493e437-e4d4-47b4-a9c5-0cead7b6bdb8.c000.snappy.parquet b/fink_mm/test/test_data/distribution_test_data/online/year=2024/month=01/day=15/part-00090-5493e437-e4d4-47b4-a9c5-0cead7b6bdb8.c000.snappy.parquet new file mode 100644 index 000000000..a3cf0bcc3 Binary files /dev/null and b/fink_mm/test/test_data/distribution_test_data/online/year=2024/month=01/day=15/part-00090-5493e437-e4d4-47b4-a9c5-0cead7b6bdb8.c000.snappy.parquet differ diff --git a/fink_mm/test/test_data/distribution_test_data/online/year=2024/month=01/day=15/part-00092-9318d2f4-d97c-42ec-b609-a7f8b3b41bbd.c000.snappy.parquet b/fink_mm/test/test_data/distribution_test_data/online/year=2024/month=01/day=15/part-00092-9318d2f4-d97c-42ec-b609-a7f8b3b41bbd.c000.snappy.parquet new file mode 100644 index 000000000..916d632f6 Binary files /dev/null and b/fink_mm/test/test_data/distribution_test_data/online/year=2024/month=01/day=15/part-00092-9318d2f4-d97c-42ec-b609-a7f8b3b41bbd.c000.snappy.parquet differ diff --git a/fink_mm/test/test_data/distribution_test_data/online/year=2024/month=01/day=15/part-00095-f744f24e-2cdd-4765-8888-f02375092f98.c000.snappy.parquet b/fink_mm/test/test_data/distribution_test_data/online/year=2024/month=01/day=15/part-00095-f744f24e-2cdd-4765-8888-f02375092f98.c000.snappy.parquet new file mode 100644 index 000000000..c1bfd78f4 Binary files /dev/null and b/fink_mm/test/test_data/distribution_test_data/online/year=2024/month=01/day=15/part-00095-f744f24e-2cdd-4765-8888-f02375092f98.c000.snappy.parquet differ diff --git a/fink_mm/test/test_data/distribution_test_data/online/year=2024/month=01/day=15/part-00096-468f377e-cbcf-4d4d-a85f-3ba873f99bca.c000.snappy.parquet b/fink_mm/test/test_data/distribution_test_data/online/year=2024/month=01/day=15/part-00096-468f377e-cbcf-4d4d-a85f-3ba873f99bca.c000.snappy.parquet new file mode 100644 index 000000000..70de7d926 Binary files /dev/null and b/fink_mm/test/test_data/distribution_test_data/online/year=2024/month=01/day=15/part-00096-468f377e-cbcf-4d4d-a85f-3ba873f99bca.c000.snappy.parquet differ diff --git a/fink_mm/test/test_data/distribution_test_data/online/year=2024/month=01/day=15/part-00097-4a1599e7-080a-4db3-a5fe-68da52c776d3.c000.snappy.parquet b/fink_mm/test/test_data/distribution_test_data/online/year=2024/month=01/day=15/part-00097-4a1599e7-080a-4db3-a5fe-68da52c776d3.c000.snappy.parquet new file mode 100644 index 000000000..273d7148d Binary files /dev/null and b/fink_mm/test/test_data/distribution_test_data/online/year=2024/month=01/day=15/part-00097-4a1599e7-080a-4db3-a5fe-68da52c776d3.c000.snappy.parquet differ diff --git a/fink_mm/test/test_data/distribution_test_data/online/year=2024/month=01/day=15/part-00099-791479a4-ceff-4b51-adc5-05e66dfc65e4.c000.snappy.parquet b/fink_mm/test/test_data/distribution_test_data/online/year=2024/month=01/day=15/part-00099-791479a4-ceff-4b51-adc5-05e66dfc65e4.c000.snappy.parquet new file mode 100644 index 000000000..fc7944dfc Binary files /dev/null and b/fink_mm/test/test_data/distribution_test_data/online/year=2024/month=01/day=15/part-00099-791479a4-ceff-4b51-adc5-05e66dfc65e4.c000.snappy.parquet differ diff --git a/fink_mm/test/test_data/distribution_test_data/online/year=2024/month=01/day=15/part-00100-a2cba83f-3ba1-4346-a646-145679c2ede4.c000.snappy.parquet b/fink_mm/test/test_data/distribution_test_data/online/year=2024/month=01/day=15/part-00100-a2cba83f-3ba1-4346-a646-145679c2ede4.c000.snappy.parquet new file mode 100644 index 000000000..e7effb932 Binary files /dev/null and b/fink_mm/test/test_data/distribution_test_data/online/year=2024/month=01/day=15/part-00100-a2cba83f-3ba1-4346-a646-145679c2ede4.c000.snappy.parquet differ diff --git a/fink_mm/test/test_data/distribution_test_data/online/year=2024/month=01/day=15/part-00101-0248af76-e7db-4d6a-b06b-cc37f5d2e979.c000.snappy.parquet b/fink_mm/test/test_data/distribution_test_data/online/year=2024/month=01/day=15/part-00101-0248af76-e7db-4d6a-b06b-cc37f5d2e979.c000.snappy.parquet new file mode 100644 index 000000000..31bc3214f Binary files /dev/null and b/fink_mm/test/test_data/distribution_test_data/online/year=2024/month=01/day=15/part-00101-0248af76-e7db-4d6a-b06b-cc37f5d2e979.c000.snappy.parquet differ diff --git a/fink_mm/test/test_data/distribution_test_data/online/year=2024/month=01/day=15/part-00102-1f71a89d-4702-48f9-bc7d-f144320be33a.c000.snappy.parquet b/fink_mm/test/test_data/distribution_test_data/online/year=2024/month=01/day=15/part-00102-1f71a89d-4702-48f9-bc7d-f144320be33a.c000.snappy.parquet new file mode 100644 index 000000000..04d558f7f Binary files /dev/null and b/fink_mm/test/test_data/distribution_test_data/online/year=2024/month=01/day=15/part-00102-1f71a89d-4702-48f9-bc7d-f144320be33a.c000.snappy.parquet differ diff --git a/fink_mm/test/test_data/distribution_test_data/online/year=2024/month=01/day=15/part-00104-ce3f3e4c-8dd8-425c-af5c-624973f343a0.c000.snappy.parquet b/fink_mm/test/test_data/distribution_test_data/online/year=2024/month=01/day=15/part-00104-ce3f3e4c-8dd8-425c-af5c-624973f343a0.c000.snappy.parquet new file mode 100644 index 000000000..e02309e37 Binary files /dev/null and b/fink_mm/test/test_data/distribution_test_data/online/year=2024/month=01/day=15/part-00104-ce3f3e4c-8dd8-425c-af5c-624973f343a0.c000.snappy.parquet differ diff --git a/fink_mm/test/test_data/distribution_test_data/online/year=2024/month=01/day=15/part-00106-40880bb7-4380-4670-8791-1a10f3da4250.c000.snappy.parquet b/fink_mm/test/test_data/distribution_test_data/online/year=2024/month=01/day=15/part-00106-40880bb7-4380-4670-8791-1a10f3da4250.c000.snappy.parquet new file mode 100644 index 000000000..51731ed6a Binary files /dev/null and b/fink_mm/test/test_data/distribution_test_data/online/year=2024/month=01/day=15/part-00106-40880bb7-4380-4670-8791-1a10f3da4250.c000.snappy.parquet differ diff --git a/fink_mm/test/test_data/distribution_test_data/online/year=2024/month=01/day=15/part-00107-5631be9a-4f88-4fa8-918f-fbc4cb41013c.c000.snappy.parquet b/fink_mm/test/test_data/distribution_test_data/online/year=2024/month=01/day=15/part-00107-5631be9a-4f88-4fa8-918f-fbc4cb41013c.c000.snappy.parquet new file mode 100644 index 000000000..31a2fb735 Binary files /dev/null and b/fink_mm/test/test_data/distribution_test_data/online/year=2024/month=01/day=15/part-00107-5631be9a-4f88-4fa8-918f-fbc4cb41013c.c000.snappy.parquet differ diff --git a/fink_mm/test/test_data/distribution_test_data/online/year=2024/month=01/day=15/part-00108-f856be88-2b97-4b94-b3bd-f06c4cd3be16.c000.snappy.parquet b/fink_mm/test/test_data/distribution_test_data/online/year=2024/month=01/day=15/part-00108-f856be88-2b97-4b94-b3bd-f06c4cd3be16.c000.snappy.parquet new file mode 100644 index 000000000..869ba8d1a Binary files /dev/null and b/fink_mm/test/test_data/distribution_test_data/online/year=2024/month=01/day=15/part-00108-f856be88-2b97-4b94-b3bd-f06c4cd3be16.c000.snappy.parquet differ diff --git a/fink_mm/test/test_data/distribution_test_data/online/year=2024/month=01/day=15/part-00110-93397a49-3da3-4905-8a4a-424e31aa6a6b.c000.snappy.parquet b/fink_mm/test/test_data/distribution_test_data/online/year=2024/month=01/day=15/part-00110-93397a49-3da3-4905-8a4a-424e31aa6a6b.c000.snappy.parquet new file mode 100644 index 000000000..d98eb91dd Binary files /dev/null and b/fink_mm/test/test_data/distribution_test_data/online/year=2024/month=01/day=15/part-00110-93397a49-3da3-4905-8a4a-424e31aa6a6b.c000.snappy.parquet differ diff --git a/fink_mm/test/test_data/distribution_test_data/online/year=2024/month=01/day=15/part-00111-c3657fab-98b1-4de4-ad3d-8b5ec8e990af.c000.snappy.parquet b/fink_mm/test/test_data/distribution_test_data/online/year=2024/month=01/day=15/part-00111-c3657fab-98b1-4de4-ad3d-8b5ec8e990af.c000.snappy.parquet new file mode 100644 index 000000000..deabdadb2 Binary files /dev/null and b/fink_mm/test/test_data/distribution_test_data/online/year=2024/month=01/day=15/part-00111-c3657fab-98b1-4de4-ad3d-8b5ec8e990af.c000.snappy.parquet differ diff --git a/fink_mm/test/test_data/distribution_test_data/online/year=2024/month=01/day=15/part-00112-0762899b-a59c-422f-b438-75e0beaa1389.c000.snappy.parquet b/fink_mm/test/test_data/distribution_test_data/online/year=2024/month=01/day=15/part-00112-0762899b-a59c-422f-b438-75e0beaa1389.c000.snappy.parquet new file mode 100644 index 000000000..e2ed5fd35 Binary files /dev/null and b/fink_mm/test/test_data/distribution_test_data/online/year=2024/month=01/day=15/part-00112-0762899b-a59c-422f-b438-75e0beaa1389.c000.snappy.parquet differ diff --git a/fink_mm/test/test_data/distribution_test_data/online/year=2024/month=01/day=15/part-00115-bb11ae46-cf49-4870-8681-f9599d04e1bf.c000.snappy.parquet b/fink_mm/test/test_data/distribution_test_data/online/year=2024/month=01/day=15/part-00115-bb11ae46-cf49-4870-8681-f9599d04e1bf.c000.snappy.parquet new file mode 100644 index 000000000..64c351b27 Binary files /dev/null and b/fink_mm/test/test_data/distribution_test_data/online/year=2024/month=01/day=15/part-00115-bb11ae46-cf49-4870-8681-f9599d04e1bf.c000.snappy.parquet differ diff --git a/fink_mm/test/test_data/distribution_test_data/online/year=2024/month=01/day=15/part-00116-3684121e-8b8a-4053-bd5e-4ab309d8bf66.c000.snappy.parquet b/fink_mm/test/test_data/distribution_test_data/online/year=2024/month=01/day=15/part-00116-3684121e-8b8a-4053-bd5e-4ab309d8bf66.c000.snappy.parquet new file mode 100644 index 000000000..ce33477d4 Binary files /dev/null and b/fink_mm/test/test_data/distribution_test_data/online/year=2024/month=01/day=15/part-00116-3684121e-8b8a-4053-bd5e-4ab309d8bf66.c000.snappy.parquet differ diff --git a/fink_mm/test/test_data/distribution_test_data/online/year=2024/month=01/day=15/part-00117-9bc831b1-e205-464c-85ee-fbaa0d9199a2.c000.snappy.parquet b/fink_mm/test/test_data/distribution_test_data/online/year=2024/month=01/day=15/part-00117-9bc831b1-e205-464c-85ee-fbaa0d9199a2.c000.snappy.parquet new file mode 100644 index 000000000..e93bf1460 Binary files /dev/null and b/fink_mm/test/test_data/distribution_test_data/online/year=2024/month=01/day=15/part-00117-9bc831b1-e205-464c-85ee-fbaa0d9199a2.c000.snappy.parquet differ diff --git a/fink_mm/test/test_data/distribution_test_data/online/year=2024/month=01/day=15/part-00120-e97f7bd9-50d6-4419-8117-4c578ea1f15f.c000.snappy.parquet b/fink_mm/test/test_data/distribution_test_data/online/year=2024/month=01/day=15/part-00120-e97f7bd9-50d6-4419-8117-4c578ea1f15f.c000.snappy.parquet new file mode 100644 index 000000000..22f477e80 Binary files /dev/null and b/fink_mm/test/test_data/distribution_test_data/online/year=2024/month=01/day=15/part-00120-e97f7bd9-50d6-4419-8117-4c578ea1f15f.c000.snappy.parquet differ diff --git a/fink_mm/test/test_data/distribution_test_data/online/year=2024/month=01/day=15/part-00121-05a5ffdf-7e1c-416b-8305-f3ebb7e96929.c000.snappy.parquet b/fink_mm/test/test_data/distribution_test_data/online/year=2024/month=01/day=15/part-00121-05a5ffdf-7e1c-416b-8305-f3ebb7e96929.c000.snappy.parquet new file mode 100644 index 000000000..ce5291e15 Binary files /dev/null and b/fink_mm/test/test_data/distribution_test_data/online/year=2024/month=01/day=15/part-00121-05a5ffdf-7e1c-416b-8305-f3ebb7e96929.c000.snappy.parquet differ diff --git a/fink_mm/test/test_data/distribution_test_data/online/year=2024/month=01/day=15/part-00123-9cf6c886-fc5c-4f56-9047-f1460358a563.c000.snappy.parquet b/fink_mm/test/test_data/distribution_test_data/online/year=2024/month=01/day=15/part-00123-9cf6c886-fc5c-4f56-9047-f1460358a563.c000.snappy.parquet new file mode 100644 index 000000000..c0833340b Binary files /dev/null and b/fink_mm/test/test_data/distribution_test_data/online/year=2024/month=01/day=15/part-00123-9cf6c886-fc5c-4f56-9047-f1460358a563.c000.snappy.parquet differ diff --git a/fink_mm/test/test_data/distribution_test_data/online/year=2024/month=01/day=15/part-00125-d7807ac1-343e-4e70-814e-d68c13e4cf28.c000.snappy.parquet b/fink_mm/test/test_data/distribution_test_data/online/year=2024/month=01/day=15/part-00125-d7807ac1-343e-4e70-814e-d68c13e4cf28.c000.snappy.parquet new file mode 100644 index 000000000..df4346180 Binary files /dev/null and b/fink_mm/test/test_data/distribution_test_data/online/year=2024/month=01/day=15/part-00125-d7807ac1-343e-4e70-814e-d68c13e4cf28.c000.snappy.parquet differ diff --git a/fink_mm/test/test_data/distribution_test_data/online/year=2024/month=01/day=15/part-00129-69d09723-422b-45c3-b771-2fe40a0f5cd6.c000.snappy.parquet b/fink_mm/test/test_data/distribution_test_data/online/year=2024/month=01/day=15/part-00129-69d09723-422b-45c3-b771-2fe40a0f5cd6.c000.snappy.parquet new file mode 100644 index 000000000..9f661d9ee Binary files /dev/null and b/fink_mm/test/test_data/distribution_test_data/online/year=2024/month=01/day=15/part-00129-69d09723-422b-45c3-b771-2fe40a0f5cd6.c000.snappy.parquet differ diff --git a/fink_mm/test/test_data/distribution_test_data/online/year=2024/month=01/day=15/part-00130-76ff6498-8e67-4a79-84e6-0bff8b0a3e5f.c000.snappy.parquet b/fink_mm/test/test_data/distribution_test_data/online/year=2024/month=01/day=15/part-00130-76ff6498-8e67-4a79-84e6-0bff8b0a3e5f.c000.snappy.parquet new file mode 100644 index 000000000..be14b5ffe Binary files /dev/null and b/fink_mm/test/test_data/distribution_test_data/online/year=2024/month=01/day=15/part-00130-76ff6498-8e67-4a79-84e6-0bff8b0a3e5f.c000.snappy.parquet differ diff --git a/fink_mm/test/test_data/distribution_test_data/online/year=2024/month=01/day=15/part-00132-7b7792da-ffe0-4905-a7bd-c25d47eec550.c000.snappy.parquet b/fink_mm/test/test_data/distribution_test_data/online/year=2024/month=01/day=15/part-00132-7b7792da-ffe0-4905-a7bd-c25d47eec550.c000.snappy.parquet new file mode 100644 index 000000000..20211f482 Binary files /dev/null and b/fink_mm/test/test_data/distribution_test_data/online/year=2024/month=01/day=15/part-00132-7b7792da-ffe0-4905-a7bd-c25d47eec550.c000.snappy.parquet differ diff --git a/fink_mm/test/test_data/distribution_test_data/online/year=2024/month=01/day=15/part-00134-cf240da3-bb15-4395-9029-55eb7a0d4817.c000.snappy.parquet b/fink_mm/test/test_data/distribution_test_data/online/year=2024/month=01/day=15/part-00134-cf240da3-bb15-4395-9029-55eb7a0d4817.c000.snappy.parquet new file mode 100644 index 000000000..75508af5b Binary files /dev/null and b/fink_mm/test/test_data/distribution_test_data/online/year=2024/month=01/day=15/part-00134-cf240da3-bb15-4395-9029-55eb7a0d4817.c000.snappy.parquet differ diff --git a/fink_mm/test/test_data/distribution_test_data/online/year=2024/month=01/day=15/part-00135-3e67f349-0038-4f91-9a17-151f88d5c20b.c000.snappy.parquet b/fink_mm/test/test_data/distribution_test_data/online/year=2024/month=01/day=15/part-00135-3e67f349-0038-4f91-9a17-151f88d5c20b.c000.snappy.parquet new file mode 100644 index 000000000..19fcb0570 Binary files /dev/null and b/fink_mm/test/test_data/distribution_test_data/online/year=2024/month=01/day=15/part-00135-3e67f349-0038-4f91-9a17-151f88d5c20b.c000.snappy.parquet differ diff --git a/fink_mm/test/test_data/distribution_test_data/online/year=2024/month=01/day=15/part-00138-8ad32ca0-953f-45f2-b4a4-d27e602e4007.c000.snappy.parquet b/fink_mm/test/test_data/distribution_test_data/online/year=2024/month=01/day=15/part-00138-8ad32ca0-953f-45f2-b4a4-d27e602e4007.c000.snappy.parquet new file mode 100644 index 000000000..6044d580f Binary files /dev/null and b/fink_mm/test/test_data/distribution_test_data/online/year=2024/month=01/day=15/part-00138-8ad32ca0-953f-45f2-b4a4-d27e602e4007.c000.snappy.parquet differ diff --git a/fink_mm/test/test_data/distribution_test_data/online/year=2024/month=01/day=15/part-00139-1f765ea2-d942-4c83-a212-30bc413d7bda.c000.snappy.parquet b/fink_mm/test/test_data/distribution_test_data/online/year=2024/month=01/day=15/part-00139-1f765ea2-d942-4c83-a212-30bc413d7bda.c000.snappy.parquet new file mode 100644 index 000000000..e49f18717 Binary files /dev/null and b/fink_mm/test/test_data/distribution_test_data/online/year=2024/month=01/day=15/part-00139-1f765ea2-d942-4c83-a212-30bc413d7bda.c000.snappy.parquet differ diff --git a/fink_mm/test/test_data/distribution_test_data/online/year=2024/month=01/day=15/part-00140-cb67e858-8c4f-44e5-b01a-2f3eb481badd.c000.snappy.parquet b/fink_mm/test/test_data/distribution_test_data/online/year=2024/month=01/day=15/part-00140-cb67e858-8c4f-44e5-b01a-2f3eb481badd.c000.snappy.parquet new file mode 100644 index 000000000..1eb981b21 Binary files /dev/null and b/fink_mm/test/test_data/distribution_test_data/online/year=2024/month=01/day=15/part-00140-cb67e858-8c4f-44e5-b01a-2f3eb481badd.c000.snappy.parquet differ diff --git a/fink_mm/test/test_data/distribution_test_data/online/year=2024/month=01/day=15/part-00141-ef9ebaff-5ce9-4765-aeee-83f7f5a24812.c000.snappy.parquet b/fink_mm/test/test_data/distribution_test_data/online/year=2024/month=01/day=15/part-00141-ef9ebaff-5ce9-4765-aeee-83f7f5a24812.c000.snappy.parquet new file mode 100644 index 000000000..821e6df76 Binary files /dev/null and b/fink_mm/test/test_data/distribution_test_data/online/year=2024/month=01/day=15/part-00141-ef9ebaff-5ce9-4765-aeee-83f7f5a24812.c000.snappy.parquet differ diff --git a/fink_mm/test/test_data/distribution_test_data/online/year=2024/month=01/day=15/part-00143-f8220c01-d8bb-4d84-b2df-1734463e2b18.c000.snappy.parquet b/fink_mm/test/test_data/distribution_test_data/online/year=2024/month=01/day=15/part-00143-f8220c01-d8bb-4d84-b2df-1734463e2b18.c000.snappy.parquet new file mode 100644 index 000000000..a261a1fe0 Binary files /dev/null and b/fink_mm/test/test_data/distribution_test_data/online/year=2024/month=01/day=15/part-00143-f8220c01-d8bb-4d84-b2df-1734463e2b18.c000.snappy.parquet differ diff --git a/fink_mm/test/test_data/distribution_test_data/online/year=2024/month=01/day=15/part-00144-5255373c-ad4e-4e48-b769-0188cb9571b2.c000.snappy.parquet b/fink_mm/test/test_data/distribution_test_data/online/year=2024/month=01/day=15/part-00144-5255373c-ad4e-4e48-b769-0188cb9571b2.c000.snappy.parquet new file mode 100644 index 000000000..ab2d0be0c Binary files /dev/null and b/fink_mm/test/test_data/distribution_test_data/online/year=2024/month=01/day=15/part-00144-5255373c-ad4e-4e48-b769-0188cb9571b2.c000.snappy.parquet differ diff --git a/fink_mm/test/test_data/distribution_test_data/online/year=2024/month=01/day=15/part-00145-08e88f8c-f714-41bc-9615-7fdffc385c1e.c000.snappy.parquet b/fink_mm/test/test_data/distribution_test_data/online/year=2024/month=01/day=15/part-00145-08e88f8c-f714-41bc-9615-7fdffc385c1e.c000.snappy.parquet new file mode 100644 index 000000000..5c6380072 Binary files /dev/null and b/fink_mm/test/test_data/distribution_test_data/online/year=2024/month=01/day=15/part-00145-08e88f8c-f714-41bc-9615-7fdffc385c1e.c000.snappy.parquet differ diff --git a/fink_mm/test/test_data/distribution_test_data/online/year=2024/month=01/day=15/part-00147-f771c7ba-d41b-42ff-ae54-e30f596f3131.c000.snappy.parquet b/fink_mm/test/test_data/distribution_test_data/online/year=2024/month=01/day=15/part-00147-f771c7ba-d41b-42ff-ae54-e30f596f3131.c000.snappy.parquet new file mode 100644 index 000000000..8a627b3e1 Binary files /dev/null and b/fink_mm/test/test_data/distribution_test_data/online/year=2024/month=01/day=15/part-00147-f771c7ba-d41b-42ff-ae54-e30f596f3131.c000.snappy.parquet differ diff --git a/fink_mm/test/test_data/distribution_test_data/online/year=2024/month=01/day=15/part-00148-d3f7c7f4-334a-4a64-9ed3-97f6a162b306.c000.snappy.parquet b/fink_mm/test/test_data/distribution_test_data/online/year=2024/month=01/day=15/part-00148-d3f7c7f4-334a-4a64-9ed3-97f6a162b306.c000.snappy.parquet new file mode 100644 index 000000000..98a7f63f6 Binary files /dev/null and b/fink_mm/test/test_data/distribution_test_data/online/year=2024/month=01/day=15/part-00148-d3f7c7f4-334a-4a64-9ed3-97f6a162b306.c000.snappy.parquet differ diff --git a/fink_mm/test/test_data/distribution_test_data/online/year=2024/month=01/day=15/part-00149-cbb1864e-6d37-4304-beb5-ee19e665547c.c000.snappy.parquet b/fink_mm/test/test_data/distribution_test_data/online/year=2024/month=01/day=15/part-00149-cbb1864e-6d37-4304-beb5-ee19e665547c.c000.snappy.parquet new file mode 100644 index 000000000..992515023 Binary files /dev/null and b/fink_mm/test/test_data/distribution_test_data/online/year=2024/month=01/day=15/part-00149-cbb1864e-6d37-4304-beb5-ee19e665547c.c000.snappy.parquet differ diff --git a/fink_mm/test/test_data/distribution_test_data/online/year=2024/month=01/day=15/part-00150-55544e54-1b6b-48c1-af28-edd79aff8df5.c000.snappy.parquet b/fink_mm/test/test_data/distribution_test_data/online/year=2024/month=01/day=15/part-00150-55544e54-1b6b-48c1-af28-edd79aff8df5.c000.snappy.parquet new file mode 100644 index 000000000..490c905fc Binary files /dev/null and b/fink_mm/test/test_data/distribution_test_data/online/year=2024/month=01/day=15/part-00150-55544e54-1b6b-48c1-af28-edd79aff8df5.c000.snappy.parquet differ diff --git a/fink_mm/test/test_data/distribution_test_data/online/year=2024/month=01/day=15/part-00153-00b3819f-ff86-42da-9713-0faee09445c7.c000.snappy.parquet b/fink_mm/test/test_data/distribution_test_data/online/year=2024/month=01/day=15/part-00153-00b3819f-ff86-42da-9713-0faee09445c7.c000.snappy.parquet new file mode 100644 index 000000000..e1f9404bb Binary files /dev/null and b/fink_mm/test/test_data/distribution_test_data/online/year=2024/month=01/day=15/part-00153-00b3819f-ff86-42da-9713-0faee09445c7.c000.snappy.parquet differ diff --git a/fink_mm/test/test_data/distribution_test_data/online/year=2024/month=01/day=15/part-00157-136c7abe-8545-4dea-bfb2-e10861a51ddd.c000.snappy.parquet b/fink_mm/test/test_data/distribution_test_data/online/year=2024/month=01/day=15/part-00157-136c7abe-8545-4dea-bfb2-e10861a51ddd.c000.snappy.parquet new file mode 100644 index 000000000..b694a7b75 Binary files /dev/null and b/fink_mm/test/test_data/distribution_test_data/online/year=2024/month=01/day=15/part-00157-136c7abe-8545-4dea-bfb2-e10861a51ddd.c000.snappy.parquet differ diff --git a/fink_mm/test/test_data/distribution_test_data/online/year=2024/month=01/day=15/part-00159-1796872d-f02c-491c-8785-97b6b02f2bea.c000.snappy.parquet b/fink_mm/test/test_data/distribution_test_data/online/year=2024/month=01/day=15/part-00159-1796872d-f02c-491c-8785-97b6b02f2bea.c000.snappy.parquet new file mode 100644 index 000000000..aaebe6dd8 Binary files /dev/null and b/fink_mm/test/test_data/distribution_test_data/online/year=2024/month=01/day=15/part-00159-1796872d-f02c-491c-8785-97b6b02f2bea.c000.snappy.parquet differ diff --git a/fink_mm/test/test_data/distribution_test_data/online/year=2024/month=01/day=15/part-00160-e8a2e673-d93c-4dee-b570-2c0b8d20a95b.c000.snappy.parquet b/fink_mm/test/test_data/distribution_test_data/online/year=2024/month=01/day=15/part-00160-e8a2e673-d93c-4dee-b570-2c0b8d20a95b.c000.snappy.parquet new file mode 100644 index 000000000..0a8a1b13b Binary files /dev/null and b/fink_mm/test/test_data/distribution_test_data/online/year=2024/month=01/day=15/part-00160-e8a2e673-d93c-4dee-b570-2c0b8d20a95b.c000.snappy.parquet differ diff --git a/fink_mm/test/test_data/distribution_test_data/online/year=2024/month=01/day=15/part-00161-669c92d1-6907-4cf1-b446-9896f7366f81.c000.snappy.parquet b/fink_mm/test/test_data/distribution_test_data/online/year=2024/month=01/day=15/part-00161-669c92d1-6907-4cf1-b446-9896f7366f81.c000.snappy.parquet new file mode 100644 index 000000000..87ca577c9 Binary files /dev/null and b/fink_mm/test/test_data/distribution_test_data/online/year=2024/month=01/day=15/part-00161-669c92d1-6907-4cf1-b446-9896f7366f81.c000.snappy.parquet differ diff --git a/fink_mm/test/test_data/distribution_test_data/online/year=2024/month=01/day=15/part-00162-860e1a03-7697-41b7-b7c6-47a97513a258.c000.snappy.parquet b/fink_mm/test/test_data/distribution_test_data/online/year=2024/month=01/day=15/part-00162-860e1a03-7697-41b7-b7c6-47a97513a258.c000.snappy.parquet new file mode 100644 index 000000000..9f2907932 Binary files /dev/null and b/fink_mm/test/test_data/distribution_test_data/online/year=2024/month=01/day=15/part-00162-860e1a03-7697-41b7-b7c6-47a97513a258.c000.snappy.parquet differ diff --git a/fink_mm/test/test_data/distribution_test_data/online/year=2024/month=01/day=15/part-00163-f3c79b17-661f-4f62-bb30-fdd97297b15b.c000.snappy.parquet b/fink_mm/test/test_data/distribution_test_data/online/year=2024/month=01/day=15/part-00163-f3c79b17-661f-4f62-bb30-fdd97297b15b.c000.snappy.parquet new file mode 100644 index 000000000..c178a06c5 Binary files /dev/null and b/fink_mm/test/test_data/distribution_test_data/online/year=2024/month=01/day=15/part-00163-f3c79b17-661f-4f62-bb30-fdd97297b15b.c000.snappy.parquet differ diff --git a/fink_mm/test/test_data/distribution_test_data/online/year=2024/month=01/day=15/part-00165-1b240dd9-ae85-4e05-81a1-7be01797fc93.c000.snappy.parquet b/fink_mm/test/test_data/distribution_test_data/online/year=2024/month=01/day=15/part-00165-1b240dd9-ae85-4e05-81a1-7be01797fc93.c000.snappy.parquet new file mode 100644 index 000000000..d89a2e2b8 Binary files /dev/null and b/fink_mm/test/test_data/distribution_test_data/online/year=2024/month=01/day=15/part-00165-1b240dd9-ae85-4e05-81a1-7be01797fc93.c000.snappy.parquet differ diff --git a/fink_mm/test/test_data/distribution_test_data/online/year=2024/month=01/day=15/part-00167-00e689a2-6cb9-49c4-961d-6e33158f7782.c000.snappy.parquet b/fink_mm/test/test_data/distribution_test_data/online/year=2024/month=01/day=15/part-00167-00e689a2-6cb9-49c4-961d-6e33158f7782.c000.snappy.parquet new file mode 100644 index 000000000..57da4b130 Binary files /dev/null and b/fink_mm/test/test_data/distribution_test_data/online/year=2024/month=01/day=15/part-00167-00e689a2-6cb9-49c4-961d-6e33158f7782.c000.snappy.parquet differ diff --git a/fink_mm/test/test_data/distribution_test_data/online/year=2024/month=01/day=15/part-00169-7ee4bb9d-5d10-4847-9c10-738d75ebf5ee.c000.snappy.parquet b/fink_mm/test/test_data/distribution_test_data/online/year=2024/month=01/day=15/part-00169-7ee4bb9d-5d10-4847-9c10-738d75ebf5ee.c000.snappy.parquet new file mode 100644 index 000000000..32c45e356 Binary files /dev/null and b/fink_mm/test/test_data/distribution_test_data/online/year=2024/month=01/day=15/part-00169-7ee4bb9d-5d10-4847-9c10-738d75ebf5ee.c000.snappy.parquet differ diff --git a/fink_mm/test/test_data/distribution_test_data/online/year=2024/month=01/day=15/part-00170-99064bab-9fb5-464a-9a24-6226a87d992e.c000.snappy.parquet b/fink_mm/test/test_data/distribution_test_data/online/year=2024/month=01/day=15/part-00170-99064bab-9fb5-464a-9a24-6226a87d992e.c000.snappy.parquet new file mode 100644 index 000000000..ffca83c4d Binary files /dev/null and b/fink_mm/test/test_data/distribution_test_data/online/year=2024/month=01/day=15/part-00170-99064bab-9fb5-464a-9a24-6226a87d992e.c000.snappy.parquet differ diff --git a/fink_mm/test/test_data/distribution_test_data/online/year=2024/month=01/day=15/part-00171-863e3394-4093-4185-bbf6-397a55c70f43.c000.snappy.parquet b/fink_mm/test/test_data/distribution_test_data/online/year=2024/month=01/day=15/part-00171-863e3394-4093-4185-bbf6-397a55c70f43.c000.snappy.parquet new file mode 100644 index 000000000..a452bae67 Binary files /dev/null and b/fink_mm/test/test_data/distribution_test_data/online/year=2024/month=01/day=15/part-00171-863e3394-4093-4185-bbf6-397a55c70f43.c000.snappy.parquet differ diff --git a/fink_mm/test/test_data/distribution_test_data/online/year=2024/month=01/day=15/part-00172-265487a7-a48f-45b5-a30d-8ca91cfd1ac6.c000.snappy.parquet b/fink_mm/test/test_data/distribution_test_data/online/year=2024/month=01/day=15/part-00172-265487a7-a48f-45b5-a30d-8ca91cfd1ac6.c000.snappy.parquet new file mode 100644 index 000000000..81a4510a1 Binary files /dev/null and b/fink_mm/test/test_data/distribution_test_data/online/year=2024/month=01/day=15/part-00172-265487a7-a48f-45b5-a30d-8ca91cfd1ac6.c000.snappy.parquet differ diff --git a/fink_mm/test/test_data/distribution_test_data/online/year=2024/month=01/day=15/part-00174-26e76315-5328-48f0-b764-2f043bcad85d.c000.snappy.parquet b/fink_mm/test/test_data/distribution_test_data/online/year=2024/month=01/day=15/part-00174-26e76315-5328-48f0-b764-2f043bcad85d.c000.snappy.parquet new file mode 100644 index 000000000..d28d9dd48 Binary files /dev/null and b/fink_mm/test/test_data/distribution_test_data/online/year=2024/month=01/day=15/part-00174-26e76315-5328-48f0-b764-2f043bcad85d.c000.snappy.parquet differ diff --git a/fink_mm/test/test_data/distribution_test_data/online/year=2024/month=01/day=15/part-00175-62827128-151b-40af-82bb-2a9eb44b04de.c000.snappy.parquet b/fink_mm/test/test_data/distribution_test_data/online/year=2024/month=01/day=15/part-00175-62827128-151b-40af-82bb-2a9eb44b04de.c000.snappy.parquet new file mode 100644 index 000000000..70213a983 Binary files /dev/null and b/fink_mm/test/test_data/distribution_test_data/online/year=2024/month=01/day=15/part-00175-62827128-151b-40af-82bb-2a9eb44b04de.c000.snappy.parquet differ diff --git a/fink_mm/test/test_data/distribution_test_data/online/year=2024/month=01/day=15/part-00177-c11a3c02-4831-42a4-8c4b-548b4d8e8c99.c000.snappy.parquet b/fink_mm/test/test_data/distribution_test_data/online/year=2024/month=01/day=15/part-00177-c11a3c02-4831-42a4-8c4b-548b4d8e8c99.c000.snappy.parquet new file mode 100644 index 000000000..d21cce694 Binary files /dev/null and b/fink_mm/test/test_data/distribution_test_data/online/year=2024/month=01/day=15/part-00177-c11a3c02-4831-42a4-8c4b-548b4d8e8c99.c000.snappy.parquet differ diff --git a/fink_mm/test/test_data/distribution_test_data/online/year=2024/month=01/day=15/part-00178-47269fba-e459-426a-b83c-fb6ff3bbeb4d.c000.snappy.parquet b/fink_mm/test/test_data/distribution_test_data/online/year=2024/month=01/day=15/part-00178-47269fba-e459-426a-b83c-fb6ff3bbeb4d.c000.snappy.parquet new file mode 100644 index 000000000..a5d94c2dc Binary files /dev/null and b/fink_mm/test/test_data/distribution_test_data/online/year=2024/month=01/day=15/part-00178-47269fba-e459-426a-b83c-fb6ff3bbeb4d.c000.snappy.parquet differ diff --git a/fink_mm/test/test_data/distribution_test_data/online/year=2024/month=01/day=15/part-00182-79c164ca-72d3-46bf-8c96-311293711f3d.c000.snappy.parquet b/fink_mm/test/test_data/distribution_test_data/online/year=2024/month=01/day=15/part-00182-79c164ca-72d3-46bf-8c96-311293711f3d.c000.snappy.parquet new file mode 100644 index 000000000..8575b8e25 Binary files /dev/null and b/fink_mm/test/test_data/distribution_test_data/online/year=2024/month=01/day=15/part-00182-79c164ca-72d3-46bf-8c96-311293711f3d.c000.snappy.parquet differ diff --git a/fink_mm/test/test_data/distribution_test_data/online/year=2024/month=01/day=15/part-00183-981ff750-09ae-44d9-8ea4-45b949ba1d10.c000.snappy.parquet b/fink_mm/test/test_data/distribution_test_data/online/year=2024/month=01/day=15/part-00183-981ff750-09ae-44d9-8ea4-45b949ba1d10.c000.snappy.parquet new file mode 100644 index 000000000..02ee33c76 Binary files /dev/null and b/fink_mm/test/test_data/distribution_test_data/online/year=2024/month=01/day=15/part-00183-981ff750-09ae-44d9-8ea4-45b949ba1d10.c000.snappy.parquet differ diff --git a/fink_mm/test/test_data/distribution_test_data/online/year=2024/month=01/day=15/part-00184-65a3e0ab-92d2-4ca7-abdd-6dd5e0985335.c000.snappy.parquet b/fink_mm/test/test_data/distribution_test_data/online/year=2024/month=01/day=15/part-00184-65a3e0ab-92d2-4ca7-abdd-6dd5e0985335.c000.snappy.parquet new file mode 100644 index 000000000..b96d7b7e4 Binary files /dev/null and b/fink_mm/test/test_data/distribution_test_data/online/year=2024/month=01/day=15/part-00184-65a3e0ab-92d2-4ca7-abdd-6dd5e0985335.c000.snappy.parquet differ diff --git a/fink_mm/test/test_data/distribution_test_data/online/year=2024/month=01/day=15/part-00185-0a7ca0c6-d6ed-4af3-91a8-3b8ac7ece3ed.c000.snappy.parquet b/fink_mm/test/test_data/distribution_test_data/online/year=2024/month=01/day=15/part-00185-0a7ca0c6-d6ed-4af3-91a8-3b8ac7ece3ed.c000.snappy.parquet new file mode 100644 index 000000000..e7a0f3166 Binary files /dev/null and b/fink_mm/test/test_data/distribution_test_data/online/year=2024/month=01/day=15/part-00185-0a7ca0c6-d6ed-4af3-91a8-3b8ac7ece3ed.c000.snappy.parquet differ diff --git a/fink_mm/test/test_data/distribution_test_data/online/year=2024/month=01/day=15/part-00187-e0df6451-21d1-4be8-a21c-a63ccddcf7ec.c000.snappy.parquet b/fink_mm/test/test_data/distribution_test_data/online/year=2024/month=01/day=15/part-00187-e0df6451-21d1-4be8-a21c-a63ccddcf7ec.c000.snappy.parquet new file mode 100644 index 000000000..18f28f396 Binary files /dev/null and b/fink_mm/test/test_data/distribution_test_data/online/year=2024/month=01/day=15/part-00187-e0df6451-21d1-4be8-a21c-a63ccddcf7ec.c000.snappy.parquet differ diff --git a/fink_mm/test/test_data/distribution_test_data/online/year=2024/month=01/day=15/part-00188-576c90e8-6206-49a5-aa39-9215cc1e5112.c000.snappy.parquet b/fink_mm/test/test_data/distribution_test_data/online/year=2024/month=01/day=15/part-00188-576c90e8-6206-49a5-aa39-9215cc1e5112.c000.snappy.parquet new file mode 100644 index 000000000..8c7d9fff5 Binary files /dev/null and b/fink_mm/test/test_data/distribution_test_data/online/year=2024/month=01/day=15/part-00188-576c90e8-6206-49a5-aa39-9215cc1e5112.c000.snappy.parquet differ diff --git a/fink_mm/test/test_data/distribution_test_data/online/year=2024/month=01/day=15/part-00189-488f7d81-eabc-4977-9e9f-b1f7c19112bd.c000.snappy.parquet b/fink_mm/test/test_data/distribution_test_data/online/year=2024/month=01/day=15/part-00189-488f7d81-eabc-4977-9e9f-b1f7c19112bd.c000.snappy.parquet new file mode 100644 index 000000000..5c4f05d53 Binary files /dev/null and b/fink_mm/test/test_data/distribution_test_data/online/year=2024/month=01/day=15/part-00189-488f7d81-eabc-4977-9e9f-b1f7c19112bd.c000.snappy.parquet differ diff --git a/fink_mm/test/test_data/distribution_test_data/online/year=2024/month=01/day=15/part-00190-aad9dbef-fcfa-4156-b10f-ee1a34809515.c000.snappy.parquet b/fink_mm/test/test_data/distribution_test_data/online/year=2024/month=01/day=15/part-00190-aad9dbef-fcfa-4156-b10f-ee1a34809515.c000.snappy.parquet new file mode 100644 index 000000000..7f700f85b Binary files /dev/null and b/fink_mm/test/test_data/distribution_test_data/online/year=2024/month=01/day=15/part-00190-aad9dbef-fcfa-4156-b10f-ee1a34809515.c000.snappy.parquet differ diff --git a/fink_mm/test/test_data/distribution_test_data/online/year=2024/month=01/day=15/part-00192-347744f8-3b04-4ee8-b8a8-ada28d126ae5.c000.snappy.parquet b/fink_mm/test/test_data/distribution_test_data/online/year=2024/month=01/day=15/part-00192-347744f8-3b04-4ee8-b8a8-ada28d126ae5.c000.snappy.parquet new file mode 100644 index 000000000..0143bd984 Binary files /dev/null and b/fink_mm/test/test_data/distribution_test_data/online/year=2024/month=01/day=15/part-00192-347744f8-3b04-4ee8-b8a8-ada28d126ae5.c000.snappy.parquet differ diff --git a/fink_mm/test/test_data/distribution_test_data/online/year=2024/month=01/day=15/part-00193-c8765b20-bed9-4a2e-b114-147f4a380387.c000.snappy.parquet b/fink_mm/test/test_data/distribution_test_data/online/year=2024/month=01/day=15/part-00193-c8765b20-bed9-4a2e-b114-147f4a380387.c000.snappy.parquet new file mode 100644 index 000000000..62c697c37 Binary files /dev/null and b/fink_mm/test/test_data/distribution_test_data/online/year=2024/month=01/day=15/part-00193-c8765b20-bed9-4a2e-b114-147f4a380387.c000.snappy.parquet differ diff --git a/fink_mm/test/test_data/distribution_test_data/online/year=2024/month=01/day=15/part-00194-17c9576a-c8ae-4d6b-96ec-41fdd9aee005.c000.snappy.parquet b/fink_mm/test/test_data/distribution_test_data/online/year=2024/month=01/day=15/part-00194-17c9576a-c8ae-4d6b-96ec-41fdd9aee005.c000.snappy.parquet new file mode 100644 index 000000000..23fa9fe69 Binary files /dev/null and b/fink_mm/test/test_data/distribution_test_data/online/year=2024/month=01/day=15/part-00194-17c9576a-c8ae-4d6b-96ec-41fdd9aee005.c000.snappy.parquet differ diff --git a/fink_mm/test/test_data/distribution_test_data/online/year=2024/month=01/day=15/part-00195-96233d03-6f98-417a-a550-a1cf0c7984bc.c000.snappy.parquet b/fink_mm/test/test_data/distribution_test_data/online/year=2024/month=01/day=15/part-00195-96233d03-6f98-417a-a550-a1cf0c7984bc.c000.snappy.parquet new file mode 100644 index 000000000..39aff771b Binary files /dev/null and b/fink_mm/test/test_data/distribution_test_data/online/year=2024/month=01/day=15/part-00195-96233d03-6f98-417a-a550-a1cf0c7984bc.c000.snappy.parquet differ diff --git a/fink_mm/test/test_data/distribution_test_data/online/year=2024/month=01/day=15/part-00196-3233496b-28fa-4c0a-8c31-3bee087448d2.c000.snappy.parquet b/fink_mm/test/test_data/distribution_test_data/online/year=2024/month=01/day=15/part-00196-3233496b-28fa-4c0a-8c31-3bee087448d2.c000.snappy.parquet new file mode 100644 index 000000000..46a56328f Binary files /dev/null and b/fink_mm/test/test_data/distribution_test_data/online/year=2024/month=01/day=15/part-00196-3233496b-28fa-4c0a-8c31-3bee087448d2.c000.snappy.parquet differ diff --git a/fink_mm/test/test_data/distribution_test_data/online/year=2024/month=01/day=15/part-00198-623b0169-2796-4aa0-b015-a9b156795081.c000.snappy.parquet b/fink_mm/test/test_data/distribution_test_data/online/year=2024/month=01/day=15/part-00198-623b0169-2796-4aa0-b015-a9b156795081.c000.snappy.parquet new file mode 100644 index 000000000..6d02fd8b6 Binary files /dev/null and b/fink_mm/test/test_data/distribution_test_data/online/year=2024/month=01/day=15/part-00198-623b0169-2796-4aa0-b015-a9b156795081.c000.snappy.parquet differ diff --git a/fink_mm/test/test_data/distribution_test_data/online/year=2024/month=01/day=15/part-00199-96e23698-6918-4adf-9ff9-2948038888cd.c000.snappy.parquet b/fink_mm/test/test_data/distribution_test_data/online/year=2024/month=01/day=15/part-00199-96e23698-6918-4adf-9ff9-2948038888cd.c000.snappy.parquet new file mode 100644 index 000000000..4e4d85d21 Binary files /dev/null and b/fink_mm/test/test_data/distribution_test_data/online/year=2024/month=01/day=15/part-00199-96e23698-6918-4adf-9ff9-2948038888cd.c000.snappy.parquet differ diff --git a/fink_mm/test/test_data/gcn_test/raw/year=2019/month=09/day=02/683476673_0 b/fink_mm/test/test_data/gcn_test/raw/year=2019/month=09/day=02/683476673_0 deleted file mode 100644 index 03d54e3de..000000000 Binary files a/fink_mm/test/test_data/gcn_test/raw/year=2019/month=09/day=02/683476673_0 and /dev/null differ diff --git a/fink_mm/test/test_data/gcn_test/raw/year=2019/month=09/day=02/683482851_0 b/fink_mm/test/test_data/gcn_test/raw/year=2019/month=09/day=02/683482851_0 deleted file mode 100644 index f6d934ae9..000000000 Binary files a/fink_mm/test/test_data/gcn_test/raw/year=2019/month=09/day=02/683482851_0 and /dev/null differ diff --git a/fink_mm/test/test_data/gcn_test/raw/year=2019/month=09/day=02/683499781_0 b/fink_mm/test/test_data/gcn_test/raw/year=2019/month=09/day=02/683499781_0 deleted file mode 100644 index 35cbda1db..000000000 Binary files a/fink_mm/test/test_data/gcn_test/raw/year=2019/month=09/day=02/683499781_0 and /dev/null differ diff --git a/fink_mm/test/test_data/gcn_test/raw/year=2019/month=09/day=02/S230518h_0_test b/fink_mm/test/test_data/gcn_test/raw/year=2019/month=09/day=02/S230518h_0_test deleted file mode 100644 index a5124dc9c..000000000 Binary files a/fink_mm/test/test_data/gcn_test/raw/year=2019/month=09/day=02/S230518h_0_test and /dev/null differ diff --git a/fink_mm/test/test_data/gcn_test/raw/year=2019/month=09/day=02/fake_683476673_0 b/fink_mm/test/test_data/gcn_test/raw/year=2019/month=09/day=02/fake_683476673_0 deleted file mode 100644 index 41490fb03..000000000 Binary files a/fink_mm/test/test_data/gcn_test/raw/year=2019/month=09/day=02/fake_683476673_0 and /dev/null differ diff --git a/fink_mm/test/test_data/gcn_test/raw/year=2019/month=09/day=02/fake_683482851_0 b/fink_mm/test/test_data/gcn_test/raw/year=2019/month=09/day=02/fake_683482851_0 deleted file mode 100644 index 679252cd2..000000000 Binary files a/fink_mm/test/test_data/gcn_test/raw/year=2019/month=09/day=02/fake_683482851_0 and /dev/null differ diff --git a/fink_mm/test/test_data/gcn_test/raw/year=2019/month=09/day=02/fake_683499781_0 b/fink_mm/test/test_data/gcn_test/raw/year=2019/month=09/day=02/fake_683499781_0 deleted file mode 100644 index 43b876a9c..000000000 Binary files a/fink_mm/test/test_data/gcn_test/raw/year=2019/month=09/day=02/fake_683499781_0 and /dev/null differ diff --git a/fink_mm/test/test_data/gcn_test/raw/year=2019/month=09/day=03/683476673_0 b/fink_mm/test/test_data/gcn_test/raw/year=2019/month=09/day=03/683476673_0 deleted file mode 100644 index 03d54e3de..000000000 Binary files a/fink_mm/test/test_data/gcn_test/raw/year=2019/month=09/day=03/683476673_0 and /dev/null differ diff --git a/fink_mm/test/test_data/gcn_test/raw/year=2019/month=09/day=03/683482851_0 b/fink_mm/test/test_data/gcn_test/raw/year=2019/month=09/day=03/683482851_0 deleted file mode 100644 index f6d934ae9..000000000 Binary files a/fink_mm/test/test_data/gcn_test/raw/year=2019/month=09/day=03/683482851_0 and /dev/null differ diff --git a/fink_mm/test/test_data/gcn_test/raw/year=2019/month=09/day=03/683499781_0 b/fink_mm/test/test_data/gcn_test/raw/year=2019/month=09/day=03/683499781_0 deleted file mode 100644 index 35cbda1db..000000000 Binary files a/fink_mm/test/test_data/gcn_test/raw/year=2019/month=09/day=03/683499781_0 and /dev/null differ diff --git a/fink_mm/test/test_data/gcn_test/raw/year=2019/month=09/day=03/S230518h_0_test b/fink_mm/test/test_data/gcn_test/raw/year=2019/month=09/day=03/S230518h_0_test deleted file mode 100644 index a5124dc9c..000000000 Binary files a/fink_mm/test/test_data/gcn_test/raw/year=2019/month=09/day=03/S230518h_0_test and /dev/null differ diff --git a/fink_mm/test/test_data/gcn_test/raw/year=2019/month=09/day=03/fake_683476673_0 b/fink_mm/test/test_data/gcn_test/raw/year=2019/month=09/day=03/fake_683476673_0 deleted file mode 100644 index 41490fb03..000000000 Binary files a/fink_mm/test/test_data/gcn_test/raw/year=2019/month=09/day=03/fake_683476673_0 and /dev/null differ diff --git a/fink_mm/test/test_data/gcn_test/raw/year=2019/month=09/day=03/fake_683482851_0 b/fink_mm/test/test_data/gcn_test/raw/year=2019/month=09/day=03/fake_683482851_0 deleted file mode 100644 index 679252cd2..000000000 Binary files a/fink_mm/test/test_data/gcn_test/raw/year=2019/month=09/day=03/fake_683482851_0 and /dev/null differ diff --git a/fink_mm/test/test_data/gcn_test/raw/year=2019/month=09/day=03/fake_683499781_0 b/fink_mm/test/test_data/gcn_test/raw/year=2019/month=09/day=03/fake_683499781_0 deleted file mode 100644 index 43b876a9c..000000000 Binary files a/fink_mm/test/test_data/gcn_test/raw/year=2019/month=09/day=03/fake_683499781_0 and /dev/null differ diff --git a/fink_mm/test/test_data/gcn_test/raw/year=2024/month=01/day=12/1208389_1705409323.9516082_0 b/fink_mm/test/test_data/gcn_test/raw/year=2024/month=01/day=12/1208389_1705409323.9516082_0 new file mode 100644 index 000000000..1ecde0333 Binary files /dev/null and b/fink_mm/test/test_data/gcn_test/raw/year=2024/month=01/day=12/1208389_1705409323.9516082_0 differ diff --git a/fink_mm/test/test_data/gcn_test/raw/year=2024/month=01/day=12/1208389_1705409349.0247345_0 b/fink_mm/test/test_data/gcn_test/raw/year=2024/month=01/day=12/1208389_1705409349.0247345_0 new file mode 100644 index 000000000..8234650bb Binary files /dev/null and b/fink_mm/test/test_data/gcn_test/raw/year=2024/month=01/day=12/1208389_1705409349.0247345_0 differ diff --git a/fink_mm/test/test_data/gcn_test/raw/year=2024/month=01/day=12/1208389_1705409349.6910033_0 b/fink_mm/test/test_data/gcn_test/raw/year=2024/month=01/day=12/1208389_1705409349.6910033_0 new file mode 100644 index 000000000..503664d0e Binary files /dev/null and b/fink_mm/test/test_data/gcn_test/raw/year=2024/month=01/day=12/1208389_1705409349.6910033_0 differ diff --git a/fink_mm/test/test_data/gcn_test/raw/year=2024/month=01/day=12/1208389_1705409352.6866326_0 b/fink_mm/test/test_data/gcn_test/raw/year=2024/month=01/day=12/1208389_1705409352.6866326_0 new file mode 100644 index 000000000..d58631e79 Binary files /dev/null and b/fink_mm/test/test_data/gcn_test/raw/year=2024/month=01/day=12/1208389_1705409352.6866326_0 differ diff --git a/fink_mm/test/test_data/gcn_test/raw/year=2024/month=01/day=12/1208389_1705409356.2537446_0 b/fink_mm/test/test_data/gcn_test/raw/year=2024/month=01/day=12/1208389_1705409356.2537446_0 new file mode 100644 index 000000000..d00f871ce Binary files /dev/null and b/fink_mm/test/test_data/gcn_test/raw/year=2024/month=01/day=12/1208389_1705409356.2537446_0 differ diff --git a/fink_mm/test/test_data/gcn_test/raw/year=2024/month=01/day=12/726724682_1705409362.784804_0 b/fink_mm/test/test_data/gcn_test/raw/year=2024/month=01/day=12/726724682_1705409362.784804_0 new file mode 100644 index 000000000..57b9c2fd8 Binary files /dev/null and b/fink_mm/test/test_data/gcn_test/raw/year=2024/month=01/day=12/726724682_1705409362.784804_0 differ diff --git a/fink_mm/test/test_data/gcn_test/raw/year=2024/month=01/day=12/726724682_1705409362.8695443_0 b/fink_mm/test/test_data/gcn_test/raw/year=2024/month=01/day=12/726724682_1705409362.8695443_0 new file mode 100644 index 000000000..20ab0fcb4 Binary files /dev/null and b/fink_mm/test/test_data/gcn_test/raw/year=2024/month=01/day=12/726724682_1705409362.8695443_0 differ diff --git a/fink_mm/test/test_data/gcn_test/raw/year=2024/month=01/day=12/726746786_1705409362.9379883_0 b/fink_mm/test/test_data/gcn_test/raw/year=2024/month=01/day=12/726746786_1705409362.9379883_0 new file mode 100644 index 000000000..74e304ef9 Binary files /dev/null and b/fink_mm/test/test_data/gcn_test/raw/year=2024/month=01/day=12/726746786_1705409362.9379883_0 differ diff --git a/fink_mm/test/test_data/gcn_test/raw/year=2024/month=01/day=12/726757170_1705409349.307716_0 b/fink_mm/test/test_data/gcn_test/raw/year=2024/month=01/day=12/726757170_1705409349.307716_0 new file mode 100644 index 000000000..5cd70fb24 Binary files /dev/null and b/fink_mm/test/test_data/gcn_test/raw/year=2024/month=01/day=12/726757170_1705409349.307716_0 differ diff --git a/fink_mm/test/test_data/gcn_test/raw/year=2024/month=01/day=12/726757170_1705409359.2014854_0 b/fink_mm/test/test_data/gcn_test/raw/year=2024/month=01/day=12/726757170_1705409359.2014854_0 new file mode 100644 index 000000000..f8f45e5a5 Binary files /dev/null and b/fink_mm/test/test_data/gcn_test/raw/year=2024/month=01/day=12/726757170_1705409359.2014854_0 differ diff --git a/fink_mm/test/test_data/gcn_test/raw/year=2024/month=01/day=12/726757170_1705409363.012987_0 b/fink_mm/test/test_data/gcn_test/raw/year=2024/month=01/day=12/726757170_1705409363.012987_0 new file mode 100644 index 000000000..060ef5607 Binary files /dev/null and b/fink_mm/test/test_data/gcn_test/raw/year=2024/month=01/day=12/726757170_1705409363.012987_0 differ diff --git a/fink_mm/test/test_data/gcn_test/raw/year=2024/month=01/day=12/726773850_1705409349.3796856_0 b/fink_mm/test/test_data/gcn_test/raw/year=2024/month=01/day=12/726773850_1705409349.3796856_0 new file mode 100644 index 000000000..bae6ec7e4 Binary files /dev/null and b/fink_mm/test/test_data/gcn_test/raw/year=2024/month=01/day=12/726773850_1705409349.3796856_0 differ diff --git a/fink_mm/test/test_data/gcn_test/raw/year=2024/month=01/day=12/726773850_1705409359.3304546_0 b/fink_mm/test/test_data/gcn_test/raw/year=2024/month=01/day=12/726773850_1705409359.3304546_0 new file mode 100644 index 000000000..5a8bb156e Binary files /dev/null and b/fink_mm/test/test_data/gcn_test/raw/year=2024/month=01/day=12/726773850_1705409359.3304546_0 differ diff --git a/fink_mm/test/test_data/gcn_test/raw/year=2024/month=01/day=12/726773850_1705409363.2712498_0 b/fink_mm/test/test_data/gcn_test/raw/year=2024/month=01/day=12/726773850_1705409363.2712498_0 new file mode 100644 index 000000000..c74b12a8f Binary files /dev/null and b/fink_mm/test/test_data/gcn_test/raw/year=2024/month=01/day=12/726773850_1705409363.2712498_0 differ diff --git a/fink_mm/test/test_data/gcn_test/raw/year=2024/month=01/day=12/726774492_1705409349.5390825_0 b/fink_mm/test/test_data/gcn_test/raw/year=2024/month=01/day=12/726774492_1705409349.5390825_0 new file mode 100644 index 000000000..bf3ff4e21 Binary files /dev/null and b/fink_mm/test/test_data/gcn_test/raw/year=2024/month=01/day=12/726774492_1705409349.5390825_0 differ diff --git a/fink_mm/test/test_data/gcn_test/raw/year=2024/month=01/day=12/726774492_1705409363.3405154_0 b/fink_mm/test/test_data/gcn_test/raw/year=2024/month=01/day=12/726774492_1705409363.3405154_0 new file mode 100644 index 000000000..2a63c00ee Binary files /dev/null and b/fink_mm/test/test_data/gcn_test/raw/year=2024/month=01/day=12/726774492_1705409363.3405154_0 differ diff --git a/fink_mm/test/test_data/gcn_test/raw/year=2024/month=01/day=12/726789369_1705409349.4469464_0 b/fink_mm/test/test_data/gcn_test/raw/year=2024/month=01/day=12/726789369_1705409349.4469464_0 new file mode 100644 index 000000000..baaa7efd6 Binary files /dev/null and b/fink_mm/test/test_data/gcn_test/raw/year=2024/month=01/day=12/726789369_1705409349.4469464_0 differ diff --git a/fink_mm/test/test_data/gcn_test/raw/year=2024/month=01/day=12/726789369_1705409359.403572_0 b/fink_mm/test/test_data/gcn_test/raw/year=2024/month=01/day=12/726789369_1705409359.403572_0 new file mode 100644 index 000000000..fb4684838 Binary files /dev/null and b/fink_mm/test/test_data/gcn_test/raw/year=2024/month=01/day=12/726789369_1705409359.403572_0 differ diff --git a/fink_mm/test/test_data/gcn_test/raw/year=2024/month=01/day=12/726789369_1705409363.4082923_0 b/fink_mm/test/test_data/gcn_test/raw/year=2024/month=01/day=12/726789369_1705409363.4082923_0 new file mode 100644 index 000000000..338e0fbf2 Binary files /dev/null and b/fink_mm/test/test_data/gcn_test/raw/year=2024/month=01/day=12/726789369_1705409363.4082923_0 differ diff --git a/fink_mm/test/test_data/gcn_test/raw/year=2024/month=01/day=12/S240112e_1705410466.6087735_0 b/fink_mm/test/test_data/gcn_test/raw/year=2024/month=01/day=12/S240112e_1705410466.6087735_0 new file mode 100644 index 000000000..27a4a3960 Binary files /dev/null and b/fink_mm/test/test_data/gcn_test/raw/year=2024/month=01/day=12/S240112e_1705410466.6087735_0 differ diff --git a/fink_mm/test/test_data/gcn_test/raw/year=2024/month=01/day=12/S240112e_1705410466.7020795_0 b/fink_mm/test/test_data/gcn_test/raw/year=2024/month=01/day=12/S240112e_1705410466.7020795_0 new file mode 100644 index 000000000..7f7d16ae9 Binary files /dev/null and b/fink_mm/test/test_data/gcn_test/raw/year=2024/month=01/day=12/S240112e_1705410466.7020795_0 differ diff --git a/fink_mm/test/test_data/gcn_test/raw/year=2024/month=01/day=12/S240112w_1705410485.1397545_0 b/fink_mm/test/test_data/gcn_test/raw/year=2024/month=01/day=12/S240112w_1705410485.1397545_0 new file mode 100644 index 000000000..d1bb099d6 Binary files /dev/null and b/fink_mm/test/test_data/gcn_test/raw/year=2024/month=01/day=12/S240112w_1705410485.1397545_0 differ diff --git a/fink_mm/test/test_data/gcn_test/raw/year=2024/month=01/day=12/S240112w_1705410486.3449316_0 b/fink_mm/test/test_data/gcn_test/raw/year=2024/month=01/day=12/S240112w_1705410486.3449316_0 new file mode 100644 index 000000000..6c570b4d2 Binary files /dev/null and b/fink_mm/test/test_data/gcn_test/raw/year=2024/month=01/day=12/S240112w_1705410486.3449316_0 differ diff --git a/fink_mm/test/test_data/gcn_test/raw/year=2024/month=01/day=12/S240112y_1705410486.4628656_0 b/fink_mm/test/test_data/gcn_test/raw/year=2024/month=01/day=12/S240112y_1705410486.4628656_0 new file mode 100644 index 000000000..2c3353626 Binary files /dev/null and b/fink_mm/test/test_data/gcn_test/raw/year=2024/month=01/day=12/S240112y_1705410486.4628656_0 differ diff --git a/fink_mm/test/test_data/gcn_test/raw/year=2024/month=01/day=12/S240112y_1705410486.5560875_0 b/fink_mm/test/test_data/gcn_test/raw/year=2024/month=01/day=12/S240112y_1705410486.5560875_0 new file mode 100644 index 000000000..dc66eef22 Binary files /dev/null and b/fink_mm/test/test_data/gcn_test/raw/year=2024/month=01/day=12/S240112y_1705410486.5560875_0 differ diff --git a/fink_mm/test/test_data/gcn_test/raw/year=2024/month=01/day=14/S240114aj_1705410582.1827586_0 b/fink_mm/test/test_data/gcn_test/raw/year=2024/month=01/day=14/S240114aj_1705410582.1827586_0 new file mode 100644 index 000000000..ba24427fa Binary files /dev/null and b/fink_mm/test/test_data/gcn_test/raw/year=2024/month=01/day=14/S240114aj_1705410582.1827586_0 differ diff --git a/fink_mm/test/test_data/gcn_test/raw/year=2024/month=01/day=14/S240114aj_1705410583.5631442_0 b/fink_mm/test/test_data/gcn_test/raw/year=2024/month=01/day=14/S240114aj_1705410583.5631442_0 new file mode 100644 index 000000000..bf9b7d7e2 Binary files /dev/null and b/fink_mm/test/test_data/gcn_test/raw/year=2024/month=01/day=14/S240114aj_1705410583.5631442_0 differ diff --git a/fink_mm/test/test_data/gcn_test/raw/year=2024/month=01/day=14/S240114b_1705410554.2402153_0 b/fink_mm/test/test_data/gcn_test/raw/year=2024/month=01/day=14/S240114b_1705410554.2402153_0 new file mode 100644 index 000000000..f81c2cbb0 Binary files /dev/null and b/fink_mm/test/test_data/gcn_test/raw/year=2024/month=01/day=14/S240114b_1705410554.2402153_0 differ diff --git a/fink_mm/test/test_data/gcn_test/raw/year=2024/month=01/day=14/S240114b_1705410555.5378928_0 b/fink_mm/test/test_data/gcn_test/raw/year=2024/month=01/day=14/S240114b_1705410555.5378928_0 new file mode 100644 index 000000000..5b6fcaa5c Binary files /dev/null and b/fink_mm/test/test_data/gcn_test/raw/year=2024/month=01/day=14/S240114b_1705410555.5378928_0 differ diff --git a/fink_mm/test/test_data/gcn_test/raw/year=2024/month=01/day=14/S240114l_1705410561.1844385_0 b/fink_mm/test/test_data/gcn_test/raw/year=2024/month=01/day=14/S240114l_1705410561.1844385_0 new file mode 100644 index 000000000..3e499fda7 Binary files /dev/null and b/fink_mm/test/test_data/gcn_test/raw/year=2024/month=01/day=14/S240114l_1705410561.1844385_0 differ diff --git a/fink_mm/test/test_data/gcn_test/raw/year=2024/month=01/day=14/S240114l_1705410562.4186833_0 b/fink_mm/test/test_data/gcn_test/raw/year=2024/month=01/day=14/S240114l_1705410562.4186833_0 new file mode 100644 index 000000000..43e85634f Binary files /dev/null and b/fink_mm/test/test_data/gcn_test/raw/year=2024/month=01/day=14/S240114l_1705410562.4186833_0 differ diff --git a/fink_mm/test/test_data/gcn_test/raw/year=2024/month=01/day=14/S240114n_1705410565.257609_0 b/fink_mm/test/test_data/gcn_test/raw/year=2024/month=01/day=14/S240114n_1705410565.257609_0 new file mode 100644 index 000000000..b99eaf164 Binary files /dev/null and b/fink_mm/test/test_data/gcn_test/raw/year=2024/month=01/day=14/S240114n_1705410565.257609_0 differ diff --git a/fink_mm/test/test_data/gcn_test/raw/year=2024/month=01/day=14/S240114n_1705410565.864688_0 b/fink_mm/test/test_data/gcn_test/raw/year=2024/month=01/day=14/S240114n_1705410565.864688_0 new file mode 100644 index 000000000..f6ab56afb Binary files /dev/null and b/fink_mm/test/test_data/gcn_test/raw/year=2024/month=01/day=14/S240114n_1705410565.864688_0 differ diff --git a/fink_mm/test/test_data/gcn_test/raw/year=2024/month=01/day=14/S240114t_1705410570.1893916_0 b/fink_mm/test/test_data/gcn_test/raw/year=2024/month=01/day=14/S240114t_1705410570.1893916_0 new file mode 100644 index 000000000..f3ec02f47 Binary files /dev/null and b/fink_mm/test/test_data/gcn_test/raw/year=2024/month=01/day=14/S240114t_1705410570.1893916_0 differ diff --git a/fink_mm/test/test_data/gcn_test/raw/year=2024/month=01/day=14/S240114t_1705410570.8703501_0 b/fink_mm/test/test_data/gcn_test/raw/year=2024/month=01/day=14/S240114t_1705410570.8703501_0 new file mode 100644 index 000000000..504e41926 Binary files /dev/null and b/fink_mm/test/test_data/gcn_test/raw/year=2024/month=01/day=14/S240114t_1705410570.8703501_0 differ diff --git a/fink_mm/test/test_data/gcn_test/raw/year=2024/month=01/day=15/10472_1705409374.659779_0 b/fink_mm/test/test_data/gcn_test/raw/year=2024/month=01/day=15/10472_1705409374.659779_0 new file mode 100644 index 000000000..34aaafda1 Binary files /dev/null and b/fink_mm/test/test_data/gcn_test/raw/year=2024/month=01/day=15/10472_1705409374.659779_0 differ diff --git a/fink_mm/test/test_data/gcn_test/raw/year=2024/month=01/day=15/727009399_1705409361.2776353_0 b/fink_mm/test/test_data/gcn_test/raw/year=2024/month=01/day=15/727009399_1705409361.2776353_0 new file mode 100644 index 000000000..d504a551b Binary files /dev/null and b/fink_mm/test/test_data/gcn_test/raw/year=2024/month=01/day=15/727009399_1705409361.2776353_0 differ diff --git a/fink_mm/test/test_data/gcn_test/raw/year=2024/month=01/day=15/727009399_1705409363.5070522_0 b/fink_mm/test/test_data/gcn_test/raw/year=2024/month=01/day=15/727009399_1705409363.5070522_0 new file mode 100644 index 000000000..b625326e5 Binary files /dev/null and b/fink_mm/test/test_data/gcn_test/raw/year=2024/month=01/day=15/727009399_1705409363.5070522_0 differ diff --git a/fink_mm/test/test_data/gcn_test/raw/year=2024/month=01/day=15/727009399_1705409366.4637592_0 b/fink_mm/test/test_data/gcn_test/raw/year=2024/month=01/day=15/727009399_1705409366.4637592_0 new file mode 100644 index 000000000..92e632741 Binary files /dev/null and b/fink_mm/test/test_data/gcn_test/raw/year=2024/month=01/day=15/727009399_1705409366.4637592_0 differ diff --git a/fink_mm/test/test_data/gcn_test/raw/year=2024/month=01/day=15/727029942_1705409361.350435_0 b/fink_mm/test/test_data/gcn_test/raw/year=2024/month=01/day=15/727029942_1705409361.350435_0 new file mode 100644 index 000000000..2ab8d471f Binary files /dev/null and b/fink_mm/test/test_data/gcn_test/raw/year=2024/month=01/day=15/727029942_1705409361.350435_0 differ diff --git a/fink_mm/test/test_data/gcn_test/raw/year=2024/month=01/day=15/727029942_1705409363.602962_0 b/fink_mm/test/test_data/gcn_test/raw/year=2024/month=01/day=15/727029942_1705409363.602962_0 new file mode 100644 index 000000000..66e2775ed Binary files /dev/null and b/fink_mm/test/test_data/gcn_test/raw/year=2024/month=01/day=15/727029942_1705409363.602962_0 differ diff --git a/fink_mm/test/test_data/gcn_test/raw/year=2024/month=01/day=15/727029942_1705409363.7163877_0 b/fink_mm/test/test_data/gcn_test/raw/year=2024/month=01/day=15/727029942_1705409363.7163877_0 new file mode 100644 index 000000000..f75e0b8de Binary files /dev/null and b/fink_mm/test/test_data/gcn_test/raw/year=2024/month=01/day=15/727029942_1705409363.7163877_0 differ diff --git a/fink_mm/test/test_data/gcn_test/raw/year=2024/month=01/day=15/727029942_1705409366.5424862_0 b/fink_mm/test/test_data/gcn_test/raw/year=2024/month=01/day=15/727029942_1705409366.5424862_0 new file mode 100644 index 000000000..ca82a5d49 Binary files /dev/null and b/fink_mm/test/test_data/gcn_test/raw/year=2024/month=01/day=15/727029942_1705409366.5424862_0 differ diff --git a/fink_mm/test/test_data/gcn_test/raw/year=2024/month=01/day=15/76008_1705409374.5029156_0 b/fink_mm/test/test_data/gcn_test/raw/year=2024/month=01/day=15/76008_1705409374.5029156_0 new file mode 100644 index 000000000..a550f4309 Binary files /dev/null and b/fink_mm/test/test_data/gcn_test/raw/year=2024/month=01/day=15/76008_1705409374.5029156_0 differ diff --git a/fink_mm/test/test_data/gcn_test/raw/year=2024/month=01/day=15/S240115ak_1705410621.6326454_0 b/fink_mm/test/test_data/gcn_test/raw/year=2024/month=01/day=15/S240115ak_1705410621.6326454_0 new file mode 100644 index 000000000..7b5ebae32 Binary files /dev/null and b/fink_mm/test/test_data/gcn_test/raw/year=2024/month=01/day=15/S240115ak_1705410621.6326454_0 differ diff --git a/fink_mm/test/test_data/gcn_test/raw/year=2024/month=01/day=15/S240115ak_1705410622.2404687_0 b/fink_mm/test/test_data/gcn_test/raw/year=2024/month=01/day=15/S240115ak_1705410622.2404687_0 new file mode 100644 index 000000000..47e4008be Binary files /dev/null and b/fink_mm/test/test_data/gcn_test/raw/year=2024/month=01/day=15/S240115ak_1705410622.2404687_0 differ diff --git a/fink_mm/test/test_data/gcn_test/raw/year=2024/month=01/day=15/S240115ao_1705410624.281796_0 b/fink_mm/test/test_data/gcn_test/raw/year=2024/month=01/day=15/S240115ao_1705410624.281796_0 new file mode 100644 index 000000000..57320e6ce Binary files /dev/null and b/fink_mm/test/test_data/gcn_test/raw/year=2024/month=01/day=15/S240115ao_1705410624.281796_0 differ diff --git a/fink_mm/test/test_data/gcn_test/raw/year=2024/month=01/day=15/S240115ao_1705410624.8843832_0 b/fink_mm/test/test_data/gcn_test/raw/year=2024/month=01/day=15/S240115ao_1705410624.8843832_0 new file mode 100644 index 000000000..79ea6d15c Binary files /dev/null and b/fink_mm/test/test_data/gcn_test/raw/year=2024/month=01/day=15/S240115ao_1705410624.8843832_0 differ diff --git a/fink_mm/test/test_data/gcn_test/raw/year=2024/month=01/day=15/S240115f_1705410598.3004456_0 b/fink_mm/test/test_data/gcn_test/raw/year=2024/month=01/day=15/S240115f_1705410598.3004456_0 new file mode 100644 index 000000000..55c87e7a4 Binary files /dev/null and b/fink_mm/test/test_data/gcn_test/raw/year=2024/month=01/day=15/S240115f_1705410598.3004456_0 differ diff --git a/fink_mm/test/test_data/gcn_test/raw/year=2024/month=01/day=15/S240115f_1705410599.5995913_0 b/fink_mm/test/test_data/gcn_test/raw/year=2024/month=01/day=15/S240115f_1705410599.5995913_0 new file mode 100644 index 000000000..cdc88eca4 Binary files /dev/null and b/fink_mm/test/test_data/gcn_test/raw/year=2024/month=01/day=15/S240115f_1705410599.5995913_0 differ diff --git a/fink_mm/test/test_data/gcn_test/raw/year=2024/month=01/day=15/S240115k_1705410600.8611884_0 b/fink_mm/test/test_data/gcn_test/raw/year=2024/month=01/day=15/S240115k_1705410600.8611884_0 new file mode 100644 index 000000000..ece4253d8 Binary files /dev/null and b/fink_mm/test/test_data/gcn_test/raw/year=2024/month=01/day=15/S240115k_1705410600.8611884_0 differ diff --git a/fink_mm/test/test_data/gcn_test/raw/year=2024/month=01/day=15/S240115k_1705410600.9664364_0 b/fink_mm/test/test_data/gcn_test/raw/year=2024/month=01/day=15/S240115k_1705410600.9664364_0 new file mode 100644 index 000000000..199dc79fc Binary files /dev/null and b/fink_mm/test/test_data/gcn_test/raw/year=2024/month=01/day=15/S240115k_1705410600.9664364_0 differ diff --git a/fink_mm/test/test_data/gcn_test/raw/year=2024/month=01/day=15/S240115n_1705410602.0768342_0 b/fink_mm/test/test_data/gcn_test/raw/year=2024/month=01/day=15/S240115n_1705410602.0768342_0 new file mode 100644 index 000000000..7885fb0b2 Binary files /dev/null and b/fink_mm/test/test_data/gcn_test/raw/year=2024/month=01/day=15/S240115n_1705410602.0768342_0 differ diff --git a/fink_mm/test/test_data/gcn_test/raw/year=2024/month=01/day=15/S240115n_1705410602.6548948_0 b/fink_mm/test/test_data/gcn_test/raw/year=2024/month=01/day=15/S240115n_1705410602.6548948_0 new file mode 100644 index 000000000..aaab95c9e Binary files /dev/null and b/fink_mm/test/test_data/gcn_test/raw/year=2024/month=01/day=15/S240115n_1705410602.6548948_0 differ diff --git a/fink_mm/test/test_data/gcn_test/raw/year=2024/month=01/day=15/S240115q_1705410604.6400092_0 b/fink_mm/test/test_data/gcn_test/raw/year=2024/month=01/day=15/S240115q_1705410604.6400092_0 new file mode 100644 index 000000000..9418c5054 Binary files /dev/null and b/fink_mm/test/test_data/gcn_test/raw/year=2024/month=01/day=15/S240115q_1705410604.6400092_0 differ diff --git a/fink_mm/test/test_data/gcn_test/raw/year=2024/month=01/day=15/S240115q_1705410605.2456036_0 b/fink_mm/test/test_data/gcn_test/raw/year=2024/month=01/day=15/S240115q_1705410605.2456036_0 new file mode 100644 index 000000000..27ee76b8a Binary files /dev/null and b/fink_mm/test/test_data/gcn_test/raw/year=2024/month=01/day=15/S240115q_1705410605.2456036_0 differ diff --git a/fink_mm/test/test_data/gcn_test/raw/year=2024/month=01/day=15/S240115x_1705410611.3398452_0 b/fink_mm/test/test_data/gcn_test/raw/year=2024/month=01/day=15/S240115x_1705410611.3398452_0 new file mode 100644 index 000000000..f289f328a Binary files /dev/null and b/fink_mm/test/test_data/gcn_test/raw/year=2024/month=01/day=15/S240115x_1705410611.3398452_0 differ diff --git a/fink_mm/test/test_data/gcn_test/raw/year=2024/month=01/day=15/S240115x_1705410611.9337041_0 b/fink_mm/test/test_data/gcn_test/raw/year=2024/month=01/day=15/S240115x_1705410611.9337041_0 new file mode 100644 index 000000000..75df80623 Binary files /dev/null and b/fink_mm/test/test_data/gcn_test/raw/year=2024/month=01/day=15/S240115x_1705410611.9337041_0 differ diff --git a/fink_mm/test/test_data/gcn_test/raw/year=2024/month=01/day=27/728023845_1706534753.0453317_0 b/fink_mm/test/test_data/gcn_test/raw/year=2024/month=01/day=27/728023845_1706534753.0453317_0 new file mode 100644 index 000000000..37f7beea6 Binary files /dev/null and b/fink_mm/test/test_data/gcn_test/raw/year=2024/month=01/day=27/728023845_1706534753.0453317_0 differ diff --git a/fink_mm/test/test_data/gcn_test/raw/year=2024/month=01/day=27/728023845_1706534757.0528302_0 b/fink_mm/test/test_data/gcn_test/raw/year=2024/month=01/day=27/728023845_1706534757.0528302_0 new file mode 100644 index 000000000..3a92726e7 Binary files /dev/null and b/fink_mm/test/test_data/gcn_test/raw/year=2024/month=01/day=27/728023845_1706534757.0528302_0 differ diff --git a/fink_mm/test/test_data/gcn_test/raw/year=2024/month=01/day=27/728023845_1706534763.244989_0 b/fink_mm/test/test_data/gcn_test/raw/year=2024/month=01/day=27/728023845_1706534763.244989_0 new file mode 100644 index 000000000..63556d9ed Binary files /dev/null and b/fink_mm/test/test_data/gcn_test/raw/year=2024/month=01/day=27/728023845_1706534763.244989_0 differ diff --git a/fink_mm/test/test_data/gcn_test/raw/year=2024/month=01/day=28/728113967_1706534763.3136504_0 b/fink_mm/test/test_data/gcn_test/raw/year=2024/month=01/day=28/728113967_1706534763.3136504_0 new file mode 100644 index 000000000..d09b7a243 Binary files /dev/null and b/fink_mm/test/test_data/gcn_test/raw/year=2024/month=01/day=28/728113967_1706534763.3136504_0 differ diff --git a/fink_mm/test/test_data/grb_prob_test.parquet b/fink_mm/test/test_data/grb_prob_test.parquet index 25dda4243..63f1cae56 100644 Binary files a/fink_mm/test/test_data/grb_prob_test.parquet and b/fink_mm/test/test_data/grb_prob_test.parquet differ diff --git a/fink_mm/test/test_data/with_hbase/fink_test.conf b/fink_mm/test/test_data/with_hbase/fink_test.conf index 699f70433..c86bde7fa 100644 --- a/fink_mm/test/test_data/with_hbase/fink_test.conf +++ b/fink_mm/test/test_data/with_hbase/fink_test.conf @@ -32,7 +32,7 @@ KAFKA_IPPORT="" KAFKA_STARTING_OFFSET="earliest" # Apache Spark mode -SPARK_MASTER="local[*]" +SPARK_MASTER="local[5]" # Should be Spark options actually (cluster resources, ...) EXTRA_SPARK_CONFIG='--driver-memory 2g --executor-memory 2g' diff --git a/fink_mm/test/test_data/ztf_test/archive/science/year=2019/month=09/day=03/merge_ztf_data_test.parquet b/fink_mm/test/test_data/ztf_test/archive/science/year=2019/month=09/day=03/merge_ztf_data_test.parquet deleted file mode 100644 index f7dac53cb..000000000 Binary files a/fink_mm/test/test_data/ztf_test/archive/science/year=2019/month=09/day=03/merge_ztf_data_test.parquet and /dev/null differ diff --git a/fink_mm/test/test_data/ztf_test/archive/science/year=2024/month=01/day=15/part-00000-e55a3e2d-0f29-423f-8621-3547113ad41b-c000.snappy.parquet b/fink_mm/test/test_data/ztf_test/archive/science/year=2024/month=01/day=15/part-00000-e55a3e2d-0f29-423f-8621-3547113ad41b-c000.snappy.parquet new file mode 100644 index 000000000..696e030e2 Binary files /dev/null and b/fink_mm/test/test_data/ztf_test/archive/science/year=2024/month=01/day=15/part-00000-e55a3e2d-0f29-423f-8621-3547113ad41b-c000.snappy.parquet differ diff --git a/fink_mm/test/test_data/ztf_test/archive/science/year=2024/month=01/day=15/part-00001-e55a3e2d-0f29-423f-8621-3547113ad41b-c000.snappy.parquet b/fink_mm/test/test_data/ztf_test/archive/science/year=2024/month=01/day=15/part-00001-e55a3e2d-0f29-423f-8621-3547113ad41b-c000.snappy.parquet new file mode 100644 index 000000000..c8978b10f Binary files /dev/null and b/fink_mm/test/test_data/ztf_test/archive/science/year=2024/month=01/day=15/part-00001-e55a3e2d-0f29-423f-8621-3547113ad41b-c000.snappy.parquet differ diff --git a/fink_mm/test/test_data/ztf_test/archive/science/year=2024/month=01/day=15/part-00002-e55a3e2d-0f29-423f-8621-3547113ad41b-c000.snappy.parquet b/fink_mm/test/test_data/ztf_test/archive/science/year=2024/month=01/day=15/part-00002-e55a3e2d-0f29-423f-8621-3547113ad41b-c000.snappy.parquet new file mode 100644 index 000000000..46144debe Binary files /dev/null and b/fink_mm/test/test_data/ztf_test/archive/science/year=2024/month=01/day=15/part-00002-e55a3e2d-0f29-423f-8621-3547113ad41b-c000.snappy.parquet differ diff --git a/fink_mm/test/test_data/ztf_test/archive/science/year=2024/month=01/day=15/part-00003-e55a3e2d-0f29-423f-8621-3547113ad41b-c000.snappy.parquet b/fink_mm/test/test_data/ztf_test/archive/science/year=2024/month=01/day=15/part-00003-e55a3e2d-0f29-423f-8621-3547113ad41b-c000.snappy.parquet new file mode 100644 index 000000000..46d773fb6 Binary files /dev/null and b/fink_mm/test/test_data/ztf_test/archive/science/year=2024/month=01/day=15/part-00003-e55a3e2d-0f29-423f-8621-3547113ad41b-c000.snappy.parquet differ diff --git a/fink_mm/test/test_data/ztf_test/archive/science/year=2024/month=01/day=15/part-00004-e55a3e2d-0f29-423f-8621-3547113ad41b-c000.snappy.parquet b/fink_mm/test/test_data/ztf_test/archive/science/year=2024/month=01/day=15/part-00004-e55a3e2d-0f29-423f-8621-3547113ad41b-c000.snappy.parquet new file mode 100644 index 000000000..fbc2611f1 Binary files /dev/null and b/fink_mm/test/test_data/ztf_test/archive/science/year=2024/month=01/day=15/part-00004-e55a3e2d-0f29-423f-8621-3547113ad41b-c000.snappy.parquet differ diff --git a/fink_mm/test/test_data/ztf_test/archive/science/year=2024/month=01/day=15/part-00005-e55a3e2d-0f29-423f-8621-3547113ad41b-c000.snappy.parquet b/fink_mm/test/test_data/ztf_test/archive/science/year=2024/month=01/day=15/part-00005-e55a3e2d-0f29-423f-8621-3547113ad41b-c000.snappy.parquet new file mode 100644 index 000000000..a2fa82a7e Binary files /dev/null and b/fink_mm/test/test_data/ztf_test/archive/science/year=2024/month=01/day=15/part-00005-e55a3e2d-0f29-423f-8621-3547113ad41b-c000.snappy.parquet differ diff --git a/fink_mm/test/test_data/ztf_test/archive/science/year=2024/month=01/day=15/part-00006-e55a3e2d-0f29-423f-8621-3547113ad41b-c000.snappy.parquet b/fink_mm/test/test_data/ztf_test/archive/science/year=2024/month=01/day=15/part-00006-e55a3e2d-0f29-423f-8621-3547113ad41b-c000.snappy.parquet new file mode 100644 index 000000000..9859e84b0 Binary files /dev/null and b/fink_mm/test/test_data/ztf_test/archive/science/year=2024/month=01/day=15/part-00006-e55a3e2d-0f29-423f-8621-3547113ad41b-c000.snappy.parquet differ diff --git a/fink_mm/test/test_data/ztf_test/archive/science/year=2024/month=01/day=15/part-00007-e55a3e2d-0f29-423f-8621-3547113ad41b-c000.snappy.parquet b/fink_mm/test/test_data/ztf_test/archive/science/year=2024/month=01/day=15/part-00007-e55a3e2d-0f29-423f-8621-3547113ad41b-c000.snappy.parquet new file mode 100644 index 000000000..873344592 Binary files /dev/null and b/fink_mm/test/test_data/ztf_test/archive/science/year=2024/month=01/day=15/part-00007-e55a3e2d-0f29-423f-8621-3547113ad41b-c000.snappy.parquet differ diff --git a/fink_mm/test/test_data/ztf_test/online/raw/year=2019/month=09/day=03/alert1.parquet b/fink_mm/test/test_data/ztf_test/online/raw/year=2019/month=09/day=03/alert1.parquet deleted file mode 100644 index e19bb90ee..000000000 Binary files a/fink_mm/test/test_data/ztf_test/online/raw/year=2019/month=09/day=03/alert1.parquet and /dev/null differ diff --git a/fink_mm/test/test_data/ztf_test/online/raw/year=2019/month=09/day=03/alert2.parquet b/fink_mm/test/test_data/ztf_test/online/raw/year=2019/month=09/day=03/alert2.parquet deleted file mode 100644 index bb391ef35..000000000 Binary files a/fink_mm/test/test_data/ztf_test/online/raw/year=2019/month=09/day=03/alert2.parquet and /dev/null differ diff --git a/fink_mm/test/test_data/ztf_test/online/science/year=2024/month=01/day=15/part-00000-e55a3e2d-0f29-423f-8621-3547113ad41b-c000.snappy.parquet b/fink_mm/test/test_data/ztf_test/online/science/year=2024/month=01/day=15/part-00000-e55a3e2d-0f29-423f-8621-3547113ad41b-c000.snappy.parquet new file mode 100644 index 000000000..696e030e2 Binary files /dev/null and b/fink_mm/test/test_data/ztf_test/online/science/year=2024/month=01/day=15/part-00000-e55a3e2d-0f29-423f-8621-3547113ad41b-c000.snappy.parquet differ diff --git a/fink_mm/test/test_data/ztf_test/online/science/year=2024/month=01/day=15/part-00001-e55a3e2d-0f29-423f-8621-3547113ad41b-c000.snappy.parquet b/fink_mm/test/test_data/ztf_test/online/science/year=2024/month=01/day=15/part-00001-e55a3e2d-0f29-423f-8621-3547113ad41b-c000.snappy.parquet new file mode 100644 index 000000000..c8978b10f Binary files /dev/null and b/fink_mm/test/test_data/ztf_test/online/science/year=2024/month=01/day=15/part-00001-e55a3e2d-0f29-423f-8621-3547113ad41b-c000.snappy.parquet differ diff --git a/fink_mm/test/test_data/ztf_test/online/science/year=2024/month=01/day=15/part-00002-e55a3e2d-0f29-423f-8621-3547113ad41b-c000.snappy.parquet b/fink_mm/test/test_data/ztf_test/online/science/year=2024/month=01/day=15/part-00002-e55a3e2d-0f29-423f-8621-3547113ad41b-c000.snappy.parquet new file mode 100644 index 000000000..46144debe Binary files /dev/null and b/fink_mm/test/test_data/ztf_test/online/science/year=2024/month=01/day=15/part-00002-e55a3e2d-0f29-423f-8621-3547113ad41b-c000.snappy.parquet differ diff --git a/fink_mm/test/test_data/ztf_test/online/science/year=2024/month=01/day=15/part-00003-e55a3e2d-0f29-423f-8621-3547113ad41b-c000.snappy.parquet b/fink_mm/test/test_data/ztf_test/online/science/year=2024/month=01/day=15/part-00003-e55a3e2d-0f29-423f-8621-3547113ad41b-c000.snappy.parquet new file mode 100644 index 000000000..46d773fb6 Binary files /dev/null and b/fink_mm/test/test_data/ztf_test/online/science/year=2024/month=01/day=15/part-00003-e55a3e2d-0f29-423f-8621-3547113ad41b-c000.snappy.parquet differ diff --git a/fink_mm/test/test_data/ztf_test/online/science/year=2024/month=01/day=15/part-00004-e55a3e2d-0f29-423f-8621-3547113ad41b-c000.snappy.parquet b/fink_mm/test/test_data/ztf_test/online/science/year=2024/month=01/day=15/part-00004-e55a3e2d-0f29-423f-8621-3547113ad41b-c000.snappy.parquet new file mode 100644 index 000000000..fbc2611f1 Binary files /dev/null and b/fink_mm/test/test_data/ztf_test/online/science/year=2024/month=01/day=15/part-00004-e55a3e2d-0f29-423f-8621-3547113ad41b-c000.snappy.parquet differ diff --git a/fink_mm/test/test_data/ztf_test/online/science/year=2024/month=01/day=15/part-00005-e55a3e2d-0f29-423f-8621-3547113ad41b-c000.snappy.parquet b/fink_mm/test/test_data/ztf_test/online/science/year=2024/month=01/day=15/part-00005-e55a3e2d-0f29-423f-8621-3547113ad41b-c000.snappy.parquet new file mode 100644 index 000000000..a2fa82a7e Binary files /dev/null and b/fink_mm/test/test_data/ztf_test/online/science/year=2024/month=01/day=15/part-00005-e55a3e2d-0f29-423f-8621-3547113ad41b-c000.snappy.parquet differ diff --git a/fink_mm/test/test_data/ztf_test/online/science/year=2024/month=01/day=15/part-00006-e55a3e2d-0f29-423f-8621-3547113ad41b-c000.snappy.parquet b/fink_mm/test/test_data/ztf_test/online/science/year=2024/month=01/day=15/part-00006-e55a3e2d-0f29-423f-8621-3547113ad41b-c000.snappy.parquet new file mode 100644 index 000000000..9859e84b0 Binary files /dev/null and b/fink_mm/test/test_data/ztf_test/online/science/year=2024/month=01/day=15/part-00006-e55a3e2d-0f29-423f-8621-3547113ad41b-c000.snappy.parquet differ diff --git a/fink_mm/test/test_data/ztf_test/online/science/year=2024/month=01/day=15/part-00007-e55a3e2d-0f29-423f-8621-3547113ad41b-c000.snappy.parquet b/fink_mm/test/test_data/ztf_test/online/science/year=2024/month=01/day=15/part-00007-e55a3e2d-0f29-423f-8621-3547113ad41b-c000.snappy.parquet new file mode 100644 index 000000000..873344592 Binary files /dev/null and b/fink_mm/test/test_data/ztf_test/online/science/year=2024/month=01/day=15/part-00007-e55a3e2d-0f29-423f-8621-3547113ad41b-c000.snappy.parquet differ diff --git a/fink_mm/test/utils_integration_test.py b/fink_mm/test/utils_integration_test.py index 88432f6a3..9ff288eee 100644 --- a/fink_mm/test/utils_integration_test.py +++ b/fink_mm/test/utils_integration_test.py @@ -63,17 +63,17 @@ def align_ztf( Examples -------- - >>> path_ztf_raw = ("fink_mm/test/test_data/ztf_test/online/raw/year=2019/month=09/day=03/") + >>> path_ztf_raw = ("fink_mm/test/test_data/ztf_test/online/science/year=2024/month=01/day=15/") >>> ztf_pdf = pd.read_parquet(path_ztf_raw) >>> random = np.random.default_rng(0) >>> align_ztf(ztf_pdf, 5, 0, 10, random) >>> ztf_pdf.loc[0]["candidate"]["ra"] - 289.4610443 + 62.2777961 >>> ztf_pdf.loc[0]["candidate"]["dec"] - -11.0504023 + 57.4803954 >>> ztf_pdf.loc[len(ztf_pdf)-1]["candidate"]["ra"] 0 @@ -82,11 +82,11 @@ def align_ztf( 10 >>> ztf_pdf.loc[0]["candidate"]["jdstarthist"] - 2458729.6881481 + 2458422.9036806 >>> ztf_pdf.loc[0]["candidate"]["jd"] - 2458729.6881481 + 2460324.7135301 >>> ztf_pdf.loc[0]["prv_candidates"][-1]["jd"] - 2458725.7316204 + 2460324.6389931 >>> ztf_pdf.loc[len(ztf_pdf)-1]["candidate"]["jdstarthist"] 5.020833333333333 @@ -262,7 +262,7 @@ def align_ztf_and_gcn( >>> path_gcn = "fink_mm/test/test_data/683571622_0_test" >>> gcn = pd.read_parquet(path_gcn) - >>> path_ztf_raw = ("fink_mm/test/test_data/ztf_test/online/raw/year=2019/month=09/day=03/") + >>> path_ztf_raw = ("fink_mm/test/test_data/ztf_test/online/science/year=2024/month=01/day=15/") >>> ztf_pdf = pd.read_parquet(path_ztf_raw) >>> random = np.random.default_rng(0) diff --git a/fink_mm/utils/fun_utils.py b/fink_mm/utils/fun_utils.py index fc9a3a134..59f75fe39 100644 --- a/fink_mm/utils/fun_utils.py +++ b/fink_mm/utils/fun_utils.py @@ -287,7 +287,7 @@ def get_observatory(obsname: str, rawEvent: str): # -> Observatory ------- >>> pdf = pd.read_parquet(grb_data) >>> type(get_observatory(pdf["observatory"].iloc[0], pdf["raw_event"].iloc[0])) - + >>> pdf = pd.read_parquet(gw_data) >>> type(get_observatory(pdf["observatory"].iloc[0], pdf["raw_event"].iloc[0])) @@ -331,7 +331,7 @@ def get_pixels(obsname: pd.Series, rawEvent: pd.Series, NSIDE: pd.Series) -> pd. >>> grb_pixs.withColumn("hpix", explode("hpix_circle"))\ .orderBy(["triggerId", "hpix"])\ .select(["triggerId", "hpix"]).head(5) - [Row(triggerId='683476673', hpix=42), Row(triggerId='683476673', hpix=42), Row(triggerId='683476673', hpix=62), Row(triggerId='683476673', hpix=62), Row(triggerId='683476673', hpix=63)] + [Row(triggerId='10472', hpix=609), Row(triggerId='727009399', hpix=177), Row(triggerId='727009399', hpix=177), Row(triggerId='727009399', hpix=177), Row(triggerId='727009399', hpix=178)] """ return pd.Series( [ @@ -757,21 +757,22 @@ def join_post_process(df_grb: DataFrame, hdfs_adress: str, root_path: str) -> Da df_grb = concat_col(df_grb, "jd") df_grb = concat_col(df_grb, "fid") - df_grb = df_grb.withColumn( - "c_rate", - compute_rate( - df_grb["candidate.magpsf"], - df_grb["candidate.jdstarthist"], - df_grb["candidate.jd"], - df_grb["candidate.fid"], - df_grb["cmagpsf"], - df_grb["cdiffmaglim"], - df_grb["cjd"], - df_grb["cfid"], - ), - ) - - df_grb = format_rate_results(df_grb, "c_rate") + # DEPRECATED + # computed by the fast transient fink science module + # df_grb = df_grb.withColumn( + # "c_rate", + # compute_rate( + # df_grb["candidate.magpsf"], + # df_grb["candidate.jdstarthist"], + # df_grb["candidate.jd"], + # df_grb["candidate.fid"], + # df_grb["cmagpsf"], + # df_grb["cdiffmaglim"], + # df_grb["cjd"], + # df_grb["cfid"], + # ), + # ) + # df_grb = format_rate_results(df_grb, "c_rate") # TODO : do something better with satellites # df_grb = add_tracklet_information(df_grb) @@ -805,7 +806,7 @@ def join_post_process(df_grb: DataFrame, hdfs_adress: str, root_path: str) -> Da df_grb["raw_event"], df_grb["ztf_ra"], df_grb["ztf_dec"], - df_grb["start_vartime"], + df_grb["jd_first_real_det"], F.lit(hdfs_adress), df_grb["gcn_status"], F.lit(root_path), @@ -838,6 +839,8 @@ def join_post_process(df_grb: DataFrame, hdfs_adress: str, root_path: str) -> Da "candidate.jdstarthist", "candidate.rb", "candidate.jd", + "candidate.magpsf", + "candidate.sigmapsf" ] df_grb = df_grb.select(cols_fink + cols_extra).filter("p_assoc != -1.0") df_grb = df_grb.withColumnRenamed("err_arcmin", "gcn_loc_error") @@ -869,7 +872,7 @@ def read_and_build_spark_submit(config, logger): >>> home_path = os.environ["HOME"] >>> path_bash_profile = os.path.join(home_path, ".bash_profile") - >>> test_str = f"if test -f '{path_bash_profile}'; then source {path_bash_profile}; fi; `which spark-submit` --master local[2] --conf spark.mesos.principal= --conf spark.mesos.secret= --conf spark.mesos.role= --conf spark.executorEnv.HOME=/path/to/user/ --driver-memory 4G --executor-memory 8G --conf spark.cores.max=16 --conf spark.executor.cores=8" + >>> test_str = f"if test -f '{path_bash_profile}'; then source {path_bash_profile}; fi; `which spark-submit` --master local[8] --conf spark.mesos.principal= --conf spark.mesos.secret= --conf spark.mesos.role= --conf spark.executorEnv.HOME=/path/to/user/ --driver-memory 4G --executor-memory 8G --conf spark.cores.max=16 --conf spark.executor.cores=8" >>> test_str == spark_str True """ diff --git a/fink_mm/ztf_join_gcn.py b/fink_mm/ztf_join_gcn.py index fa3d6a510..83539c81e 100644 --- a/fink_mm/ztf_join_gcn.py +++ b/fink_mm/ztf_join_gcn.py @@ -79,7 +79,7 @@ def ztf_grb_filter(spark_ztf, ast_dist, pansstar_dist, pansstar_star_score, gaia >>> spark_filter = ztf_grb_filter(sparkDF, 5, 2, 0, 5) >>> spark_filter.count() - 32 + 53 """ spark_filter = ( spark_ztf.filter( @@ -321,7 +321,7 @@ def print_logs(): df_join["rb"], df_join["gcn_loc_error"], df_join["p_assoc"], - df_join["rate"], + df_join["mag_rate"], ), ) @@ -357,7 +357,7 @@ def ztf_pre_join( "cutoutDifference", "year", "month", - "day" + "day", ) ztf_dataframe = ztf_grb_filter( @@ -380,7 +380,7 @@ def gcn_pre_join( gcn_dataframe: DataFrame, NSIDE: int, test: bool, -) -> DataFrame: +) -> Tuple[DataFrame, DataFrame]: gcn_dataframe = gcn_dataframe.drop("year").drop("month").drop("day") # compute pixels for gcn alerts @@ -396,12 +396,20 @@ def gcn_pre_join( remove_skymap(gcn_dataframe.observatory, gcn_dataframe.raw_event), ) + gcn_rawevent = gcn_dataframe.select(["triggerId", "raw_event"]).withColumnRenamed( + "triggerId", "gcn_trigId" + ) + gcn_dataframe = gcn_dataframe.select( + [col for col in gcn_dataframe.columns if col != "raw_event"] + ) + gcn_dataframe = gcn_dataframe.withColumn("hpix", explode("hpix_circle")) gcn_dataframe = gcn_dataframe.withColumnRenamed("ra", "gcn_ra").withColumnRenamed( "dec", "gcn_dec" ) - return gcn_dataframe + + return gcn_dataframe, gcn_rawevent def ztf_join_gcn( @@ -470,41 +478,36 @@ def ztf_join_gcn( ... ztf_datatest, ... gcn_datatest, ... grb_dataoutput, - ... "20190903", + ... "20240115", ... 4, 100, 5, 7, "127.0.0.1", 5, 2, 0, 5, False, True ... ) - >>> datajoin = pd.read_parquet(grb_dataoutput + "/online").sort_values(["objectId", "triggerId", "gcn_ra"]).reset_index(drop=True).sort_index(axis=1) - - >>> datajoin = datajoin.drop("t2", axis=1) - + >>> datajoin = pd.read_parquet(grb_dataoutput + "/online") >>> datajoin = datajoin.reindex(sorted(datajoin.columns), axis=1) >>> list(datajoin.columns) - ['DR3Name', 'Plx', 'anomaly_score', 'candid', 'cdsxmatch', 'day', 'delta_mag', 'delta_time', 'diff_vartime', 'e_Plx', 'event', 'fid', 'fink_class', 'from_upper', 'gcn_dec', 'gcn_loc_error', 'gcn_ra', 'gcn_status', 'gcvs', 'instrument', 'jd', 'jd_first_real_det', 'jdstarthist', 'jdstarthist_dt', 'lc_features_g', 'lc_features_r', 'lower_rate', 'mag_rate', 'mangrove', 'month', 'mulens', 'nalerthist', 'objectId', 'observatory', 'p_assoc', 'rate', 'raw_event', 'rb', 'rf_kn_vs_nonkn', 'rf_snia_vs_nonia', 'roid', 'sigma_rate', 'snn_sn_vs_all', 'snn_snia_vs_nonia', 'start_vartime', 'timestamp', 'triggerId', 'triggerTimeUTC', 'upper_rate', 'vsx', 'x3hsp', 'x4lac', 'year', 'ztf_dec', 'ztf_ra'] + ['DR3Name', 'Plx', 'ackTime', 'anomaly_score', 'candid', 'cdsxmatch', 'day', 'delta_time', 'e_Plx', 'event', 'fid', 'fink_class', 'from_upper', 'gcn_dec', 'gcn_loc_error', 'gcn_ra', 'gcn_status', 'gcvs', 'instrument', 'jd', 'jd_first_real_det', 'jdstarthist', 'jdstarthist_dt', 'lc_features_g', 'lc_features_r', 'lower_rate', 'mag_rate', 'magpsf', 'mangrove', 'month', 'mulens', 'nalerthist', 'objectId', 'observatory', 'p_assoc', 'raw_event', 'rb', 'rf_kn_vs_nonkn', 'rf_snia_vs_nonia', 'roid', 'sigma_rate', 'sigmapsf', 'snn_sn_vs_all', 'snn_snia_vs_nonia', 't2', 'timestamp', 'triggerId', 'triggerTimeUTC', 'upper_rate', 'vsx', 'x3hsp', 'x4lac', 'year', 'ztf_dec', 'ztf_ra'] >>> len(datajoin) - 36 + 204 >>> ztf_join_gcn( ... DataMode.OFFLINE, ... ztf_datatest, ... gcn_datatest, ... grb_dataoutput, - ... "20190903", + ... "20240115", ... 4, 100, 5, 7, "127.0.0.1", 5, 2, 0, 5, False, True ... ) - >>> datajoin = pd.read_parquet(grb_dataoutput + "/offline").sort_values(["objectId", "triggerId", "gcn_ra"]).reset_index(drop=True).sort_index(axis=1) - - >>> datajoin = datajoin.drop("t2", axis=1) - + >>> datajoin = pd.read_parquet(grb_dataoutput + "/offline") >>> datajoin = datajoin.reindex(sorted(datajoin.columns), axis=1) >>> list(datajoin.columns) - ['DR3Name', 'Plx', 'anomaly_score', 'candid', 'cdsxmatch', 'day', 'delta_mag', 'delta_time', 'diff_vartime', 'e_Plx', 'event', 'fid', 'fink_class', 'from_upper', 'gcn_dec', 'gcn_loc_error', 'gcn_ra', 'gcn_status', 'gcvs', 'instrument', 'is_grb_bronze', 'is_grb_gold', 'is_grb_silver', 'is_gw_bronze', 'jd', 'jd_first_real_det', 'jdstarthist', 'jdstarthist_dt', 'lc_features_g', 'lc_features_r', 'lower_rate', 'mag_rate', 'mangrove', 'month', 'mulens', 'nalerthist', 'objectId', 'observatory', 'p_assoc', 'rate', 'raw_event', 'rb', 'rf_kn_vs_nonkn', 'rf_snia_vs_nonia', 'roid', 'sigma_rate', 'snn_sn_vs_all', 'snn_snia_vs_nonia', 'start_vartime', 'timestamp', 'triggerId', 'triggerTimeUTC', 'upper_rate', 'vsx', 'x3hsp', 'x4lac', 'year', 'ztf_dec', 'ztf_ra'] + ['DR3Name', 'Plx', 'ackTime', 'anomaly_score', 'candid', 'cdsxmatch', 'day', 'delta_time', 'e_Plx', 'event', 'fid', 'fink_class', 'from_upper', 'gcn_dec', 'gcn_loc_error', 'gcn_ra', 'gcn_status', 'gcvs', 'instrument', 'is_grb_bronze', 'is_grb_gold', 'is_grb_silver', 'is_gw_bronze', 'jd', 'jd_first_real_det', 'jdstarthist', 'jdstarthist_dt', 'lc_features_g', 'lc_features_r', 'lower_rate', 'mag_rate', 'magpsf', 'mangrove', 'month', 'mulens', 'nalerthist', 'objectId', 'observatory', 'p_assoc', 'raw_event', 'rb', 'rf_kn_vs_nonkn', 'rf_snia_vs_nonia', 'roid', 'sigma_rate', 'sigmapsf', 'snn_sn_vs_all', 'snn_snia_vs_nonia', 't2', 'timestamp', 'triggerId', 'triggerTimeUTC', 'upper_rate', 'vsx', 'x3hsp', 'x4lac', 'year', 'ztf_dec', 'ztf_ra'] + >>> len(datajoin) - 72 + 382 """ logger = init_logging() @@ -528,7 +531,7 @@ def ztf_join_gcn( ztf_dataframe = ztf_pre_join( ztf_dataframe, ast_dist, pansstar_dist, pansstar_star_score, gaia_dist, NSIDE ) - gcn_dataframe = gcn_pre_join(gcn_dataframe, NSIDE, test) + gcn_dataframe, gcn_rawevent = gcn_pre_join(gcn_dataframe, NSIDE, test) # join the two streams according to the healpix columns. # A pixel id will be assign to each alerts / gcn according to their position in the sky. @@ -537,8 +540,16 @@ def ztf_join_gcn( ztf_dataframe.hpix == gcn_dataframe.hpix, ztf_dataframe.candidate.jdstarthist > gcn_dataframe.triggerTimejd, ] + # multi_messenger join to combine optical stream with other streams df_join_mm = gcn_dataframe.join(F.broadcast(ztf_dataframe), join_condition, "inner") + # combine the multi-messenger join with the raw_event removed previously to save memory + df_join_mm = gcn_rawevent.join( + F.broadcast(df_join_mm), + [df_join_mm.triggerId == gcn_rawevent.gcn_trigId], + "inner", + ).drop("gcn_trigId").dropDuplicates(["objectId", "triggerId", "gcn_status"]) + df_join_mm = join_post_process(df_join_mm, hdfs_adress, gcn_datapath_prefix) # re-create partitioning columns if needed. @@ -586,39 +597,35 @@ def launch_join(arguments: dict, data_mode, test: bool = False): -------- >>> launch_join({ ... "--config" : None, - ... "--night" : "20190903", - ... "--exit_after" : 100, + ... "--night" : "20240115", + ... "--exit_after" : 180, ... "--verbose" : False ... }, DataMode.STREAMING, True) - >>> datajoin = pd.read_parquet("fink_mm/test/test_output/online").sort_values(["objectId", "triggerId", "gcn_ra"]).reset_index(drop=True).sort_index(axis=1) - - >>> datajoin = datajoin.drop("t2", axis=1) - + >>> datajoin = pd.read_parquet("fink_mm/test/test_output/online") >>> datajoin = datajoin.reindex(sorted(datajoin.columns), axis=1) >>> list(datajoin.columns) - ['DR3Name', 'Plx', 'anomaly_score', 'candid', 'cdsxmatch', 'day', 'delta_mag', 'delta_time', 'diff_vartime', 'e_Plx', 'event', 'fid', 'fink_class', 'from_upper', 'gcn_dec', 'gcn_loc_error', 'gcn_ra', 'gcn_status', 'gcvs', 'instrument', 'jd', 'jd_first_real_det', 'jdstarthist', 'jdstarthist_dt', 'lc_features_g', 'lc_features_r', 'lower_rate', 'mag_rate', 'mangrove', 'month', 'mulens', 'nalerthist', 'objectId', 'observatory', 'p_assoc', 'rate', 'raw_event', 'rb', 'rf_kn_vs_nonkn', 'rf_snia_vs_nonia', 'roid', 'sigma_rate', 'snn_sn_vs_all', 'snn_snia_vs_nonia', 'start_vartime', 'timestamp', 'triggerId', 'triggerTimeUTC', 'upper_rate', 'vsx', 'x3hsp', 'x4lac', 'year', 'ztf_dec', 'ztf_ra'] + ['DR3Name', 'Plx', 'ackTime', 'anomaly_score', 'candid', 'cdsxmatch', 'day', 'delta_time', 'e_Plx', 'event', 'fid', 'fink_class', 'from_upper', 'gcn_dec', 'gcn_loc_error', 'gcn_ra', 'gcn_status', 'gcvs', 'instrument', 'jd', 'jd_first_real_det', 'jdstarthist', 'jdstarthist_dt', 'lc_features_g', 'lc_features_r', 'lower_rate', 'mag_rate', 'magpsf', 'mangrove', 'month', 'mulens', 'nalerthist', 'objectId', 'observatory', 'p_assoc', 'raw_event', 'rb', 'rf_kn_vs_nonkn', 'rf_snia_vs_nonia', 'roid', 'sigma_rate', 'sigmapsf', 'snn_sn_vs_all', 'snn_snia_vs_nonia', 't2', 'timestamp', 'triggerId', 'triggerTimeUTC', 'upper_rate', 'vsx', 'x3hsp', 'x4lac', 'year', 'ztf_dec', 'ztf_ra'] + >>> len(datajoin) - 36 + 204 >>> launch_join({ ... "--config" : None, - ... "--night" : "20190903", - ... "--exit_after" : 100, + ... "--night" : "20240115", + ... "--exit_after" : 180, ... "--verbose" : False ... }, DataMode.OFFLINE, True) - >>> datajoin = pd.read_parquet("fink_mm/test/test_output/offline").sort_values(["objectId", "triggerId", "gcn_ra"]).reset_index(drop=True).sort_index(axis=1) - - >>> datajoin = datajoin.drop("t2", axis=1) - + >>> datajoin = pd.read_parquet("fink_mm/test/test_output/offline") >>> datajoin = datajoin.reindex(sorted(datajoin.columns), axis=1) >>> list(datajoin.columns) - ['DR3Name', 'Plx', 'anomaly_score', 'candid', 'cdsxmatch', 'day', 'delta_mag', 'delta_time', 'diff_vartime', 'e_Plx', 'event', 'fid', 'fink_class', 'from_upper', 'gcn_dec', 'gcn_loc_error', 'gcn_ra', 'gcn_status', 'gcvs', 'instrument', 'is_grb_bronze', 'is_grb_gold', 'is_grb_silver', 'is_gw_bronze', 'jd', 'jd_first_real_det', 'jdstarthist', 'jdstarthist_dt', 'lc_features_g', 'lc_features_r', 'lower_rate', 'mag_rate', 'mangrove', 'month', 'mulens', 'nalerthist', 'objectId', 'observatory', 'p_assoc', 'rate', 'raw_event', 'rb', 'rf_kn_vs_nonkn', 'rf_snia_vs_nonia', 'roid', 'sigma_rate', 'snn_sn_vs_all', 'snn_snia_vs_nonia', 'start_vartime', 'timestamp', 'triggerId', 'triggerTimeUTC', 'upper_rate', 'vsx', 'x3hsp', 'x4lac', 'year', 'ztf_dec', 'ztf_ra'] + ['DR3Name', 'Plx', 'ackTime', 'anomaly_score', 'candid', 'cdsxmatch', 'day', 'delta_time', 'e_Plx', 'event', 'fid', 'fink_class', 'from_upper', 'gcn_dec', 'gcn_loc_error', 'gcn_ra', 'gcn_status', 'gcvs', 'instrument', 'is_grb_bronze', 'is_grb_gold', 'is_grb_silver', 'is_gw_bronze', 'jd', 'jd_first_real_det', 'jdstarthist', 'jdstarthist_dt', 'lc_features_g', 'lc_features_r', 'lower_rate', 'mag_rate', 'magpsf', 'mangrove', 'month', 'mulens', 'nalerthist', 'objectId', 'observatory', 'p_assoc', 'raw_event', 'rb', 'rf_kn_vs_nonkn', 'rf_snia_vs_nonia', 'roid', 'sigma_rate', 'sigmapsf', 'snn_sn_vs_all', 'snn_snia_vs_nonia', 't2', 'timestamp', 'triggerId', 'triggerTimeUTC', 'upper_rate', 'vsx', 'x3hsp', 'x4lac', 'year', 'ztf_dec', 'ztf_ra'] + >>> len(datajoin) - 72 + 382 """ config = get_config(arguments) logger = init_logging()