From 54047359018c271a87cb2f236168bb9ab0eb2ffd Mon Sep 17 00:00:00 2001 From: FusRoman <46221629+FusRoman@users.noreply.github.com> Date: Mon, 17 Jul 2023 16:57:19 +0200 Subject: [PATCH] Issue/80/refactoring into fink mm (#81) * start the refactoring * still refactoring * refactoring the last occurence of fink_grb * pep8 refactoring and requirements * refactoring workflows * restore fink_grb_bronze topics * test * rename fink-mm package * print grb_data * remove print * add gcn_test data * update gitignore * change columns name * update test data for distribution * fix get_assoc_proba test * pep8 requirements * update test data for online, test only online in ci * run all test * fink_added_value are part of the fink_mm output now * pep8 requirements * fix parquet reading issue * sort colums name before test * fix test I hope * test * add fink added value to the hbase column filter for the offline mode * new column filters for hbase * pep8 * remove drop t2 in offline mode doctest * new test data for offline mode * -1.0 for offline data test * test * test * test * test * pep8 * test * drop t2 columns * drop t2 columns * run all tests * load gcn data of the previous day for the online mode * fix append in the online mode * print test * test * restore online as previous * all test * pep8 --- .flake8 | 14 +- .github/workflows/linter.yml | 4 +- .github/workflows/run_test.yml | 67 ++++---- .gitignore | 8 +- README.md | 36 ++--- ...42a7-a2be-1f56fc5c6571.c000.snappy.parquet | Bin 16833 -> 0 bytes ...4e73-adc6-a480974420d3.c000.snappy.parquet | Bin 16468 -> 0 bytes ...4442-9cb1-9834fb0b6c1e.c000.snappy.parquet | Bin 16522 -> 0 bytes ...42a7-a2be-1f56fc5c6571.c000.snappy.parquet | Bin 7243 -> 0 bytes ...4e73-adc6-a480974420d3.c000.snappy.parquet | Bin 6902 -> 0 bytes ...4442-9cb1-9834fb0b6c1e.c000.snappy.parquet | Bin 6982 -> 0 bytes {fink_grb => fink_mm}/__init__.py | 0 .../conf/distribute_for_test.conf | 6 +- .../conf/fink_mm.conf | 6 +- .../conf/fink_mm_schema_version_1.0.avsc | 0 .../conf/fink_mm_schema_version_1.1.avsc | 0 .../conf/fink_mm_schema_version_1.2.avsc | 0 {fink_grb => fink_mm}/conf/integration.conf | 8 +- {fink_grb => fink_mm}/conftest.py | 58 +++---- .../distribution/__init__.py | 0 .../distribution/apply_filters.py | 29 +++- .../distribution/distribution.py | 26 ++-- .../fink_grb_cli.py => fink_mm/fink_mm_cli.py | 22 +-- {fink_grb => fink_mm}/gcn_stream/__init__.py | 0 .../gcn_stream/gcn_reader.py | 4 +- .../gcn_stream/gcn_stream.py | 14 +- {fink_grb => fink_mm}/init.py | 22 +-- .../observatory/Fermi/Fermi.py | 4 +- .../observatory/Fermi/__init__.py | 0 .../observatory/Fermi/fermi.json | 0 .../observatory/IceCube/IceCube.py | 4 +- .../observatory/IceCube/__init__.py | 0 .../observatory/IceCube/icecube.json | 0 .../observatory/Integral/Integral.py | 4 +- .../observatory/Integral/__init__.py | 0 .../observatory/Integral/integral.json | 0 {fink_grb => fink_mm}/observatory/LVK/LVK.py | 6 +- .../observatory/LVK/__init__.py | 0 .../observatory/LVK/lvk.json | 0 .../observatory/Swift/Swift.py | 4 +- .../observatory/Swift/__init__.py | 0 .../observatory/Swift/swift.json | 0 {fink_grb => fink_mm}/observatory/__init__.py | 14 +- .../observatory/observatory.py | 10 +- .../observatory_schema_version_1.1.json | 8 +- {fink_grb => fink_mm}/offline/__init__.py | 0 .../offline/spark_offline.py | 70 +++------ {fink_grb => fink_mm}/online/__init__.py | 0 {fink_grb => fink_mm}/online/ztf_join_gcn.py | 77 +++++++-- {fink_grb => fink_mm}/test/__init__.py | 0 .../test/display_distribution.py | 0 .../test/hypothesis/__init__.py | 0 .../test/hypothesis/join_schema.py | 2 +- .../test/hypothesis/observatory_schema.py | 0 {fink_grb => fink_mm}/test/prep_ztf_data.py | 14 +- .../test/test_data/683571622_0_test | Bin .../test/test_data/S230518h_0_test | Bin .../VODB/fermi/voevent_number=193.xml | 0 .../VODB/fermi/voevent_number=2750.xml | 0 .../VODB/fermi/voevent_number=2842.xml | 0 .../VODB/fermi/voevent_number=699.xml | 0 .../VODB/icecube/voevent_number=3028.xml | 0 .../VODB/icecube/voevent_number=45412.xml | 0 .../VODB/icecube/voevent_number=825.xml | 0 .../VODB/integral/voevent_number=18790.xml | 0 .../VODB/integral/voevent_number=18791.xml | 0 .../VODB/integral/voevent_number=737.xml | 0 .../test/test_data/VODB/lvk/initial.txt | 0 .../test/test_data/VODB/lvk/test.txt | 0 .../test/test_data/VODB/lvk/update.txt | 0 .../VODB/swift/voevent_number=17834.xml | 0 .../VODB/swift/voevent_number=391.xml | 0 .../VODB/swift/voevent_number=392.xml | 0 .../VODB/swift/voevent_number=393.xml | 0 .../VODB/swift/voevent_number=405.xml | 0 .../VODB/swift/voevent_number=4554.xml | 0 .../VODB/swift/voevent_number=8582.xml | 0 ...42a7-a2be-1f56fc5c6571.c000.snappy.parquet | Bin 0 -> 16811 bytes ...4e73-adc6-a480974420d3.c000.snappy.parquet | Bin 0 -> 16446 bytes ...4442-9cb1-9834fb0b6c1e.c000.snappy.parquet | Bin 0 -> 16500 bytes .../raw/year=2019/month=09/day=03/683476673_0 | Bin .../raw/year=2019/month=09/day=03/683482851_0 | Bin .../raw/year=2019/month=09/day=03/683499781_0 | Bin .../year=2019/month=09/day=03/S230518h_0_test | Bin .../month=09/day=03/fake_683476673_0 | Bin .../month=09/day=03/fake_683482851_0 | Bin .../month=09/day=03/fake_683499781_0 | Bin .../test/test_data/grb_prob_test.parquet | Bin .../test/test_data/grb_samples.parquet | Bin .../test/test_data/join_raw_datatest.parquet | Bin .../test/test_data/offline_datatest.parquet | Bin 0 -> 37374 bytes ...42a7-a2be-1f56fc5c6571.c000.snappy.parquet | Bin 0 -> 52108 bytes ...4e73-adc6-a480974420d3.c000.snappy.parquet | Bin 0 -> 51364 bytes ...4442-9cb1-9834fb0b6c1e.c000.snappy.parquet | Bin 0 -> 51400 bytes .../test/test_data/voevent_number=8727.xml | 0 .../test/test_data/voevent_number=9897.xml | 0 .../with_hbase/fink-broker_2.11-1.2.jar | Bin .../test/test_data/with_hbase/fink_test.conf | 4 +- ...rk-hbase2.2_spark3_scala2.11_hadoop2.7.jar | Bin ...ed-hbase2.2_spark3_scala2.11_hadoop2.7.jar | Bin .../test/test_data/with_hbase/ztf.jd.json | 0 .../test_data/ztf_alert_samples_fid_1.parquet | Bin .../test_data/ztf_alert_samples_fid_2.parquet | Bin .../test/test_data/ztf_alerts_sample.parquet | Bin .../day=03/merge_ztf_data_test.parquet | Bin .../year=2019/month=09/day=03/alert1.parquet | Bin .../year=2019/month=09/day=03/alert2.parquet | Bin .../month=01/day=01/grb_join_output.parquet | Bin .../test/utils_integration_test.py | 12 +- {fink_grb => fink_mm}/utils/__init__.py | 0 {fink_grb => fink_mm}/utils/application.py | 19 +-- {fink_grb => fink_mm}/utils/fun_utils.py | 146 ++++++++++++------ {fink_grb => fink_mm}/utils/grb_prob.py | 8 +- {fink_grb => fink_mm}/utils/monitoring.py | 4 +- run_test.sh | 22 +-- scheduler/gen_avro_schema.py | 8 +- scheduler/grb2distribution.sh | 6 +- scheduler/science2grb.sh | 6 +- scheduler/science2grb_offline.sh | 6 +- setup.py | 28 ++-- 120 files changed, 465 insertions(+), 345 deletions(-) delete mode 100644 fink_grb/test/test_data/distribution_test_data/online/year=2019/month=09/day=03/part-00043-d0a99157-fe17-42a7-a2be-1f56fc5c6571.c000.snappy.parquet delete mode 100644 fink_grb/test/test_data/distribution_test_data/online/year=2019/month=09/day=03/part-00064-73d02dc2-dbe0-4e73-adc6-a480974420d3.c000.snappy.parquet delete mode 100644 fink_grb/test/test_data/distribution_test_data/online/year=2019/month=09/day=03/part-00069-60c7f1c4-fdac-4442-9cb1-9834fb0b6c1e.c000.snappy.parquet delete mode 100644 fink_grb/test/test_data/online/year=2019/month=09/day=03/part-00043-d0a99157-fe17-42a7-a2be-1f56fc5c6571.c000.snappy.parquet delete mode 100644 fink_grb/test/test_data/online/year=2019/month=09/day=03/part-00064-73d02dc2-dbe0-4e73-adc6-a480974420d3.c000.snappy.parquet delete mode 100644 fink_grb/test/test_data/online/year=2019/month=09/day=03/part-00069-60c7f1c4-fdac-4442-9cb1-9834fb0b6c1e.c000.snappy.parquet rename {fink_grb => fink_mm}/__init__.py (100%) rename {fink_grb => fink_mm}/conf/distribute_for_test.conf (92%) rename fink_grb/conf/fink_grb.conf => fink_mm/conf/fink_mm.conf (92%) rename fink_grb/conf/fink_grb_schema_version_1.0.avsc => fink_mm/conf/fink_mm_schema_version_1.0.avsc (100%) rename fink_grb/conf/fink_grb_schema_version_1.1.avsc => fink_mm/conf/fink_mm_schema_version_1.1.avsc (100%) rename fink_grb/conf/fink_grb_schema_version_1.2.avsc => fink_mm/conf/fink_mm_schema_version_1.2.avsc (100%) rename {fink_grb => fink_mm}/conf/integration.conf (91%) rename {fink_grb => fink_mm}/conftest.py (77%) rename {fink_grb => fink_mm}/distribution/__init__.py (100%) rename {fink_grb => fink_mm}/distribution/apply_filters.py (73%) rename {fink_grb => fink_mm}/distribution/distribution.py (87%) rename fink_grb/fink_grb_cli.py => fink_mm/fink_mm_cli.py (71%) rename {fink_grb => fink_mm}/gcn_stream/__init__.py (100%) rename {fink_grb => fink_mm}/gcn_stream/gcn_reader.py (97%) rename {fink_grb => fink_mm}/gcn_stream/gcn_stream.py (93%) rename {fink_grb => fink_mm}/init.py (88%) rename {fink_grb => fink_mm}/observatory/Fermi/Fermi.py (93%) rename {fink_grb => fink_mm}/observatory/Fermi/__init__.py (100%) rename {fink_grb => fink_mm}/observatory/Fermi/fermi.json (100%) rename {fink_grb => fink_mm}/observatory/IceCube/IceCube.py (98%) rename {fink_grb => fink_mm}/observatory/IceCube/__init__.py (100%) rename {fink_grb => fink_mm}/observatory/IceCube/icecube.json (100%) rename {fink_grb => fink_mm}/observatory/Integral/Integral.py (93%) rename {fink_grb => fink_mm}/observatory/Integral/__init__.py (100%) rename {fink_grb => fink_mm}/observatory/Integral/integral.json (100%) rename {fink_grb => fink_mm}/observatory/LVK/LVK.py (98%) rename {fink_grb => fink_mm}/observatory/LVK/__init__.py (100%) rename {fink_grb => fink_mm}/observatory/LVK/lvk.json (100%) rename {fink_grb => fink_mm}/observatory/Swift/Swift.py (93%) rename {fink_grb => fink_mm}/observatory/Swift/__init__.py (100%) rename {fink_grb => fink_mm}/observatory/Swift/swift.json (100%) rename {fink_grb => fink_mm}/observatory/__init__.py (92%) rename {fink_grb => fink_mm}/observatory/observatory.py (97%) rename {fink_grb => fink_mm}/observatory/observatory_schema_version_1.1.json (89%) rename {fink_grb => fink_mm}/offline/__init__.py (100%) rename {fink_grb => fink_mm}/offline/spark_offline.py (85%) rename {fink_grb => fink_mm}/online/__init__.py (100%) rename {fink_grb => fink_mm}/online/ztf_join_gcn.py (85%) rename {fink_grb => fink_mm}/test/__init__.py (100%) rename {fink_grb => fink_mm}/test/display_distribution.py (100%) rename {fink_grb => fink_mm}/test/hypothesis/__init__.py (100%) rename {fink_grb => fink_mm}/test/hypothesis/join_schema.py (99%) rename {fink_grb => fink_mm}/test/hypothesis/observatory_schema.py (100%) rename {fink_grb => fink_mm}/test/prep_ztf_data.py (85%) rename {fink_grb => fink_mm}/test/test_data/683571622_0_test (100%) rename {fink_grb => fink_mm}/test/test_data/S230518h_0_test (100%) rename {fink_grb => fink_mm}/test/test_data/VODB/fermi/voevent_number=193.xml (100%) rename {fink_grb => fink_mm}/test/test_data/VODB/fermi/voevent_number=2750.xml (100%) rename {fink_grb => fink_mm}/test/test_data/VODB/fermi/voevent_number=2842.xml (100%) rename {fink_grb => fink_mm}/test/test_data/VODB/fermi/voevent_number=699.xml (100%) rename {fink_grb => fink_mm}/test/test_data/VODB/icecube/voevent_number=3028.xml (100%) rename {fink_grb => fink_mm}/test/test_data/VODB/icecube/voevent_number=45412.xml (100%) rename {fink_grb => fink_mm}/test/test_data/VODB/icecube/voevent_number=825.xml (100%) rename {fink_grb => fink_mm}/test/test_data/VODB/integral/voevent_number=18790.xml (100%) rename {fink_grb => fink_mm}/test/test_data/VODB/integral/voevent_number=18791.xml (100%) rename {fink_grb => fink_mm}/test/test_data/VODB/integral/voevent_number=737.xml (100%) rename {fink_grb => fink_mm}/test/test_data/VODB/lvk/initial.txt (100%) rename {fink_grb => fink_mm}/test/test_data/VODB/lvk/test.txt (100%) rename {fink_grb => fink_mm}/test/test_data/VODB/lvk/update.txt (100%) rename {fink_grb => fink_mm}/test/test_data/VODB/swift/voevent_number=17834.xml (100%) rename {fink_grb => fink_mm}/test/test_data/VODB/swift/voevent_number=391.xml (100%) rename {fink_grb => fink_mm}/test/test_data/VODB/swift/voevent_number=392.xml (100%) rename {fink_grb => fink_mm}/test/test_data/VODB/swift/voevent_number=393.xml (100%) rename {fink_grb => fink_mm}/test/test_data/VODB/swift/voevent_number=405.xml (100%) rename {fink_grb => fink_mm}/test/test_data/VODB/swift/voevent_number=4554.xml (100%) rename {fink_grb => fink_mm}/test/test_data/VODB/swift/voevent_number=8582.xml (100%) create mode 100644 fink_mm/test/test_data/distribution_test_data/online/year=2019/month=09/day=03/part-00043-d0a99157-fe17-42a7-a2be-1f56fc5c6571.c000.snappy.parquet create mode 100644 fink_mm/test/test_data/distribution_test_data/online/year=2019/month=09/day=03/part-00064-73d02dc2-dbe0-4e73-adc6-a480974420d3.c000.snappy.parquet create mode 100644 fink_mm/test/test_data/distribution_test_data/online/year=2019/month=09/day=03/part-00069-60c7f1c4-fdac-4442-9cb1-9834fb0b6c1e.c000.snappy.parquet rename {fink_grb => fink_mm}/test/test_data/gcn_test/raw/year=2019/month=09/day=03/683476673_0 (100%) rename {fink_grb => fink_mm}/test/test_data/gcn_test/raw/year=2019/month=09/day=03/683482851_0 (100%) rename {fink_grb => fink_mm}/test/test_data/gcn_test/raw/year=2019/month=09/day=03/683499781_0 (100%) rename {fink_grb => fink_mm}/test/test_data/gcn_test/raw/year=2019/month=09/day=03/S230518h_0_test (100%) rename {fink_grb => fink_mm}/test/test_data/gcn_test/raw/year=2019/month=09/day=03/fake_683476673_0 (100%) rename {fink_grb => fink_mm}/test/test_data/gcn_test/raw/year=2019/month=09/day=03/fake_683482851_0 (100%) rename {fink_grb => fink_mm}/test/test_data/gcn_test/raw/year=2019/month=09/day=03/fake_683499781_0 (100%) rename {fink_grb => fink_mm}/test/test_data/grb_prob_test.parquet (100%) rename {fink_grb => fink_mm}/test/test_data/grb_samples.parquet (100%) rename {fink_grb => fink_mm}/test/test_data/join_raw_datatest.parquet (100%) create mode 100644 fink_mm/test/test_data/offline_datatest.parquet create mode 100644 fink_mm/test/test_data/online/year=2019/month=09/day=03/part-00043-d0a99157-fe17-42a7-a2be-1f56fc5c6571.c000.snappy.parquet create mode 100644 fink_mm/test/test_data/online/year=2019/month=09/day=03/part-00064-73d02dc2-dbe0-4e73-adc6-a480974420d3.c000.snappy.parquet create mode 100644 fink_mm/test/test_data/online/year=2019/month=09/day=03/part-00069-60c7f1c4-fdac-4442-9cb1-9834fb0b6c1e.c000.snappy.parquet rename {fink_grb => fink_mm}/test/test_data/voevent_number=8727.xml (100%) rename {fink_grb => fink_mm}/test/test_data/voevent_number=9897.xml (100%) rename {fink_grb => fink_mm}/test/test_data/with_hbase/fink-broker_2.11-1.2.jar (100%) rename {fink_grb => fink_mm}/test/test_data/with_hbase/fink_test.conf (96%) rename {fink_grb => fink_mm}/test/test_data/with_hbase/hbase-spark-hbase2.2_spark3_scala2.11_hadoop2.7.jar (100%) rename {fink_grb => fink_mm}/test/test_data/with_hbase/hbase-spark-protocol-shaded-hbase2.2_spark3_scala2.11_hadoop2.7.jar (100%) rename {fink_grb => fink_mm}/test/test_data/with_hbase/ztf.jd.json (100%) rename {fink_grb => fink_mm}/test/test_data/ztf_alert_samples_fid_1.parquet (100%) rename {fink_grb => fink_mm}/test/test_data/ztf_alert_samples_fid_2.parquet (100%) rename {fink_grb => fink_mm}/test/test_data/ztf_alerts_sample.parquet (100%) rename {fink_grb => fink_mm}/test/test_data/ztf_test/archive/science/year=2019/month=09/day=03/merge_ztf_data_test.parquet (100%) rename {fink_grb => fink_mm}/test/test_data/ztf_test/online/raw/year=2019/month=09/day=03/alert1.parquet (100%) rename {fink_grb => fink_mm}/test/test_data/ztf_test/online/raw/year=2019/month=09/day=03/alert2.parquet (100%) rename {fink_grb => fink_mm}/test/test_data/ztfxgcn_test/online/year=2023/month=01/day=01/grb_join_output.parquet (100%) rename {fink_grb => fink_mm}/test/utils_integration_test.py (95%) rename {fink_grb => fink_mm}/utils/__init__.py (100%) rename {fink_grb => fink_mm}/utils/application.py (95%) rename {fink_grb => fink_mm}/utils/fun_utils.py (92%) rename {fink_grb => fink_mm}/utils/grb_prob.py (95%) rename {fink_grb => fink_mm}/utils/monitoring.py (96%) diff --git a/.flake8 b/.flake8 index 76928aec4..41c7c49cb 100644 --- a/.flake8 +++ b/.flake8 @@ -10,10 +10,10 @@ exclude = build, dist per-file-ignores = - ../Fink_GRB/fink_grb/online/ztf_join_gcn.py:W503,E402 - ../Fink_GRB/fink_grb/offline/spark_offline.py:W503,W605 - ../Fink_GRB/fink_grb/utils/fun_utils.py:F811 - ../Fink_GRB/fink_grb/distribution/distribution.py:W503 - ../Fink_GRB/fink_grb/observatory/__init__.py:E402 - ../Fink_GRB/setup.py:W503 - ../Fink_GRB/fink_grb/observatory/LVK/LVK.py:W503 \ No newline at end of file + ../Fink_MM/fink_mm/online/ztf_join_gcn.py:W503,E402 + ../Fink_MM/fink_mm/offline/spark_offline.py:W503,W605 + ../Fink_MM/fink_mm/utils/fun_utils.py:F811 + ../Fink_MM/fink_mm/distribution/distribution.py:W503 + ../Fink_MM/fink_mm/observatory/__init__.py:E402 + ../Fink_MM/setup.py:W503 + ../Fink_MM/fink_mm/observatory/LVK/LVK.py:W503 \ No newline at end of file diff --git a/.github/workflows/linter.yml b/.github/workflows/linter.yml index 18a4e2fa1..f33670f12 100644 --- a/.github/workflows/linter.yml +++ b/.github/workflows/linter.yml @@ -24,6 +24,6 @@ jobs: run: | python -m pip install --upgrade pip pip install flake8 - - name: Flake8 Fink GRB + - name: Flake8 Fink MM run: | - flake8 ../Fink_GRB \ No newline at end of file + flake8 ../Fink_MM \ No newline at end of file diff --git a/.github/workflows/run_test.yml b/.github/workflows/run_test.yml index a7cbbca14..3fcf8940d 100644 --- a/.github/workflows/run_test.yml +++ b/.github/workflows/run_test.yml @@ -12,7 +12,7 @@ on: env: NIGHT: 20190903 - CONFIGFILE_PATH: fink_grb/conf/integration.conf + CONFIGFILE_PATH: fink_mm/conf/integration.conf SECRET_ID: ${{ secrets.GCN_ID }} SECRET_SECRET: ${{ secrets.GCN_SECRET }} @@ -35,13 +35,13 @@ jobs: - name: Set up env [1/3] run: | echo "ROOTPATH=`pwd`" >> $GITHUB_ENV - echo "FINK_GRB=$GITHUB_WORKSPACE" >> $GITHUB_ENV + echo "FINK_MM=$GITHUB_WORKSPACE" >> $GITHUB_ENV echo "JAVA_HOME=$(dirname $(dirname $(readlink -f $(which java))))" >> $GITHUB_ENV - name: Set up env [2/3] run: | - echo "PYTHONPATH="${PYTHONPATH}:${SPARKLIB}:${FINK_GRB}"" >> $GITHUB_ENV + echo "PYTHONPATH="${PYTHONPATH}:${SPARKLIB}:${FINK_MM}"" >> $GITHUB_ENV - name: Install fink-broker run: | @@ -50,7 +50,7 @@ jobs: cd fink-broker pip install . echo "FINK_HOME=${HOME}/fink-broker" >> $GITHUB_ENV - cd $FINK_GRB + cd $FINK_MM - name: Set up env [3/3] run: | @@ -67,7 +67,7 @@ jobs: run: | echo "GITHUB_PATH: $GITHUB_PATH" echo "PATH: $PATH" - echo "FINK_GRB: $FINK_GRB" + echo "FINK_MM: $FINK_MM" echo "SPARK_HOME: $SPARK_HOME" echo "SPARKLIB: $SPARKLIB" echo "PYTHONPATH: $PYTHONPATH" @@ -76,24 +76,27 @@ jobs: echo "FINK_PACKAGES: $FINK_PACKAGES" echo "FINK_JARS: $FINK_JARS" echo `python -V` + echo + echo + export - name: Start services run: | cd $USRLIBS source scripts/start_services.sh --kafka-version ${KAFKA_VERSION} --hbase-version ${HBASE_VERSION} - cd $FINK_GRB + cd $FINK_MM - name: Call raw2science run: | - fink start raw2science -c ${FINK_GRB}/fink_grb/test/test_data/with_hbase/fink_test.conf --night $NIGHT --exit_after 90 + fink start raw2science -c ${FINK_MM}/fink_mm/test/test_data/with_hbase/fink_test.conf --night $NIGHT --exit_after 90 - name: Merge data run: | - fink start merge -c ${FINK_GRB}/fink_grb/test/test_data/with_hbase/fink_test.conf --night $NIGHT + fink start merge -c ${FINK_MM}/fink_mm/test/test_data/with_hbase/fink_test.conf --night $NIGHT - name: Push Hbase data run: | - fink start index_archival -c ${FINK_GRB}/fink_grb/test/test_data/with_hbase/fink_test.conf --night $NIGHT --index_table jd_objectId + fink start index_archival -c ${FINK_MM}/fink_mm/test/test_data/with_hbase/fink_test.conf --night $NIGHT --index_table jd_objectId - name: Check HBase table run: | @@ -104,11 +107,11 @@ jobs: echo "PYTHONPATH="${SPARK_HOME}/python/test_coverage:$PYTHONPATH"" >> $GITHUB_ENV echo "COVERAGE_PROCESS_START="${ROOTPATH}/.coveragerc"" >> $GITHUB_ENV python -m pip install . --upgrade - python -m pip install -U fink_filters + pip install -U fink_filters - name: Run test suites run: | - coverage run --source=${ROOTPATH} --rcfile=${ROOTPATH}/.coveragerc -m pytest --doctest-modules fink_grb + coverage run --source=${ROOTPATH} --rcfile=${ROOTPATH}/.coveragerc -m pytest --doctest-modules fink_mm curl -s https://codecov.io/bash | bash run-integration-test: @@ -128,12 +131,12 @@ jobs: - name: Set up env [1/3] run: | echo "ROOTPATH=`pwd`" >> $GITHUB_ENV - echo "FINK_GRB=$GITHUB_WORKSPACE" >> $GITHUB_ENV + echo "FINK_MM=$GITHUB_WORKSPACE" >> $GITHUB_ENV echo "JAVA_HOME=$(dirname $(dirname $(readlink -f $(which java))))" >> $GITHUB_ENV - name: Set up env [2/3] run: | - echo "PYTHONPATH="${PYTHONPATH}:${SPARKLIB}:${FINK_GRB}"" >> $GITHUB_ENV + echo "PYTHONPATH="${PYTHONPATH}:${SPARKLIB}:${FINK_MM}"" >> $GITHUB_ENV echo "NIGHT=`date +"%Y%m%d" -d "now"`" >> $GITHUB_ENV - name: Install fink-broker @@ -143,7 +146,7 @@ jobs: cd fink-broker pip install . echo "FINK_HOME=${HOME}/fink-broker" >> $GITHUB_ENV - cd $FINK_GRB + cd $FINK_MM - name: Set up env [3/3] run: | @@ -162,70 +165,70 @@ jobs: echo "COVERAGE_PROCESS_START="${ROOTPATH}/.coveragerc"" >> $GITHUB_ENV python -m pip install . --upgrade python -m pip install -U fink_filters - mkdir fink_grb/ci_gcn_test - mkdir fink_grb/ci_join_test + mkdir fink_mm/ci_gcn_test + mkdir fink_mm/ci_join_test - name: Launch GCN stream # don't put --test for the gcn_stream command otherwise it will miss the gw alerts shell: bash run: | - fink_grb gcn_stream start --config $CONFIGFILE_PATH --test > fink_grb_gcnstream_${NIGHT}.log 2>&1 & + fink_mm gcn_stream start --config $CONFIGFILE_PATH --test > fink_mm_gcnstream_${NIGHT}.log 2>&1 & sleep 180s - name: Start services run: | cd $USRLIBS source scripts/start_services.sh --kafka-version ${KAFKA_VERSION} --hbase-version ${HBASE_VERSION} - cd $FINK_GRB + cd $FINK_MM - name: Generate fake gcn counterparts run: | - python fink_grb/test/prep_ztf_data.py - mv fink_grb/test/test_data/ztf_test/online/raw/year=2019 fink_grb + python fink_mm/test/prep_ztf_data.py + mv fink_mm/test/test_data/ztf_test/online/raw/year=2019 fink_mm - name: Check data run: | echo "GCN ALERTS" - ls -dlth fink_grb/ci_gcn_test/*/*/*/* + ls -dlth fink_mm/ci_gcn_test/*/*/*/* echo "" echo "ZTF ALERTS" echo "--- ONLINE SCIENCE" - ls -dlth fink_grb/test/test_data/ztf_test/online/science/*/*/*/* + ls -dlth fink_mm/test/test_data/ztf_test/online/science/*/*/*/* echo "--- ARCHIVE SCIENCE" - ls -dlth fink_grb/test/test_data/ztf_test/archive/science/*/*/*/* + ls -dlth fink_mm/test/test_data/ztf_test/archive/science/*/*/*/* # - name: Call raw2science # run: | - # fink start raw2science -c ${FINK_GRB}/fink_grb/test/test_data/with_hbase/fink_test.conf --night $NIGHT --exit_after 90 + # fink start raw2science -c ${FINK_MM}/fink_mm/test/test_data/with_hbase/fink_test.conf --night $NIGHT --exit_after 90 # - name: Merge data # run: | - # fink start merge -c ${FINK_GRB}/fink_grb/test/test_data/with_hbase/fink_test.conf --night $NIGHT + # fink start merge -c ${FINK_MM}/fink_mm/test/test_data/with_hbase/fink_test.conf --night $NIGHT - name: Push Hbase data run: | - fink start index_archival -c ${FINK_GRB}/fink_grb/test/test_data/with_hbase/fink_test.conf --night $NIGHT --index_table jd_objectId + fink start index_archival -c ${FINK_MM}/fink_mm/test/test_data/with_hbase/fink_test.conf --night $NIGHT --index_table jd_objectId - name: Run online join streaming run: | - fink_grb join_stream online --config $CONFIGFILE_PATH --night $NIGHT --exit_after 180 + fink_mm join_stream online --config $CONFIGFILE_PATH --night $NIGHT --exit_after 180 - name: Check online stream run: | - ls -lth fink_grb/ci_join_test/online/*/*/* + ls -lth fink_mm/ci_join_test/online/*/*/* - name: Run offline stream run: | - fink_grb join_stream offline --night $NIGHT --config $CONFIGFILE_PATH --test + fink_mm join_stream offline --night $NIGHT --config $CONFIGFILE_PATH --test - name: Check offline stream run: | - ls -lth fink_grb/ci_join_test/offline/*/*/* + ls -lth fink_mm/ci_join_test/offline/*/*/* # - name: Online Distribution # run: | - # fink_grb distribute --config $CONFIGFILE_PATH --night $NIGHT --exit_after 60 + # fink_mm distribute --config $CONFIGFILE_PATH --night $NIGHT --exit_after 60 # - name: Check Distribution # run: | - #python fink_grb/test/display_distribution.py + #python fink_mm/test/display_distribution.py diff --git a/.gitignore b/.gitignore index e4272e910..11c9b0bf8 100644 --- a/.gitignore +++ b/.gitignore @@ -136,5 +136,9 @@ gcn_test inspect_testdata.py tmp_alert build_egg.sh -launch_offline_grb_locally.sh -science2grb_locally.sh \ No newline at end of file +launch_offline_mm_locally.sh +science2mm_locally.sh +.vscode +clean_it.sh +export_unit_test.sh +it_test.sh diff --git a/README.md b/README.md index 2c8b8e274..51f634851 100644 --- a/README.md +++ b/README.md @@ -1,7 +1,7 @@ -# Fink_GRB +# Fink_MM -[![PEP8](https://github.com/FusRoman/Fink_GRB/actions/workflows/linter.yml/badge.svg)](https://github.com/FusRoman/Fink_GRB/actions/workflows/linter.yml) -[![Sentinel](https://github.com/FusRoman/Fink_GRB/actions/workflows/run_test.yml/badge.svg)](https://github.com/FusRoman/Fink_GRB/actions/workflows/run_test.yml) +[![PEP8](https://github.com/FusRoman/Fink_MM/actions/workflows/linter.yml/badge.svg)](https://github.com/FusRoman/Fink_MM/actions/workflows/linter.yml) +[![Sentinel](https://github.com/FusRoman/Fink_MM/actions/workflows/run_test.yml/badge.svg)](https://github.com/FusRoman/Fink_MM/actions/workflows/run_test.yml) Correlation of the [Fink](https://fink-broker.org/) alerts with multi-messenger instruments for real-time purpose * Available Instruments @@ -15,24 +15,24 @@ Correlation of the [Fink](https://fink-broker.org/) alerts with multi-messenger * Install from GitHub with pip ```console -toto@linux:~$ pip install git+https://github.com/FusRoman/Fink_GRB@v0.6-beta +toto@linux:~$ pip install git+https://github.com/FusRoman/Fink_MM@v0.6-beta ``` -* Download the default config file from GitHub:fink_grb/conf/fink_grb.conf and move it in a custom location +* Download the default config file from GitHub:fink_mm/conf/fink_mm.conf and move it in a custom location Update your configuration file with your custom parameters. Please note that the configuration file's paths must not end with a '/'. -* Once Fink_GRB has been installed via pip, the command `fink_grb` become available. Type `fink_grb --help` to see the help message and show what you can do with Fink_GRB. +* Once Fink_MM has been installed via pip, the command `fink_mm` become available. Type `fink_mm --help` to see the help message and show what you can do with Fink_MM. -### Setup the Fink_GRB daemons +### Setup the Fink_MM daemons * Start listening to GCN ```console -toto@linux:~$ fink_grb gcn_stream start --config /config_path +toto@linux:~$ fink_mm gcn_stream start --config /config_path ``` The above command will start a daemon that will store the GCN issued from the instruments registered in the system. The GNC will be stored at the location specified in the configuration file by the entry named 'online_gcn_data_prefix'. The path can be a local path or a hdfs path. In the latter case, the path must start with hdfs://IP:PORT///your_path where IP and PORT refer to the hdfs driver. -> :warning: The GCN stream need to be restarted after each update of Fink_GRB. Use the `ps aux | grep fink_grb` command to identify the process number of the gcn stream and kill it then restart the gcn stream with the same command as above. +> :warning: The GCN stream need to be restarted after each update of Fink_MM. Use the `ps aux | grep fink_mm` command to identify the process number of the gcn stream and kill it then restart the gcn stream with the same command as above. ### Schedulers -Fink_GRB has multiples script in the scheduler folder to launch the different services. +Fink_MM has multiples script in the scheduler folder to launch the different services. * science2grb.sh will launch the online services that will cross-match in real-time the alerts issued from the ZTF/LSST with incoming GCN. (latency: ZTF/LSST latency + 30 seconds max) * science2grb_offline.sh launch the offline services that will cross-match all the latest alerts from ZTF/LSST with the GCN within the time window (in days) specified in your config file. (latency: 1 day) * grb2distribution.sh launch the distribution services that will send the outputs of the online services in real-time to the registered users of the [fink-client](https://github.com/astrolabsoftware/fink-client). (latency: ZTF/LSST latency + 30 seconds + Network latency to reach fink-client) @@ -40,9 +40,9 @@ Fink_GRB has multiples script in the scheduler folder to launch the different se #### **Modify the scripts** Download the multiple scripts from GitHub:scheduler/ These scripts use some paths that have to be modified before the deployment. -* Two variables names `FINK_GRB_CONFIG` and `LOG_PATH` are common to all script. The first is the location of your configuration file, and the second is where to store log files. Either you modify the value of these variables directly in the scripts, science2grb.sh and science2grb_offline.sh or you remove the declaration in these scripts and export these variables within your ~/.bashrc or ~/.bash_profile. +* Two variables names `FINK_MM_CONFIG` and `LOG_PATH` are common to all script. The first is the location of your configuration file, and the second is where to store log files. Either you modify the value of these variables directly in the scripts, science2grb.sh and science2grb_offline.sh or you remove the declaration in these scripts and export these variables within your ~/.bashrc or ~/.bash_profile. ```console -export FINK_GRB_CONFIG="path/to/conf/fink_grb.conf" +export FINK_MM_CONFIG="path/to/conf/fink_mm.conf" export LOG_PATH="path/to/store/log" ``` @@ -69,7 +69,7 @@ The module output is pushed into the folder specified by the config entry named The output could be local or in a HDFS cluster. Two folders are created inside; one called 'online' and one called 'offline'. Inside these two folders, the data are repartitions following the same principle: 'year=year/month=month/day=day'. At the end of the path, you will find ```.parquet``` files containing the data. -### Fink_GRB Output Schema +### Fink_MM Output Schema |Field |Type |Contents | |-------------------|------|----------------------------------------------------------------------------------| @@ -85,11 +85,11 @@ Two folders are created inside; one called 'online' and one called 'offline'. In |instrument_or_event|String|Triggering instruments (GBM, XRT, ...) | |platform |String|Triggering platform (Fermi, Swift, IceCube, ...) | |triggerId |String|Unique GCN identifier ([GCN viewer](https://heasarc.gsfc.nasa.gov/wsgi-scripts/tach/gcn_v2/tach.wsgi/) to retrieve quickly the notice)| -|grb_ra |float |GCN Event right ascension | -|grb_dec |float |GCN Event declination | -|grb_loc_error |float |GCN error location in arcminute | +|gcn_ra |float |GCN Event right ascension | +|gcn_dec |float |GCN Event declination | +|gcn_loc_error |float |GCN error location in arcminute | |triggerTimeUTC |String|GCN TriggerTime in UTC | -|grb_proba |float |Serendipitous probability to associate the alerts with the GCN event | +|p_assoc |float |Serendipitous probability to associate the alerts with the GCN event | |fink_class |String|[Fink Classification](https://fink-broker.readthedocs.io/en/latest/science/classification/) | | | |Field available only for the online mode | @@ -99,7 +99,7 @@ Two folders are created inside; one called 'online' and one called 'offline'. In |start_vartime |float |first variation time at 5 sigma of the object (in jd, only valid for 30 days) | |diff_vartime |float |difference between start_vartime and jd (if above 30, start_vartime is not valid) | -### Fink-client topics related to Fink_GRB +### Fink-client topics related to Fink_MM The connection to the distribution stream is made by the [fink-client](https://github.com/astrolabsoftware/fink-client). Three topics are available : diff --git a/fink_grb/test/test_data/distribution_test_data/online/year=2019/month=09/day=03/part-00043-d0a99157-fe17-42a7-a2be-1f56fc5c6571.c000.snappy.parquet b/fink_grb/test/test_data/distribution_test_data/online/year=2019/month=09/day=03/part-00043-d0a99157-fe17-42a7-a2be-1f56fc5c6571.c000.snappy.parquet deleted file mode 100644 index 939b58d540373350e5073d3ac24c250d8b92de6b..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 16833 zcmc&cZERcDb&s-T)3$6Yp2<8VN~&n~v(eZVKO|Z5Y*>$!NRgIkitMkIrpCLd+xc1hA*&juWP$&&Ed+Ax!PRR0g8I^bQ?ucnaPlM zFv~5kELIx6cFI}JRq<=FQa{yBQI-C7YI`-0kmd56i*j|kx{kSMSG#M6Ye#24b+oOy z9`Lj}TC7xn_`cp2_@o`%S}*-m_d|Oq=J}RRS6fG4JGG;Ob}n#pgb}22v{4<}`I00I98!emO^StrBFVV4W4_1bbtbF+V)%}&BN8i7*@Fa|@Z?ydI-`tg<&pz{X z?Pu`)vr`{l`C4~n$gHk{p`8j1RM!l(QupOuovz)Ax~sL~bR|pt2Wc;J=zt=gfr^O5H#-|S%4j{WHBPak=LS^JBt|8?p2_cLp; z*f%~m@X(Ppv%1Pk+NsPzVa!k~b^otTRd*+HuV84W@`g;5cNJ<7|JmPBJ+NmkU#(>;wb@d&M!X=Y z@yiF8>koh$W?D8JEY&Exies*Q#0vx4e{jR$=(=?lWa;|C&);=8+n&j7|4)b0Ipp`Z zI-M_Hxzb8|hJN^ycX7tc2%dINi zQsIcMUvF`LxrMpuV*d3Y=xU~8L)}tWUBoG7+c1~C-=UFyPD+Q8r{1EO*XaB9)IQe% zWxwpCw}Jh_fpMeo{nXKI0H*N+W@tCWtDWj$w7Vo$Yn3ITAW8wX>VDS2+bh_FUIkw}OYTC0vf}#B}I&Yh9`YdltBAp|YH!ul@ml?|^Th&ubyqt|pRJUk&oT->u5Gt5zw^%oHkex$sqo`&9>X=P~B` zV;0a+t#)H%sYqG*xxZz;gs|7`wsU_2lM73=7mEL|7JQi)Yn(#bMJh?`A6d& zYmd6;#(yRLu7xTJhIT4*k(n84RONq;s;cZnRg|+&(&fD)%qvGg7c(In^6TpY5>V7L z%R^ru`S!0qdi*5|T@(ySs$?!QGeeEKyl_m_WiRT&FXW~RmC8be1ihruoli4wd>S+| zle1xBsZqb{l(HQjo_!Dm@p-(1$35`J-{t8&Z!7Ne_o#+zB#FiSyz@ zU3A0i#RWcF>AKjc)(S$GRN`lA1!K+tgYJ|U(QfM{Kub*)&r;0m{i?rq7fa&&bdJwf ztHf77b+~^4zUpIc^rElUC2gtIldeqjdejTthu-_BV-3EkzynamtX=)M)!9a!`TK_* zv}bMPv4;rCJ@Vih=%%~zN!txE(i<#UL+c_bJrn@iYKGkYE}hc2Fkj$n*=Zp=L$riz z!;3AzV_ys+~RkWA71LiWN_V82 zrF)}Dir2v6UnV1RsP+@%7QSxR4x~! zTq*6kx6d+vepYswo|FxP^|jdWI`o5}s>NP%ahP6)+XEy-;tolX_XEs#0-%V(U427E z%uwsP%)%Y!a!@fh+{{R*gVCQ+t)+z;+;KUqD$j?< z>zVvA2Xl~>=1!q7L$FlqYa!a&TMJifUIj|zld4HsKDi!~M*50IlEQHQYbmM6s4TF0 z^ysEFQ?vX&A(kUn}6wLw(2g* z{1&c*%vN02Hu>JfY{h+r+2kt%vn3@wvsEQFvqdE^vlS((`$tFIG~;w%K6((_y!4JT z{A`%5(tk@o)m;+vg_Y^t0>31P)$UVWPdwE<4={M^u4KiTLie$*ZZRtq03a^$Jd8-? z1w#*cUns+?*GJw9tT{y)3rt!`E@UerQAYZ5@AUw zH<}3P7hM!hDAcm~Y%L4ZpXx4%Fuf$s;M%Fl(D8xpr!~UprRY|E6-FvLS#+5Dk_Zj< z^f3@j*0Njb<(N6Da09=RSnKa4F3{=U1Z&hcF_W$rW@akfz^uqGWG&fg5bH67maR2M z8L`1E;htn>Cj$+vnl=;f-ISW20nO|TNCR7q?evT_h`SlJ!kIZrunlH3XJJVRw!w_l zU%2U_e_m&P!`+@xQWoFp~Z!9;>&s=7n2IR(aZvKE}=Bq<(ELW*nU}mYp4g56m z{~K)#xglZAadWZ_vTXj2W6fBH-e8UOKd_kjoACw#?sEg7`sBwDGix2n!1aC|tu~9y zeDyd3|5pCwILmAjYnOi_SuF_{r1=HXAgjoFy;*AvrDak7Z{8tyGQszrsC0l7=p#pfK$_OgK)HpH?sY;cCa0s$Sqj=+&7esp478p9`{Lhuomt+TKWz6lZdCX3)R z4Sxgmbh2+gor0yq9{32Hj0pTvCYeI_FG&2q^poU@cqu2jc@Y$@lD&*sDlfU$LXyqpV0`Xj?54SCE<zy*$q2#6*<~0v#~` zashAepr?=u5L`q|EM=3i2 z-j@TJ!G@)LvagIXfV`+b*x6ezBs4lIe1ZgD%cSD7 zK*N)6+HGZUEQq?OeA7PUV=tEuZ)I1All*YnBgcT3O9lG0GK0N5u{ledk~k@X-yx<{ zTZv^)h97Kz%$7cm=<#yg#Be2-c_}Z%7qY3C6gQS}Ww3>LzZ(4fiDYb~OLqQ?mPib5 zoh*y?ek)6PV(VmqScrLYVB-wzSGybqdwDfK8ufC?#FFe^tGM!V0zb#WUa`vwO0JXe zpU7`~9`dh!OeK6xxvgw=f4Sc}PeHED`4Wx15b|*TXrGZF9qB z$j9Y1NXLg8Sis= zSF*@4tKj!WJ~RmD8q|Au|H3=q*7+9hVHx2Q$Z0um>+Fd(OrWi5aL={5&zR;|#Us|< zP=iB0wY~e{T{S^`5u1y1P|xM1J!w^6dGqMkv_Hy|x}*g6YOLkMd!N|(k-1t)L zVssGu4PpF<)}dDsn1EgcjG;kserA|ugT2t6U}srqJ%n(~R&m^e?TQ#11pExPEpj*> z!7(g^u@BoG=a5vOj-)1V%+_H%oP!s35kE%p1DYuCAWt?nd9tTx0A5DU;cQ%r1N@Ml zdH4ztJK{ZsOCdB6?77g>11})km@GnRbnB5`z!-OAQhs^0rw7q9Y_Ci&z^)J! zY041!$Pl!KN4_V4O<}L0@3EhAt|Ie~u-pZi-o!X!WL3#44yArkSeH*ge|5cRSWozn!qu-k>Mg-5d<`d+YJd;Wpl z`cdA-LVqR(}$;&04h1mU@U+;;-^0!&J=TG?|`*tF!84GTY6?-|HA`= z03>aEjQNYC5~X(tcvxS@#0_z+5P?GgSAvm2Z;5^#4*(L(iV>7}Gt!6oB;J&sDGreo z_?av{L(yI3Y-Mq&P&=M0myc($N#7+m;k%^vJp3DG_^0I?@`w7~35vP^|9Qjz1A=4$ AW&i*H diff --git a/fink_grb/test/test_data/distribution_test_data/online/year=2019/month=09/day=03/part-00064-73d02dc2-dbe0-4e73-adc6-a480974420d3.c000.snappy.parquet b/fink_grb/test/test_data/distribution_test_data/online/year=2019/month=09/day=03/part-00064-73d02dc2-dbe0-4e73-adc6-a480974420d3.c000.snappy.parquet deleted file mode 100644 index cbc4b1f36ff037401a9dc62ecd8c6d89ee972b2e..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 16468 zcmc&+eQaCR6}Q{8b=wrG4jZRrEM)Xijndf8N8=7Qo}JjSo1~4K`U|n_z2}cS$M41U zXB~vbrh)i_A;$K{geHa%LJT3nHpCEOTZIrp6O*P&WBX%&Z2ZBbjXw|+2_cYn&VBFt zeK$Z8^OQRG-E;5xosWCZJ@+{388#L44tPhrH_v-_cwL8Fu7kG^x?HaOax56;_)4o< zZ+KnaLtfu878eb`Q9ii|%3X;Lzkj*X|*|XPvJU zg=V7Q-QgSXz2Nq}>}KW|&t?pV5ZdM4;ewBY@XO^QqFhHM_6&=I4?od(W`JmR9NSD8a}{uD;bT>SQv4p; zLJ->4dSSAD@xO9FTShxehBfr;AoIz75Z}tff%E3#AN|kY{yy6zQT|FCxkrT9Ixg&?%8_3EXEKzhlilfI%4N3VIAN4+4um4yRiTj{Uy9-;wtkAEkC z$0G}7(@(jM?u2eXewf62WJrti`$W0Xthc11tSprc-;dnPtA6HTKZbgZgRLAM?;w?9 z08b*zZUh!CKN9pi?cp0V_+7;|MPBvQm^ThHj~_;0oMj6MryGUbJF*Kn$BK1Hv{8N+ zA^4SFCJ7$>x+cN?b-q!ox4Gtey-Swa18(2LZsw(7=EuV*iII=3OozQUQkf0}_5#+_ z^!UVdXnHbeBiG&@(XTa4@?H9dCg0v>T`ZT2b@e2E-0geX&AfXf^W2RnoyJ1X%~ndo z#$Eg29n2*-!%jc{^WuYV+MQ)|gx^EQ_+aT#ADhnUV-?TZEDaG%hV-d{SYtrPOM>I|LpeFQ8Q&f5D zX6B)rK@}?>2kx7z0s=rBZQuFi>`NbhcIz)}RG}0SP03gUv-X-)`QWIg$^lfRx-P5~ z>-F`zF0j6H3-jbHppccD1BR_aQ{IzwFWj(tJ;)IX1jDxl;Aea!FmW;vfS==mK)_~a zT<o{?BhO<@JaFR4u z-}~OL{`YH}bC6Q}9@;{y(ze!X(&^nX&2xLvbG7<9ug?zOKF&OS9K2@bn@rlQ}PR>Xf3n=9k2D8Gz+T>Owu1v&L^6$qcnE#H0 zg{_<%_;0R6D*KV^{^^hYWL1Jv{2tmu5ZcyyO-j5tttl~3=bA;G{7;==E>D5{RwfRt zHJDB_K07b0a956Ol1jKM% ze0Igb9Y|3Ok1c8~W3Ne(ORT2IU_mUEZ1i~TPUg`&RfiavIY4Z#$1B%f{>kG%-DSxl z{T@ouS_oq8HRTrCAw=_fMr?f0a{u;5MW-Dli72bSXRq z!)_(D*7xXBU46fnV$xnBpOU?)@~H`s@@C8-6e&z6uAY;2aH#@o$AV#4FZwDdB#6!1N_$XZn>=Gkr>VnSP|CzTe*D8(=)X4@R%UDyn~X4ZaoQ8va-P_l$~i zp?G0MSg*DuxiNZj40wBR*B+UqrFhDfuB z&EUOv!;8Xig)lFwY>iw>VzF9SG4?59tu(tegd1(iqOe|2Bxp&sZjYdR6^W8b#U@wa znj8$jXS688a8WMf+;hvZ+onhF)d^#iVw8TJiChe_7${dI5!&nCV<4ES<*4#yq!m%? zH{pqLb83Qkz@UGxTc^GSOu0>1!L)u8SgE?s*}~}vn*l;e)E-eKY%wHUi>z>JrwOWO zGx>{~Qup7^q@nNu_U#fwb?JteP2VCg3j?4lMhD ze6eJO>~(J~n(j*^E2fFKwaC4%hpe~;($=CXwE-)n)^Ebot^Y5zG1U5lJ>vF68-(nA zQDP5lAaAk8#&0mJ_*S|}fUDdBUh&BaYap3W zuU1_w)>{#2?-qk;m31qmk+#LahH9M^+d|%Aczp@mif*QEF?g@;Y(+MZOt6sV7Lks# zj1iMkrY7!^8+Rk!oKcXJyk5k=X&{+$Rpjejy^D!*t;@mpz8##^F4i03x{STS;Boj? zb`-MLQB@2*@%Z@h$y2;B6;%PC$7rV(i2V)bJ^>RW^uJTf$nh-nXgfzLzXU^f6Rv~kt zF&||`VLY`e$V<@2cGUN_5KT-aW)h7=G`UfTr#r}JAsZ|pJr@c$3YlOl8&a_I*&b#q zmkCyJEH9@Ud=&BWwS4BnS~g^Z8;^nPv5o9_YEp=!OsrVQOxA$^TK+7-1{s5$Vmcfs zW+Q}eLJEtVl&%z_gx<_&Q>(xukUIp4sB%+>qsaoaZsS+>6njp^<*rJ`2YSr9$Dw$XCRevm&3Uk$t zIwr_VRg{6+JIH4vlX@K~j^dACwGfYiPgAQsyUhg}sEZD_f_~?mifc`?RZ&h!+41D6qJ!DTtZ1Oi5mjpJz&E)rSwD9EGvR{%CjaH73+-I?i`fg- z(wdV@J({hZsx8e1R#^;w&7=dlOlq|dpY84M(%DWKVnJ_TCZyV$vfoqdTy{~lt<`^= zl!AR9>lWgxHDO#?H+JjFWDA@9YVz~vkx@>HPPP)H1lgN?lLhSMBnv0?*)9<4!FOGV ziB;IIPB9Ah@?t=MJr1!@@vq&wlG80Nv)Il<+{IXD6BE=}r^bxW<2NZC>S0>qKE%^( zz9+Yx|E?yMdj?Viw)Zm+XI4BmGhx= z_X_8ZWlX>9mmg1xSBmony4^aHO#9z@_L$=B=Qjh48wX# zwgr)#GZL3cH}bKtE)EyExoQ~Bz{L`*&n`dS=eQ=Px?H9p1NJ!%;tAwQGfr`?Fqf?5 zLxrkPg}f@0PN!!V9Aofy>|te%MY^z;9rF##nEtIr@a<$B^6y42ySU+GLpbAGTrgbW zA>W1E6Z7hR#g{e6zXQ2+xS7icPVvt ~W(^(HOS2=@h^6f6fZ6%NHqdx^H+{})r zJ3>6&b@V6fZ9W|i3UiUFy2tJOX^L&u{LN%j<9){AOjGl>QbEG}tcrINTu;^*19?C$ zn+#;f^Kiezy{p){Z@!gMwY-$yTu%FOeX}RAA>_hvckMX&+cL&>J!0;i4q!Z0_}KUx z?y7KSMPGy#W6hQj3@G`RU4E5@dtfwK5u`N9OYnXOxqO4mBw=49C-FTYM<$^S=ZCVl z3sSl(gsN>`#QSRE!ot~!&U}Rx@g4wU={+n2aesr$gjX=w&Qh2>3pW?=TVGLs<4W zg{&eSWVM9vYzN+F1Zc45@na4@pojtwVzP_NC&tF6p)q?3N8?m{!4KtGgI^J1Yupcf z7JDPnu`^?1&;Z$%r`7p0WD-han2-D-aHr10J+`+$$e==dV{8n`Wp+a47hx|D5-G|M z`N%714Uc6%Vm;?nUByqZ{27(s(gIRcRY?W4MtgE1xqG!iS#D7sX)|(%*_a82mNG~r=V;oGU&n*G| z*)yR3L~p)>|8cEH2tL#qDp|%wY^-k!-`{VRsPc^ zd?*gWqg%hEo~P)Ky3d}tRk2rsojn6fAJ^Z~Is5g3y>q?%EtyBo>; zraKWkk9eVqia)E^+afQntwAG$ohc%XajP{Amol@ZW-}TY*0MK zKM|s;&OTWBjGRZ^5@1thKhCUCJas>k1+QC=WS2vY0PI|WaCgnfglA)7=Kt+k!)RkzvB8KwKBY`bY{+~q<719rB$VVrSck1*K!y1o4BW7; zDHAvZa3&ZT^j7$H@Bko-nHqtvH#2`2Pu3f)tKuuN0zZ?fcewl`HLiZXRcziS)M~eJ gSfn3Oi|`{#c^>{`8UB)dOMYB09e26T!2c)ke+-z4>;M1& diff --git a/fink_grb/test/test_data/distribution_test_data/online/year=2019/month=09/day=03/part-00069-60c7f1c4-fdac-4442-9cb1-9834fb0b6c1e.c000.snappy.parquet b/fink_grb/test/test_data/distribution_test_data/online/year=2019/month=09/day=03/part-00069-60c7f1c4-fdac-4442-9cb1-9834fb0b6c1e.c000.snappy.parquet deleted file mode 100644 index a64643316a4b9bc4bd4cc29334307d6e6957aaa7..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 16522 zcmc&cYiwKBd6%*+Q;ysiuE|^_YNjg5tqVUyQV*M6viMRGDKe65(K25}C)@`~k>b6g z`7{*->x!)iiZ1Zlq9}$f3-V*=hOH}#A}ER?G13+Y3bzP01TBIf2#T&l*PtkxpfHMT z*!P`tFYnumYl*$EFV8vOIp6zy=R4mue3?&&0|x^a1EFR3VU9D*#N(q3!xYxz;iw>X zuD83*xns=1PCLK^jt7FL1OC9Vz;IxAe2zIeYM#ZKdYv|)7?~J?x8LVGaD=<%8xM?* z%^qWh$5{WiSS?Deq#PIu`h&0gg5URX>zsc#gkz_X!N3p$uVe7TFwma}JgW13?D1$X z_sQmsXOEuMe(fByetib(*XIP8lP;q+?q54N9yq9RKP(BVT%o+b<^%S*Zy(@p4~^di z^4%r*Mr!&K(FYnAU;V*<{pZ`k#zm(j`V7{u&p}C?qc%z2Iy@dYqD!*VDs40bVg)M6 ztzqs9!yt(hjR*4GB{{tOmE`xPK7RA%fB5AON8g{g>6Ao=uzr0m5_68)B>Bw|Lz2TN ziCmOSqP#Q0y)XizIPrL3cNOJW;KXuZh-?f0IJz7@P7!nn>(^UIL~k3Tu(-b|{0#TI zqlQ}@C}}?RZC~)FkNd?a_tGe^@5JGOd3W|t{`VU{nRzwFJK5JEtY2>-5xs4U+Sq^p zm|=nYs&ca>G+LXLW{X&Xit+RNxo_PMVmNVmP;eC^6o@QC$id*m9{kZQpKsyaCxMrn zUwY$b_{+R`=;u#@R3`$g5uI57k8a`f?vt!vpNlk|qX0efDC*ho8#*8}+>0j+v4$EV zk>*Wb@Esp_TVPr!us_V64BenC@^mJsplUsabKlj zirV72Aoq3vxObxPKM*_$79dVdF0K^lq6}v~xF((heFpDo6dvA>y=fIJQ+H5u2 zN>QaQvlINRk9&5Edy~bZz`)?jhd(f?^KlT*E61(`6by?1vv%Hm$RK~H*wHC(2J74x zAL9P!K_rZ;TuH*|r;{9+I0TsE#fDOG5&jSXu=K~q!P_4*IJj?HY!(|Gp|#!U5jXps zFZi5~`|QWK*T<0)D;` t=FJ^K>*c0<`ANPtTl>oSzN5@HNs0vPRR!-}4g&e}%It&Nh7p z>(}Q5!S$V^X4Vgka$kPLaQ)$OL$sc4e>KT{|Ks5PPBb3KcbDWZ-uY|)msZAZJ{+vB zyr(|mB8d)R{rX%a<{Y(2@-@zohA2l-l-jnmQEW7}8>Ykh>nFKie*y$@V)LNkD$rcu zg1#3X+I#@`h=jt?Ga>jhJrSC@5DLMcv!PJPWoJC#F_rZKn5Bf1{PSmwUGoub*Q{5R z;??zeBIa*>!QaE7eu}$s3b&3IR&R2y0$Wa!=H|P9{*S-KwJn z^v#fAxe>Hny|FFQY>^%)FHUj4JquQIV)DT6Dnu~wpl-GMP96pxuBuz=c2~8_0cty} zfmpp^kBbFZHR zIh;s5P`k=uaeJT+&u?7ZUWX91kvW>ONZS~NZr>QNm0x(=;QvUev8`;h>-8dyR@Cor z{3-X&1!@f&Dh~p?OQGjvPAljTBX2_@&QY5b|N5jM#Zi)(Z*(9%hmb_fK?V8yKjYqg z0t9hl^PsW2AixH5^ItYR%z+T9cx>s~vX0sWc{yeXGAdU}B^Noq&U62FQ8S1YnFop8 z<@nD1@Bhv7e|v~=8bpV%e!Yc6^tLf-ljFyW#$9g&?Epqm>Lz2nyudxT075u1dEj>w zV(f#lS%w{MN}EMR2!xox=wm}hz2z}xc^udZ9E3rwjKKtsV24%$Xh*+H(|LF=rQ_x+ z9vy+(k=Cff93*8*>2+c_OjB+1h&lB&j~gkcK1%qb*__6omJrdNHK(9l*iK+GCgUK} z7&eXx%d~zJrc_8EqfJvx&^WMcFFI!NRH{hB=#QOP{=)I`(_jgP3Hryz$39^9r#{B* zd)%-6 z;p>^uWWUQkJy}uZ;`I$_yVh3J=H!KmPdz=k1!K^fYzS((IC*+vQWcaU45;l|4SFQg zf^LYkx9iZHAU#Xl`N?NY%viaY)IWbB77H&H$USj{`g-Ox zFs4a4Nk3XTDH`K8dKI-bH$yC7k-tA|65m0L-X@&HjBy*WQf*ssC1(QcMi5G*?i4k^ z9+QM?k&~P@Xd~6vnR@rCG<{ZclCuGAWKFuW5;g(uq|}D!q)5m1m{8S4lXPs42^rs; zoRV2#Ho|uo4jlV|d}wl#><{lPHPc5WCrul0Zz=ab7dh!#KzmD-)&`s;jd2@2lmGug z8$+u_Hir5euJxrAAJpsN0 zcX3sx*l1R^RU8e6&%#&QNyuI&H8#w^v(sm1C!b;1Q9hl%l)XR;-R!NiQ}|5tX`biT z;4eN;@%#)QEAw%lFZ1(dG8T(j;9VSC?8BGVR%jpI1Pg&DdA`fTI`}0>;Fm0dcOL%6 zy7}zvRz3$y7rO9@ol7dUb|I0O$wgK>;!>t1sw6Vkq?53E#igVg>_Ih)9p07yL=}k#pgE79V^*hphaxS^B)=bW)cH~5+i+EOYVHx51 zNVF+u!|hyz(k|rsG~4-XxQ25@HPaO5kzS!*$X?&dMO^46;=p@+CpVp*l@iDkUy-x3 zb-=$>xRONLz+<>u%tS-QT#VpND$$CdWUBH!fwu~|^d{g4<*xw0OU+6lJ6jba>AI*$ z1YahTPAspND!wYjPqEHcfB#^|lmhqAVu5 zRkXX{&6fo(x#$&qA+%bha)E7H(o)ULb6r%|BCBpXP0WVH#P#U%Jo=iUtD>4#a?`0z zDudm|e8mD+W8i<_V_~x$3r#T+zY|Z2n22u53;EDp=xI()6iToks(0iIY?JR1|8cXQ zjV{PdvjpC zfWEwV5tISz1!6tet_Lx(2K&`3M!{ZQ4N0)aAr?~my7`rwX$#raP66UB#yXdnpv5{Z zW_%dCDVa#0rX%h{Jk1sQe7o82o?^d8oPt=pQ^=(v5MMizy5*&_PF`KDNooQ1PPA8~ zXOCNK*pJvAl9ZZSh-7+qICmU#2Bp8kbgFWvIB$Ww`I%zd|IV|=7H>b)d8kTC>W*=q z@b$-t^Qf-~oDbnB_*1GQRmeG`2-!@t5RaPTu-wbnqHqSTmcT!I;`D&ynwstjSy=_z zR|SYCkS8s8#W{H?RWC&3npA_lDx1k<7FRrD@P6!J>SM7U>}Ahd|L=dt0LsPkb7coJE-`w1^IUXjoc`)wDhCW=~sebLMY0om%fR9%q`Czm;SK z^RpV>P0*j5F$VI0d@dErO&8#PhkKXmd0@6x()EH;*qu-Jc>~iY(IMo*aChx`+1oM4 z)gLkU&V(?YQamp9hPx`zK z!54kD;3r0Ojr&2&;b?4r>hjbSG=R7D^V<3avIx1ctVet?xKkJ49^2m^WRl$3nVLdy znV-?{#rW$4M2a$mKhg!M;c@Iotmiz{(C8<5@v??*Z3Q8ksHBNqS}#fz;Cu_0u@5Rzdk8;EDU73JB-WYM_+Loc_$T@{@t?v4a9?i1U9Ti&<0|%7jrD!` zV_5A%)52p~4|OkCZ$8^VZ*$0RZ#`^INH`|IQG_Cr^R z_4DEk#=%VH>Kf3$av9{G>96;&Kl*x%(1$uhH3#`O#Qh!&^3lHh^D`Phy8W>TXO^FM zZ9muS@|CHM#{VT7eJBpXWAa~8&!h6A?2G44QGF%(#mnIO=znWh-Tehem-_izs(_gJ z1zdk#qaX9I*D}tu-HG^Rq!+1b^cShV9sHti#cB{=+A<=EufzOPYrD`R{sH;X4&;zw z$Q{YrGSWM@qtm1Ii4jpv`oYp?>>A3J1fA0TxUi+u)Al2Ac#}VpT@Ez@&}Rb{Vf;w) z3lI~iK0Wpy>N;Vqi||Xv`pft_%HDvQty(;-T`fuG4ST`mp*0K|L{O3202EW|9(DcFk>io&(4<~N?f#cEB&28v(Gaor@h+jM3@=KyF zr^Wnf!vgcBjfi?2q-%v>E`faU_us}NHJ$44)~Bw&aJE$)?)ukiZVcx6i>`P7EULp# z9zRyN0>4*=uAg}&pboRKm>SHR7$g=}`q{DU`uTfr9{8G6{7w504qPpC7aw`(g@emp zso49Mv;WxigjD>|*?*q=W~)>j8F}rQfel-WY%HcH=1m(C3oCVwrA@eduizr3!TU(w z$Uoo!F-Df5Z3F>Wf-{8G6d_sP?e@K?$6O?6{Nml0?hIlOn1hkfF zO&^HIR@m0}FF$eh$yYvpoxI~^dzh)gyg4F^VUDS^9Q|Qo#3-RH^Yx&7EPE)MJDe3^ z_-HO2RsD}1$rqH2KOW5{Qi*6mDX*(A>Zv!Ju;}apShS#?)sM`_&AqK3w8NKgdW-Oj zcDL8^eDUnH<%Y|vEVp3Wk0+fHje^F*VMz@US_9r5Yke)am|?c2VWfAQdr#h=`K`Nfge)x{OxOZ$H}d1LWQq1QX+ zp8xkE8;hyIyy-(?VWrNoSkp}6N9dYEnoW&S^I&5>FR`amuv@c;%%g`bB2z;TC4$aM z*F)DMbl88jtPrgkCFID+m0b!mTkKfy8E>nncRu_6<)s#cCQocZO#B|J7TadPWSiTy zc1D+X)%&0)KNVFEjppMMN+v4ENbqcNVLlg|RN{q)5?kQ;+#smaUkfBOlF8GBO){0x}jFK%7A|k3)W-YU( zTr97sv(Z9MJz`ok6x7uCxT5O8Y4MD!F&YntLGnu7x@li!pbE;0!VgMXWWt7Jb zewpAB(*|NXrRHMrU|Mj=RQAwlJRQyFOC|{=U5JimqT{-YTZMf>A2ue{TxN7;YDxhX zjahs8(OFmm;L&BCL@Jr2qB_D#v|yTatHW%w@+v^;q~G$;ZM7y zc{jkbO*MHZSSfOVzfecv&2qEgU#J6owoxYdtkem=2rF)`4k=&_`%O-~$Vo*Wu-OKg zV6##u?9~PuyY-c@aI7`}2fOzW4p!>K@m3S%fS1f%nt7M>Rue!&0o++v?6CAp%@*iM z>&h%Of6k~+w9-1aYy?{LbDa1cPCC&V(Yj%_C3c8(p^d`nn}~*#77NlsTLjal*ui(M zo%&k091YD}rJ=sfiOZbyMn}Zb^;(9IeCNj4f#ud+ zG)Hy%R;W=dapIpi>DFD4C${$`)x}Dks%sfq5%cA)H#S!b{;~l0Y|l&ZS*a8LxlW32 z->xkz=Q@Ff?RW_bD|KS|1k+TM9RM25iVL<^=@T4Tvt36_8gMpy*4(eEvvt<*@)lX zfSU2!rA`zp-ISu))*V<_R=RHK=v2!C6Jkr*c_b z8w&OXLkQo)SmdYEL1=QPXOu!v%^ePAltL74cv1XE&(T2P$dnQo3XncDUI_FCk|`yf z$Ond=I2y=CGel+f!l0zasq8q+voq;*G?oS{a94T*CURit=&@tHB?P^BrvB+%CEj{N zMZFWb8A5r->_&rLJpi;s)VZ?QEtdcaQRxGyvFd#%EumM+OK&HXAL^u{)N9w7>fJ5( zpXxzM7pCeV^tOWfSGtr&-|+bWO8271ny zm4-pCu+@9hTW|Fkwxh6m80+1-dX&<2oI+Sd>u&hpIEL`U#zAv$M)eyflfN(6CkOld zJ+YZoI^mBF3?`BT$-%+Uwpdt6hQpyyLJrH}L1nufAB^2Mc%K|oVu`+c;BNswg#ITV NbhRC!{qWz2{{>8*%Od~) diff --git a/fink_grb/test/test_data/online/year=2019/month=09/day=03/part-00064-73d02dc2-dbe0-4e73-adc6-a480974420d3.c000.snappy.parquet b/fink_grb/test/test_data/online/year=2019/month=09/day=03/part-00064-73d02dc2-dbe0-4e73-adc6-a480974420d3.c000.snappy.parquet deleted file mode 100644 index accfd1053de50d6ba9110adcca3192698e583b46..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 6902 zcmb`MUuYCZ9LIO>E|*Pmi5lH?FFp0R)+5q*>=})@aHVRg{X?Z95sIa~?%nMrcey{w z-X@|VwNOEdlwf`EL9Fy4*t}?4X`xsuMT$?Y2&K~cU?Etrc@V6n4+{Ow?#|8Z&EDC& zRzm#E?>F=L{bqJ{=DWIg=f1e;6T{*&uZbSf$8jr@0gmGijPH&o)O2CCs8>Z!Tq&*@ zf?ht+FA5<$tK{4T(clc%_iPv z^lM-B3|{mc~E{Dpzq!}E_}2ncojieF9pRNaVHk~$%|h;c>L@cfWk1?v2$jnte1k~ zj<^%ddqfTgMayeqxHEFl)ZHK}We!0A}1M)Nl;c)9^K_ zPAqv>e#_J?;D}ta5S{flqeBe-+;>@a9I^K! z^s#F<`x7rT8$MKg*0HtY2ss52o%J@OqYAGsrW(|aB36~i!nHfS^>kSb*;^(sG++L8 z>3Sc>72e;-9-IL>Dmz+`$wwHT1gcZP7G}H$U-Gu&@#p= z*^O{)`@rGPsO4-v%Zc5naoEVEbk%BfqDEHBSpB>?+^MIKts1IsOy;YG)zhP=ZKD7n zl%_jht{VDmNh=#D5ps2Cb5yIRlCEm{oN83`L%2tmq30(iG~FuBtY<<`lX_6YtOtd9 zQL9+Vq%>Wx=vIn88!(o(UvP0m`aS zmo2{7OK{c!W0H!J5nb#}%0!2{j6K&!#?r@;NjS$#3w-okU(!~gjl`%6_u2}=B}bY` z{hpV8;G@@8Bq^aunXruRTGS=Wsa4i9;`~9`Zyl3niUnR;@K{=xq%M!_-dgfnkvqw(&N`Cst%cdM0u;Sntk>=BT7VV7H%vFV z<;)bH^U~LRbRn3mY0sJg30Pt3GJ!h-)C6ibCN+UO126$}#vn;bEHnYu>o$S2A(Fwo zgDLpg5Wra#ir}o*4Ss7Kfm63SmEzVqt9~faS+3|4F?Cu6;TkhFY-~=`tNBVp_Twi+3G^GKwumQV<*VhEDX`SJv`%d^Fz znl3^w7_JRPFgY@IQs`OU#lJ69fANEO0kKX*SX@5BNCDcBo_UY6Bg+70t)|Y9xqOmxNrwgwbjlwR@HzS}g;8%(g$8eE+u3a9oA$W9YB!_9?Xw zbAq5R~q=)UBDIAZ*l~_DHn4ZlSvte~Kk; zOLD&?LRhmT2I4}WkBvf1Z>kIstwA?@dpcd7HW(6=94pA7W=+p zC-G@g`q_CB#DhfA!tkU`tpIOZq7JFsDnJYJfb-ccJGm9|fDz#@fULBufkeq?0eSJ` z#aB^40*r_6>XGo!StDD6P2k+BYWXaeM3Z4g<_`eb`qhv<(h3W*J=zM+_~5|$!F7Wh zVjOI1#f4}S;Aw}kSJSXNVBechg)iTm9De@G_tc|vlMc@!i9jNW0UnXJ0t?*%zN(Yu zjo;oDkMH!)tdPfd-YN7t&=3z2Ni3$}tqanCQxFbM4S)R1nLGbHal(Oyc#QpQ#9|uW z%F)E7A*(ja_AUkQx@au1HVU7CV08UZGzyUWvRznibZ`b;M`N#S19f;a0D%u1Ca*6lV-(M z4bE;(@Ez`4i)P?=_}VT13^3TPI-`h2gwZl$@D#{6Y6T13oYmJ}`JTJGYx3TOD>uKm zb79^63p@>pKq863G`y9gVG99H_?veq1o2p#5@M`b81d8!-I`@4XC_w#htm|Jllah z+yB7-oUb&YX!}M=qkGl7B1Hwsx7v-wVrpC+)wiWr!}0G0L-VLoNN3Z6)P_5+>zNTv zF*_}|lU7H~I8TNIEgDakbVbwmW=pym>DE$KV;q1mOomn)8rsG3&POo|Q*V0SjJ?%Dk` z#>JtnICMfQreI=5a+z#l-^gfADU~WAX*H)SBY9=a9OBg!T{W9+8LgNfDNjtOz+x~P zvmcp)B>*1N=1XTY8ER_Atk?^o`3XT5{Q>_26cz)L&%Z%z_WEfZB)2Te-@D{lS6EEO z7nZaBez`p=wJ@zx8i?Py02I8*I%KQ^{0UD$Y9sLG=3wAYcmSRk zs`#1Ut%3vmO)rHf+l|4$=>>dVp<;aA>V$u~nXV4rzy|hdmwet8KHUu1ygbF&ywwSN zzJ=z_J`xPad<$^!A{BG+Rws^At&{^EGG=K`wE{G307KClEkn~{Xx{1seZP%?9%onP zej7mZCnkpGtxnKqd~}~P2Z53OJ1+TMSNM!CX^2%`u{vy$xpoR?9wG)(PS(S7?MXy6 ztHapy9ds;v;qx6yOR?4xt`4}@J1H(bwwU@{?@ThHlxe~;I_p*kmg&W` zM$P6?$WTl#hC1=m6xWHjI@Rf6H-&Gp4p#>MVK?COniS*nRww*RODMkmxMo-`Eddr@ zl42I#>cnzVrlqL501TSTZuzP^d{R!<6qGd`kVO6J)S-@d5nTv+85!A`htXXGi@G2X%VJ-=pwYy>&>9}$|aw3g{OOx6&YG-00FO29VT#l88d;(RmoKG_A;0N96XPv zPaPP}E~gB}<;#GdT@K*9{=}=pTb)+t$Vv*%T<1&yj;u8Ehbhck>JGOs1wKygRws(t zAf;%x2?T~^HV7>I8H!nWs}sw^Rg}dhIT(_MtE{QmB`KMT-ReYgt~VexTX#McF$Tjm zmwdw&KG&PHi%)wGm=+SMeDUSmy|M_IAXL=GBFcmUiZx=0*T|Hrm9oVGY8r^dA_D?i z!$p)Pa}m(FlVw$pXvO`Jys9f{MOV-dzJnqCz=Rqa3SlWas)zbRnXH;imqJ4?9}E?g zJa!Q|3{D!HEsR0GP|oF)R1TtGxY{2g$f2QwhYs~u5KR3{$LEUmj5akD9i1+gG36t( zTN=G_08mAAaDBNm?*SB|-U(=_#>b8`sn^R(Ur;DN?321mZ@keoKAd?4Y8<5U&eS-B zsVwODdY96Y4*w0H@+#LjjQ#D8;?!GKJRjqAkwVnFB=icV81<%6xo0Uvy=maL*v3cG z7vIJ)Y*%69FzoAf<0zFkatgtk7Ci7@IV7O%) F{{o%wfJ6WQ diff --git a/fink_grb/__init__.py b/fink_mm/__init__.py similarity index 100% rename from fink_grb/__init__.py rename to fink_mm/__init__.py diff --git a/fink_grb/conf/distribute_for_test.conf b/fink_mm/conf/distribute_for_test.conf similarity index 92% rename from fink_grb/conf/distribute_for_test.conf rename to fink_mm/conf/distribute_for_test.conf index b1f94331b..e9fd14a23 100644 --- a/fink_grb/conf/distribute_for_test.conf +++ b/fink_mm/conf/distribute_for_test.conf @@ -4,7 +4,7 @@ secret= [PATH] # Prefix path on disk where are save GCN live data. used by the joining stream -online_gcn_data_prefix=fink_grb/test/test_data/gcn_test +online_gcn_data_prefix=fink_mm/test/test_data/gcn_test # same path as online_gcn_data_prefix without the URI part. used by the gcn_stream monitor hdfs_gcn_storage=/user/roman.le-montagner/gcn_storage/raw @@ -13,10 +13,10 @@ hdfs_gcn_storage=/user/roman.le-montagner/gcn_storage/raw # They can be in local FS (/path/ or files:///path/) or # in distributed FS (e.g. hdfs:///path/). # Be careful though to have enough disk space! -online_ztf_data_prefix=fink_grb/test/test_data/distribution_test_data/ +online_ztf_data_prefix=fink_mm/test/test_data/distribution_test_data/ # Prefix path on disk to save GRB join ZTF data (work for both online and offline). -online_grb_data_prefix=fink_grb/test/test_data +online_grb_data_prefix=fink_mm/test/test_data # Path where are store the hbase catalog in order to query the hbase database hbase_catalog=/home/roman.le-montagner/fink-broker/catalogs_hbase/ztf.jd.json diff --git a/fink_grb/conf/fink_grb.conf b/fink_mm/conf/fink_mm.conf similarity index 92% rename from fink_grb/conf/fink_grb.conf rename to fink_mm/conf/fink_mm.conf index 65f580952..fb661a781 100644 --- a/fink_grb/conf/fink_grb.conf +++ b/fink_mm/conf/fink_mm.conf @@ -4,7 +4,7 @@ secret= [PATH] # Prefix path on disk where are save GCN live data. used by the joining stream -online_gcn_data_prefix=fink_grb/test/test_data/gcn_test/raw +online_gcn_data_prefix=fink_mm/test/test_data/gcn_test/raw # same path as online_gcn_data_prefix without the URI part. used by the gcn_stream monitor hdfs_gcn_storage=/user/roman.le-montagner/gcn_storage/raw @@ -13,10 +13,10 @@ hdfs_gcn_storage=/user/roman.le-montagner/gcn_storage/raw # They can be in local FS (/path/ or files:///path/) or # in distributed FS (e.g. hdfs:///path/). # Be careful though to have enough disk space! -online_ztf_data_prefix=fink_grb/test/test_data/ztf_test/online +online_ztf_data_prefix=fink_mm/test/test_data/ztf_test/online # Prefix path on disk to save GRB join ZTF data (work for both online and offline). -online_grb_data_prefix=fink_grb/test/test_output +online_grb_data_prefix=fink_mm/test/test_output # Path where are store the hbase catalog in order to query the hbase database hbase_catalog=/home/roman.le-montagner/fink-broker/catalogs_hbase/ztf.jd.json diff --git a/fink_grb/conf/fink_grb_schema_version_1.0.avsc b/fink_mm/conf/fink_mm_schema_version_1.0.avsc similarity index 100% rename from fink_grb/conf/fink_grb_schema_version_1.0.avsc rename to fink_mm/conf/fink_mm_schema_version_1.0.avsc diff --git a/fink_grb/conf/fink_grb_schema_version_1.1.avsc b/fink_mm/conf/fink_mm_schema_version_1.1.avsc similarity index 100% rename from fink_grb/conf/fink_grb_schema_version_1.1.avsc rename to fink_mm/conf/fink_mm_schema_version_1.1.avsc diff --git a/fink_grb/conf/fink_grb_schema_version_1.2.avsc b/fink_mm/conf/fink_mm_schema_version_1.2.avsc similarity index 100% rename from fink_grb/conf/fink_grb_schema_version_1.2.avsc rename to fink_mm/conf/fink_mm_schema_version_1.2.avsc diff --git a/fink_grb/conf/integration.conf b/fink_mm/conf/integration.conf similarity index 91% rename from fink_grb/conf/integration.conf rename to fink_mm/conf/integration.conf index 69ee5c947..56755f829 100644 --- a/fink_grb/conf/integration.conf +++ b/fink_mm/conf/integration.conf @@ -4,19 +4,19 @@ secret=$SECRET_SECRET [PATH] # Prefix path on disk where are save GCN live data. used by the joining stream -online_gcn_data_prefix=fink_grb/ci_gcn_test +online_gcn_data_prefix=fink_mm/ci_gcn_test # same path as online_gcn_data_prefix without the URI part. used by the gcn_stream monitor -hdfs_gcn_storage=fink_grb/ci_gcn_test +hdfs_gcn_storage=fink_mm/ci_gcn_test # Prefix path on disk where are save ZTF live data. # They can be in local FS (/path/ or files:///path/) or # in distributed FS (e.g. hdfs:///path/). # Be careful though to have enough disk space! -online_ztf_data_prefix=fink_grb/test/test_data/ztf_test/online +online_ztf_data_prefix=fink_mm/test/test_data/ztf_test/online # Prefix path on disk to save GRB join ZTF data (work for both online and offline). -online_grb_data_prefix=fink_grb/ci_join_test +online_grb_data_prefix=fink_mm/ci_join_test # Path where are store the hbase catalog in order to query the hbase database hbase_catalog=$FINK_HOME/catalogs_hbase/ztf.jd.json diff --git a/fink_grb/conftest.py b/fink_mm/conftest.py similarity index 77% rename from fink_grb/conftest.py rename to fink_mm/conftest.py index fcee57958..59d0e9306 100644 --- a/fink_grb/conftest.py +++ b/fink_mm/conftest.py @@ -6,9 +6,9 @@ from pandas.testing import assert_frame_equal from astropy.time import Time -from fink_grb.gcn_stream.gcn_reader import load_voevent_from_path, load_json_from_path -from fink_grb.observatory import voevent_to_class, json_to_class -from fink_grb.init import init_logging +from fink_mm.gcn_stream.gcn_reader import load_voevent_from_path, load_json_from_path +from fink_mm.observatory import voevent_to_class, json_to_class +from fink_mm.init import init_logging from scipy import special from math import sqrt @@ -36,10 +36,10 @@ def init_test(doctest_namespace): def init_fermi(doctest_namespace): doctest_namespace[ "fermi_gbm_voevent_path" - ] = "fink_grb/test/test_data/VODB/fermi/voevent_number=193.xml" + ] = "fink_mm/test/test_data/VODB/fermi/voevent_number=193.xml" doctest_namespace[ "fermi_lat_voevent_path" - ] = "fink_grb/test/test_data/VODB/fermi/voevent_number=2842.xml" + ] = "fink_mm/test/test_data/VODB/fermi/voevent_number=2842.xml" fermi_gbm = voevent_to_class( load_voevent_from_path(doctest_namespace["fermi_gbm_voevent_path"], logger) @@ -56,13 +56,13 @@ def init_fermi(doctest_namespace): def init_swift(doctest_namespace): doctest_namespace[ "swift_bat_voevent_path" - ] = "fink_grb/test/test_data/VODB/swift/voevent_number=392.xml" + ] = "fink_mm/test/test_data/VODB/swift/voevent_number=392.xml" doctest_namespace[ "swift_xrt_voevent_path" - ] = "fink_grb/test/test_data/VODB/swift/voevent_number=4554.xml" + ] = "fink_mm/test/test_data/VODB/swift/voevent_number=4554.xml" doctest_namespace[ "swift_uvot_voevent_path" - ] = "fink_grb/test/test_data/VODB/swift/voevent_number=8582.xml" + ] = "fink_mm/test/test_data/VODB/swift/voevent_number=8582.xml" swift_bat = voevent_to_class( load_voevent_from_path(doctest_namespace["swift_bat_voevent_path"], logger) @@ -83,13 +83,13 @@ def init_swift(doctest_namespace): def init_integral(doctest_namespace): doctest_namespace[ "integral_weak_voevent_path" - ] = "fink_grb/test/test_data/VODB/integral/voevent_number=737.xml" + ] = "fink_mm/test/test_data/VODB/integral/voevent_number=737.xml" doctest_namespace[ "integral_wakeup_voevent_path" - ] = "fink_grb/test/test_data/VODB/integral/voevent_number=18790.xml" + ] = "fink_mm/test/test_data/VODB/integral/voevent_number=18790.xml" doctest_namespace[ "integral_refined_voevent_path" - ] = "fink_grb/test/test_data/VODB/integral/voevent_number=18791.xml" + ] = "fink_mm/test/test_data/VODB/integral/voevent_number=18791.xml" integral_weak = voevent_to_class( load_voevent_from_path(doctest_namespace["integral_weak_voevent_path"], logger) @@ -114,13 +114,13 @@ def init_integral(doctest_namespace): def init_icecube(doctest_namespace): doctest_namespace[ "icecube_cascade_voevent_path" - ] = "fink_grb/test/test_data/VODB/icecube/voevent_number=825.xml" + ] = "fink_mm/test/test_data/VODB/icecube/voevent_number=825.xml" doctest_namespace[ "icecube_bronze_voevent_path" - ] = "fink_grb/test/test_data/VODB/icecube/voevent_number=3028.xml" + ] = "fink_mm/test/test_data/VODB/icecube/voevent_number=3028.xml" doctest_namespace[ "icecube_gold_voevent_path" - ] = "fink_grb/test/test_data/VODB/icecube/voevent_number=45412.xml" + ] = "fink_mm/test/test_data/VODB/icecube/voevent_number=45412.xml" icecube_cascade = voevent_to_class( load_voevent_from_path( @@ -143,9 +143,9 @@ def init_icecube(doctest_namespace): def init_LVK(doctest_namespace): doctest_namespace[ "lvk_initial_path" - ] = "fink_grb/test/test_data/VODB/lvk/initial.txt" - doctest_namespace["lvk_update_path"] = "fink_grb/test/test_data/VODB/lvk/update.txt" - doctest_namespace["lvk_test_path"] = "fink_grb/test/test_data/VODB/lvk/test.txt" + ] = "fink_mm/test/test_data/VODB/lvk/initial.txt" + doctest_namespace["lvk_update_path"] = "fink_mm/test/test_data/VODB/lvk/update.txt" + doctest_namespace["lvk_test_path"] = "fink_mm/test/test_data/VODB/lvk/test.txt" lvk_initial = json_to_class( load_json_from_path(doctest_namespace["lvk_initial_path"], logger) @@ -172,11 +172,11 @@ def init_spark(doctest_namespace): doctest_namespace["Time"] = Time - grb_data = "fink_grb/test/test_data/gcn_test/raw/year=2019/month=09/day=03" - gw_data = "fink_grb/test/test_data/S230518h_0_test" - join_data = "fink_grb/test/test_data/join_raw_datatest.parquet" + grb_data = "fink_mm/test/test_data/gcn_test/raw/year=2019/month=09/day=03" + gw_data = "fink_mm/test/test_data/S230518h_0_test" + join_data = "fink_mm/test/test_data/join_raw_datatest.parquet" alert_data = ( - "fink_grb/test/test_data/ztf_test/online/science/year=2019/month=09/day=03/" + "fink_mm/test/test_data/ztf_test/online/science/year=2019/month=09/day=03/" ) doctest_namespace["grb_data"] = grb_data @@ -184,13 +184,15 @@ def init_spark(doctest_namespace): doctest_namespace["join_data"] = join_data doctest_namespace["alert_data"] = alert_data - ztf_datatest = "fink_grb/test/test_data/ztf_test/online" - gcn_datatest = "fink_grb/test/test_data/gcn_test/raw" - join_data_test = "fink_grb/test/test_data/online" + ztf_datatest = "fink_mm/test/test_data/ztf_test/online" + gcn_datatest = "fink_mm/test/test_data/gcn_test/raw" + join_data_test = "fink_mm/test/test_data/online" + offline_join_data_test = "fink_mm/test/test_data/offline_datatest.parquet" doctest_namespace["ztf_datatest"] = ztf_datatest doctest_namespace["gcn_datatest"] = gcn_datatest doctest_namespace["join_data_test"] = join_data_test + doctest_namespace["offline_data_test"] = offline_join_data_test fink_home = os.environ["FINK_HOME"] hbase_catalog = fink_home + "/catalogs_hbase/ztf.jd.json" @@ -219,19 +221,19 @@ def init_spark(doctest_namespace): doctest_namespace["AlertConsumer"] = AlertConsumer doctest_namespace["tabulate"] = tabulate - doctest_namespace["ztfxgcn_test"] = "fink_grb/test/test_data/distribution_test_data/" + doctest_namespace["ztfxgcn_test"] = "fink_mm/test/test_data/distribution_test_data/" doctest_namespace["headers"] = headers doctest_namespace["maxtimeout"] = maxtimeout doctest_namespace["myconfig"] = myconfig doctest_namespace["topics"] = topics - from fink_grb.init import get_config, init_logging + from fink_mm.init import get_config, init_logging from scipy import special from math import sqrt from pyspark.sql.functions import explode - path_data_fid_1 = "fink_grb/test/test_data/ztf_alert_samples_fid_1.parquet" - path_data_fid_2 = "fink_grb/test/test_data/ztf_alert_samples_fid_2.parquet" + path_data_fid_1 = "fink_mm/test/test_data/ztf_alert_samples_fid_1.parquet" + path_data_fid_2 = "fink_mm/test/test_data/ztf_alert_samples_fid_2.parquet" doctest_namespace["special"] = special doctest_namespace["sqrt"] = sqrt diff --git a/fink_grb/distribution/__init__.py b/fink_mm/distribution/__init__.py similarity index 100% rename from fink_grb/distribution/__init__.py rename to fink_mm/distribution/__init__.py diff --git a/fink_grb/distribution/apply_filters.py b/fink_mm/distribution/apply_filters.py similarity index 73% rename from fink_grb/distribution/apply_filters.py rename to fink_mm/distribution/apply_filters.py index 47ee8fb21..cb7a83af8 100644 --- a/fink_grb/distribution/apply_filters.py +++ b/fink_mm/distribution/apply_filters.py @@ -18,6 +18,31 @@ def apply_filters( username, password, ): + """ + Apply the user defined filters the the output of the Fink_MM package + + Parameters + ---------- + df_stream : spark streaming dataframe + input streaming dataframe save by the ztf_join_gcn online + schema : dictionnary + schema describing the data send to the kafka stream + tinterval : integer + interval between processing batch + checkpointpath_grb : str + path where are stored the kafka checkpoint + kafka_broker_server : str + IP adress of the kafka broker + username : str + username + password : password + password + + Returns + ------- + spark streaming dataframe + same as input but filtered by the user defined filters + """ df_grb_bronze = ( df_stream.withColumn( "f_bronze", @@ -36,7 +61,7 @@ def apply_filters( df_stream["fink_class"], df_stream["observatory"], df_stream["rb"], - df_stream["grb_proba"], + df_stream["p_assoc"], ), ) .filter("f_silver == True") @@ -50,7 +75,7 @@ def apply_filters( df_stream["fink_class"], df_stream["observatory"], df_stream["rb"], - df_stream["grb_proba"], + df_stream["p_assoc"], df_stream["rate"], ), ) diff --git a/fink_grb/distribution/distribution.py b/fink_mm/distribution/distribution.py similarity index 87% rename from fink_grb/distribution/distribution.py rename to fink_mm/distribution/distribution.py index b0e8c616c..2bf5a1ddc 100644 --- a/fink_grb/distribution/distribution.py +++ b/fink_mm/distribution/distribution.py @@ -6,17 +6,17 @@ from fink_utils.broker.sparkUtils import init_sparksession, connect_to_raw_database from importlib_resources import files -import fink_grb +import fink_mm -from fink_grb.utils.fun_utils import ( +from fink_mm.utils.fun_utils import ( read_and_build_spark_submit, read_grb_admin_options, read_additional_spark_options, ) -import fink_grb.utils.application as apps -from fink_grb.init import get_config, init_logging, return_verbose_level -from fink_grb.utils.fun_utils import build_spark_submit -from fink_grb.distribution.apply_filters import apply_filters +import fink_mm.utils.application as apps +from fink_mm.init import get_config, init_logging, return_verbose_level +from fink_mm.utils.fun_utils import build_spark_submit +from fink_mm.distribution.apply_filters import apply_filters def grb_distribution( @@ -72,14 +72,14 @@ def grb_distribution( 1 """ spark = init_sparksession( - "science2grb_distribution_{}{}{}".format(night[0:4], night[4:6], night[6:8]) + "science2mm_distribution_{}{}{}".format(night[0:4], night[4:6], night[6:8]) ) logger = init_logging() - schema_path = files("fink_grb").joinpath( - "conf/fink_grb_schema_version_{}.avsc".format( - fink_grb.__distribution_schema_version__ + schema_path = files("fink_mm").joinpath( + "conf/fink_mm_schema_version_{}.avsc".format( + fink_mm.__distribution_schema_version__ ) ) with open(schema_path, "r") as f: @@ -148,7 +148,7 @@ def launch_distribution(arguments): Examples -------- >>> launch_distribution({ - ... "--config" : "fink_grb/conf/distribute_for_test.conf", + ... "--config" : "fink_mm/conf/distribute_for_test.conf", ... "--night" : "20190903", ... "--exit_after" : 30 ... }) @@ -227,14 +227,14 @@ def launch_distribution(arguments): stdout, stderr = process.communicate() if process.returncode != 0: # pragma: no cover logger.error( - "Fink_GRB distribution stream spark application has ended with a non-zero returncode.\ + "Fink_MM distribution stream spark application has ended with a non-zero returncode.\ \n\t cause:\n\t\t{}\n\t\t{}\n\n\n{}\n\n".format( stdout, stderr, spark_submit ) ) exit(1) - logger.info("Fink_GRB distribution stream spark application ended normally") + logger.info("Fink_MM distribution stream spark application ended normally") return diff --git a/fink_grb/fink_grb_cli.py b/fink_mm/fink_mm_cli.py similarity index 71% rename from fink_grb/fink_grb_cli.py rename to fink_mm/fink_mm_cli.py index fd8fc4847..284b3f7b5 100644 --- a/fink_grb/fink_grb_cli.py +++ b/fink_mm/fink_mm_cli.py @@ -1,10 +1,10 @@ """ Usage: - fink_grb gcn_stream (start|monitor) [--restart] [options] - fink_grb join_stream (offline|online) --night= [--exit_after=] [options] - fink_grb distribute --night= [--exit_after=] [options] - fink_grb -h | --help - fink_grb --version + fink_mm gcn_stream (start|monitor) [--restart] [options] + fink_mm join_stream (offline|online) --night= [--exit_after=] [options] + fink_mm distribute --night= [--exit_after=] [options] + fink_mm -h | --help + fink_mm --version Options: gcn_stream used to manage the gcn stream. @@ -24,7 +24,7 @@ """ from docopt import docopt -from fink_grb import __version__ +from fink_mm import __version__ def main(): @@ -35,27 +35,27 @@ def main(): if arguments["gcn_stream"]: if arguments["start"]: - from fink_grb.gcn_stream.gcn_stream import start_gcn_stream + from fink_mm.gcn_stream.gcn_stream import start_gcn_stream start_gcn_stream(arguments) elif arguments["monitor"]: - from fink_grb.utils.monitoring import gcn_stream_monitoring + from fink_mm.utils.monitoring import gcn_stream_monitoring gcn_stream_monitoring(arguments) elif arguments["join_stream"]: if arguments["online"]: - from fink_grb.online.ztf_join_gcn import launch_joining_stream + from fink_mm.online.ztf_join_gcn import launch_joining_stream launch_joining_stream(arguments) elif arguments["offline"]: - from fink_grb.offline.spark_offline import launch_offline_mode + from fink_mm.offline.spark_offline import launch_offline_mode launch_offline_mode(arguments) elif arguments["distribute"]: - from fink_grb.distribution.distribution import launch_distribution + from fink_mm.distribution.distribution import launch_distribution launch_distribution(arguments) diff --git a/fink_grb/gcn_stream/__init__.py b/fink_mm/gcn_stream/__init__.py similarity index 100% rename from fink_grb/gcn_stream/__init__.py rename to fink_mm/gcn_stream/__init__.py diff --git a/fink_grb/gcn_stream/gcn_reader.py b/fink_mm/gcn_stream/gcn_reader.py similarity index 97% rename from fink_grb/gcn_stream/gcn_reader.py rename to fink_mm/gcn_stream/gcn_reader.py index 3681b006a..531dcb039 100644 --- a/fink_grb/gcn_stream/gcn_reader.py +++ b/fink_mm/gcn_stream/gcn_reader.py @@ -4,7 +4,7 @@ from lxml.objectify import ObjectifiedElement import json from logging import Logger -from fink_grb.observatory import voevent_to_class, json_to_class +from fink_mm.observatory import voevent_to_class, json_to_class def load_voevent_from_path( @@ -105,7 +105,7 @@ def parse_xml_alert(gcn: bytes, logger: Logger, logs: bool) -> pd.DataFrame: Examples -------- - >>> f = open('fink_grb/test/test_data/voevent_number=9897.xml').read().encode("UTF-8") + >>> f = open('fink_mm/test/test_data/voevent_number=9897.xml').read().encode("UTF-8") >>> parse_xml_alert(f, logger, False) observatory instrument event ... year month day 0 Fermi GBM ... 2022 08 30 diff --git a/fink_grb/gcn_stream/gcn_stream.py b/fink_mm/gcn_stream/gcn_stream.py similarity index 93% rename from fink_grb/gcn_stream/gcn_stream.py rename to fink_mm/gcn_stream/gcn_stream.py index 93879a3ee..a96f61d64 100644 --- a/fink_grb/gcn_stream/gcn_stream.py +++ b/fink_mm/gcn_stream/gcn_stream.py @@ -8,10 +8,10 @@ from pyarrow.fs import FileSystem -import fink_grb.gcn_stream.gcn_reader as gr -from fink_grb.init import get_config, init_logging, return_verbose_level -from fink_grb.utils.fun_utils import get_hdfs_connector -from fink_grb.observatory import TOPICS, TOPICS_FORMAT +import fink_mm.gcn_stream.gcn_reader as gr +from fink_mm.init import get_config, init_logging, return_verbose_level +from fink_mm.utils.fun_utils import get_hdfs_connector +from fink_mm.observatory import TOPICS, TOPICS_FORMAT from fink_client.scripts.fink_datatransfer import my_assign @@ -70,12 +70,12 @@ def load_and_parse_gcn( Examples -------- - >>> f = open('fink_grb/test/test_data/voevent_number=9897.xml').read().encode("UTF-8") + >>> f = open('fink_mm/test/test_data/voevent_number=9897.xml').read().encode("UTF-8") >>> with tempfile.TemporaryDirectory() as tmp_dir_gcn: ... load_and_parse_gcn(f, "gcn.classic.voevent.FERMI_GBM_FIN_POS", tmp_dir_gcn, logger, False, False) ... base_gcn = pd.read_parquet(tmp_dir_gcn + "/year=2022/month=08/day=30/683571622_0") ... base_gcn = base_gcn.drop(columns="ackTime") - ... test_gcn = pd.read_parquet("fink_grb/test/test_data/683571622_0_test") + ... test_gcn = pd.read_parquet("fink_mm/test/test_data/683571622_0_test") ... assert_frame_equal(base_gcn, test_gcn) >>> json_str = open(lvk_initial_path, 'r').read() @@ -83,7 +83,7 @@ def load_and_parse_gcn( ... load_and_parse_gcn(json_str, "igwn.gwalert", tmp_dir_gcn, logger, False, False) ... base_gcn = pd.read_parquet(tmp_dir_gcn + "/year=2023/month=05/day=18/S230518h_0") ... base_gcn = base_gcn.drop(columns="ackTime") - ... test_gcn = pd.read_parquet("fink_grb/test/test_data/S230518h_0_test") + ... test_gcn = pd.read_parquet("fink_mm/test/test_data/S230518h_0_test") ... assert_frame_equal(base_gcn, test_gcn) """ diff --git a/fink_grb/init.py b/fink_mm/init.py similarity index 88% rename from fink_grb/init.py rename to fink_mm/init.py index 65405c590..29cf99654 100644 --- a/fink_grb/init.py +++ b/fink_mm/init.py @@ -8,7 +8,7 @@ import logging import pathlib -import fink_grb +import fink_mm def return_verbose_level(config, logger): @@ -29,7 +29,7 @@ def return_verbose_level(config, logger): Examples -------- - >>> c = get_config({"--config" : "fink_grb/conf/fink_grb.conf"}) + >>> c = get_config({"--config" : "fink_mm/conf/fink_mm.conf"}) >>> logger = init_logging() >>> return_verbose_level(c, logger) @@ -48,9 +48,9 @@ def return_verbose_level(config, logger): return logs -def init_fink_grb(arguments): +def init_fink_mm(arguments): """ - Initialise the fink_grb environment. Get the config specify by the user with the + Initialise the fink_mm environment. Get the config specify by the user with the --config argument or the default if not provided. Parameters @@ -64,11 +64,11 @@ def init_fink_grb(arguments): Examples -------- - >>> init_fink_grb({"--config" : None}) - >>> os.path.isdir("fink_grb/test/test_data/gcn_test/raw") + >>> init_fink_mm({"--config" : None}) + >>> os.path.isdir("fink_mm/test/test_data/gcn_test/raw") True - >>> os.path.isdir("fink_grb/test/test_output/grb") + >>> os.path.isdir("fink_mm/test/test_output/grb") True """ config = get_config(arguments) @@ -92,7 +92,7 @@ def init_fink_grb(arguments): def get_config(arguments): """ - Get, read and return the configuration file of fink_grb + Get, read and return the configuration file of fink_mm Parameters ---------- @@ -106,7 +106,7 @@ def get_config(arguments): Examples -------- - >>> c = get_config({"--config" : "fink_grb/conf/fink_grb.conf"}) + >>> c = get_config({"--config" : "fink_mm/conf/fink_mm.conf"}) >>> type(c) >>> c.sections() @@ -132,7 +132,7 @@ def get_config(arguments): ) exit(1) else: - config_path = files("fink_grb").joinpath("conf/fink_grb.conf") + config_path = files("fink_mm").joinpath("conf/fink_mm.conf") config.read(config_path) return config @@ -186,7 +186,7 @@ def init_logging() -> logging.Logger: """ # create logger - logger = logging.getLogger(fink_grb.__name__) + logger = logging.getLogger(fink_mm.__name__) logger.setLevel(logging.DEBUG) # create console handler and set level to debug diff --git a/fink_grb/observatory/Fermi/Fermi.py b/fink_mm/observatory/Fermi/Fermi.py similarity index 93% rename from fink_grb/observatory/Fermi/Fermi.py rename to fink_mm/observatory/Fermi/Fermi.py index a1633e9a6..c13ac2c6e 100644 --- a/fink_grb/observatory/Fermi/Fermi.py +++ b/fink_mm/observatory/Fermi/Fermi.py @@ -3,8 +3,8 @@ from lxml.objectify import ObjectifiedElement -from fink_grb.observatory import OBSERVATORY_PATH -from fink_grb.observatory.observatory import Observatory, BadInstrument +from fink_mm.observatory import OBSERVATORY_PATH +from fink_mm.observatory.observatory import Observatory, BadInstrument class Fermi(Observatory): diff --git a/fink_grb/observatory/Fermi/__init__.py b/fink_mm/observatory/Fermi/__init__.py similarity index 100% rename from fink_grb/observatory/Fermi/__init__.py rename to fink_mm/observatory/Fermi/__init__.py diff --git a/fink_grb/observatory/Fermi/fermi.json b/fink_mm/observatory/Fermi/fermi.json similarity index 100% rename from fink_grb/observatory/Fermi/fermi.json rename to fink_mm/observatory/Fermi/fermi.json diff --git a/fink_grb/observatory/IceCube/IceCube.py b/fink_mm/observatory/IceCube/IceCube.py similarity index 98% rename from fink_grb/observatory/IceCube/IceCube.py rename to fink_mm/observatory/IceCube/IceCube.py index a4d2fa6c7..e619a4411 100644 --- a/fink_grb/observatory/IceCube/IceCube.py +++ b/fink_mm/observatory/IceCube/IceCube.py @@ -8,8 +8,8 @@ import astropy.units as u -from fink_grb.observatory import OBSERVATORY_PATH -from fink_grb.observatory.observatory import ( +from fink_mm.observatory import OBSERVATORY_PATH +from fink_mm.observatory.observatory import ( Observatory, BadInstrument, voevent_df_schema, diff --git a/fink_grb/observatory/IceCube/__init__.py b/fink_mm/observatory/IceCube/__init__.py similarity index 100% rename from fink_grb/observatory/IceCube/__init__.py rename to fink_mm/observatory/IceCube/__init__.py diff --git a/fink_grb/observatory/IceCube/icecube.json b/fink_mm/observatory/IceCube/icecube.json similarity index 100% rename from fink_grb/observatory/IceCube/icecube.json rename to fink_mm/observatory/IceCube/icecube.json diff --git a/fink_grb/observatory/Integral/Integral.py b/fink_mm/observatory/Integral/Integral.py similarity index 93% rename from fink_grb/observatory/Integral/Integral.py rename to fink_mm/observatory/Integral/Integral.py index 96a8250d7..8c28853f9 100644 --- a/fink_grb/observatory/Integral/Integral.py +++ b/fink_mm/observatory/Integral/Integral.py @@ -1,8 +1,8 @@ import voeventparse as vp import os.path as path -from fink_grb.observatory import OBSERVATORY_PATH -from fink_grb.observatory.observatory import Observatory, BadInstrument +from fink_mm.observatory import OBSERVATORY_PATH +from fink_mm.observatory.observatory import Observatory, BadInstrument class Integral(Observatory): diff --git a/fink_grb/observatory/Integral/__init__.py b/fink_mm/observatory/Integral/__init__.py similarity index 100% rename from fink_grb/observatory/Integral/__init__.py rename to fink_mm/observatory/Integral/__init__.py diff --git a/fink_grb/observatory/Integral/integral.json b/fink_mm/observatory/Integral/integral.json similarity index 100% rename from fink_grb/observatory/Integral/integral.json rename to fink_mm/observatory/Integral/integral.json diff --git a/fink_grb/observatory/LVK/LVK.py b/fink_mm/observatory/LVK/LVK.py similarity index 98% rename from fink_grb/observatory/LVK/LVK.py rename to fink_mm/observatory/LVK/LVK.py index a5501b00a..dc27cb8bb 100644 --- a/fink_grb/observatory/LVK/LVK.py +++ b/fink_mm/observatory/LVK/LVK.py @@ -12,9 +12,9 @@ import json from healpy.pixelfunc import pix2ang, ang2pix -from fink_grb.observatory import OBSERVATORY_PATH -from fink_grb.observatory.observatory import Observatory -from fink_grb.test.hypothesis.observatory_schema import voevent_df_schema +from fink_mm.observatory import OBSERVATORY_PATH +from fink_mm.observatory.observatory import Observatory +from fink_mm.test.hypothesis.observatory_schema import voevent_df_schema class LVK(Observatory): diff --git a/fink_grb/observatory/LVK/__init__.py b/fink_mm/observatory/LVK/__init__.py similarity index 100% rename from fink_grb/observatory/LVK/__init__.py rename to fink_mm/observatory/LVK/__init__.py diff --git a/fink_grb/observatory/LVK/lvk.json b/fink_mm/observatory/LVK/lvk.json similarity index 100% rename from fink_grb/observatory/LVK/lvk.json rename to fink_mm/observatory/LVK/lvk.json diff --git a/fink_grb/observatory/Swift/Swift.py b/fink_mm/observatory/Swift/Swift.py similarity index 93% rename from fink_grb/observatory/Swift/Swift.py rename to fink_mm/observatory/Swift/Swift.py index a1097c3a5..b5fe50569 100644 --- a/fink_grb/observatory/Swift/Swift.py +++ b/fink_mm/observatory/Swift/Swift.py @@ -1,8 +1,8 @@ import voeventparse as vp import os.path as path -from fink_grb.observatory import OBSERVATORY_PATH -from fink_grb.observatory.observatory import Observatory, BadInstrument +from fink_mm.observatory import OBSERVATORY_PATH +from fink_mm.observatory.observatory import Observatory, BadInstrument class Swift(Observatory): diff --git a/fink_grb/observatory/Swift/__init__.py b/fink_mm/observatory/Swift/__init__.py similarity index 100% rename from fink_grb/observatory/Swift/__init__.py rename to fink_mm/observatory/Swift/__init__.py diff --git a/fink_grb/observatory/Swift/swift.json b/fink_mm/observatory/Swift/swift.json similarity index 100% rename from fink_grb/observatory/Swift/swift.json rename to fink_mm/observatory/Swift/swift.json diff --git a/fink_grb/observatory/__init__.py b/fink_mm/observatory/__init__.py similarity index 92% rename from fink_grb/observatory/__init__.py rename to fink_mm/observatory/__init__.py index 2085b9490..04f4e0b7f 100644 --- a/fink_grb/observatory/__init__.py +++ b/fink_mm/observatory/__init__.py @@ -1,4 +1,4 @@ -import fink_grb +import fink_mm import importlib.util from importlib_resources import files import sys @@ -27,7 +27,7 @@ def __get_observatory_class(): """ get all the observatory class into a dictionnary with the observatory name as key """ - all_obs = glob(path.join(path.dirname(fink_grb.__file__), "observatory/*/*.py")) + all_obs = glob(path.join(path.dirname(fink_mm.__file__), "observatory/*/*.py")) # remove the multiple __init__.py all_obs = [p for p in all_obs if path.basename(p) != "__init__.py"] @@ -55,7 +55,7 @@ def __get_topics(): instr_format: dict a key value map (observatory name => file format) """ - p = files("fink_grb").__str__() + "/observatory/*/*.json" + p = files("fink_mm").__str__() + "/observatory/*/*.json" res = [] topic_format = {} instr_format = {} @@ -77,8 +77,8 @@ def __get_topics(): OBSERVATORY_PATH = "observatory" -OBSERVATORY_SCHEMA_VERSION = fink_grb.__observatory_schema_version__ -OBSERVATORY_JSON_SCHEMA_PATH = files("fink_grb").joinpath( +OBSERVATORY_SCHEMA_VERSION = fink_mm.__observatory_schema_version__ +OBSERVATORY_JSON_SCHEMA_PATH = files("fink_mm").joinpath( "observatory/observatory_schema_version_{}.json".format(OBSERVATORY_SCHEMA_VERSION) ) __OBS_CLASS = __get_observatory_class() @@ -128,9 +128,9 @@ def __get_detector(voevent): return instr_name -# The fink_grb.observatory import have to be after the OBSERVATORY_JSON_SCHEMA_PATH definiton +# The fink_mm.observatory import have to be after the OBSERVATORY_JSON_SCHEMA_PATH definiton # to avoid a circular import issue -from fink_grb.observatory import observatory +from fink_mm.observatory import observatory from lxml.objectify import ObjectifiedElement diff --git a/fink_grb/observatory/observatory.py b/fink_mm/observatory/observatory.py similarity index 97% rename from fink_grb/observatory/observatory.py rename to fink_mm/observatory/observatory.py index 7147ce0b9..ff3d4c956 100644 --- a/fink_grb/observatory/observatory.py +++ b/fink_mm/observatory/observatory.py @@ -16,9 +16,9 @@ from fink_utils.science.utils import ra2phi, dec2theta -from fink_grb.observatory import OBSERVATORY_JSON_SCHEMA_PATH -from fink_grb.test.hypothesis.observatory_schema import voevent_df_schema -from fink_grb.utils.grb_prob import p_ser_grb_vect +from fink_mm.observatory import OBSERVATORY_JSON_SCHEMA_PATH +from fink_mm.test.hypothesis.observatory_schema import voevent_df_schema +from fink_mm.utils.grb_prob import p_ser_grb_vect class BadInstrument(Exception): @@ -49,7 +49,7 @@ def __init__(self, instr_file: str, voevent: ObjectifiedElement): """ - instr_path = files("fink_grb").joinpath(instr_file) + instr_path = files("fink_mm").joinpath(instr_file) with open(instr_path, "r") as f: instr_data = json.loads(f.read()) @@ -386,4 +386,4 @@ def association_proba( # command to call to run the doctest : -# pytest --doctest-modules fink_grb/observatory/observatory.py -W ignore::DeprecationWarning +# pytest --doctest-modules fink_mm/observatory/observatory.py -W ignore::DeprecationWarning diff --git a/fink_grb/observatory/observatory_schema_version_1.1.json b/fink_mm/observatory/observatory_schema_version_1.1.json similarity index 89% rename from fink_grb/observatory/observatory_schema_version_1.1.json rename to fink_mm/observatory/observatory_schema_version_1.1.json index be91ce04f..cfa469671 100644 --- a/fink_grb/observatory/observatory_schema_version_1.1.json +++ b/fink_mm/observatory/observatory_schema_version_1.1.json @@ -1,16 +1,16 @@ { "$schema": "https://json-schema.org/draft/2020-12/schema", - "$id": "fink_grb/instruments/instrument_schema_version_1.0.json", - "title": "Instrument", + "$id": "fink_mm/observatory/observatory_schema_version_1.1.json", + "title": "Observatory", "description": "The description of an astronomical observatory sending alerts on the General Coordinates network", "type": "object", "properties": { "name": { - "description": "name of the instrument", + "description": "name of the observatory", "type": "string" }, "//gcn_description": { - "description": "URL where to find the GCN description for this instrument", + "description": "URL where to find the GCN description for this Observatory", "type": "string" }, "gcn_file_format": { diff --git a/fink_grb/offline/__init__.py b/fink_mm/offline/__init__.py similarity index 100% rename from fink_grb/offline/__init__.py rename to fink_mm/offline/__init__.py diff --git a/fink_grb/offline/spark_offline.py b/fink_mm/offline/spark_offline.py similarity index 85% rename from fink_grb/offline/spark_offline.py rename to fink_mm/offline/spark_offline.py index 545ceaf0a..52acc129f 100644 --- a/fink_grb/offline/spark_offline.py +++ b/fink_mm/offline/spark_offline.py @@ -11,7 +11,7 @@ from fink_utils.spark.partitioning import convert_to_datetime -from fink_grb.utils.fun_utils import ( +from fink_mm.utils.fun_utils import ( build_spark_submit, join_post_process, read_and_build_spark_submit, @@ -19,9 +19,9 @@ read_grb_admin_options, read_additional_spark_options, ) -import fink_grb.utils.application as apps -from fink_grb.init import get_config, init_logging, return_verbose_level -from fink_grb.utils.fun_utils import get_pixels +import fink_mm.utils.application as apps +from fink_mm.init import get_config, init_logging, return_verbose_level +from fink_mm.utils.fun_utils import get_pixels def ztf_grb_filter(spark_ztf, ast_dist, pansstar_dist, pansstar_star_score, gaia_dist): @@ -170,16 +170,20 @@ def spark_offline( ... False ... ) - >>> datajoin = pd.read_parquet(grb_dataoutput + "/offline").sort_values(["objectId", "triggerId", "grb_ra"]).reset_index(drop=True) - >>> datajoin = datajoin.drop("grb_proba", axis=1) + >>> datajoin = pd.read_parquet(grb_dataoutput + "/offline").sort_values(["objectId", "triggerId", "gcn_ra"]).reset_index(drop=True) + >>> datatest = pd.read_parquet(offline_data_test).sort_values(["objectId", "triggerId", "gcn_ra"]).reset_index(drop=True) - >>> datatest = pd.read_parquet(join_data_test).sort_values(["objectId", "triggerId", "grb_ra"]).reset_index(drop=True) - >>> datatest = datatest.drop(["delta_mag", "rate", "from_upper", "start_vartime", "diff_vartime", "grb_proba"], axis=1) + >>> cols = ['t2_AGN', 't2_EB', + ... 't2_KN', 't2_M-dwarf', 't2_Mira', 't2_RRL', 't2_SLSN-I', 't2_SNII', + ... 't2_SNIa', 't2_SNIa-91bg', 't2_SNIax', 't2_SNIbc', 't2_TDE', + ... 't2_mu-Lens-Single'] + >>> datatest = datatest.drop(cols, axis=1) + >>> datajoin = datajoin.drop(cols, axis=1) >>> assert_frame_equal(datatest, datajoin, check_dtype=False, check_column_type=False, check_categorical=False) """ spark = init_sparksession( - "science2grb_offline_{}{}{}".format(night[0:4], night[4:6], night[6:8]) + "science2mm_offline_{}{}{}".format(night[0:4], night[4:6], night[6:8]) ) logger = init_logging() low_bound = start_window - TimeDelta(time_window * 24 * 3600, format="sec").jd @@ -218,34 +222,6 @@ def spark_offline( .filter(~col("jd_objectId").startswith("schema_")) # remove key column ) - ztf_alert = ztf_alert.select( - "jd_objectId", - "objectId", - "candid", - "ra", - "dec", - "jd", - "fid", - "rb", - "jdstarthist", - "jdendhist", - "ssdistnr", - "distpsnr1", - "sgscore1", - "neargaia", - "cdsxmatch", - "roid", - "mulens", - "snn_snia_vs_nonia", - "snn_sn_vs_all", - "rf_snia_vs_nonia", - "ndethist", - "drb", - "classtar", - "rf_kn_vs_nonkn", - "tracklet", - ) - ztf_alert = ztf_alert.filter( ztf_alert["jd_objectId"] >= "{}".format(low_bound) ).filter(ztf_alert["jd_objectId"] < "{}".format(start_window)) @@ -271,8 +247,8 @@ def spark_offline( "dec", "ztf_dec" ) - grb_alert = grb_alert.withColumnRenamed("ra", "grb_ra").withColumnRenamed( - "dec", "grb_dec" + grb_alert = grb_alert.withColumnRenamed("ra", "gcn_ra").withColumnRenamed( + "dec", "gcn_dec" ) join_condition = [ @@ -328,11 +304,15 @@ def launch_offline_mode(arguments): ... } ... ) - >>> datajoin = pd.read_parquet("fink_grb/test/test_output/offline").sort_values(["objectId", "triggerId", "grb_ra"]).reset_index(drop=True) - >>> datajoin = datajoin.drop("grb_proba", axis=1) + >>> datajoin = pd.read_parquet("fink_mm/test/test_output/offline").sort_values(["objectId", "triggerId", "gcn_ra"]).reset_index(drop=True) + >>> datatest = pd.read_parquet(offline_data_test).sort_values(["objectId", "triggerId", "gcn_ra"]).reset_index(drop=True) - >>> datatest = pd.read_parquet(join_data_test).sort_values(["objectId", "triggerId", "grb_ra"]).reset_index(drop=True) - >>> datatest = datatest.drop(["delta_mag", "rate", "from_upper", "start_vartime", "diff_vartime", "grb_proba"], axis=1) + >>> cols = ['t2_AGN', 't2_EB', + ... 't2_KN', 't2_M-dwarf', 't2_Mira', 't2_RRL', 't2_SLSN-I', 't2_SNII', + ... 't2_SNIa', 't2_SNIa-91bg', 't2_SNIax', 't2_SNIbc', 't2_TDE', + ... 't2_mu-Lens-Single'] + >>> datatest = datatest.drop(cols, axis=1) + >>> datajoin = datajoin.drop(cols, axis=1) >>> assert_frame_equal(datatest, datajoin, check_dtype=False, check_column_type=False, check_categorical=False) """ @@ -406,14 +386,14 @@ def launch_offline_mode(arguments): stdout, stderr = process.communicate() if process.returncode != 0: # pragma: no cover logger.error( - "Fink_GRB offline crossmatch application has ended with a non-zero returncode.\ + "Fink_MM offline crossmatch application has ended with a non-zero returncode.\ \n\t cause:\n\t\t{}\n\t\t{}".format( stdout, stderr ) ) exit(1) - logger.info("Fink_GRB offline crossmatch application ended normally") + logger.info("Fink_MM offline crossmatch application ended normally") return diff --git a/fink_grb/online/__init__.py b/fink_mm/online/__init__.py similarity index 100% rename from fink_grb/online/__init__.py rename to fink_mm/online/__init__.py diff --git a/fink_grb/online/ztf_join_gcn.py b/fink_mm/online/ztf_join_gcn.py similarity index 85% rename from fink_grb/online/ztf_join_gcn.py rename to fink_mm/online/ztf_join_gcn.py index 7da43918e..43e151f78 100644 --- a/fink_grb/online/ztf_join_gcn.py +++ b/fink_mm/online/ztf_join_gcn.py @@ -9,11 +9,13 @@ from pyspark.sql import functions as F from pyspark.sql.functions import explode, col +# from astropy.time import Time + from fink_utils.science.utils import ang2pix from fink_utils.spark.partitioning import convert_to_datetime from fink_utils.broker.sparkUtils import init_sparksession, connect_to_raw_database -from fink_grb.utils.fun_utils import ( +from fink_mm.utils.fun_utils import ( build_spark_submit, join_post_process, read_and_build_spark_submit, @@ -21,9 +23,9 @@ read_additional_spark_options, read_grb_admin_options, ) -import fink_grb.utils.application as apps -from fink_grb.init import get_config, init_logging, return_verbose_level -from fink_grb.utils.fun_utils import get_pixels +import fink_mm.utils.application as apps +from fink_mm.init import get_config, init_logging, return_verbose_level +from fink_mm.utils.fun_utils import get_pixels def ztf_grb_filter(spark_ztf, ast_dist, pansstar_dist, pansstar_star_score, gaia_dist): @@ -135,6 +137,28 @@ def ztf_grb_filter(spark_ztf, ast_dist, pansstar_dist, pansstar_star_score, gaia # return pd.Series(ipix_disc) +def check_path_exist(spark, path): + """Check we have data for the given night on HDFS + + Parameters + ---------- + path: str + Path on HDFS (file or folder) + + Returns + ---------- + out: bool + """ + # check on hdfs + jvm = spark._jvm + jsc = spark._jsc + fs = jvm.org.apache.hadoop.fs.FileSystem.get(jsc.hadoopConfiguration()) + if fs.exists(jvm.org.apache.hadoop.fs.Path(path)): + return True + else: + return False + + def ztf_join_gcn_stream( ztf_datapath_prefix, gcn_datapath_prefix, @@ -198,13 +222,17 @@ def ztf_join_gcn_stream( ... 4, 100, 5, 5, 2, 0, 5 ... ) - >>> datatest = pd.read_parquet(join_data_test).sort_values(["objectId", "triggerId", "grb_ra"]).reset_index(drop=True) - >>> datajoin = pd.read_parquet(grb_dataoutput + "/online").sort_values(["objectId", "triggerId", "grb_ra"]).reset_index(drop=True) + >>> datatest = pd.read_parquet(join_data_test).sort_values(["objectId", "triggerId", "gcn_ra"]).reset_index(drop=True).sort_index(axis=1) + >>> datajoin = pd.read_parquet(grb_dataoutput + "/online").sort_values(["objectId", "triggerId", "gcn_ra"]).reset_index(drop=True).sort_index(axis=1) + + >>> datatest = datatest.drop("t2", axis=1) + >>> datajoin = datajoin.drop("t2", axis=1) + >>> assert_frame_equal(datatest, datajoin, check_dtype=False, check_column_type=False, check_categorical=False) """ logger = init_logging() spark = init_sparksession( - "science2grb_online_{}{}{}".format(night[0:4], night[4:6], night[6:8]) + "science2mm_online_{}{}{}".format(night[0:4], night[4:6], night[6:8]) ) scidatapath = ztf_datapath_prefix + "/science" @@ -225,6 +253,26 @@ def ztf_join_gcn_stream( gcn_rawdatapath = gcn_datapath_prefix # connection to the gcn stream + # last_time = (Time(f"{night[0:4]}-{night[4:6]}-{night[6:8]}") - 1).strftime("%Y%m%d") + # path_last_night = gcn_rawdatapath + "/year={}/month={}/day={}".format( + # last_time[0:4], last_time[4:6], last_time[6:8] + # ) + # list_path_gcn_data = [ + # gcn_rawdatapath + # + "/year={}/month={}/day={}".format(night[0:4], night[4:6], night[6:8]) + # ] + # if check_path_exist( + # spark, + # path_last_night, + # ): + # list_path_gcn_data.append(path_last_night) + + # print() + # print("----") + # print(list_path_gcn_data) + # print("-----") + # print() + df_grb_stream = connect_to_raw_database( gcn_rawdatapath + "/year={}/month={}/day={}".format(night[0:4], night[4:6], night[6:8]), @@ -258,8 +306,8 @@ def ztf_join_gcn_stream( "ztf_dec", col("candidate.dec") ) - df_grb_stream = df_grb_stream.withColumnRenamed("ra", "grb_ra").withColumnRenamed( - "dec", "grb_dec" + df_grb_stream = df_grb_stream.withColumnRenamed("ra", "gcn_ra").withColumnRenamed( + "dec", "gcn_dec" ) # join the two streams according to the healpix columns. @@ -335,8 +383,11 @@ def launch_joining_stream(arguments): ... "--exit_after" : 100 ... }) - >>> datatest = pd.read_parquet(join_data_test).sort_values(["objectId", "triggerId", "grb_ra"]).reset_index(drop=True) - >>> datajoin = pd.read_parquet("fink_grb/test/test_output/online").sort_values(["objectId", "triggerId", "grb_ra"]).reset_index(drop=True) + >>> datatest = pd.read_parquet(join_data_test).sort_values(["objectId", "triggerId", "gcn_ra"]).reset_index(drop=True).sort_index(axis=1) + >>> datajoin = pd.read_parquet("fink_mm/test/test_output/online").sort_values(["objectId", "triggerId", "gcn_ra"]).reset_index(drop=True).sort_index(axis=1) + + >>> datatest = datatest.drop("t2", axis=1) + >>> datajoin = datajoin.drop("t2", axis=1) >>> assert_frame_equal(datatest, datajoin, check_dtype=False, check_column_type=False, check_categorical=False) """ @@ -408,14 +459,14 @@ def launch_joining_stream(arguments): stdout, stderr = process.communicate() if process.returncode != 0: # pragma: no cover logger.error( - "Fink_GRB joining stream spark application has ended with a non-zero returncode.\ + "Fink_MM joining stream spark application has ended with a non-zero returncode.\ \n\t cause:\n\t\t{}\n\t\t{}".format( stdout, stderr ) ) exit(1) - logger.info("Fink_GRB joining stream spark application ended normally") + logger.info("Fink_MM joining stream spark application ended normally") return diff --git a/fink_grb/test/__init__.py b/fink_mm/test/__init__.py similarity index 100% rename from fink_grb/test/__init__.py rename to fink_mm/test/__init__.py diff --git a/fink_grb/test/display_distribution.py b/fink_mm/test/display_distribution.py similarity index 100% rename from fink_grb/test/display_distribution.py rename to fink_mm/test/display_distribution.py diff --git a/fink_grb/test/hypothesis/__init__.py b/fink_mm/test/hypothesis/__init__.py similarity index 100% rename from fink_grb/test/hypothesis/__init__.py rename to fink_mm/test/hypothesis/__init__.py diff --git a/fink_grb/test/hypothesis/join_schema.py b/fink_mm/test/hypothesis/join_schema.py similarity index 99% rename from fink_grb/test/hypothesis/join_schema.py rename to fink_mm/test/hypothesis/join_schema.py index 3fa2e6f7b..3453d94eb 100644 --- a/fink_grb/test/hypothesis/join_schema.py +++ b/fink_mm/test/hypothesis/join_schema.py @@ -1,6 +1,6 @@ from pandera import DataFrameSchema, Column, Check, Index -# minimum set of column required by fink_grb for candidate field in the ztf alerts +# minimum set of column required by fink_mm for candidate field in the ztf alerts fink_candidate_schema = DataFrameSchema( columns={ "ssdistnr": Column( diff --git a/fink_grb/test/hypothesis/observatory_schema.py b/fink_mm/test/hypothesis/observatory_schema.py similarity index 100% rename from fink_grb/test/hypothesis/observatory_schema.py rename to fink_mm/test/hypothesis/observatory_schema.py diff --git a/fink_grb/test/prep_ztf_data.py b/fink_mm/test/prep_ztf_data.py similarity index 85% rename from fink_grb/test/prep_ztf_data.py rename to fink_mm/test/prep_ztf_data.py index db54eb386..ddbbfda57 100644 --- a/fink_grb/test/prep_ztf_data.py +++ b/fink_mm/test/prep_ztf_data.py @@ -1,6 +1,6 @@ import pandas as pd import numpy as np -import fink_grb.test.utils_integration_test as it +import fink_mm.test.utils_integration_test as it import glob from astropy.time import Time from pathlib import Path @@ -24,7 +24,7 @@ def generate_data_offline(ztf_pdf, gcn_pdf): new_gcn = it.align_ztf_and_gcn(ztf_pdf, gcn_pdf, past_time, random) gcn_past_str = ( - "fink_grb/ci_gcn_test/year={:04d}/month={:02d}/day={:02d}/".format( + "fink_mm/ci_gcn_test/year={:04d}/month={:02d}/day={:02d}/".format( past_time.to_datetime().year, past_time.to_datetime().month, past_time.to_datetime().day, @@ -46,14 +46,14 @@ def dictify(x, type): # If no gcn exist today, create some with the current date today = Time.now() gcn_today_data_path = ( - "fink_grb/ci_gcn_test/year={:04d}/month={:02d}/day={:02d}/".format( + "fink_mm/ci_gcn_test/year={:04d}/month={:02d}/day={:02d}/".format( today.to_datetime().year, today.to_datetime().month, today.to_datetime().day ) ) new_path_gcn_today = Path(gcn_today_data_path) new_path_gcn_today.mkdir(parents=True, exist_ok=True) - path_gcn = glob.glob("fink_grb/ci_gcn_test/*/*/*/*") + path_gcn = glob.glob("fink_mm/ci_gcn_test/*/*/*/*") for p in path_gcn: gcn_pdf = pd.read_parquet(p) gcn_pdf = it.get_xml_notices(gcn_pdf).reset_index(drop=True) @@ -67,7 +67,7 @@ def dictify(x, type): # create fake ztf counterparts for the gcn of the current date path_ztf_raw = ( - "fink_grb/test/test_data/ztf_test/archive/science/year=2019/month=09/day=03/" + "fink_mm/test/test_data/ztf_test/archive/science/year=2019/month=09/day=03/" ) ztf_pdf = pd.read_parquet(path_ztf_raw) @@ -75,12 +75,12 @@ def dictify(x, type): generate_data_offline(ztf_pdf, gcn_pdf) archive_path_ztf_data = Path( - "fink_grb/test/test_data/ztf_test/archive/science/year={:04d}/month={:02d}/day={:02d}/".format( + "fink_mm/test/test_data/ztf_test/archive/science/year={:04d}/month={:02d}/day={:02d}/".format( today.to_datetime().year, today.to_datetime().month, today.to_datetime().day ) ) online_path_ztf_data = Path( - "fink_grb/test/test_data/ztf_test/online/science/year={:04d}/month={:02d}/day={:02d}/".format( + "fink_mm/test/test_data/ztf_test/online/science/year={:04d}/month={:02d}/day={:02d}/".format( today.to_datetime().year, today.to_datetime().month, today.to_datetime().day ) ) diff --git a/fink_grb/test/test_data/683571622_0_test b/fink_mm/test/test_data/683571622_0_test similarity index 100% rename from fink_grb/test/test_data/683571622_0_test rename to fink_mm/test/test_data/683571622_0_test diff --git a/fink_grb/test/test_data/S230518h_0_test b/fink_mm/test/test_data/S230518h_0_test similarity index 100% rename from fink_grb/test/test_data/S230518h_0_test rename to fink_mm/test/test_data/S230518h_0_test diff --git a/fink_grb/test/test_data/VODB/fermi/voevent_number=193.xml b/fink_mm/test/test_data/VODB/fermi/voevent_number=193.xml similarity index 100% rename from fink_grb/test/test_data/VODB/fermi/voevent_number=193.xml rename to fink_mm/test/test_data/VODB/fermi/voevent_number=193.xml diff --git a/fink_grb/test/test_data/VODB/fermi/voevent_number=2750.xml b/fink_mm/test/test_data/VODB/fermi/voevent_number=2750.xml similarity index 100% rename from fink_grb/test/test_data/VODB/fermi/voevent_number=2750.xml rename to fink_mm/test/test_data/VODB/fermi/voevent_number=2750.xml diff --git a/fink_grb/test/test_data/VODB/fermi/voevent_number=2842.xml b/fink_mm/test/test_data/VODB/fermi/voevent_number=2842.xml similarity index 100% rename from fink_grb/test/test_data/VODB/fermi/voevent_number=2842.xml rename to fink_mm/test/test_data/VODB/fermi/voevent_number=2842.xml diff --git a/fink_grb/test/test_data/VODB/fermi/voevent_number=699.xml b/fink_mm/test/test_data/VODB/fermi/voevent_number=699.xml similarity index 100% rename from fink_grb/test/test_data/VODB/fermi/voevent_number=699.xml rename to fink_mm/test/test_data/VODB/fermi/voevent_number=699.xml diff --git a/fink_grb/test/test_data/VODB/icecube/voevent_number=3028.xml b/fink_mm/test/test_data/VODB/icecube/voevent_number=3028.xml similarity index 100% rename from fink_grb/test/test_data/VODB/icecube/voevent_number=3028.xml rename to fink_mm/test/test_data/VODB/icecube/voevent_number=3028.xml diff --git a/fink_grb/test/test_data/VODB/icecube/voevent_number=45412.xml b/fink_mm/test/test_data/VODB/icecube/voevent_number=45412.xml similarity index 100% rename from fink_grb/test/test_data/VODB/icecube/voevent_number=45412.xml rename to fink_mm/test/test_data/VODB/icecube/voevent_number=45412.xml diff --git a/fink_grb/test/test_data/VODB/icecube/voevent_number=825.xml b/fink_mm/test/test_data/VODB/icecube/voevent_number=825.xml similarity index 100% rename from fink_grb/test/test_data/VODB/icecube/voevent_number=825.xml rename to fink_mm/test/test_data/VODB/icecube/voevent_number=825.xml diff --git a/fink_grb/test/test_data/VODB/integral/voevent_number=18790.xml b/fink_mm/test/test_data/VODB/integral/voevent_number=18790.xml similarity index 100% rename from fink_grb/test/test_data/VODB/integral/voevent_number=18790.xml rename to fink_mm/test/test_data/VODB/integral/voevent_number=18790.xml diff --git a/fink_grb/test/test_data/VODB/integral/voevent_number=18791.xml b/fink_mm/test/test_data/VODB/integral/voevent_number=18791.xml similarity index 100% rename from fink_grb/test/test_data/VODB/integral/voevent_number=18791.xml rename to fink_mm/test/test_data/VODB/integral/voevent_number=18791.xml diff --git a/fink_grb/test/test_data/VODB/integral/voevent_number=737.xml b/fink_mm/test/test_data/VODB/integral/voevent_number=737.xml similarity index 100% rename from fink_grb/test/test_data/VODB/integral/voevent_number=737.xml rename to fink_mm/test/test_data/VODB/integral/voevent_number=737.xml diff --git a/fink_grb/test/test_data/VODB/lvk/initial.txt b/fink_mm/test/test_data/VODB/lvk/initial.txt similarity index 100% rename from fink_grb/test/test_data/VODB/lvk/initial.txt rename to fink_mm/test/test_data/VODB/lvk/initial.txt diff --git a/fink_grb/test/test_data/VODB/lvk/test.txt b/fink_mm/test/test_data/VODB/lvk/test.txt similarity index 100% rename from fink_grb/test/test_data/VODB/lvk/test.txt rename to fink_mm/test/test_data/VODB/lvk/test.txt diff --git a/fink_grb/test/test_data/VODB/lvk/update.txt b/fink_mm/test/test_data/VODB/lvk/update.txt similarity index 100% rename from fink_grb/test/test_data/VODB/lvk/update.txt rename to fink_mm/test/test_data/VODB/lvk/update.txt diff --git a/fink_grb/test/test_data/VODB/swift/voevent_number=17834.xml b/fink_mm/test/test_data/VODB/swift/voevent_number=17834.xml similarity index 100% rename from fink_grb/test/test_data/VODB/swift/voevent_number=17834.xml rename to fink_mm/test/test_data/VODB/swift/voevent_number=17834.xml diff --git a/fink_grb/test/test_data/VODB/swift/voevent_number=391.xml b/fink_mm/test/test_data/VODB/swift/voevent_number=391.xml similarity index 100% rename from fink_grb/test/test_data/VODB/swift/voevent_number=391.xml rename to fink_mm/test/test_data/VODB/swift/voevent_number=391.xml diff --git a/fink_grb/test/test_data/VODB/swift/voevent_number=392.xml b/fink_mm/test/test_data/VODB/swift/voevent_number=392.xml similarity index 100% rename from fink_grb/test/test_data/VODB/swift/voevent_number=392.xml rename to fink_mm/test/test_data/VODB/swift/voevent_number=392.xml diff --git a/fink_grb/test/test_data/VODB/swift/voevent_number=393.xml b/fink_mm/test/test_data/VODB/swift/voevent_number=393.xml similarity index 100% rename from fink_grb/test/test_data/VODB/swift/voevent_number=393.xml rename to fink_mm/test/test_data/VODB/swift/voevent_number=393.xml diff --git a/fink_grb/test/test_data/VODB/swift/voevent_number=405.xml b/fink_mm/test/test_data/VODB/swift/voevent_number=405.xml similarity index 100% rename from fink_grb/test/test_data/VODB/swift/voevent_number=405.xml rename to fink_mm/test/test_data/VODB/swift/voevent_number=405.xml diff --git a/fink_grb/test/test_data/VODB/swift/voevent_number=4554.xml b/fink_mm/test/test_data/VODB/swift/voevent_number=4554.xml similarity index 100% rename from fink_grb/test/test_data/VODB/swift/voevent_number=4554.xml rename to fink_mm/test/test_data/VODB/swift/voevent_number=4554.xml diff --git a/fink_grb/test/test_data/VODB/swift/voevent_number=8582.xml b/fink_mm/test/test_data/VODB/swift/voevent_number=8582.xml similarity index 100% rename from fink_grb/test/test_data/VODB/swift/voevent_number=8582.xml rename to fink_mm/test/test_data/VODB/swift/voevent_number=8582.xml diff --git a/fink_mm/test/test_data/distribution_test_data/online/year=2019/month=09/day=03/part-00043-d0a99157-fe17-42a7-a2be-1f56fc5c6571.c000.snappy.parquet b/fink_mm/test/test_data/distribution_test_data/online/year=2019/month=09/day=03/part-00043-d0a99157-fe17-42a7-a2be-1f56fc5c6571.c000.snappy.parquet new file mode 100644 index 0000000000000000000000000000000000000000..7e386d86cb5da9c08bafaaa8d1eff1fc65a3ee98 GIT binary patch literal 16811 zcmc&+4Qw0dc|OXLP1~}qcs}NoD5;{^XQQz#lKP40*|3h3M3I(giMIF`vVqgjv|Mvw;SHgvJkpP&r_7Xvn+4T3ETYSsN&H>Aeu6+^HEXtEidks8C$2Hj#f z%iiz1JKnz)*P=?;>}IAH5-|pB0EW9FF`zYypte^#Mud# zCz-0PESDPYF2Y`B%jjphR6o{55T)TRqP?6(Bei0lB*-4J_b^G3UF24BYtJxosI$2q z=#+A|Tq*(c-GgoLP1&||T=;R{Lpurjg|;5DvwNtE*xF6m3#ojDtpw6!n}c$^+va$# zjeg!n+t*{;eHauW$TkALy5NT(;QlExEbHYw{FR;W{#)s#3%h=?c>5o0c4Bo`7xD5g z`|-Lo{H@n^e-DxM>QMHVr5Ti6p5q`6nT(pLcD192Y?oBq$|i+$PE_nq+Z-3#=*#W& zn_GJB0l`AcruUq0{r#7z(3S^FqX$<$dg0QZ(&(c%?-ZVb`|?|DKl^7#Y4r2YoU42Q zzaJd?=;Al}N~1<~WeR1Nc_4L-qh{*9vb~4gA*;JuS(q;+u^&O*OKtRdJAHl|sB7e9 zfv~>1%$x7L9*)s#KX~BP@U6=JwbOwY&R*-L*AD;u(k~x*l3x3(OaFP{kN41Pq0qO! zF!IpBHKV%nO3E(JLBbeE&D8zBPDR~qNZoXXRq1EJ)G$H#Wzbjs9>q{lncL zzLAFo=k>+Ee4Kn|@T!+FiZ4?ryL^Q}Q_fH~ejH5tR=gVU#U*2*kO1G6# zSl4g1IlkIPUnl8*+6THC`B-o_)m0O5ve`DwWjDK3(&xo=D0=E0ioQ(Ux2JZKd*%JI zjoJeC^LwZD!VeRNw!kojzF>xWL%i0bJdAeagmR@+9pF#x$R6t$Fm;U4gef@uzM^FFtpJy)oWEnjp_mPx3*~&90yFAAMyVp1h zqVL!*CEJd*J@k7Al-S>zWd-^9_>VTnbsK$cKmFojyU1ZK(BM~n9~V=jyj!Alg;s<#Z=0Q0NmB7!5wv@(xLU)RDy(^!loxM({u{sgHhLcF$`_eJ+2t!_ zRry*O)#>!?fa1Ar#rb5pTo5ZT>Dlk1ue!izMm`qYO;rezAvrl5cxf9*FefYtg}UH? z%QFQoS?WF0C|5FkuUOzGD;a&p0CR4W7EuoKCBREb70(j%8^emXb}ZzCrFoVE^{}UY zX>7nKwo%Tt z@y8y*B**xJYoMFvyeDopL=mrG$r`dQVx=900d*~d&wkGxQyjQGo#86Uc|MuNTEd~> zr8fG55&FX?z=cM379bYNs?JES{pEL05&qj(uAN|)e;EDf$|s-neC_WhZ(sS&$6x!4 z(4IS2uJ?R(=12M4SNaFOz4!U^|8vEtu}q=t@){(JanwxXn71u^5!i4d*f(>R|L|6ln@p{V$WCgprd4e={Yz?{p6l z)YdY)nBkM8lOQ_}v?)EO1H@Dhh()%;pwz7)$Q|fL>}7~I@=0HuhYro+xcdAlj=dIR^u2j78D3(vGB_;W!YD~P-)-$3Mx~IRIll%~s#8nPbqX7BO%d6E9U6LBiK;E^$l~r=l4ZmmkiQbH}3fl)?~_mWEx{OE^_7xBHq&>ww@*m&85j_%90zx}?t)t< zzAhBu(&fh28GL>2b8sUvoX6ZV5??Kwgo5L#g4PfHY!U(2}>%bPcLedmYMOY6InziPL5@CZO z;f!R2lLmEAO`8e#Zc5b)KqH)vsDoA6P7ACe+|8(^lo3%J+hCx)i6zCc4F*!)eHk^= z(saQ0mJSU20l)MzLN_$i}ZG(X|jXNW@fxN-+>Rz@HT~FO$@MZ(rh^!&$ zV4=j#AUay5&6YA{c<@P~{5eF|R};7#FJ;ibwZK&}mrIqBr3NaJi;X0_jIN_qwM?m; zD+p-RdkRGq4VRF%$U3i{LvBf4%j1bZ99a zgQa6j@a26Xz;o4vKQb6|hia+GNF^mq=T7ImsZ1kSO2XVgzK{(F!NnAx<^s8THoy-p zrlNCbPBk4}DNp&B96JzRWQAE6W9rg7EnsCp@t;h=+v5d5S4M%w_zM zRfH2ADCbiAL?v+s*7}EBDSy2j@CEr~G*V7Xc&hQJi$n8LLZqDX0UyB&asjr>9M#|iUC(D<{eVY4EAz{@}@S$ak2n@hnP}qB@~?r&eH&y zO?@2D(xtc!sr|^O{GP?T$&}z%YVp{tJ9H0r$_QaT<_>f5#$DxFN<^n|{FwPqhil@$ zL^?Gi#tGQ3Ryhjx@@jfA=t@N+Rms0*apg+!Ts{SR#VRMrxlW{iD!*}Q$iLPx z6>&G^HnZ9N<$m)#1-X`WM;d8{{SS44?%AWux1Z{Hm~e-xcg^#nT)&4rkL>}^htLAV zQzJDHz_FCzIWFZ7FEr~TUr>m*nwp1h@8t+63I z%ZEx(dzX=XR=EY_ttMTbdE4ut^Qe*1!kM_)E@!Sm;6MSNQIeF80n`TdJ zL#S`zY+AS4zlJ%c@rcf#5hu!zA|I3I9GvmstcQHz4oy_5tjmdg0(F}D3@h3*0(HAD zm}mJ2@P%gsoSWhNRZd2OurHvF!RLDK5M0AEM%>#_do@@$S4-vK{Ot>@OrII7Pvw~$ zI=iXP+HT0>tI4RRih1P6tJ94^S3q#0Rs(7>Fg$8VjMt%8!OTG~f*C@simBN#hVh~H z1QS8`zAW624Wc&1Dr#2*>p=9D(ftg%XVCrF9AZ^KZI4OR)(|@)W9UAF`W?Okd@(Hg z8pHh#gvq#PYW@9)JX^q>kx4YzY_-5DG&+gw7+Hn$P6e+YPmT2Vj{sqs@y)DC>!vTh)87wNA2YTp@f#n*LV-5$!I$`oLH6<@{6>1h19c;ur@;4! zR;o$(S#Mq9=Vey#Dl96psq}%q5yNQm37BD^x!#x6t;haRt;h1PP}?T3{LHET1xfyC zo%~+hUsB1x1jt@>?nj~wH`mWJ*H7R@hV{sGUd{6loLENko|+lZ*x!#htL%?iO*WtD z%u-F7pY_g3_+BsSu1QTWmA@%y;wU! zZ^7l^Sjgopnl+Px^(cMq@5gl>sxO&9?Fmue?AVeHU+*t$3XDGxTm<`_?uSqZ zP#@zRdtAgHx3D+%SF>h={u3-?7_@$7M#7(+!-IzXf?V&{o`V?QD&Tnv625G2y}Z!R z1bf929P}XI3j70jOY7kwd-}O5JgXYe`x4g%kbaMk$as=}<`93CfAB6#@_^-=f+nlz zkD(@>xSz35s(8Y!(#c>BIGsYRSzrl#U=QN;(*8BXZy6zLdm#+$e|h~_bG=WDe{5RS z{(+do@juJ35c4=+q4qC)lJfnmC@*%pN?tMkasLd*j}vT)*Jpim626=-kR{dj!K`qP z3&#(l#*wlxj$H@$0jAN!4`8A7`49Tkup)$duf+sh!g^393*BX$xM2i7*fSuscUI!h zGB|I7vzkVb;!V#VI}zed?xl(hft9qBpn8kR(sDIZIl>l;N0O*9-zzoVd&QPK{EKDy TXXG35hxqPMf|!B-rs4kq96kG< literal 0 HcmV?d00001 diff --git a/fink_mm/test/test_data/distribution_test_data/online/year=2019/month=09/day=03/part-00064-73d02dc2-dbe0-4e73-adc6-a480974420d3.c000.snappy.parquet b/fink_mm/test/test_data/distribution_test_data/online/year=2019/month=09/day=03/part-00064-73d02dc2-dbe0-4e73-adc6-a480974420d3.c000.snappy.parquet new file mode 100644 index 0000000000000000000000000000000000000000..c5e3abc7c2ab146fdc6a8cb41f5401fdd51da0c8 GIT binary patch literal 16446 zcmc&+YiuLc6}ES?yWZU_thU%VkkT}T>8iVIY$p!!77Al0b{rCK;@$W$s7+_a@63#i zUx|ZIX;lzE=&GevRaMm%p$fGK3HlISReh`wLi}i}3aPX|s;cb|`lI|nJS2ob>bZC3 zdLDOX1FRjataIm{d(U?s_nv$1nP^wpbkIBEo$%he?A_;eop8C1-#O}Xxws4QV1yTH z9jV#!y1Xa6zSA)5xc8v<;P|ZTW?Pp%!H?FhPTMo>3*WkrpA3gc( zyZ7HiYd|P|4{0F~($;$|8a#Js+ z?;$M&LfU#SOg1k5-yGAI(f*QY4Lv=|ymu7Dr+GMV-d+5Y|NZOVW?qQ0wD^SL_mCC> zA#J_aBL3y$x(ANd3avKZY*(tSw!sTOy`FjZI#7e=<$z$T1`a>4$8q@i(;w02^}9is zhulB^_uaHQWQ5;C#vm}d*Qn0^eax$$($(456tEUAyM2FgGcTQB9ytM8&@3Dn+iG## zYY0g{z0>=5duI@e-$PmmgtYZuqx29+U-22FuNlM93tr}NFGx?baA0gJ{dL|GB%mJh z?+5Tibj51=S=Y(^(CtTG7V*A4rpNgsRi)K#cI2X>E|neMH{8r`{mex_3iUb%TRA-5 zQ6k3(I*F*=2rOKFL@@5Oi#O}=2Z~*ST=6xSmu_M%-GqSAEL%u8-3a92i37knUTn%$ z8|4pRf?xe*k>K%9>k=HT3$0?a%eU*z9$sdTxP2Gh%=6>SH^-4AWN6$aE}l z7_esNrf23tbK#(kT!;HazuvUS_sVB<`3|?6)pEJm)K20{Zr@XG=CxaxXKq2#=`4)g zY^5}9+|94w#k>M%*tuuFSA6tkyR(dp@O#J@A1r;k*C;w{+?OZyrGKz2D&#CvH}B=o zGVe~n@~7E2FyCF1ul?&`&v#eGuH53Qt-Pgt&gN7m6u*azMPPKVMU#KrrfYHtX+q94 zRh2*9$~<-}s6z8`;J&*mAOOVC?mbT~Jpa}Qr+#3g3ZdX=ipL@_y4Rx0nL-M;tS%nwg7FMa`S9Vf2NaJCAY zPLfvT^{@Q+e?PN12MNXRAuVK8($;$|I=wcfd+tzUgKxF!#>DE?Gt5(Gz-Kfc2ky2i z_`Ej}pB+7U2qainHkJC8V&kJj8p%%@7I|L^={`78sw$fsq69)?`#AjMsZD@;(*T?R&UTl-m4;%$KpaYWDs8 z8RoTVurSTZf&cDGq;v1M9-e#W53~}5;`fji0wHa^*P_I8bGi~EO}H zKh4B}_3rWmI(*)+S>%L*&4tHkM#62q7aolX!E;~K(qfu02lXeF>%e)a$ zy_4v*8Z-493 zcMecFq~Aj*dJBQjy%s(GIj`U74kJH+Q{;xh8Gnv3mtvp<&B=kkof2d3kA+?SgDtUA zlzDH!} zF>n}Dn}&#>;hQ#>+Gkp)su^{c-*&_DV<*N>gCATj-^keb*!%wB)Vi3V*P@uA*NT|o z*K(NQ)>@dosf7%;MlswP0yCHr0Q2#$^2{Lb;LPAJ%)`IPGQ+=MGJAFL`y}pO%<%6< z%z&>m%wAkKm?2*Wm_13?nY~JmOz ze??*sY$6X>WAnQU8lR?H1h~cxq`haLfei!;;Y^e&jYxV245nAqX-G3|z`&+D9gR(q2Mlj4UDN1R>VU!f^=2B`M6$p_id#fh<{*kr9Q*@^hbuyM9gXP}vfHiY z7+V#m(-l!!gFbdkd+v&{9BP%WRYF$!T5L^ z>__Bct3qh$elD8|oX@URk)E6^NkSrB66AOrN2*jF!B3Tq*ds;PQ` zZ`$IbWR$rfC~KjVolO(rppe*(EXUAV(`{8$(sFKksiNv&^)XvD(UmByKUiZyqaV4J z5Q<+-CRs>CDuwxc;3L>+wvgaTupeqyl?!~6@8R`h=RXTB=x^~KCvSK2xsPTqSWBBu zGWBV;c9J@}4QN?Re$8eA`E0sUNG$aCcj^3A8Dc?yUnb?orn=wLbv~EUY)ku(m(#HC z%n(Dh=~&H zSEm>SdpQ*lVUI&BRQ+qWu9QrN&!)N@#9b8YY+`~I>$I5hVf-d%LVZjs?n6AyaecY% z{C7>U-yu#xtli;qOCgA_T~XO|vROB;q$E+{VDChFMfL2liw(nw?Ez7i6fTtMUFF=N z#_W~-xKOHdwK#90+pRO%vj6F`#}aQp)Oo1nL*c8&c`Vl_5$BP;LU29=Be0&9x?&Zd zGcupew77W05QhsjL5jc`m@2{g>7?-b_>i%ShIRFFgo z@~UhmlUZ1CjKRa$!(u#A5vNmK?aZcasYFZpTxv%MIm`^)ovDnv~mpjB}QH}?+ zJfD^e?kolEY_f}W7QDga9WvJwLcvlY7v17=>VCBIrzl6-x#`T7n8@@T{W+Ytr{-kw z2%Kk20km(O_N5^}6lZa|H( zU5{`Fg}W%kN5^~$-SObA2fnzF4`f5(ot)|u+r5&5ygezWdR#UhKxYHo*Wv!P-N?hf zKzR(hE5jWQ+DqMny0=3R%Xj!}q=WDLwT12F8rxZkEy3Ll#`f=o9Gpw3h7g*;JXX%n z>@3&Tt0lFPfr<>Qj0zG9TTrWD*P#}{hEb_vd2OC$W2ijAX3%r24A1j3s7$eg$`wmn zK#aD~^D25~(ewN|VpT_Fk4040P%9$y=sAVz9kDumu_F4K$Mp_`$%ax8; zqRxJ)1!ke%MSK>a{m{nq7lgT~sW~95u(8!0ZT+My(1kAFZDx+xal^Cvg4- z_FG|--8)o$%k!HSe5=2(DM-!?_B%HPOC3OCY;^vViof7sZ|pDQJpiV$ESkT%s^PD# z<5fWI7v%axq$AwgY&Px`t2eZIu`5nP4yV-Uco358!R?hKKB#Fx^$b ztVq98a|BQG&pP67@DJW)aIXf7EJKl1^+&jmr|xGglwm#LQt9GS6*%3CO0&Qc_`n{- z^R@j;t=|?xzMor61@Kv#kah|z?0+(UzCS;vu77M=!~TI7h}ZubE5e$`@d}lH;lq>X zHC0~hbc4L=`fmi_e8lS~05-+*%dvG0pTrAfNuzx*D^goSG*lM`WuYIt4xW>2uaBR^ zLR;e>)Tv=c3f1Hq95i7*C{u>&G7j9(10U=e5IVZ1@fTSfH^EtbBWUZ*${#xs)*G(V zE=XwM!UtGsQ(gXv2H(8DQ*7TZHX66{s4zdF72YS*l05u@GW^x}Kz>|Lo^iQW;eQVJ EKcY;8FaQ7m literal 0 HcmV?d00001 diff --git a/fink_mm/test/test_data/distribution_test_data/online/year=2019/month=09/day=03/part-00069-60c7f1c4-fdac-4442-9cb1-9834fb0b6c1e.c000.snappy.parquet b/fink_mm/test/test_data/distribution_test_data/online/year=2019/month=09/day=03/part-00069-60c7f1c4-fdac-4442-9cb1-9834fb0b6c1e.c000.snappy.parquet new file mode 100644 index 0000000000000000000000000000000000000000..b871d35acf4df0d44cd243e6ebef88ae6bb7592a GIT binary patch literal 16500 zcmc&+dyE^$d0(ES6Hh+Hf-2H`_Qk!N?TrFGj^rKBqUzLIl1CotJL-Jl zyQKI$o?$dK+Q10hz;%McC{j1@AAu7f4;VpU_>cJDrhpMBK+wWyU>JrGBx%zcMPM6+ zV5C65ncd|+W>;rL}wow>9OI=ggXC0xt{qEShrYLzwfoN@= zYEl2AQ+fpMsTJBCzSXHz+Z`MQ#f)Fw%lz`oQh|HoVdQ7GYvM?OVqQ9< zn>EoAur+VHyzjY~=l+Cw<1|=9%5acuYmI46z5JNx7xqEXIJ8@9ArP&tQ$ca#oO94$9Ed7 z0iI@Gc6q<*VxImq^YdAxiCK=VPWO8r)O0%On*yxmON&dF0+)h*8@;B6Oj>VR^m}nm z*KexRs+P;embz2F=<+3pW!S90cb55|&w@duGzSb@ zgO)v)wYBg-9kwp=q=<` zl4`N(6`vltm7bu;>(8B@xc=F$`W{* zl;j})U}L(sCQhHaaqIQBe)HWAUk`lzI%$m7gLZ3u5Ex0d81s#gZp_gZ-zggGc=i%= z^HH#a6yhM-)(+F82Xy%S&SuhU9PBpSM|Bo%>r{9g1_KuIHy+dVKT&Emr0s6ASyXbB zGVh=LGv=MkN;E8@8~_fsLOYR>QP4R0sSSaURErfaKdxJG5}(VrdvG|1goMMP81llO zGynN{FocxmfN`)Ppa!-7oo$D3AdcdA)Y80VrdkYnEvy?dU8t5yHgr{&!KV8vpdQ&J45EQ9t2*zts<||P!f|TSS z-_D4c-_Haoda^B6iW2YfQJ(2XCiH5{Bh=a~sO32dM74-Pc}}1Qr4FDxXjf^aAG(z) zbmIk2p}^fpZPcI-f-)TFH84b04%$YF@!)Hu>jzGaiuF^@a;kpnaY%VGpC}Y9Ec>rf zlD3Vh>gt<>v6G&I6%3rll%^pgsQ9MErO-_CRB@vo@<-0BJ$rigJj8*byzZIVncvgn zsf{sXuURo;uO%_#uknnz2E!aqjbywv3ude}1!goQ0p<_C+B2iPlQW~gK#%`o%Z&fR z$sE>2@Q-m9W5$11Vn%$uIh1P$Gv@2S;T+eQ!#YkghjiR!4&ylL{o%aV&A7eqK6D1v zQ|ZYjyf@?9^sDrf^HsS}+}#!%^{ynh=P%EF;mP?Oph0WC#mnX5{Q0?gnU{({kh}Fd zJmGT|Jo<3E(S+s_ZWd9~|I`=ZL6NtOg%?$^W+|m=v0m6V(-gI~JA)>IoA1h1u~ASo z=t|AOkfFS^M4Cy(4qxCqJam6@z9>WYs$526x30%7T$+E%pp03IdF{m~l48=u#JMYt zkiVWf4~nT)&MU7gNkpA)!K=!hell?=o!F&Qtxk->yBL>pn zG?A8>c^2S1O9#Yyz^|A{$RT;OXolBGB&LNqTIAu2A`;ic8ZD|)7a$>Zx&_ZL|No$c zq1GPk5yz8l5ajTci9N81Jz|T^?=nbyl5bJqE*Fsc-hl)*Ff5378)UsIM53B`7T~)y zby)f*VNEOx>fM@)iu5F+MjbJjUQs6@&AbrmmqFcEm1|Qa&Nn{ht z0t-jnBI2V_=4`2PmY2RDx4($^*3$$&j<<^FUpC-Va=j|F_|^a&lA8k_-v0K{s9v$v zt~O*u_4_ZvyV-d-xz4L<7?~FrF9zqIqUdQhoxYmAtQ5HE{fi6e6DX%ymfeKE=<@{2 zF0tV<8)Mlr8!2O2IBeqg5piY6UuD;oXZXfaFh9YveHO;SFBSs7co2N&;cvK~&jxq$ zIT#u(!&msxgjDZx@yt>#km?DmnT{Z@SFcvXLUE96@z6I>Ym^gmvLZ-@dZOAdC!}CS z$ZnxN-9mP^y%u4s;$pfY%9}uA`|5K~j3ky5(am-ulH4oAGkqjyJ?AeVem)Rt7qb3t zE}&p@xglmZpY_*KUqQ~ag$Tmqnp}2wCl|26jmJRu*j{ci9Tek8C$?J12Ad#%hr5eEDmj?`pfsWrH;#kZuZ+h~;H6>G<09 zaBUxsK=KBcP5G{6H>=1_PO8_1c)BD=u@3074Epr)QnQ|4OeZ-7 zUyVnqR9LO|)jobfsv{jVfBS`8IB2v{!%^bluNUGmh-tbq^xLYhjchUC)==DeXT3~( z(Z;VZ7ksIjVi&}wBd*qsIyVJ*Gmx_LX*}o`;=7@>2%2lUuZnV7$}J`asDJa%E_veuS6+<*XE(=0{(K+g_jQ5 zFOcgYb_2+Xby%-XISSTt$|u4ahg_({*KS_PnJ%AA^*G48DA(EK1U1*GIpdSWP09p@ zm_*)(e468idfUbCu5!Oao`PJv$K{d%$ge$7-f{9-FE6L+qRheC2@Q(M-eZ>=#*y27 zqEwf;KxS~8eTV3CSpMSzsqXFayoql&&m_zGC-)vpzWqe|p_~r{Z=2__UVn@{kNg#Y z{lOoC`IPL5RlLtgd^XeOVj)8wF4Tm22=>5K3FhZOSUlvsCZ`8{wjcxc77zIZ&Ph?H zJXcsvHn~8dF4o~(mCa-_E9;IqcpPh3jD;%VVydU^*`zP!a4DZl?Fj*Ay~eUnL5>ro zPAMPQ1^pCkn=>Qm8*IV(yp43kKDJvYW68(qy0DtA#u zcxK=j&++6v^xDX4NxZsUV>tj6r z#6Cp65^}85*?{O{J0IZ=3U^V+kB;XlbjO3c9>n5$-j@vo_i{>1Y}ZN-&g}^)HQ=%_ zAKDw>z7F@V-DV!v1vW1}ZYJ5-Lcn^r2S4Zb2=A4Wd%T+GdnxBd9#VX3%q_ z4A0ReRHoQN<%(nU(MmF};@#}azE}|I5gW@b3 zPd7P<)7J;?8D3u4SXcmVgpKa0@>Zf)DX<1UxQSb=4}QYLdPKZdPsJ~X`>MP!yNgF* zRgq8C57doVmV+1ynUG&x4cA3*!{Ts!q-+_F<6{_)^}%@*>(4&1P*U|@x9A_n^(BM; zJAfSCf!d5Op9yvHS*+UfaQwz_d<+jF#-q@MO?w*zv5oY7Vq?+d|2Q(z;D0P?b+~7< zJ3Y03IlQIfhr_74rWU~r_FhFqoAU@nrWRUQIJcsWSN)AAm0^4ejaSZwhBF%KwIRTG zG=CCo_mk)!$LBZj-#VM_HfTY7 z0B`3YJmk-~>8=WHMfRP$q~WRY*+TLS@xiMM?$zLtH7K$w@dys_l=Y02GR!AjDqT%h zLDCbbGz%g@4y-{uUR}S${B0xT8{BHjhxgKWeXqd6`q#!shvOs4{Ku{}tRKjMc>Zs) zBFuT5uTc3HKD_q4spyM?ZqQem|4kq4k9hw0z^8b8IkKhVYxx3s(&!&7N~^6Q9;yq2 zu|UV6gXaW07~&_e($@S3b!r%qLN&Q22Td3c#+0GDj1xCd;DaLrN{2U9`67$+CIoBf z1a-by<>Mg2e8Y9xl{%WZ@BvZkP>P;w@~s=)V&{U`Y+m3|VSY|6yw52mdH4s)@UO;4 P@`rlqB1LV$|2Ode7WbMl literal 0 HcmV?d00001 diff --git a/fink_grb/test/test_data/gcn_test/raw/year=2019/month=09/day=03/683476673_0 b/fink_mm/test/test_data/gcn_test/raw/year=2019/month=09/day=03/683476673_0 similarity index 100% rename from fink_grb/test/test_data/gcn_test/raw/year=2019/month=09/day=03/683476673_0 rename to fink_mm/test/test_data/gcn_test/raw/year=2019/month=09/day=03/683476673_0 diff --git a/fink_grb/test/test_data/gcn_test/raw/year=2019/month=09/day=03/683482851_0 b/fink_mm/test/test_data/gcn_test/raw/year=2019/month=09/day=03/683482851_0 similarity index 100% rename from fink_grb/test/test_data/gcn_test/raw/year=2019/month=09/day=03/683482851_0 rename to fink_mm/test/test_data/gcn_test/raw/year=2019/month=09/day=03/683482851_0 diff --git a/fink_grb/test/test_data/gcn_test/raw/year=2019/month=09/day=03/683499781_0 b/fink_mm/test/test_data/gcn_test/raw/year=2019/month=09/day=03/683499781_0 similarity index 100% rename from fink_grb/test/test_data/gcn_test/raw/year=2019/month=09/day=03/683499781_0 rename to fink_mm/test/test_data/gcn_test/raw/year=2019/month=09/day=03/683499781_0 diff --git a/fink_grb/test/test_data/gcn_test/raw/year=2019/month=09/day=03/S230518h_0_test b/fink_mm/test/test_data/gcn_test/raw/year=2019/month=09/day=03/S230518h_0_test similarity index 100% rename from fink_grb/test/test_data/gcn_test/raw/year=2019/month=09/day=03/S230518h_0_test rename to fink_mm/test/test_data/gcn_test/raw/year=2019/month=09/day=03/S230518h_0_test diff --git a/fink_grb/test/test_data/gcn_test/raw/year=2019/month=09/day=03/fake_683476673_0 b/fink_mm/test/test_data/gcn_test/raw/year=2019/month=09/day=03/fake_683476673_0 similarity index 100% rename from fink_grb/test/test_data/gcn_test/raw/year=2019/month=09/day=03/fake_683476673_0 rename to fink_mm/test/test_data/gcn_test/raw/year=2019/month=09/day=03/fake_683476673_0 diff --git a/fink_grb/test/test_data/gcn_test/raw/year=2019/month=09/day=03/fake_683482851_0 b/fink_mm/test/test_data/gcn_test/raw/year=2019/month=09/day=03/fake_683482851_0 similarity index 100% rename from fink_grb/test/test_data/gcn_test/raw/year=2019/month=09/day=03/fake_683482851_0 rename to fink_mm/test/test_data/gcn_test/raw/year=2019/month=09/day=03/fake_683482851_0 diff --git a/fink_grb/test/test_data/gcn_test/raw/year=2019/month=09/day=03/fake_683499781_0 b/fink_mm/test/test_data/gcn_test/raw/year=2019/month=09/day=03/fake_683499781_0 similarity index 100% rename from fink_grb/test/test_data/gcn_test/raw/year=2019/month=09/day=03/fake_683499781_0 rename to fink_mm/test/test_data/gcn_test/raw/year=2019/month=09/day=03/fake_683499781_0 diff --git a/fink_grb/test/test_data/grb_prob_test.parquet b/fink_mm/test/test_data/grb_prob_test.parquet similarity index 100% rename from fink_grb/test/test_data/grb_prob_test.parquet rename to fink_mm/test/test_data/grb_prob_test.parquet diff --git a/fink_grb/test/test_data/grb_samples.parquet b/fink_mm/test/test_data/grb_samples.parquet similarity index 100% rename from fink_grb/test/test_data/grb_samples.parquet rename to fink_mm/test/test_data/grb_samples.parquet diff --git a/fink_grb/test/test_data/join_raw_datatest.parquet b/fink_mm/test/test_data/join_raw_datatest.parquet similarity index 100% rename from fink_grb/test/test_data/join_raw_datatest.parquet rename to fink_mm/test/test_data/join_raw_datatest.parquet diff --git a/fink_mm/test/test_data/offline_datatest.parquet b/fink_mm/test/test_data/offline_datatest.parquet new file mode 100644 index 0000000000000000000000000000000000000000..70c952bc4e11885f438bc873010490983221933f GIT binary patch literal 37374 zcmdTt3vgT2m6GGc3OK=}NGPmJjT*~(=r{JO*?ze zyYJ~ey^o#LlE|t-diQt!_uS7p&mCrhR%5gAfhOZo{g82sQMW~>JEq^;s?+HrV{WS> z8Y?Z8<}2!+W9!!LHr$<^lb^gFJ|M=fOdBc=HaKqB?j-I?Cf8f)H?Qs{rk(efA51^4=Cj!KsFnMJ}3`mt+qT~xUAiHnIMnl;`CHL z%9+5+O{UkI$io}Sdo3UjB}5H87bK5B;K9%KU30_PmZAH+KiqTO*{8nsKhL+m-*NVb zS3VScx47->qbGlU>=;bu@|?JvpZio`SwR|CTIWQk-v-FGPU)r zQXWE!*&yIRI+eBB@_2W%D32DDMXFrZ=6?lYeg`kF~Y007;d! z4&$~VoYLT}+V*~Tukls;Gfrmv&;RwO_zMCz8LyF$X%Kpt4MHE3gR&OLzI;1RcFTrM zP2|a|L^)rcNfe6Fd@+?S6ggSHpf|muC*S=Dd1njAS_w@9hblvZ#v#EMZ996;A2#oQ zX?Mr|Q>RV)Kk>;c_W$z>t^4o%<(2!}KW^TC@|#=tw;tKFUtv|vIKteFeNh);N!yA( zxVqhF;MLNUk8w=BtvCHjPkwD8_n1H{A`}@+RVLbuR|y7VXcy!GgjO4kL{dVZYDo#T zG9!BB8c`Qb8R`$8(3{Tc$+KI@@vT)V5I|@ER;7Nkaf?8Gi+LR&_c;A>;(K)4*Fm)z zzhvay`%!V7x-88Wiur|HlBJfiZ2G*OJk?H)wc|=9LQtinYT5x3T(qhr2q1K3Y>-Z| zV;>XIuTPc)%qi16`O&rHfol;lj#5Pl;04w8ZTYcSIv83o6$6DhcS zE>=k9OVQ$7zRa1`U3$}3^yG;Sa#sgZQVmCyrfZDX3N&r9Yy@0AeO>lGTVJAX$vnT4IHfl|tS9eul1Dp{IuV2vnkto5j_A<$jwg@&=+*J9Z+^ey!~q~hr#oE2EsBg8p@-QZ^g(3GT4=3S%;8-kGn|A^* zna|JVC2PdHJILuBz>*T41{PJ8dW<&;GvZHDR|6$Bi`B8q0{^-?EcP2M7WlW@VzH>q zkE=Dhnzb4Ly41V8U=ioeW`6FBrE|&gF&D??ul1&LuoUhgFYm?)qy<$Qohrjs>q#N? z?x(-~%fH5jugV*m4ML0fgi=-3%6NLeOVnJ;{8Y41n4^0rx~TljMjp3;&XjO8psTXr zGX{i>%Js)ufP!&$mYrKTvYfyDhy|i_K!`wOM-XxSX<4w&w<2KUiXq6`Ff`Ej@j=MVwzv)eH=*j*2$&>qFN2m!Dwk;$L4jfJ$>#}N6X{eUVHlV7kd9TY<%tM<7a*QPB>qCy6Ao6 z_LHYxeHyswG!h9n2BF1l5c=X=RMrAtS014I_;oEu$d?A}-korj@TyCk@tdOTTrQd| zPZi>Gd8)A!O{NE$$P-TT1OxM42~`7^D&w%c2)_FA?UH#9JXLuw=?oO6B2X{&z0bNu z3)>h^6qa()Vm!sE=3c$&q@KJvKpq*Wn#&bPsNJH+#1he!6%5#Up#K z`&e=BJFlJI``(>D-n-=sXZI>i3VWKHaU7bISW-8svtH4p)+eW`jp~^}@|FjTN(3SU zsY-thWqbahDBI?lc!|15D%KMR$y>gQ5Np+Hg;Ogq$D%iv>$@52&D^(4(rF@WSi)3A0A#1QapUe*p^fPkO zeszdEb*Mq4oezqnU0ERwBsY~1PU%!$4+Y8lBMlQ)3rT6j%-S zk=qu(cZIgf(_!>ke0NDL)JQ$dk+A3&H{=$wNtVt7x)J&yM1B+kho}Uj0ep2c6hPIC zgQGW#9Bj@{PtCGZrNR_D2b(RZtRe(z5L}Kb^*0ZN>H)cPS@I|mg5ITn%2Yq zGb-NqHRtE}3%^J7rpNT;`?rt}roiXp5Hq?-IlK?SeC|nzLf=$iS&%x)Jeti)tmj;e zJRe)L#H-vfa*v;y&4c@AL`JTpjHsK>(P{E}@)Bp}2N{u>4Mp1&;|WOA@o<{_Hg$<} za3&{mupT%Va7q|>BumcCUg8WqIbY4dL8%5#&yn-&CCGvmz5{NuvR;l!X^RLr#6>5@+FOpVupRknW6L z`5atteNKOgbMfAt5-wV)CZf9>*4T{1qTc)>?3ccHsk3s|mn5thcvk4$5uM$?y<1P7 zyZcgSe}8!_pP@E89~PhPp_QfK2kUzMz=YtZM28z`W-2a6) z*zi%FQ-v0@L3lzW!n0TlgQa}D`=+M{En|aXsJ32fKnlgnM)xFp=X|4Fmd8Luf=KY9R+{yVl*mw2$%wqLa{3zu&PQ?TB_Re)2hnWxvCh8(;~l>qXp%bi0afJY(WIMqF!+)AT84cxVi?{ zJ)s6Cqqw$Qf7Q^@E$t9K73L}DuIZGeZjU~r`_P%TZEfFrzOPP6+obMk(WLH)#-#qK zyERU?ZBP;{scAMUscW_z}Yl>YqkP8kO8fK7{-_Qvdw7 zI_II0y5?ThI2o#8$w#EN`9h@D2|T1`IX9&C`7)$Fxh|yEc`BsVIVPm;IUuBAIU1ya zITW={bs&vOMIen#6(DsF<&%1c+DW}b;iMslE#G^ zNn=8Rq%om7(uhzRX>h2EtTkjsY8YiAH4O}rn#OWSgJLbD{;?3!fLH~odo029*o0{t zF_?~qufo?@%^T+7?;=DC<`24W>rAtWyBr=UOqPI1taJltw9)9GY3!DYaUpebEb zMM#i`%FYEg9iK~35*Bjvr_pZLqJ0!$Z zQ_(5>K@5aaMN3u7nkYhn-LVr0<2mo7uc|3AiuE#lX|~v7=TuO|f2CT&z5*GI11XV- z^)h7B*}14HI!VK72;qfPb&PyNjZs2CONoxJl%WEdb=g!!Nvl`lO!D^&l;~uQGPDxe zu~j;hG^~U=A5&tab!v#zIOGsr2#-oc8BV%th}5j|0yz@j@K++NZj&LtICr3!58SK$ zN|crA+G3Nu#;?RFYpgBi$_x5Ryj4xL#Y&^PN|a)~44=eW{-}w;M~>Ao*2mi*%F0XX z>WHhFYs_)=+v7_7%61t6Yn(v*o8C&qk_H*ZwJKSR=_|2Tx5*H%kyZY6WF=BbLv1la zN`ev_Un#>Rap)>DXLV0smGhuPs%Tb2zXtzbc@bBMx~fTrc8wb4UZ7QCtW?XetyRYO zMywK>P%A^WM%F5CZ;J1`Dp6~6suLyEueh#2jk05%C=%XmU4a&9mpV})B=4+1g|thJ zso>sERY6^;m*HEh_pW}OREb#GUV8$n-U3x6LD8euM8I2|Dv0@78M3vUQmPWTlo%Dw zGSq9;w&JBpEfOT}RbrPKlL0C3@VcWCw^(0$gab}he6+gu=nks0N2_a(j(_t{l|@Ng z9T3CIdg@sDhT5Zqm+Mq9^7XYxNZ*B1#VIz`0WBm~sw3qa>Vi^Tn~>yqJj)|B_h6ZjcB-N zQa*`)5S0t^WYe*HG+)LT&wM!we=c$nF_)70LVAwHTB~(8{4vK)2)qbRUD~+2Yj=0& z9XfL>6AT`jxRJhHV}5md4}Q94atyBT$S`)sIm5UaW`=RiaCJ^+Rr?}V4piFv znHVj@HwT4l_cF{P1LNSA0|LL?Ao!ky|D21H6Wz0uVL!tR&qSfz>&;~sBA$>vYzvfP zgP~%K^`{S|ov~!umybeUZ)R@B%lcBWTq5gDFV1*#-Kp5bIQCgcOe__KTueIN6->q1 z5vXGp`En`l^7eT92kd+~OUrIYcobpqxU!Dhjjv7fZOV@B*RXw&C7-sq7#8q#50~o+|aie zAIwfC2D6JCT$isjnVZi}c8$>XiI8P-BA7~e29PI^M`&Tv=9mTES&sK?Xgrv<*uAW^ zns!l66}r4RM>?7dWfB9~St_f@d}Ly2Hf$4Q2K3l`w?-xcmeC1{KkjjZ{N2mpu3&c@ zXa*Uk6BFI@NCVJ|{DYjWi^-6LM*)wEYcEE^!4$x-OjhK!i4cTh4fU0}b10n23AXNh{U z%Lz@I=4Yv22+l>r0ctm@b|sUAa=yy+d_gppCn>*5S+Tw}$Lk8T_?Nv}zJsUbBIUJ$ zj8>ziGN)=I7qEqZB@VD6Fkh+8l|BmQ@^WI(XN^sS7I^)t*_Abx%VuIQS126t*D{|V z_&VBtL3+z3!2fF7RLEALTTOp!^Zn|63Vdzc7AhxlZcD7o*R3Tp;B`3?_SsKh%KN;WU^zOGV1b|UDD2@vpHHfZm0u)scz8J~2*ZtTs>T;g1K#Gl4_gzMI?o zE7yvk+q*nQ_w2ZjYZP50vB4S6$ExVE(ze3hY=!+0ohy|+K=t`RV~ubx#BA1749*#G zI7h{E^z5NR8=O}h#mR{!*gFR@u%6JhQ*9p9$KEoqceBSP;H+TNl%JxHnjGt~jzvO_ zitbcomknE;g(Tfeso9xqZ>G6U2)e(}b;8EaIptmw)`$e01GzJ9gU)fFyM;)XuLS#~ zY5%O%Ic^OIYl8YYDhGC%OTao3$%dDGfss&POgN8zSagr#Sv>6X2gid>DJ?2Ghq=iw~)xB-W z8VgBiz%!%#ZbD*Pcoziwqxm?0PtqtK5DUXy51c==*21dx#`(=M+*RaZA8UiVCh+y$ z`Iz?k67CO*3*oNFGT3%0mMbnacu#?}F2mU@CDM>hX~{F5i@=%%_f4S+Z}a?}jM}`a z%U&GjxAxrz>nlZ9xGTeZr^;HbJR?i*J}$`LgDgOIa5mud=Z1Smt$U!r0@%YcZxboF ztJGWvWapFG+y}8d{tMMJ&sTK9J)K~2&2Ydy=|F>N9w~<{?!ht5xl`5NF#AjN`=*2A zn&&KwKkIOhP?{S`)2PL+wZ}!8hIlzPl%}y!>ukd2E=N|>p`{oZOr+wu$q&K4tn=|4 zwQH6A1j;QC4g07+Y~1>Rct^))sgKa0tJ1Ev*4D>|zbe_h-?#~|ZETy#LvojXBC)K{kt;ac=4bDTXY#Cyw zMP3%lyCP*DHCxBnE5u;oK3y~RkGTpV3*Ms#GQk+c^AgmDUQm3kJ2K(sW4+4re{v#} zifiw4@hm08(pAnd()@r6qNNDuXiHSmr@ZUKb73WRdO>%%!BkSq zuR@Lq-=?OsHOj@BzIU_FXA5?Sn#{-i(VFD_0l`|9K?le*%I7U01WTaP{zLy81*?I5r|>>F+Zq(4s(rI zW{zPzBanl{Sg`EGa)9wMjBC~fxkX847Ltf&pv=Uv+>5zILyM4G;^yJYu=O z0J%)e0wy?lG55(ifMqw94~_IgCkr40?2ymoW}pm@Q0~R_r=dk4&E-eZLRfae{{S8i zEg>i`rbz)dC?CY~Fbo9x;0xpd9}h>E1s@S7!(?oG_VmH;fD3waL$QF5^aC^izmwrv zW_UQZXAgqIbT0S=gz|`&gO7}hsa-rM54Cq}i3$Dr@e!e{H68~IX2E`dPx!Dxe|~&c zD38o>_(2iy;U2js)G0G4@nKx;P#5Lb&n#8Ok89IENc}DjEnxp0%YwYzR9>7sCGr9t zGfujG;Gh5}-*E?|#$kgqh?QsvP{1$xK^-%2e_>~aIrw&2f6mY({W*a^>|0ptIVcB* zAqCGi#`8DO&k@B5L84D5);jUrrrfKb6asmEycgsR@KE0)es%dE>w|cXg1s5^aF#N( z|4hGEpr2WSJ}|%VOF_S|7tW!L0cvr4h5rj0ALewxL@MK%xgEn)-u!s3g9JZ^u(IDB zwRHO@eBAV4e0e*y4=)oQ=K0a$A7#Vw&z+f=r3!wRSB4*x6s7!grYhhc9qBLg{awt^ z7>}RVA3B<<`3OFX>Thh!9XK>Jj%6mj15^dSF6NMYyxcyyd`aQl4v>4h1%3yH*_HJz z@N41jBA8@}??1!z@%Wt@`a&PfAitG+79i`^%6Ml%HeN0-&f(yAxH|$!{Sy^_eHos9 zL0)oxx%r6vx<|OCp&o(XrPcVw{-|HP+k@w!QeLp{MH#)E5+wG;&2r(+RdzQCK%KcA zC0<{y3{Q_RU!l25-{`8HaPQrX_u1SXC&XC`*eO?`#~J70Ihho;a7nRJd*fUIcOT6) zyT}rd2srb#w;ca%Ds1v4POQNQmYIGu{l)vu9uxdd84+ zDLqv&)Dc%7j^81phZCZN9?pI}T9FKb{#tpG|m<4;N|8z2eF@b)K^&OWoXC;4x@uPh9^a=Fx>$`lt<5Ul4 zaWUToaQ7m8@O((@+8nQ zR)VR3{LU~Vd%!f|NtmB&gfO zgef4tD2|uNTQPq)dC^=>L>I#T-TkZJgEI-^tjb@-WgPL9D)=<)7knn;nMdE01{)OC ze*qtFPni{$3^vK7!J(ZoizX8sziwIWaJx zvKf3Ig{lq(?XTdc=N}gmCD||2_e(PPehI{Dp=kyWWI%cx4_^F-dKM5e@Xf&oSBHiI z&b6{B5Y{7Zh(sT#M6fsR-Ho6x!Zjm)Vdzaa>C7GT(fq9o$>Of~{QNHNg`E!mg`5uh Zst)|$UcmpA&Hsykx^K+rbi+*0`hO!q!At-E literal 0 HcmV?d00001 diff --git a/fink_mm/test/test_data/online/year=2019/month=09/day=03/part-00043-d0a99157-fe17-42a7-a2be-1f56fc5c6571.c000.snappy.parquet b/fink_mm/test/test_data/online/year=2019/month=09/day=03/part-00043-d0a99157-fe17-42a7-a2be-1f56fc5c6571.c000.snappy.parquet new file mode 100644 index 0000000000000000000000000000000000000000..94d2a0dea5880e842c854824dc06fa26ef16b49b GIT binary patch literal 52108 zcmeHwYjm5{m8j%6i4qbTxX37N+Pb;)lsis{9)5`%NNuis&<7f9O>kLD6-M%b-& z4TtN~bvxbx1OqC|y=Q;)`(LyL*Ik<5_mR2ZA364({JsyodVcB}crQF!_tT#> zpC+0)ONDC>*vQ_dH=`A;{C_| zRFK-5VwT&5o z(4}{q^^zWI9aT=t4is$s038=w8ziN(S;QRa34v%+q4K6iOtzWLHWRsFn+V*H}MZHYM$V#GTVuy>vDFZT#GCe`U68ITWw| zwb^3X=W?yL*zf%Emw$zM`+oY%UtO)0ILvXmetosgg6&{gL`0hkDH>DxQiDWqT|OoX z+pTrvNh^8!5+GCCjzR0XWGaMB(q7`_8DdgStmVZBgbtOchZH{i$zu0Xj=!I z2zYwc$hVpshtvjn~M4L_;@lecnZhO4JXXx?1#w1A42S; zCgiQ@V%ccDtOtDuaDKM!o&4-ofOHh zfrd=)?{fcLhZHf#W)aIGFm0-ul+%|>q-;b|#9@!o^8820v5x>P+J+38U!E4gz+^f! zv+u5shko~mrlUHvh&j{%(y|Clo2sVehFuaZTagxiDn6FX=cn>i=ov+4|BO8FXF!p* zIRhZN6t!Bf6xHE<6PE%aPP@as+YbL+jrQg%?RNNgh23t~QI1OuirS1apmn)vd3uke zI2Q_vGn~#Q4-9)LnP<%n=gj23E6B6EQG1M9H3p|kuwpPNOq~AYSAY9;91m5Vi8>@mWe$(e##6>bZ^6W6?rkiaB3wM0wRg9(F*VY1=VqU6+JzYfv-}AHHQHU~nKe znVXu;HNfN06d%nu4$T*e$!sI@?W4t{+F}4ZHwu+(4f<7}Ey+}DF_HUPrQX_{O6MlW z;yjQ?d+HT)!|P^pPYZdh8GCBkrgf>h#u_O*GtB^f-^+j4z6gI!o{QF7EQ`nfxZbkG zbk(=tXSUfFJ3jPYn$^&8=_1gknCdZQ!yNVmRTh!E*pxCA5;VTdIk(h-n-WRB7#+(- z$0?OCU$~`?yx2}&YlAM-HfI3DfLPh|=)~Xt!+ulX+{ve|iXVJpR>mV}PR6UAMT_sUmpGRTA zNiLflJ+YTO@jroBZA%92>k_L_IpmwkbVf(z#2mp|LK%^xSYAqna&j|rYM+Gv=2U(v zJ64*WPBKd=_;FX0<5vq^rfkXp!17d7-3xuqD^anP&KSmKVA=rYkahcI3v61U{3s_r=Nmk8OCNT*Udbe!%Wk9L$hMBbPI zh~+!vk@vjum&d=j-lVnUVvfxsJ|PhCSxQyWaim+CgKolpfKJIy%R1wHC;4$F5TR|! zpnY8;8m#XX$KMMsm)UYP9j!96D%w{S#%FpY3%x0xD9mM}#rOo%W^=jp53o7 z(B+nNaXYlm`lea5$&2Ps?*M>-Xgb>H8+0``JKXLzSF_V)cew$|W2(`D_@kfWu| z-qzOLXlie5Zn3vH?9f)5)85*wws*jb9TFZ|h&EX+V5w>ih|pO5R|X{3Z3OCqQ9jWDUP#u(~X^Tbo4&+dlrOOYZ;Zg^x8}{lFC;z1nwf_tjr`{I5Uy zwC{55Y`q>!*y=GKWKH&_N89lOJFFHV{I#y1pQ8ftxVhnnW^(s1`T5Weo2&s%tIj3n;yJ_VNTGkRL4$@l6ywqfsR!Xtur5&&{{WBU>(h{ zI`VPy?8n|6tnL_-uv$OonkXI*@rJLka*>FMxsV>z%U^^g*r$|X0$UuQ2jG(=NzXyr@Uu>?d1 z@j03sr_Z20x0$W&yKohxynta5OWHO|EkLDi+jjWge)P+sI@2x;FfE$buiJk30sD39 zOypRyOlH`Uh4%APGs$`$;MJ#LiXc6KSy+T%%DRS!>&W2*c`1hE>x0zQFPn8o4=lA{ z#A3@XGhiV`Xn_{UlSM1)X#fHlVnW50-XLMiE*71rTi<`D$wY3S*kP>`Y*&4LN{9rX zFgM(9CU=gLS5sgHr24AXbkVG>^Ydm#a(=euQ)7i(IyyE}7|Tt;gUok$B~2bmSJweT zcH#%vt(wZR^c8qnO27K%oJ7k-gNNS4pdZl9I(cXKYv zCuIvEx0oA_n#qY7@^%TdSZYGm;Jcv3*DXjqz7Vu{GAHQpWKJe6XXnV{a~1QtX7;EK z7t(IOwoJ+8bw%eoE)bMaaO0=Rv!7by6dXHbR*+EF;gL~r(`U%@|7(p?aQu1|1^eZF za5JpnocgSJjni=AMkNg!dv_;hqxqDqD{lXsnY?ntS|{Y;zfclV-(AFZ6;B*dQn79jwv=GhjuG+T=gs7X&#!eNp8A54hz&zML&4o$GCCfErM;VPUh8z6 zx>ZR>{ZOz=PQ_DSG?U|BTK|Vn0!g>rsHPv{8!dG8TZ_! zBxBvMuR~_3j{a{mdG*e%YJe4zU$k*8Mgl7Qbg`6R z2=+IG$lLdt$&2@z|A*c#(iMZe}T}5P>bRezLZ~-=(-3NXd?M{E)A-Cc%;dpu zR0!>r2MnRDkCzIiY#H4B517d<4^#;5i3bhAZBOTl$^00E9^AXcA>aL|A>;-%1Zx4iTcAoGLvIJsu1j{pBRGO zkWD54V!1C*{KQNS|D-~AkNt}wyzLC$STt6c;!DLO?BSeAqi{Fq^6aw}!oK?#hOjrmM#&<& z!28SOo_-DnujeWRdDFieg4{ZrOpi|#!ADWpkY}C;NA!7pu(e$!cq+ znY{VRDoXCr-x@MiJ574=H8{%hnm%7wEUM>E8$w)mn)JkJSdcwkAp&o|Zis;KH0gV| zfBf|dA)h^C2zj|_(j#Zg3y!ar*WUw@kP$XS4TwnC_<-Y|r^)HLbD8{o^mQ6cPS z-&`SVbyQ!`GE{PUE28E@0#|(U(=6!cbP4g7tfzvzs<7PbTJ$*z2+k?(X4&n zJhy8fL_oC%GH4FnN)9!e_F)_sT2iG#im)}eu6z#vjze$|U*F_Qb1-%^@O~WE_79Eowf<7ltG)Gvj1l!>w>D!b^WHP>Bk#{Fw+{xvDLduj7ps`iio`NiC!H)aUES z{hwd!)c)`mLuw7?&5XK7z5rKue4%pcR_u@6@mGMTzKu}MvR{Vt^545v-!!Zk+FQ37 zLR)HSIXypbG z_I($7U^$fDeq%CiHJi*^Ev7p7zYhM_!~ga0e*^sA2>&hcf74dWUff)cp9|O@T#JN* z2G$ z{Fr@gj7Cu2s%!X09l8B0b>xOSkPj<{sAn>_pD(tUY#R#ki6pFPL`>E#AFh*bmgCN$Hrvcj4$k#7xp6Ffx^n+bs{WXelrJ4fx9b&hYXZKP_hdbL|~ZAE<%u} z$S)?4r%M+Ch^Z7mcGO>*229OGkblupEb_>!I@7w_0bC64k;I(jDJfGiVFRf%a9NW;@eCM!O%Gd_$6 zdZI=whE?$_i~OCPB2lK``3)hZ1tdf~=m1yLZ0OkwxV6;V=uro_gbDKL(FeUeN&w9j zo38@|=*a`d3eBc|maN?$FLIOan-{j4vovAXHkfp1TDB>g<*N1@Bet+{*Sdq{;s##! z1p8oxhy6l|xF?e((~XTnCD%qdz8}Y3`}}ePI{ztVd5Yq z7`xq4ndO~UsjSxJlk%vSCB5pnK~kpVr$}kav7V-@#W|1SjD}qj=h}%f!Z8+Q2KT?j zSXdOtrD$1KD%OfwItjPEw20n&j@P9l<2kK37SqeOQvz_5>uU>9Iyq#j}c}@&kyZ zil+)mHI53B8lC?mH9piws(ez9RDS%9RQpUFsnS6?QpMA64^>eQGB=WDti7L9)5R&MO-~wo`NPVCN@xHST03*qMYR zE%>ROc=J9~$(48>N~F?BK0)u@m(m!Zu_9d74~aWVxpaIgk=zNTbEWL`d^v;t0x+g$ zlf`HvT8u*RPwhaw@_IDZO?VASO`0 zll=sk7NV4{f(Q49wKi+%FDJ{$*8pS3F0{a;bQQ1^KNZylC(Bq4ApFjsK13m-${^wV zh!&iXr~*~ynNpO>$xBft``K76I8~+!tc-U`z_N^`K<8sxh%BwjfHFFqWNB3fB>ilv zmNI3Y3h+BK2O52#zn7{7Sx&AlG}&*XYGJA}s|&gOtEXDHimd8FWrwJ=Af#oly%?nO9}NibIWB*qZDrgO~pdq87S3 zx5~iFzg?(>tjJP<1s6Ao`B+P7EybMH=8xqHpTPX3yovG;5o*CJvQ*$J)F$CK{IoD- z8P$adXIZt-ghUl2sdVB&AWeT;qyefSBg+e!)5od#?C-aPzL zn-;d1s)7sCAw69vCoiQ5>YrHC0jl~DHY}k+pwb~!^V?`zEOd+11P$S+xe*Go>?(t2 zzeT2nE2UQ%p!9QMTA<2272vXDDMh=Syo|ufeho|uSov#Vn6Cm}sjwC4u%V0|u{0GR zNmG?%O6g0{srbbNEt#5Z74S-_RQxi67Q7}~1zw`@PoZ7jdVXo(G`pI>(UgMFDEn6% z@6^f_Nbfre+tDCp9)5jdDo+EMd^(no=I1evIXxfE=ci_o@)`Jf2)KP2Qyq>g>{r-# zLU2(OX?f-qt}9x0UTd;#;RXl$My_NRw%C3=w+BB9T#@6r0r-!f{T$cKdB!GSEH~f`GPCSMmmG`(Qvmt z8sHZaOm6h#O2rGUzN-3pML5Q@T}J&kKboexxoWsXvrF)3mH7g)u#o5k8#xjx34Nkt zqxe$XH8>AC&V$ZVKI+L;Vx(ny)CsyWMER*>69ikwXk1ahMZ%FXOdUDr6WKD}di__C z?KiMfGo!g6AJ0X=zPjg=j8+|oR5wqa*=Si@wlTc zvs(PijXGPFvh#|3Jvme9zmA-#?Cpxi!$c;ww4JATy$g1p!iDi669##TXWjG6Pl-%} z4>cKfN_IHn43@%97y}({kk9U!_$u2qup`qEXM&IOgVSJ7z)vz9Gwty#pNjxJp#Wcu zghB@f<_&FdEp&J`?DQ|hoo+T}XSKYH``r_9S8!(37_XX)Q()stQHMJd1K$RGU<02q zG9AknQ=`r~z>)D-Pfu!Mj{~6sjPPr()oW0Rh( zt4)7`9>zj$N4(R+3;L)tRz&7wPDd&c_RL1Y%umtl&zji19P`P-uE<0@hkDEA7&>Du z^YQ#{C-vcQ{(|$V)$7~I=tv~H0z7rgtsXzA?L%G_nhC38s_EIjn&nlL3Fc8T8mb>Kr%njWuQT_*e@&GwAPH7-sW( zeH%DAIMi!rV;|0mR|~HLfw^WQyI0OzLMM6`n11Wb;RXBCD~#_e>O(&4@D!3pa+VFH z-UXa;qpk>0iLn-~4F(^@DU54+GGB|a-Whip%6B<$wd5!13r@vdoRPjY_c|vQ1iP_X z{Amsb<|YI`N5@u;+rbwxj&0Op{*rY-Y$`Dyb43itai%Q}D-)V`wn;i!mBK{4fr)ggwAG%t9~o;a4PN>kYmBav0C!9r)F;NzaB|}*R zU5g5ROLfQDY#0}-jqbpV*AWe%C!zH&y@Kqk~6T??jrOTtf>Su5bH9u*|1hL z19rOr>jVoBC!AmHxmYX^OhvMxi9}~;J`#p!!};|b#9AUe_}`9rHUxGlZrHa30`s{MO$- z8+NoC^03nNh%l^eb!CI|$g`NM3)j40ePV8UG}jGt+DW>`2kT|<4E_S^b0@?`7osEX z5?zDKbjC&Jn+>s3yI!dSfJ&?`0gEmci(IWEW6Hn zU_G^)8-|wR_}+u>{o_NB56y5~+5^k5{sDOBCh@(=1IxI*vxCq`yaSqw<9o+AymR!u z9Tsr8z&PY^cF5tHxORl!59K?2(0&NtJ^1dM^g(;wu;$D4&GtaiDOf}1upzc6?2EUi z_UwTd4>v&*C-EDb7e2J8LH}cs@zxnypW}d9_>2ng9$y#$abCK zl)LpYciYJa`yYRhl+y{jUh{B#&H?ki|;-J%FWUKsLXKJR&FzViK^_Y< zm+ALTmf_FQs>pOXd@6MiTZR~qA7b6zjz|`Cly}d=*piEc+z>zO&P2ek9vw;W;RVzl z*qTsU5cykBwO>%RAHjF2eLt+>xC;r0!-Qe~KmvSIh{w#tP*el?0^@fV*V8AoFVG*r zMr8D(4siho4|@nEur6Q5$1~d2ui!V9l+zo;cfe-?`WoYI(?Dk?g8b}Q0QgN^<@$_Y zy8T7#)0TbnLU}LOGc4fWA;=3tlF18`*Wze+VmcNM6=7^I#9SWQUk8!iCRKgZHhFyy z%AyDSf_Uv(HiYw5M0*pm4?svo93f%WG z2>v^ti;aNT2YgV!DXzj#ZU4|Fb0FI#(CvBHn*d`{3$D@3fFBFI^B%zZcEkrIbAW0F zS|fa+-}psY8JL4HG&BLS4QnN#5*19&*FoX;wJGYuW*_)S^>Ga1r21LlKhQBZ1N@ih z>0dy8G`E53LTORo)%4It%k(%S(+NM!*8-qt4BRZ%k1dA}(yGQM8=6dv1kny=qJhbI zE|4h!eX5bZ_TPDEjz z5q5QByrFC_hM^^9kGTP8SYe)S1`h@yI{|2y-tbeFI4+N*8 z!>s4KvAojBaQaGM>;NCQ@7svNz%=fjIf?b z9M5~H1`WVFl!vCK(ToLfJcpqj-do^->p^}%5%|zS-Gc;zmj)lTA05Fg|1i+gG|;{R z`X=tkvR2SCY!>Q_HC&=x)?s#74_W|eyfuEv%20|3}6=i3M@w^Y= zw~x0g?WtORseH)B^Ij}J)ZeMFFYk0bAHe#q1=yD;lV|7}1^!+H?ExQX5XZFCXfBk6 zxf$3@K^Ikao!N442jVxY!jC#ZRAvwac0GOx<1O!=fc+Y{7ts!5j>_lpaoFA-ReJ&2 z3dNT`smJeDEI&Uyub_{K7Ud_cB#fWvRQ0J6$o$I`wzrb;HR$uCf&<=J zY_ARJ!IlJjN0E^A2eonm2aL}x%wDMdH0X2mtHKLiv!c(&^;7=LwxbduiOe5Rk5BsJ ztt$xn=s;g_3hPJuL5XGfY2+EXD2-2SyoEiQF(?V@q*%0bU-|B2K=c7eC7NL0KaJz>%*Ja zzq>R zpC5K8`9~{B{fPsW!FV0$52OC8>g)0!YzFu%hU2wozzy~*gYC)WTf=`OMLk}3E&%+d zW~`5ha&r`aiiFG_P*v8%?XW#>-pT0i;@Zma9aPns+fDh`hpnb?eD>49myXwNG;_Ul z{ymCjB8u<6aRf`J@(!%DAM1Ol|4)^iAw{(g;+v4S0Px*iZqXj<<4@to!Uu}Oflw(v z?-)jZCl&xb=#lVG^~VJMF@J{&iOhb{uJjLf3=a6x0z5Cbe^{VjnD1-sH|m;aczBXe zk7D0&ZQ%5y{pjNCDtH=sL(=`hzM&C6^`H7?aL|LaabLTNUQ|-K{imkwfW4)AbjaS) zJJOxQ{Ajs8S8$8=l<`ZgA5?y6eZ+zDjtJ$woIfMrZ$vNcFQk{5*WR8oy%YTc{!7h| z{lVS?_U7UcG}Bhb*E_4im$`0;8vD?x(gD7^m-ED$MgBUMgNJoH9MiDZoVGGBlM(pK zbu@|gNS{AcwT6crlL;rpP{utap}d##`2_ePe<%YP015?V*3&Z#8pfC5c}7+6)bvq# zA3Okcd)mtMjrR-q3jEZS*A9Q@@VGBq#&3T{i63oQ?r)%|dFElSdmdvA!unwzec8ZF zPL#iR8DAV-(Z7a|%wEu`D4c(QxUaC!8}bAFLVX&&1lEi1OUXk3{1?<&ko#mAKG)u~ z0(&K{6S8$2e<6lQ?r&=mpFwFYm?YiOBo;UlnNiUe?o;ocU_j|aLFcS zzr0+ZPr$E(dd7Ho2R<_S!3rMOC&p74oF~)D=)s^SX8DBo4jNNd#2;x@+8!+`TRX>^ zoTnc`e|A-UHa`*+1Pf7w={v*|_!z@~7V(c^eLDUeps4}cDl0(S$1jNYMd%!v<3_N2 z%dm*ApEjzYm)cLZHv!x3Lj0QA1%7!v%pX+SFRBW{9t)@h{R8;5sPNO(m(IvSlJoUq zd2c^_ElW=#lgP37gB#ml@U^l25cOZ(f0Q1!UkCZoofpf4e<{LSYW#tHKAg|nB--z; zVX?je|4=nO9S1VOK0Is0bs~OE1+l)sKehf+RTy;+E+oNUPk_&ZsOAtqcSeM_pY<0q zNG_i!mL9qn4dT-iu`p(P0(+P}1v{Mq*A>S}d^dGgqiao@@);7dDmMKdZJvP$+xCH_D_KBzlZZ# zur3?!hP^*n%IC!T@Bs1Z72#jrqtAJJK-7`mF0MrdU(;W-zbX9rQNU$*21Fgjahx02 z1ELR4qP@}Wukyb4Nx?SvL;ZyX0e**|pK5)K})&e2Z@xc&N%4{|wsWu>Q;HdO3tQQ-)_?e<**ZQNhz-epr9>4U6ztyj3PjPOVl3(`JV)P2PLupg^?=& zKGX8@PtO?Q84oT9F%3a}c$_2B{?M0Ss!E;ooFJ7SfC*hTjzc)>I+WkuGJH0^dl5Cc z{iO5%49vB_24V+~W55%b7#D_eEY3zQu)jbzZ68b*m8${C2@~W^(>``0$eVZ`6`P~* zWmP%uDwC~oI+{ONN)~s=r>A#E@sM((a0t1P9Z!Z|Gu~k`{nx(=Ibd42-el_M1|9z& D=`m^{ literal 0 HcmV?d00001 diff --git a/fink_mm/test/test_data/online/year=2019/month=09/day=03/part-00064-73d02dc2-dbe0-4e73-adc6-a480974420d3.c000.snappy.parquet b/fink_mm/test/test_data/online/year=2019/month=09/day=03/part-00064-73d02dc2-dbe0-4e73-adc6-a480974420d3.c000.snappy.parquet new file mode 100644 index 0000000000000000000000000000000000000000..374cc133ca26dbafb8cfa0828fd339b7917c4ccd GIT binary patch literal 51364 zcmeHweSBlpb)e<(%y@FPPq^IH==ojo0wc63~F&7O{q zj>xRrYLCTp6~0*N>FBw%r}qjdc1h2Uo*nxScO2Ry&o?XHpz^K*o$%XbGIe#3Gp7AL zd-e@o(y?QoxqB&|OC`$QWKU;rSMS57-p5U3jC60*;64Q1(bL%hKbOEi9UT;v4~ZCe z@6YZzc3vy(1ATB(n_X$4$LwOyr(z8K*JENsQ%rSS1PY6VOmQqy|Ji1$=U?)`(C?HIJaF$pMI_@^a7%rSS1PY6VOmQtazT=<{ZD~Zvzv_ghX?;+m){$F%E4*T7jw+r;u8W9pQTh4`j1~C^}t2BWT_l0mKU<6vaA;#_yBqLVjx4? zmH~u~$p9p7G=252Kc*!^%rSS1PY6VOmQo=_-V0~{g+xYYF)qXYu}jI(O96Xr69x@$ zjQs=G^t`0s31W`9TYN$w;h!v3jV8TL838G59L+|&cuYnw1=SQqs)-ArDBze*-=yPF%!9eKY8cZ5EErPx;S0l^I;LE zy_Q`7>+s0H;D~Kx$f|?au6m|fdQ{=})UQkU?J5_unM|rEjKKGrdha)p7e7qy`7mN9 zH6d?J7fb3dz@SY1g~d;Ol)M0A*W^QYrfz*&ZwwPln7hR?y&(G9R5|L~I?3Y)CDGrJ zNfg8}>}gZ)^Ct4ZZ;&?*fcR^hF=%{aO8(&0Te?5*+jr)}y*b~{3csl{8jCsRZm}!^ z)26B@dG;d`B|DK4aV%q`eDk-+d%p>!XxlMpePdDp0kESh*Pa}G>}S91JE230m_yAU zEsMaksVY)lJ0y{^7fIol646w#xKyMP&j@g-kIuB)KWwCxzQu1i91&*fsDU36$CU@%u$EG(@SboywgkYyGhRQNt- zllowHI$KzbCU`)a^8dK0_iht;!AkD2BLBq}q^5L{Q}7p{-B|nOht8-mW9|UvNKeQC zvAmS3!uPHriNCv&DZU(w=3^O(F7vvN4w4rKfWz9B4BFqAh(PG)9k-19{M%Y0#2jV?^jS)U zSGhxQ-@lgd-<>Wl<)fA57sMRNOG6;qR23Cp z{#}WRz0~iHu7L9m21wQ&_k5JRc?}SvZOs72#)JSG9cNBQ4Y~t!7~}CthuZ;sft|_TE3-uJIww-C~aPgg~^ZDmtEXNi(@! z*bmSt`DIyWJn1CIoIr%OC4=^LiD;tre8eMJ&s~XRX)PZsCl)Bi%y>TMCMVqIVLY$x zafz1wz$f|~l?0_pUaA4488L|%!+lOq!)}axr-EJ9FYr^4z+yJ6(_bBEl#b12jchzK?yD*jgLch!2R3!Y= zx4)-bSf1H7#jnwRxx>_Z!bIMmATLhrU#F#wQM>fhXRgtrhNJG7D~)wF}z+3lKX-e2CpB7<#-M9G3JLdypD#+^P%&=>qb-0 zQPV`vZaN#i_N|Xy4Dviyg)x6()Wt;$U>mw1#lKWY?S_ACK5k})rlK3hSMt#$tZ3nL ztdOD4pg33c*sg#z@%nOiw^+`+TdEILE(_tR!S)khIPnEs$2+in+xA-@vA}=i!B5mF z40Y$B>EhB#s*49uUDR+52E8x&Pmg8$XVO>x0wPEQLh^6tp8A65nB_ zzo@Fx8$8J!r{llBzo%2MNL|GxVX=~#q?1u{?)O0JrOK+Nba7LA+gMwRAF`hQi7s8o zjnKGGDio6A`*vLNEG{XgqoqPN7F{Vt3rp}I8<<~6cMn)0Hf;pv0?=|>TB7BH zr9#1wm~%<;Omd48b1EwlvzHNr{SxDOS+Cr=Kwi&maY7#BB|>)OD}1WJ1{*ewxPOuS zQ?6kO6#8od5!Vjem}>6AUsO8jO{EXVmn8k&U5N21*;>7z#RYOM4_YiWp=$5~NdEiR zNj$z7w0N-~=41@v~Yg`J>eE??<>;d3T(@^f3Aly^U`BxO62!lv#_%ilO|B99;6 z>SP@K4_Y$f@|~T3`XduL`$t=yj63gElCf>pH7c`IH-EuIzI$S;6LIf6+45ebMcbeO7~Q!B9P*oh2@oDDz=;qXO_5F=Lefq%uCl=V zg}iwZ&=OWF4(;U=(R3J5@dPl91RY5G@s=Lk0q@`Y1| zATNUhf~jI@3AXiPMIJgD79onjC@Q^(z}=@zVQ4^^jA#e^j8{$`1(VJ5HADaO~s1QbhaEVrjnIJDj8LD@|Pbnkz)@vh`>{)0H_~l}% zAgS=luLIPtHwg9YHw>X(Du)VID_fE{e(W1C#d)kj*iZi1CSfbNee};v}K8uRhTrCeM7!5R+eD2a`cdbf|7^j(^KU z-uY&Oh`jP`LqzuGV{6$w?1w1z?$ozU1pCZi7=rD}r($fW54G(0UjUJ38iaQL ze=&r%7op)1glw#kkVO5bPnpQ`Pc;bk&i`r%c1J#y1c>#%eC0bP^5l0Kg!j%_LwNfb zyl5<5TH-6^6dX5P$)a#eD17s*iQId(K@`qCZHU5e2q?rW=`@U^Z15CDQ`9y2IQle< zqfa-8${qjB5S0%KV53^k@TF%=+>>xf_vYst1bOBKLy&t{Q`yWy z8GICl4Y~CNa76!yKG@c70i5_tz*SuMRLn|>s#ytJUbywUCi2q%YeHebe-yqbd|H=@mat?FHUzy0;FEvn;)c_tbAMwZul!Ae2t4{jLj;VcN$>qFtSbF&gODG1!w~XD)10BgqEEPVs!fFy;T#k_cW~4H0bKP>xiq(8R{QOr18;sVSnbWh zYHXSIiC?V4N8Op&O}{Xa*WbMmd_4Z1As^+_H^#}E{x2AubB%LSEgI0bI!A!S6Kflv zT~mt2jLHYzhjH|mjgRuJrSj%acN!_cCMThc+IK$HNxt(bjO(Yp06+Do zt#MS4s{5-)4XHJl=Q8S^zonDhcT3~cZQg(S3?QnzMN?GHiqGBJNnW|Nv$5ToWB=X| z+Iq9%7jJ_o)7u(^ee|=2uytm|&)p6Y3b*TnR457@(=!+;#2jc_$;M@2+K!` z7e8kxIyzHnCN^h}b&|V|q1b4fHvp7RBi6m#kth-V)W1^OqY!fX9Z zFa!+W{du5Jytm_n(P$R-<<_D+#D()wOEgMXCRl*#ZO1#wGk+-1rE1gwlpbCC-YXAv zn0J&C3n|zuh;;Pq`ADa9&B;ePCNNH>rwbB=TSPi~cH;}Xkb}L5cU!P>cufRLm*2R+ zQsD9e;UNcQ5R@$9UId28ED~OxB9Ba$r%MsxVk*Ut1zQXF3GvG8O&Rjbuuuia%3BiS zOa*kMC{B4HDOi%hpmB%nQ5M1{z*j`Wu{1?s8DpaqqxF#wP27Iz{wru7Q=Un|P$CY< zGBJ)wOzVa;49w~1!6z06!-$|!Ct@+Iit!%ucNS@(Ou-ljA!P_8M2rZ4J9l>|qr$@xAw@jR$dc)yK%o*Gtz%EZ`81&f9`ceNW9(5n zZp7Z%nH}jlo?y>-dW1cxj$p7vWdvxlF@2}R6?^{AY~cM=cNLQ3AdE2Gb1+2q*GQYbHh2MRCJ?B zpHxygE0fac3`lxaZvRN>f`cz57`w+(nT?%GsjSvDlk%uHB)#gmD^jN9Pe^IXv74r= z#kq*$jD}qj=hlfb!Z8+Q2KS%ESlDdFEnZnyD)wMmI?g()tUWNV_gpx)668k=EZmM4EhE z5NXGCKBUFh?U1%yheO(RT@7i|buy$K*S(PD-q1oW%&jP-xi^`RX5T(Sns#9bY1SPd zqVVdV_LEkJ-jh~` z#*?;$u9J3zmXmgbev>wZW|OvrPLsBTHk0;+9+UQj29x%L?vi$d){-`dzLGYErZy|y zl(Zy>lr%3glr$dhNtzJDNm>+`NtzRFNm>#@NtzoMNm>!?NM3GQ1S4s7SR!e5tRZP_ z03m5oXGWSBJVq`^ za2IJw#1?5@7#3+fwu&?%V2W&sfF{zMC?(R8&?C~^cp}n@AR+QV(;|OJv%`5vvtx8f zYXfgcOQUH>6GLQ3yF*w=6XQ}y`{PE)mWb;hTPp5?T+p}(@<_wM4y4%;45Ue63FLx> z4Yd0>fA5J(Zx`wAJ$e0y@Pl#Y9m{x|LdpD#4G5bN}Wa=Q4E>!Z% z)p`c|ek!KtQ{`APR*pgO>kp<1P#mru!@6^`?kh(Q-f)A2j1r53;x|~a6$LJeoQ5z$ zN^iIV5ECdr$iASeg(#(~;KAivhX=Lv*OO)BYk;xX3N0`xT?H)7FU54h$uc$q2;Xzl zhbUw;86*UWXu%1IDo}NvDMhKCydGt;Z+>dQsWMeyWxP`YmSwC5x)|3&WNA$Xl+)oP zOKUP9>EoDM%9MF3z~^TUH2Oflim3%zPi`(W*%vUiFjbk&gln>K`Qf_3|JAlsD-V`ZZdfJ z*Y~v0)wxXuUjJmC7P2Bs1r}V~6y{?srL`1u4iA33Q2GSsujftFzkjC%ugFq?Z%~_r zPt$2($}*Y@5dv1V(1b)4B&xFmHY~2>tSnNXkv`B7N9&w1rxuJ_Hp_n^x-F{ZYw2JcN(nm zdyDW5Gc9Z}RRtHOLwdSUPhL+G)E^(y0jl~u8J18X(C84V`D&OJ3*90uK|_cqH$p*{ z-DL3WD_vT+QhJjCN*~(N0#)Xz0M{i;Dcbeq4Fpd1oh&V2<#)6&Uj@8TVJp*NL%Djy z(o}#XO;wU9rLRY);&TI9GBw#M;EhtL_^f~yye3-(UZU|=pgF0CTvEAVXvxVjiqt=6k7 zS6L3i;-Vj+ zWjJSsra2sn{8daGt>;fp2iZIP(pG4`m*ZABs0aVjM&Mss34X)yzhgB#KeQMQ`2eB? z>N|$LdA<_y1P4Pl|4Mu;SdJHb*~zRU?&a<27?h2K7SeG~&=#3rOZ(jRw3oM(yw34l z+_q+o&ku=Z_(auv*y}DsdqWHH`8jN(lAK>FO*px1B+v74Paqx7yUURoDCZfn#yzXp zj)0FpogCv=X`XZ38qf34et`F8S2NI_H6C&dg$4pc2@kf*W#hK->mu`h3(L3iNsk+7 z3M_bYOPN%4oR8=ID-jRSg*Z9KgTc{odD1y9OV}&{DNa4o)0V}taeK(PV-o~h$7tMCzePfkI!qlo=acz5-g^DlknK0H zQ!C+upHCDbU|;Rk6r)v#XZCEu7OW=oZcBV%e8|u?w8FL*0@c`jvJeT48;?8MGVS7D zA#593&(16I_2f*Y|2lG}vbUQW50kn0`gWe;bph-=g$v_DE(G$F$lI&TPl+sp549Mw zNp?75^H)MP7z3?#kk9dzM4Rmz*pcOkEy*YNz%tkq@RJP3OiLoq7a~AU(8HG_!Qk9< z)zAjFLWfsFHg_#yv$HWfujOUJZC^+X_*cTlc(r7l0vlI}S?#$v_%`4J8~BWo<#@iF z4%^lMN5*44J!y$O_5@2X)~Dn1?t-Bls>|s4w5{=ps^GT^e!fnhf-M%tCOuo%n*Ibm zj0f%3#F&E@^igN5h*aY?YdRTntVTl2Ptohome{=l^T|R3k%dG7^_I;sbjDidcipu1?N-k>)XZHd?ddKJax;pkDt``A+O2_=I3bpoANm*FQX@qI0uEfDA-3se+uV# zK}!t!!k-Jo;{|_`>7;TFYv9M^z#kt3J+_2x86)3VQ$~-Et*|ozx3@OS=J)zGa4|6B zv#_xb=fv&8YtFMaXk_>5dF$weug3ISXAUpeAD=M3Z>kUZkkwI28Od2bn7#lw6~Y4% zpc2B)R6#(G;~z)-&HdF_&)l*_-A7~qWbZO~_1s0ntXUHlpJd)5{N zKS#$_Wh~%}7_V)#V*Zl1!rD}_8Xt%lj^$vtDv^Qlm1LfcS9<*y$8`pGxfBceE%O87 z8d@+JGPI}XKR*S3+;|NQ#(r^)Mh`z+hgruS;5tmr$9(ur&6N{*SUEYTU-=hL^t{SY*F2J~6gSpHKJ{#E%psliH zaONqG7hv7lNSA>wnCHS=7x^28wQEY(I%7k9T~fOyr}9lVH8zxCy&Tp~4Ci2q{I>Nu zt{Zw_U1a^WLRhC>gE?$q33b<)Z@9)Rj+feXLWO>9W(@#!!d34zx!0lPnKFM|g7uqx z1pM-3Xq*Q-ZrIOide&L*QSJ|E?Ta#h6S_j@6i$5~FJ!gD`b2a@U*Ysn4Uq6G)+g?gsvA;E(0 z4t)1>upjDNbVJ#cV-7Y=Gq}H+!1rMX{7kGuqSFT(v|I?^ozw8{S{;RL*Hs7Xr;c;8 z&{6{5C-8kLGXwe1498_1unp^;hIeie-}@Y}jq6(tKqHA!Xexp4qZxST==%t4;BuY} z7n$c?&?#~eU|Kl(_YSp9*> zJYR^71$fv4E`z*QFwa{7rU0H?#^@0lNnNU@wMGj=}yqZ^u63tO$S2 z1wRx%_CC^szY=_u8P0)g4!Cbx1^BQRA6W()hw=@1IjC-UA`A`kD2aYCGY9fD2 zs`g8&_9OT%weN;K9D6AV>o6fWKad396xL%_;wY+~Vu|tF%S}uQ?F;mKun`&ks6(8` z%EK9g1+2^0@o|KQrxg6gl5%?e_zw6iKwsm!+cMCZiy%KoYXHB`Td&XfrQ2V$K5f}m z70NrgiCF>vQ9)i1l1yHhyq3eEh2?lCScb8^6d!QV{<;q7?Nil9ZIjn`pe#DTFGyBm zFpk2$Njd?3U)TowbsjhN6|U7T$E_n4pL@j0P391v#bMFDFr%aHe@X?eJ;OYK%1;Pz z669(D_7h-V$qnma_OvhONc+Ho;@`yNK!3k2SKfqvlwd4^aeo}nR^Yjh0Qm2GAwCac z?{Pu>zJv-twf#ez%z5-`0Y4Ua=bXd(7Q_c7bDC-fS|fa+zxYF0 z=~;s@G`Ik=4SOZQ3KdMwHA>-64lC-zVITOD>f;#1N%ix-5+n%SkuP*F2zS4BRT#k8gwz(yGQMA6!h%`_T^OVxGk+=gC!ozl$n- z)b@|EDmH-g2YW4s=Z@fyNBDLZ)Tj1Ou)kFGW%dtAbHJQBfcBzpCt@(q2n~$mdPCh_ z%tA}d9&^*svYbC^{lej!3lG3r?mUXBYXtG1oD}K{{p-O(vi_m=+~Z%0&9a^!$MQB8 z)THGzn*c4Ffp^z5j^8|v=N_sSOZYy58gnEABdlW)$8#UmplNuA z^3c>WnlTTK=PE6UYxJ0)KQ+cOZe_rNN)tkB(rLdlu;Fn;zMOeNNhfu9G_=(S%8T{;IaPh^1i5|J9nVWsE&yE<_$k_FAVgtbQFhiD&nFT7NM=N7Pu22E z_1X)R@A)obPx(jr9j=a4u0M&(Jjr{GABe0X|Rw$Fy{~5X{5e3~Z*Liz>U$ zY`Jq3@mp2lN1Y%lGYA5^9)E=KmbWj!c?~>^Xn`?D<@2~WY_DI{o`<$V@ug4d@%u2A zug+E#^fA$*{G^qH@e_SKx@QHx1DqqI&%^I)e2b)c>bS&XA&72kV=VR|EKVuU)i<`uJ1$@z7j())TBGs@7Tbcj6w7Sj+1W)t8^*rjhQaAa`{Cs* zDtH=sL(<*;$(eaK^`9nJaL|LajB7+iFDj|r{!`O7XBip~&sc_h^Wz1~kJamQCA(-( z8Nbx}LFJd$M;tikyinfBxpM;kM)cDDLVB5b^-a|2U6>N^UvGZw_WR~6gXI}$X1I>8 zb5(^ebKMX%_D82mbNslEbHoQl{@ONzhjmA-%W&45wlckv6Zp%G_KEgLpFdQ!W@oI6 zNgJ%8WE>Ttypwae1o$FF)<4o#@FFF!YX)b`l!4I=AiDxaGk!)lz^|m zPhEK(b&t(vT=_bFM{-L1Xv=bc14YeIg|qHOTx$^a4~yu_dR7Xe{3Yu6;^>P0HT=o! z1)YjQ`~$4}3g^5b-!moDr>mF1dhvZddGLV$f;tOwpQ^*>M*22kuf%;qwvXd3#SzIP z!$YEcBvkM<{s4B}m|u5q5}O*TEG}7=w4< zPbNQD!2##QcnX7aa8VPpT*CV(T~k)9KhmnSJz7+@caAkV#}q96Syc7e{76s` zEJP8e@35Z0#~J>sh<_aG)A47Hrg~_rtN?8ve_*{YLg&aFH;?6qW<`A6v{4Pc)PAzF z3D~w5@#`BA_~mdge^70|s456&ET9tf58yYX!cSLU#v-*8=kj5B=M?;GNKZ1CEU@(l zJGNhQ4YU3b^X}E5#ABjU&tW2e4H-_(f6 z{3F~m;>PMR2XlBHF&GR$pF6z$V6PYI^_hHY>SOGg6$D)^fIqWw+bSHpnI>H2e;Q+mF&>;B)-(k9 z;c<>g`$J!TsVa@pI6*2u028__gF`s$I+R~u9X=c1ors#;e$x4W4(3{51F?fMxZnv) zObA0c7H1wgbDJdX&*ZgpgvUq<-$3OqqAO||4pYG_G;sVzH52c=fnE(I) literal 0 HcmV?d00001 diff --git a/fink_mm/test/test_data/online/year=2019/month=09/day=03/part-00069-60c7f1c4-fdac-4442-9cb1-9834fb0b6c1e.c000.snappy.parquet b/fink_mm/test/test_data/online/year=2019/month=09/day=03/part-00069-60c7f1c4-fdac-4442-9cb1-9834fb0b6c1e.c000.snappy.parquet new file mode 100644 index 0000000000000000000000000000000000000000..0e6d6beffae02ae323a27a43b24a9a4026ddca46 GIT binary patch literal 51400 zcmeHwYj~UGm7wG}iIXO6$|9q**-psNxAZZ=l6;B9L$~=PS+X2kzQmR#b-P~OE%}pd zbg?Zl>}Dv#q!cKVc3FmpN&93NmZ6j~3`5&Ag$$)^Hp8$i+Y6@W*6= z_q~>@M7(!h@2rGATfFP&&TECgYK*NNVhL-9Sfb~;ylYjKLXLnrKds<70m}Bh_ z%OWsss)~}gw@Z|4MM@IMxQvw7wvlgc15&i@7__cS$}Y?0BbGL*79D#~6XSIDr>GV{dx;6n>}-U8Tbn=oit7yFM| zE)z9jt926q_qj&ZtnW8ny$O=7_@zScx!sce-=4`AONDYSnP;L>?Y!Ac-m{XkRvZGP z7Ig9GurWscDM}@)j|6Ic0BMY%&val2_EfgwL%x*Pz-f1T9>>;Q3ASTLo zbaDEGFP*ey`(eqq|Rksxv$KoBVI93=pJk%>afjLH(BNL|ypg+!cU_!)ABx zx52-jy|&)#Y&Q6Jt<7fB(Tpn$idu~_moVxg6 zOn!bg&I8hv|HsUo_nOIZ2YJ(u{1;o0n$ks1VYUF;k&o@So`$~%U-{eJr_~rzy`UxN z6D@XLHfCOgmohB`11;)7q8{+FzTP@_TKU?YiN; zr+)gzw}11L-rcG55bGuU_ro9j?C1rGcaDbdT0CZ>@F=$$s4u#MI zbojijBlKbpWg9IcFpEA*sqk9s3{d3Tub1%Oo+>QlX3L9*%Z z0IW@g=#goALClf7HUy$gRZ(%z=Orq3Qh$7Q37m2;NVGc`AxHlOdFgXNh_*EY7;6&( zXqZkvG;7cun8O&4Pa?OJsVYJqa!G{jNMuqe9Xjsi$Xf@64pBB{0Ag)APJZmd9moG* zvq{?_Vve;#d_o}Nvy`f$mP9R_3Mt*o1mI?&uW$AtOxV@=pl)atpFzw6|~^Ix$}aVJTy#B`H_ruG_1>W z#YE)EZ!BqDr*=%W((lgpt7UIBOgHT?x0}cB_ zLzOG7I1S%+&mV4U8^Kk(gPS*PJNlTd_K&>y#TucZjvPQJEG#A4d4SlSDJ65#6X^fJ z&hNE#o@yhf{w2Bl3rN2{R9yvA7$!hOJDMR6Oji00qm1kglI2d(6aRXTrA;tQ?S%zl zfs$FKXJ^UrSx|n%rgU-BapH9F!ycd?(Ookg!O>8g3Mq1Y?~Y4e!>&SVwwTXEXP1hz z`2~284Z63+$j_oRR}NSqHf;pv643HUN}}cC#eCk7nBz(E>%;~p=1fK+W+x*C`z6Zr zvR?V#9636-!3lYpmk8OCEAz=b8*`XveRiJQJ6|^i3jH;Mh-*i0OtrS-H|nqSrqYKe z79{=Mo{#cL+4{Vo#d&f(4_YiWp=$6FNdAd}#N&^G7SHDe9iGq2q~+KRCGu`@gHv#FML|K^pi4%;L(Am-r43HNsT-999Fq6I z!&UM@WrGv&Vl>DQpVQ$hh-sX7cK>jZVhH|6WT*OumhC-`!?%{@-nM zGEUyBBxBQ*XHaIT9{NKwdHMfu8X zF}_fc@BW0UY5u8Vnjv_+RRx2*LbeK)##2*7% z!m`EHJNfu*Dj6-63(4YadK)ah^O;gP!4Mqp43f7_n8|Y|%%9aeT)1wKNB_hS9KLZ~mo ziH^q~s1xeD4;n&kkI!WsH5#9M5Il+p>xA~=Lx#{c$IHcXt_JSjhs@-IKdlqoqYoQ` z+nvdml7(4VR*b_*q70ve6DRp}a@KCEq43_f%;e>7)rrCrj~JrRF<&l}7K$0R{D3|B z;kV7?wQtu6@!m%bAzlH(n~WA_Q<>6iA(<%0lZjbHC%^t3GkNAabs})zV}=Oqk|9v! z7ZxQQKKEy4^6a103Hh$a4Iy``Aht%6CfZMLswZO;4p_ZNbOCa(u>xA~oj|`#h zL}+-N3(wRE@1g%_2yZuoHye!= z7x;2131($P81&fPlhO5DWDK5r&2JEvcXdrO;Ok6-3HsDu8-i{x&L@{^-G0ko!=eA@>V$UBe>Q}68Js-H zgA1C_>F?K`H zg$;S~MQ}u4)Cb$#y@2O`3b=|Zpo&>ZK{YF3%M4Hc#7yq|N&Qp7bP^XK|(T{i;9UN%HPe-88VFJSojMV(MT{G}mOdXVUm07znRJFueMNfZ~5}o zH_sVDTyvWA=s7bvcCJnY-g?6j0pn@XV{gELvN!64{OX&Akk^_fJ@BTP9DTE1_-`4) z*PkXm@D@OQt4^qA&l^HrZJKoKJos{dS10VJf4xrF%4yPVzlH_g|5+#OlW!ZsUjH=d z=C{q{-2bW*ljq+t#AMymq`TjNIJ5s#j@SGAYK~F5vGPs1QO@$QUU{o7MU!VEEfU~|ym*!YPF!Q^>qyHmV z@Abi|Z@Ko!`>XI%cSd&R?_nbN_m`5NZ+>9NPx*9?arC8sFq3=!p?;1wchtZ0NSl#} ztTAI`qyC2Zfkq{?yUi08!m7n}Twd{KA*p$YWn_t8cgF+*b^ttu{-3_$zJX z&ac!7`;|KkVe8D2@4uss+ND}Q|&d=IrXX(<#TjoxfKOFg58+r5?61Z+ydSm474@!L|>y~1C zE(x0n5tC)dr`x2fk$A1_al$E(ZJr8sdhmEy-jum${tC}#Gi3m0=iOc zr@W99I>}(r=tK4>3*-~vD`MhUnj*Z6u~CZD`t;=^w_mpB8Wc2!gcK4b;(#m@xMG_M|$i!4j2c9gqsef?nFv%jlOD_V`mMK*$rLwKS=OVzEm~rmIoXD_bs- z-i?=Lq%yVD5Gl!Mxk4(TSx=C%3?}?i0mY15O4gY&8)ectt(3Xuq*E%o)~rw}shqY+ z>2zu&y(;&Iq;$dKmlBNqWvR^C-lkMm>#Iq5)IE}3b^I48Q*tn*H02;p)79c!KygOH zE{SvVL>b{2i!y`zPhu?WM&ll^tSc3pv@9LjNyV6PPEUmCkj*yaST`GsFUwzBbXhsA z*fJ#Q5m`i5DY9(D*1Bt^&W_!Cc7MP(=jJ)m^c&$w({E}c&A&md$v2#l*4#kW+?&Cg zdvh0Q%*9)ET%bi7dGi%%{54h0zdnjI|5_)~s_T|6#f?X#`8N`g245o7lFNNalP}$A z&80Y`RhQO~7F{YsT5;(MY3w~M&Alz9!8e?cM&C+88g`KgY1G{xq;c1EkjCAx(V9Cf zNQ3X4AT7KXg0%A92GXi~6-d+XJs?fFtAMoZE`jF9`jckI;*(~_%9EDFvXhp?nv+(< zf|C}-YLk}4Qj?a%I+NDLB9qp{3X|5v@{(4>+L9K>!nQI-wNZhmq$$y)q;Vmlr2e>1 z(tu!2(xeDY(wMMI(v(0V(Hqz+WG}54eGt$5)GIB|xyhu|*w@BmSu}J+v zR-^%uQ)EL#Hj&1JE0LzeAd$uf7LjH|3z1hF76L>X9oIt|9jrr|8-YWb8dgIZ7%S8A z7#7mNz!cK@fDy7G0y@Zs3b-IoT*`n5Qh$^KX>{lUX;3@@xnywytv~~eQ z5T$e#Jh+9czgJ6tEm=ms1{jN>&;pavRlrjGLR1%=EMqNz@M~`R5QU5egM?rZEjS@j z1**m~r6|>sSEEe!%b!|ss!SDF8Sj*UWf`l1F2uADSz3buWpy~o(i#j%`e{rpWy(Ai z;EOW{8hxO@iKzuyOKvPQ+3#O!VX8743%T~omRh)qtj0oR5m8!@Qo0JBO#c5-iXlW@ z=|gOewLy@z-+0sqtjKOK#mXOF)WX;1s}QJDfrKAQ)B=`es36wc$x=L=7OFB&1-MR9 zYrj>f1*^zXL9Lfu{8d3ML@~9&z@(o5)PhvzH5jlWLQxA_ligtO^55FiLRaTD7Xg;7N#ttu@E6J zRSQi>R6(LT+vC9EO3unMR%0ps$v7>rT5^NI;cvicp^2#~V0HEz#O-Q935kt`M1QPJ z3#*pgU}zX^tOHFo}+?^znS(0`X%(?7pZP5B~*7Btnqsb@T+E8*kY;* zE=-5?bfK2KnkJ}!j!Xxr>IciPgbIOrhfvM$h-tCVEz%S;gt&1d6lB>A2G4%COAA*@ zZ!kdVr?<2~m3b<_HOW$nb}e}gfs_4mmKLz`m$fio1-xEiE74&?xq8IXRDdK+Rgx*C zuSTchcL%g&YO+

!nih`vO|cNk-;-F~g@TH8KYTogrG zo_TH0wSD_;GFf+UlapgJ*Rk7Itk11<ybPYSimY`FUH>?;g&^94q$NOrKbW zk5~Ntes2ld>zj+sOk*46#LP-@#LZuq$pXB|S5)+5t`urT7pp zr%`}2!SZKiqeF*RwLCpOfgVac(BdNR z*HQ%Z0QhVR*Ys?~IpRh=7xj_@`JJqS&Qub$yH*eZ!zouT{%qosbdoaTgPZzSHDFhl_aB8hiA@A+!3rKa$Z}kXSmPMHZ;Sw=O-)CnM6Jk8a5txv}IbwzkJxyx0;<- zJwYE|iUfnx6BR=n+zcIF z4mrHlxWmcD?3|XDaj$bO-V;~~8{^fKaSCi)Icj%iW8m9>4{YEwMiyhaQY!3N0UQ~R z_4K4E_ShFJ!dRb*&3N;Ma;Pq&=hHUFC#rzoF8KL6eG0Z%7@PELU32;q^e`55+T%km zUeHIKu_985Iqa!K$h8~^F+WAGKbvCr^2{d-^+e|4dDL4r$IuyTnUCjnI;ana^B0^? zwXSdHqcf4*I`Gsj*E)Vu+lRa=#hIU@?QhEGpuCKpJmMS_=AvL94gD#c;{|O|=!-yh zG8W4R5=_{8=#+ovEd~AlDne_UrQ*3^(Zv*Ei50Bc|*oSlC zR^c`6Tj@2jd$qh(bYis1^jl{RFW8?^VSHa#AMzo)tC%#Bvs^HB32@4Xdm=z3uC-`w zF!(4AVO-Ob`DTpuj(CrueAn_?B|k||U?JYa8R=W^sAH}w*o{{4r#IkRnG^gR9b1*Q zfiGgbw$Y6FOU@2!Q;ABfCt^63gWW1edWM%0IW}JD^W= zo?iU?6#Q}HH8dFe#Wfl|{BRv+6?=f|Fx64!!>?&`~H40ORf7v{Ri-!QCQQ@U0e8*1y4+BG?qZ@R9rp#j=+}DI08l3!C6F;u7E5JASp;2+3Vlm;$I)xJ zF4i2~ff=*G7@P1I&mok19a^3#^VcP3V5^-a=r7n)31nfd%g|=SUeOZR?IP?GRAHU4 z((1Wb%oj*Sa>2R8P_PmS!L#A~dIi>6B0Tut_ING`b}4Syw@Q9Bgr7xNzsW_wFHeMq zd9dS#{j8>EmGvIw{*czbDD&5$D|Alb*7xy3b|)nqC z!QPfX7pNf5Vm*G`^Md_}mBny=80NI|bdL}A%itOO1=i;dSR1WIXPjla2bUdO8Od_x zQP|T{&LtvXg93{&&~4l&96j7y9m!5-Qn)|rw!xQY!3*1>c^B-3`X*)|!G`ZHd=GH2 zAL^d>LfNEi8a7SSxW5|5_X95Y8d-)!_b6=8av^+oPr$opc@VZ;mtC-*I?PQ$OL2T3 z!T0g>VaSJOI4@;Ds?dp0HICchZhZG3 zP;Lpo{E)$6$3a#6;;_G68r{U4#8{!b`+(<$?rz{CH|Rkga{vuK^pQTX`aO{uJ|7*L z>BvMUJRcYg8g;giG9S`5&nt?z9@X` zeWVA!CHN>aoD0_+aNo27@L?}LvIsT~<|GhgfhwD`d2Oh{>UWwLiT0w&25R9NgfOI*64T7*WjpOX zd@6OYwhU`LURdiMwnuWHqr9^MV@o~~bi(@Ca5e&db$BMhhpMPOa5SM@75Q6KwO>@V zAHjF2eJ||cIEx8bhY7*?fdu%bupYA%Ls9h=ij3cWZe&boU!dQIjmYRn9pZd;9?lTV zVO_q4k1Kp&T)}TFDW^Ap?|{!7^fj)#Edrg{2=a5V3h=x9wfc--y8T7#)0RCIp}dfE}YbhL>Ta1N*B^cX_u^t!guN#owE>(ThHhFy)%AyPWf25pHhKq&oED*@)H7_1i6}n z{RG%o^1^zUGc}rZrAEPm;%8*6r@PycEv-X8iZB+zxIYYMEAZUMB>3-qJ~jhl@AE+Y zuDA+6wf#ez%z05y@ zG&l#c4SOZQG8IhDGf3f&9Z=MV!#?nl>f;#1N%eEUf1qP!3HUG3GhRi0^d11!h0>zF ztLdSQ*649W7872WulYdF7`SDuA6p9_q*aYiE;yf<37{R!Mt$=Y&X+9%f9F;BsO=wR zRkR1^56)^7&mF;sNBB+;)Tj1Ou)kFGW%dtAbHSW?675CJPDEjz5$YMn^@f_gn1Ys= zJ?18$WjTM;`h~+c7w&~q2qgniFBTVw{# zq@k@wP=2i6&8g~RC&=x)?s#4t_Wk2hsydd#aXSDj%}( zd=$$c9v@QJmx~?GC$N4`70xBfD8L;O&JyAv^3lO163k>vt7Z^{Eob{L2)!x1RAe;Bln_6Ygbf?*P(+EeZCHA|dM! zYUO-(7@zx?y-@pU(C6q^g{uB#(P!hvDSws+P>GO4<`1aHC;joZCjk0rM_+LP>qo{x ziDmd{X~ow6zEQR9QAN}`A1Gt5VZg5Y0+ z5AT>;98Gh}PG~Bo!bdY+P<%>uSkGGmf2xYFoVN<_yTVu>-o*aZ^~Z620Q`EH|Hv1q z|44oQffecn3}>Oe?rBwfYI~<1uQ@E=t-{|&5tQjazRSkzek@;^vMc#VD@px{1C_ye z?HdoF{;TTi@*ivl_$!9vwQIr&_A86+$>dwZe1 zD;>i6F6#eNC1*%ct%LPV$g2W;r{5{sLw)=y{8(tZH02AH;}!c9`a3Zn=)s7De`-7? z@Q?XBR7hm@i*}`Va&U6OlM&##xkFO|{la`-W4}??TvJo?d?t*2!yN#pAMJ;qv#H=| zlv&rv6lfS`==u3|a_zL{g zmDfS<&{W!!tKoMjtHh7CEcZ81)La!f>t4XM24VlOfWEA6DKE-jyoN81uIOLGM`kbR zR21SLVBJ?Z=MDM3aiKn4y#&^a@2km!5BwL@S&;i=4L&!}wGMkF?h~?o9B(m(NFF-S zC(1`$1z+P2VAl-=boVB)slFOM?gbS-YX5`^f$qL6&Eb+=%znAKF^_=XAoYwfcn3Z* z`N0Y=F|1F=pJ|%vqph+6w0-=7^}Yz5BXisgmhYPq@%7S1HS|*Z$<8KV z+kV8aYe3+a%fkm=?)%{25Vdr&_AHxN)JouL)yw%1ZIOoH8++Cvm_D_lR4fuzu=^C8Q2F5Vf zh#Nxux{6|bfq!cKrK%8iOjeWNuP4CgK~(dIpEE1MJH+}686=la6iXL9iw5h{bFmO+ zyL{cuo`Rjug6oRopTHlD{lWHJ>a%;;2s5ehx+P z&;bFypNo$R?Fsfr<1b;?`4nd|VWmK-zo4qGEAK-%r=jMYb1w<&eQD3A3V&Vx!&+(W zj4jeXCe^ReZ_t0#{wmH816vOuzLhKtr853&^o`0(?OZfBJR+3$bHQ-|{|Tnw)QHIZ zBiu9MhH5bf(|8^+7@UMYclo=)UeDF)Gx^rk$Nurr^Y;*+1^cq0VL1DPr92L-4-c?j zy)OJ~XY@Ju2#7k;>*xAZ@HPEK`K+Cd0zXHE`l9}- z`AL^@RPkh>|F!y{!@_)p`5$V3iTZ;e|MOt)pd|LcFmn08XIfkSX^bJpcyIw&(-7o` z$2lVH4}JNisx(C71gZQ0Oz5&S4&kipP<}^i@Y(q8M%3i?lg|IMFxLVbh#j291y5jN zTo}r+I2*aZ{sP^!eK1{At|lZWOprHC``C#fZ(=+uHb>#hs&d@tOxC@N(ZUVoWNCkV madCeXgOv9QLCAYqcryGn@g9@upZ?X!0h8}bCet`KY5)K7!j;Vc literal 0 HcmV?d00001 diff --git a/fink_grb/test/test_data/voevent_number=8727.xml b/fink_mm/test/test_data/voevent_number=8727.xml similarity index 100% rename from fink_grb/test/test_data/voevent_number=8727.xml rename to fink_mm/test/test_data/voevent_number=8727.xml diff --git a/fink_grb/test/test_data/voevent_number=9897.xml b/fink_mm/test/test_data/voevent_number=9897.xml similarity index 100% rename from fink_grb/test/test_data/voevent_number=9897.xml rename to fink_mm/test/test_data/voevent_number=9897.xml diff --git a/fink_grb/test/test_data/with_hbase/fink-broker_2.11-1.2.jar b/fink_mm/test/test_data/with_hbase/fink-broker_2.11-1.2.jar similarity index 100% rename from fink_grb/test/test_data/with_hbase/fink-broker_2.11-1.2.jar rename to fink_mm/test/test_data/with_hbase/fink-broker_2.11-1.2.jar diff --git a/fink_grb/test/test_data/with_hbase/fink_test.conf b/fink_mm/test/test_data/with_hbase/fink_test.conf similarity index 96% rename from fink_grb/test/test_data/with_hbase/fink_test.conf rename to fink_mm/test/test_data/with_hbase/fink_test.conf index 66d1f5c72..0600453c4 100644 --- a/fink_grb/test/test_data/with_hbase/fink_test.conf +++ b/fink_mm/test/test_data/with_hbase/fink_test.conf @@ -70,8 +70,8 @@ FINK_ALERT_SCHEMA=${FINK_HOME}/schemas/template_schema_ZTF_3p3.avro # in distributed FS (e.g. hdfs:///path/). # Be careful though to have enough disk space! FS_KIND=local -ONLINE_DATA_PREFIX=${FINK_GRB}/fink_grb/test/test_data/ztf_test/online -AGG_DATA_PREFIX=${FINK_GRB}/fink_grb/test/test_data/ztf_test/archive +ONLINE_DATA_PREFIX=${FINK_MM}/fink_mm/test/test_data/ztf_test/online +AGG_DATA_PREFIX=${FINK_MM}/fink_mm/test/test_data/ztf_test/archive # The name of the HBase table SCIENCE_DB_NAME="ztf" diff --git a/fink_grb/test/test_data/with_hbase/hbase-spark-hbase2.2_spark3_scala2.11_hadoop2.7.jar b/fink_mm/test/test_data/with_hbase/hbase-spark-hbase2.2_spark3_scala2.11_hadoop2.7.jar similarity index 100% rename from fink_grb/test/test_data/with_hbase/hbase-spark-hbase2.2_spark3_scala2.11_hadoop2.7.jar rename to fink_mm/test/test_data/with_hbase/hbase-spark-hbase2.2_spark3_scala2.11_hadoop2.7.jar diff --git a/fink_grb/test/test_data/with_hbase/hbase-spark-protocol-shaded-hbase2.2_spark3_scala2.11_hadoop2.7.jar b/fink_mm/test/test_data/with_hbase/hbase-spark-protocol-shaded-hbase2.2_spark3_scala2.11_hadoop2.7.jar similarity index 100% rename from fink_grb/test/test_data/with_hbase/hbase-spark-protocol-shaded-hbase2.2_spark3_scala2.11_hadoop2.7.jar rename to fink_mm/test/test_data/with_hbase/hbase-spark-protocol-shaded-hbase2.2_spark3_scala2.11_hadoop2.7.jar diff --git a/fink_grb/test/test_data/with_hbase/ztf.jd.json b/fink_mm/test/test_data/with_hbase/ztf.jd.json similarity index 100% rename from fink_grb/test/test_data/with_hbase/ztf.jd.json rename to fink_mm/test/test_data/with_hbase/ztf.jd.json diff --git a/fink_grb/test/test_data/ztf_alert_samples_fid_1.parquet b/fink_mm/test/test_data/ztf_alert_samples_fid_1.parquet similarity index 100% rename from fink_grb/test/test_data/ztf_alert_samples_fid_1.parquet rename to fink_mm/test/test_data/ztf_alert_samples_fid_1.parquet diff --git a/fink_grb/test/test_data/ztf_alert_samples_fid_2.parquet b/fink_mm/test/test_data/ztf_alert_samples_fid_2.parquet similarity index 100% rename from fink_grb/test/test_data/ztf_alert_samples_fid_2.parquet rename to fink_mm/test/test_data/ztf_alert_samples_fid_2.parquet diff --git a/fink_grb/test/test_data/ztf_alerts_sample.parquet b/fink_mm/test/test_data/ztf_alerts_sample.parquet similarity index 100% rename from fink_grb/test/test_data/ztf_alerts_sample.parquet rename to fink_mm/test/test_data/ztf_alerts_sample.parquet diff --git a/fink_grb/test/test_data/ztf_test/archive/science/year=2019/month=09/day=03/merge_ztf_data_test.parquet b/fink_mm/test/test_data/ztf_test/archive/science/year=2019/month=09/day=03/merge_ztf_data_test.parquet similarity index 100% rename from fink_grb/test/test_data/ztf_test/archive/science/year=2019/month=09/day=03/merge_ztf_data_test.parquet rename to fink_mm/test/test_data/ztf_test/archive/science/year=2019/month=09/day=03/merge_ztf_data_test.parquet diff --git a/fink_grb/test/test_data/ztf_test/online/raw/year=2019/month=09/day=03/alert1.parquet b/fink_mm/test/test_data/ztf_test/online/raw/year=2019/month=09/day=03/alert1.parquet similarity index 100% rename from fink_grb/test/test_data/ztf_test/online/raw/year=2019/month=09/day=03/alert1.parquet rename to fink_mm/test/test_data/ztf_test/online/raw/year=2019/month=09/day=03/alert1.parquet diff --git a/fink_grb/test/test_data/ztf_test/online/raw/year=2019/month=09/day=03/alert2.parquet b/fink_mm/test/test_data/ztf_test/online/raw/year=2019/month=09/day=03/alert2.parquet similarity index 100% rename from fink_grb/test/test_data/ztf_test/online/raw/year=2019/month=09/day=03/alert2.parquet rename to fink_mm/test/test_data/ztf_test/online/raw/year=2019/month=09/day=03/alert2.parquet diff --git a/fink_grb/test/test_data/ztfxgcn_test/online/year=2023/month=01/day=01/grb_join_output.parquet b/fink_mm/test/test_data/ztfxgcn_test/online/year=2023/month=01/day=01/grb_join_output.parquet similarity index 100% rename from fink_grb/test/test_data/ztfxgcn_test/online/year=2023/month=01/day=01/grb_join_output.parquet rename to fink_mm/test/test_data/ztfxgcn_test/online/year=2023/month=01/day=01/grb_join_output.parquet diff --git a/fink_grb/test/utils_integration_test.py b/fink_mm/test/utils_integration_test.py similarity index 95% rename from fink_grb/test/utils_integration_test.py rename to fink_mm/test/utils_integration_test.py index 565739545..88432f6a3 100644 --- a/fink_grb/test/utils_integration_test.py +++ b/fink_mm/test/utils_integration_test.py @@ -6,8 +6,8 @@ from astropy.time import Time -from fink_grb.utils.fun_utils import get_observatory -from fink_grb.observatory import INSTR_FORMAT +from fink_mm.utils.fun_utils import get_observatory +from fink_mm.observatory import INSTR_FORMAT def get_copy_of_row(pdf: pd.DataFrame, index_row: int) -> dict: @@ -63,7 +63,7 @@ def align_ztf( Examples -------- - >>> path_ztf_raw = ("fink_grb/test/test_data/ztf_test/online/raw/year=2019/month=09/day=03/") + >>> path_ztf_raw = ("fink_mm/test/test_data/ztf_test/online/raw/year=2019/month=09/day=03/") >>> ztf_pdf = pd.read_parquet(path_ztf_raw) >>> random = np.random.default_rng(0) @@ -200,7 +200,7 @@ def align_xml_gcn( Examples -------- - >>> path_gcn = "fink_grb/test/test_data/683571622_0_test" + >>> path_gcn = "fink_mm/test/test_data/683571622_0_test" >>> gcn = pd.read_parquet(path_gcn) >>> trigger_time = '2023-06-06 07:33:30.511' @@ -259,10 +259,10 @@ def align_ztf_and_gcn( Examples -------- >>> today = Time.now() - >>> path_gcn = "fink_grb/test/test_data/683571622_0_test" + >>> path_gcn = "fink_mm/test/test_data/683571622_0_test" >>> gcn = pd.read_parquet(path_gcn) - >>> path_ztf_raw = ("fink_grb/test/test_data/ztf_test/online/raw/year=2019/month=09/day=03/") + >>> path_ztf_raw = ("fink_mm/test/test_data/ztf_test/online/raw/year=2019/month=09/day=03/") >>> ztf_pdf = pd.read_parquet(path_ztf_raw) >>> random = np.random.default_rng(0) diff --git a/fink_grb/utils/__init__.py b/fink_mm/utils/__init__.py similarity index 100% rename from fink_grb/utils/__init__.py rename to fink_mm/utils/__init__.py diff --git a/fink_grb/utils/application.py b/fink_mm/utils/application.py similarity index 95% rename from fink_grb/utils/application.py rename to fink_mm/utils/application.py index 5ee63b96b..fb8531860 100644 --- a/fink_grb/utils/application.py +++ b/fink_mm/utils/application.py @@ -5,10 +5,10 @@ from astropy.time import Time -import fink_grb -import fink_grb.offline.spark_offline as offline -import fink_grb.online.ztf_join_gcn as online -import fink_grb.distribution.distribution as distrib +import fink_mm +import fink_mm.offline.spark_offline as offline +import fink_mm.online.ztf_join_gcn as online +import fink_mm.distribution.distribution as distrib class Application(Flag): @@ -43,7 +43,7 @@ def build_application(self, logger, **kwargs): """ if self == Application.OFFLINE: application = os.path.join( - os.path.dirname(fink_grb.__file__), + os.path.dirname(fink_mm.__file__), "offline", "spark_offline.py prod", ) @@ -75,9 +75,8 @@ def build_application(self, logger, **kwargs): return application elif self == Application.ONLINE: - application = os.path.join( - os.path.dirname(fink_grb.__file__), + os.path.dirname(fink_mm.__file__), "online", "ztf_join_gcn.py prod", ) @@ -101,9 +100,8 @@ def build_application(self, logger, **kwargs): return application elif self == Application.DISTRIBUTION: - application = os.path.join( - os.path.dirname(fink_grb.__file__), + os.path.dirname(fink_mm.__file__), "distribution", "distribution.py prod", ) @@ -127,7 +125,6 @@ def run_application(self): Run the application """ if self == Application.OFFLINE: - hbase_catalog = sys.argv[2] gcn_datapath_prefix = sys.argv[3] grb_datapath_prefix = sys.argv[4] @@ -159,7 +156,6 @@ def run_application(self): ) elif self == Application.ONLINE: - ztf_datapath_prefix = sys.argv[2] gcn_datapath_prefix = sys.argv[3] grb_datapath_prefix = sys.argv[4] @@ -188,7 +184,6 @@ def run_application(self): ) elif self == Application.DISTRIBUTION: - grbdata_path = sys.argv[2] night = sys.argv[3] exit_after = sys.argv[4] diff --git a/fink_grb/utils/fun_utils.py b/fink_mm/utils/fun_utils.py similarity index 92% rename from fink_grb/utils/fun_utils.py rename to fink_mm/utils/fun_utils.py index 81a672bf8..18ac7c803 100644 --- a/fink_grb/utils/fun_utils.py +++ b/fink_mm/utils/fun_utils.py @@ -12,10 +12,10 @@ from fink_filters.classification import extract_fink_classification from fink_utils.spark.utils import concat_col -from fink_grb.observatory import obsname_to_class, INSTR_FORMAT -from fink_grb.observatory.observatory import Observatory -from fink_grb.gcn_stream.gcn_reader import load_voevent_from_file, load_json_from_file -from fink_grb.init import init_logging +from fink_mm.observatory import obsname_to_class, INSTR_FORMAT +from fink_mm.observatory.observatory import Observatory +from fink_mm.gcn_stream.gcn_reader import load_voevent_from_file, load_json_from_file +from fink_mm.init import init_logging # from fink_broker.tracklet_identification import add_tracklet_information @@ -369,7 +369,7 @@ def get_association_proba( >>> sparkDF = spark.read.format('parquet').load(join_data) >>> df_proba = sparkDF.withColumn( - ... "grb_proba", + ... "p_assoc", ... get_association_proba( ... sparkDF["observatory"], ... sparkDF["raw_event"], @@ -379,31 +379,31 @@ def get_association_proba( ... ), ... ) - >>> df_proba.select(["objectId", "triggerId", "grb_proba"]).show() - +------------+---------+---------+ - | objectId|triggerId|grb_proba| - +------------+---------+---------+ - |ZTF19abvxqrw|683482851| -1.0| - |ZTF19aarcrtb|683482851| -1.0| - |ZTF18abrhuke|683482851| -1.0| - |ZTF19aarcsqv|683482851| -1.0| - |ZTF18abrfzni|683482851| -1.0| - |ZTF18abthehu|683482851| -1.0| - |ZTF18abrhqed|683482851| -1.0| - |ZTF18abcjaer|683482851| -1.0| - |ZTF19aarcsra|683482851| -1.0| - |ZTF18abdlhrp|683482851| -1.0| - |ZTF18abrgwwe|683482851| -1.0| - |ZTF19abvxscp|683482851| -1.0| - |ZTF18abthswi|683482851| -1.0| - |ZTF19abvxvpj|683482851| -1.0| - |ZTF19abvxscg|683482851| -1.0| - |ZTF19abrvetd|683482851| -1.0| - |ZTF19abvxwnh|683482851| -1.0| - |ZTF19abvxwnw|683482851| -1.0| - |ZTF19abvxwyv|683482851| -1.0| - |ZTF19abagehm|683482851| -1.0| - +------------+---------+---------+ + >>> df_proba.select(["objectId", "triggerId", "p_assoc"]).show() + +------------+---------+-------+ + | objectId|triggerId|p_assoc| + +------------+---------+-------+ + |ZTF19abvxqrw|683482851| -1.0| + |ZTF19aarcrtb|683482851| -1.0| + |ZTF18abrhuke|683482851| -1.0| + |ZTF19aarcsqv|683482851| -1.0| + |ZTF18abrfzni|683482851| -1.0| + |ZTF18abthehu|683482851| -1.0| + |ZTF18abrhqed|683482851| -1.0| + |ZTF18abcjaer|683482851| -1.0| + |ZTF19aarcsra|683482851| -1.0| + |ZTF18abdlhrp|683482851| -1.0| + |ZTF18abrgwwe|683482851| -1.0| + |ZTF19abvxscp|683482851| -1.0| + |ZTF18abthswi|683482851| -1.0| + |ZTF19abvxvpj|683482851| -1.0| + |ZTF19abvxscg|683482851| -1.0| + |ZTF19abrvetd|683482851| -1.0| + |ZTF19abvxwnh|683482851| -1.0| + |ZTF19abvxwnw|683482851| -1.0| + |ZTF19abvxwyv|683482851| -1.0| + |ZTF19abagehm|683482851| -1.0| + +------------+---------+-------+ only showing top 20 rows """ @@ -720,7 +720,7 @@ def join_post_process(df_grb, with_rate=True, from_hbase=False): Returns ------- df_grb: PySpark DataFrame - df_grb with the output schema, see conf/fink_grb_schema_version_1.0.avsc + df_grb with the output schema, see conf/fink_mm_schema_version_1.0.avsc Examples -------- @@ -773,7 +773,7 @@ def join_post_process(df_grb, with_rate=True, from_hbase=False): # refine the association and compute the serendipitous probability df_grb = df_grb.withColumn( - "grb_proba", + "p_assoc", get_association_proba( df_grb["observatory"], df_grb["raw_event"], @@ -783,6 +783,66 @@ def join_post_process(df_grb, with_rate=True, from_hbase=False): ), ) + fink_added_value = [ + "cdsxmatch", + "DR3Name", + "Plx", + "e_Plx", + "gcvs", + "vsx", + "x3hsp", + "x4lac", + "mangrove", + "roid", + "rf_snia_vs_nonia", + "snn_snia_vs_nonia", + "snn_sn_vs_all", + "mulens", + "nalerthist", + "rf_kn_vs_nonkn", + "t2", + "anomaly_score", + "lc_features_g", + "lc_features_r", + ] + if from_hbase: + fink_added_value = [ + "DR3Name", + "Plx", + "anomaly_score", + "cdsxmatch", + "e_Plx", + "gcvs", + "mangrove_2MASS_name", + "mangrove_HyperLEDA_name", + "mangrove_ang_dist", + "mangrove_lum_dist", + "mulens", + "rf_kn_vs_nonkn", + "rf_snia_vs_nonia", + "roid", + "snn_sn_vs_all", + "snn_snia_vs_nonia", + "t2_AGN", + "t2_EB", + "t2_KN", + "t2_M-dwarf", + "t2_Mira", + "t2_RRL", + "t2_SLSN-I", + "t2_SNII", + "t2_SNIa", + "t2_SNIa-91bg", + "t2_SNIax", + "t2_SNIbc", + "t2_TDE", + "t2_mu-Lens-Single", + "tracklet", + "vsx", + "x3hsp", + "x4lac", + ] + column_to_return = [ "objectId", "candid", @@ -796,13 +856,13 @@ def join_post_process(df_grb, with_rate=True, from_hbase=False): "event", "observatory", "triggerId", - "grb_ra", - "grb_dec", - col("err_arcmin").alias("grb_loc_error"), + "gcn_ra", + "gcn_dec", + col("err_arcmin").alias("gcn_loc_error"), "triggerTimeUTC", - "grb_proba", + "p_assoc", "fink_class", - ] + ] + fink_added_value if with_rate: column_to_return += [ @@ -814,7 +874,7 @@ def join_post_process(df_grb, with_rate=True, from_hbase=False): ] # select a subset of columns before the writing - df_grb = df_grb.select(column_to_return).filter("grb_proba != -1.0") + df_grb = df_grb.select(column_to_return).filter("p_assoc != -1.0") return df_grb @@ -837,7 +897,7 @@ def read_and_build_spark_submit(config, logger): Examples -------- - >>> config = get_config({"--config" : "fink_grb/conf/fink_grb.conf"}) + >>> config = get_config({"--config" : "fink_mm/conf/fink_mm.conf"}) >>> logger = init_logging() >>> read_and_build_spark_submit(config, logger) 'spark-submit --master local[2] --conf spark.mesos.principal= --conf spark.mesos.secret= --conf spark.mesos.role= --conf spark.executorEnv.HOME=/path/to/user/ --driver-memory 4G --executor-memory 8G --conf spark.cores.max=16 --conf spark.executor.cores=8' @@ -904,7 +964,7 @@ def read_prior_params(config, logger): Examples -------- - >>> config = get_config({"--config" : "fink_grb/conf/fink_grb.conf"}) + >>> config = get_config({"--config" : "fink_mm/conf/fink_mm.conf"}) >>> logger = init_logging() >>> read_prior_params(config, logger) ('5', '2', '0', '5') @@ -954,7 +1014,7 @@ def read_additional_spark_options(arguments, config, logger, verbose, is_test): Examples -------- - >>> arguments = {"--config" : "fink_grb/conf/fink_grb.conf"} + >>> arguments = {"--config" : "fink_mm/conf/fink_mm.conf"} >>> config = get_config(arguments) >>> logger = init_logging() >>> res = read_additional_spark_options(arguments, config, logger, True, True) @@ -1065,7 +1125,7 @@ def read_grb_admin_options(arguments, config, logger, is_test=False): Examples -------- >>> arguments = { - ... "--config" : "fink_grb/conf/fink_grb.conf", + ... "--config" : "fink_mm/conf/fink_mm.conf", ... "--night" : "20221014", ... "--exit_after" : "120" ... } @@ -1073,7 +1133,7 @@ def read_grb_admin_options(arguments, config, logger, is_test=False): >>> logger = init_logging() >>> read_grb_admin_options(arguments, config, logger, False) - ('20221014', '120', 'fink_grb/test/test_data/ztf_test/online', 'fink_grb/test/test_data/gcn_test/raw', 'fink_grb/test/test_output', '30', '4', '/home/roman.le-montagner/fink-broker/catalogs_hbase/ztf.jd.json', 7, 'localhost:9092', 'toto', 'tata') + ('20221014', '120', 'fink_mm/test/test_data/ztf_test/online', 'fink_mm/test/test_data/gcn_test/raw', 'fink_mm/test/test_output', '30', '4', '/home/roman.le-montagner/fink-broker/catalogs_hbase/ztf.jd.json', 7, 'localhost:9092', 'toto', 'tata') >>> res = read_grb_admin_options(arguments, config, logger, True) diff --git a/fink_grb/utils/grb_prob.py b/fink_mm/utils/grb_prob.py similarity index 95% rename from fink_grb/utils/grb_prob.py rename to fink_mm/utils/grb_prob.py index 2a599b757..5f849b871 100644 --- a/fink_grb/utils/grb_prob.py +++ b/fink_mm/utils/grb_prob.py @@ -43,10 +43,10 @@ def p_ser_grb_vect( -------- # alerts from the object ZTF21aagwbjr - >>> ztf_alerts = pd.read_parquet("fink_grb/test/test_data/ztf_alerts_sample.parquet") + >>> ztf_alerts = pd.read_parquet("fink_mm/test/test_data/ztf_alerts_sample.parquet") # gcn contains the notice 634112970 (GRB210204270) - >>> grb_alerts = pd.read_parquet("fink_grb/test/test_data/grb_samples.parquet") + >>> grb_alerts = pd.read_parquet("fink_mm/test/test_data/grb_samples.parquet") >>> proba = grb_alerts.apply( ... lambda x: p_ser_grb_vect( @@ -191,8 +191,8 @@ def grb_assoc( ... ]) >>> grb_prob = df_grb.toPandas() - >>> grb_prob.to_parquet("fink_grb/test/test_data/grb_prob_test.parquet") - >>> grb_test = pd.read_parquet("fink_grb/test/test_data/grb_prob_test.parquet") + >>> grb_prob.to_parquet("fink_mm/test/test_data/grb_prob_test.parquet") + >>> grb_test = pd.read_parquet("fink_mm/test/test_data/grb_prob_test.parquet") >>> assert_frame_equal(grb_prob, grb_test) """ grb_proba = np.ones_like(ztf_ra.values, dtype=float) * -1.0 diff --git a/fink_grb/utils/monitoring.py b/fink_mm/utils/monitoring.py similarity index 96% rename from fink_grb/utils/monitoring.py rename to fink_mm/utils/monitoring.py index 3e0d4b179..c115e66a7 100644 --- a/fink_grb/utils/monitoring.py +++ b/fink_mm/utils/monitoring.py @@ -3,10 +3,10 @@ # from terminaltables import DoubleTable, AsciiTable # import datetime # import pytz -# from fink_grb.init import get_config, init_logging +# from fink_mm.init import get_config, init_logging # import pandas as pd # import pyarrow.parquet as pq -# from fink_grb.utils.fun_utils import get_hdfs_connector, return_verbose_level +# from fink_mm.utils.fun_utils import get_hdfs_connector, return_verbose_level def gcn_stream_monitoring(arguments): # pragma: no cover diff --git a/run_test.sh b/run_test.sh index ea55b5ac4..4ace91730 100755 --- a/run_test.sh +++ b/run_test.sh @@ -27,9 +27,9 @@ python -m pip install . # echo "\n" # Run the test suite -for filename in fink_grb/online/*.py +for filename in fink_mm/online/*.py do - if [ $filename != "fink_grb/online/ztf_join_gcn.py" ]; then + if [ $filename != "fink_mm/online/ztf_join_gcn.py" ]; then echo $filename # Run test suite + coverage coverage run \ @@ -38,7 +38,7 @@ do fi done -for filename in fink_grb/utils/*.py +for filename in fink_mm/utils/*.py do echo $filename # Run test suite + coverage @@ -47,25 +47,25 @@ do --rcfile ${ROOTPATH}/.coveragerc $filename done -echo "fink_grb/init.py test" +echo "fink_mm/init.py test" coverage run \ --source=${ROOTPATH} \ - --rcfile=${ROOTPATH}/.coveragerc fink_grb/init.py + --rcfile=${ROOTPATH}/.coveragerc fink_mm/init.py -echo "fink_grb/online/ztf_join_gcn.py test" +echo "fink_mm/online/ztf_join_gcn.py test" coverage run \ --source=${ROOTPATH} \ - --rcfile=${ROOTPATH}/.coveragerc fink_grb/online/ztf_join_gcn.py "test" + --rcfile=${ROOTPATH}/.coveragerc fink_mm/online/ztf_join_gcn.py "test" -echo "fink_grb/offline/spark_offline.py test" +echo "fink_mm/offline/spark_offline.py test" coverage run \ --source=${ROOTPATH} \ - --rcfile=${ROOTPATH}/.coveragerc fink_grb/offline/spark_offline.py "test" + --rcfile=${ROOTPATH}/.coveragerc fink_mm/offline/spark_offline.py "test" -echo "fink_grb/distribution/distribution.py test" +echo "fink_mm/distribution/distribution.py test" coverage run \ --source=${ROOTPATH} \ - --rcfile=${ROOTPATH}/.coveragerc fink_grb/distribution/distribution.py "test" + --rcfile=${ROOTPATH}/.coveragerc fink_mm/distribution/distribution.py "test" coverage combine diff --git a/scheduler/gen_avro_schema.py b/scheduler/gen_avro_schema.py index 57c331f4c..43396faad 100755 --- a/scheduler/gen_avro_schema.py +++ b/scheduler/gen_avro_schema.py @@ -8,7 +8,7 @@ import shutil # This script is used to generate the avro schema used by the distribution -# The schema will be pushed into fink_grb/conf/ and its name will be 'fink_grb_schema_version_*SCHEMA_VERSION*.avro' +# The schema will be pushed into fink_mm/conf/ and its name will be 'fink_mm_schema_version_*SCHEMA_VERSION*.avro' # where *SCHEMA_VERSION* is the schema version SCHEMA_VERSION = "1.2" @@ -75,14 +75,14 @@ def readschemafromavrofile(fn: str) -> dict: spark = SparkSession.builder.appName("fink_test").config(conf=conf).getOrCreate() -df = spark.read.format("parquet").load("fink_grb/test/test_data/online/") +df = spark.read.format("parquet").load("fink_mm/test/test_data/online/") df = df.drop("year").drop("month").drop("day").drop("timestamp") df.show() -path_for_avro = "fink_grb_schema_version_{}.avro".format(SCHEMA_VERSION) +path_for_avro = "fink_mm_schema_version_{}.avro".format(SCHEMA_VERSION) df.coalesce(1).limit(1).write.format("avro").save(path_for_avro) @@ -98,7 +98,7 @@ def readschemafromavrofile(fn: str) -> dict: shutil.copy( "/tmp/{}".format(path_for_avro.replace(".avro", ".avsc")), - "fink_grb/conf/{}".format(path_for_avro.replace(".avro", ".avsc")), + "fink_mm/conf/{}".format(path_for_avro.replace(".avro", ".avsc")), ) schema = json.loads(schema_) diff --git a/scheduler/grb2distribution.sh b/scheduler/grb2distribution.sh index e7d5010ae..c90dbbda7 100755 --- a/scheduler/grb2distribution.sh +++ b/scheduler/grb2distribution.sh @@ -8,8 +8,8 @@ YEAR=${NIGHT:0:4} MONTH=${NIGHT:4:2} DAY=${NIGHT:6:2} -FINK_GRB_CONFIG="path/config/file" -FINK_GRB_LOG="path/to/store/log" +FINK_MM_CONFIG="path/config/file" +FINK_MM_LOG="path/to/store/log" # same entries as in the .conf @@ -29,7 +29,7 @@ while true; do # LEASETIME must be computed by taking the difference between now and max end LEASETIME=$(( `date +'%s' -d '17:00 today'` - `date +'%s' -d 'now'` )) - nohup fink_grb distribute --config ${FINK_GRB_CONFIG} --night ${NIGHT} --exit_after ${LEASETIME} > ${FINK_GRB_LOG}/grb_distribution_${YEAR}${MONTH}${DAY}.log + nohup fink_mm distribute --config ${FINK_MM_CONFIG} --night ${NIGHT} --exit_after ${LEASETIME} > ${FINK_MM_LOG}/grb_distribution_${YEAR}${MONTH}${DAY}.log break fi if [[ $LEASETIME -le 0 ]] diff --git a/scheduler/science2grb.sh b/scheduler/science2grb.sh index d7eaa130a..f53e29693 100755 --- a/scheduler/science2grb.sh +++ b/scheduler/science2grb.sh @@ -8,8 +8,8 @@ YEAR=${NIGHT:0:4} MONTH=${NIGHT:4:2} DAY=${NIGHT:6:2} -FINK_GRB_CONFIG="path/config/file" -FINK_GRB_LOG="path/to/store/log" +FINK_MM_CONFIG="path/config/file" +FINK_MM_LOG="path/to/store/log" # same entries as in the .conf @@ -34,7 +34,7 @@ while true; do # LEASETIME must be computed by taking the difference between now and max end LEASETIME=$(( `date +'%s' -d '17:00 today'` - `date +'%s' -d 'now'` )) - nohup fink_grb join_stream online --config ${FINK_GRB_CONFIG} --night ${NIGHT} --exit_after ${LEASETIME} > ${FINK_GRB_LOG}/fink_grb_online_${YEAR}${MONTH}${DAY}.log 2>&1 + nohup fink_mm join_stream online --config ${FINK_MM_CONFIG} --night ${NIGHT} --exit_after ${LEASETIME} > ${FINK_MM_LOG}/fink_mm_online_${YEAR}${MONTH}${DAY}.log 2>&1 break fi fi diff --git a/scheduler/science2grb_offline.sh b/scheduler/science2grb_offline.sh index 7a5dada20..e17e24c54 100755 --- a/scheduler/science2grb_offline.sh +++ b/scheduler/science2grb_offline.sh @@ -3,12 +3,12 @@ source ~/.bash_profile # Config -FINK_GRB_CONFIG="path/to/conf/fink_grb.conf" -FINK_GRB_LOG="path/to/store/log" +FINK_MM_CONFIG="path/to/conf/fink_mm.conf" +FINK_MM_LOG="path/to/store/log" NIGHT=`date +"%Y%m%d" -d "now"` YEAR=${NIGHT:0:4} MONTH=${NIGHT:4:2} DAY=${NIGHT:6:2} -nohup fink_grb join_stream offline --night=${NIGHT} --config ${FINK_GRB_CONFIG} > ${FINK_GRB_LOG}/fink_grb_offline_${YEAR}${MONTH}${DAY}.log & +nohup fink_mm join_stream offline --night=${NIGHT} --config ${FINK_MM_CONFIG} > ${FINK_MM_LOG}/fink_mm_offline_${YEAR}${MONTH}${DAY}.log & diff --git a/setup.py b/setup.py index cdaf698e2..7cb435323 100644 --- a/setup.py +++ b/setup.py @@ -3,35 +3,35 @@ from glob import glob import os.path as path from setuptools import setup, find_packages -import fink_grb +import fink_mm with open("README.md", "r") as fh: long_description = fh.read() setup( - name="fink-grb", - version=fink_grb.__version__, - description="Correlation of Fink alerts with notices from gamma ray monitors", + name="fink-mm", + version=fink_mm.__version__, + description="Correlation of Fink alerts with notices from the General Coordinate Network (GCN)", author="Roman Le Montagner", author_email="roman.le-montagner@ijclab.in2p3.fr", - url="https://github.com/FusRoman/Fink_GRB", + url="https://github.com/FusRoman/Fink_MM", long_description=long_description, long_description_content_type="text/markdown", packages=find_packages(), package_data={ - "fink_grb": [ - "conf/fink_grb.conf", - "conf/fink_grb_schema_version_{}.avsc".format( - fink_grb.__distribution_schema_version__ + "fink_mm": [ + "conf/fink_mm.conf", + "conf/fink_mm_schema_version_{}.avsc".format( + fink_mm.__distribution_schema_version__ ), "observatory/observatory_schema_version_{}.json".format( - fink_grb.__observatory_schema_version__ + fink_mm.__observatory_schema_version__ ), ] + [ - path.relpath(el, start="fink_grb") - for el in glob("fink_grb/observatory/*/*.json") + path.relpath(el, start="fink_mm") + for el in glob("fink_mm/observatory/*/*.json") ] }, install_requires=[ @@ -57,11 +57,11 @@ "pandera==0.14.5", "astropy_healpix==0.7", ], - entry_points={"console_scripts": ["fink_grb=fink_grb.fink_grb_cli:main"]}, + entry_points={"console_scripts": ["fink_mm=fink_mm.fink_mm_cli:main"]}, license="Apache-2.0 License", platforms="Linux Debian distribution", project_urls={ - "Source": "https://github.com/FusRoman/Fink_GRB", + "Source": "https://github.com/FusRoman/Fink_MM", }, python_requires=">=3.7", )