From 2d9f7e6f8f7623e12986a143c5d1bfaccb0bb3dc Mon Sep 17 00:00:00 2001 From: FusRoman <46221629+FusRoman@users.noreply.github.com> Date: Wed, 12 Jul 2023 10:50:37 +0200 Subject: [PATCH] Issue/78/new updated filters (#79) * add a file containing functions applying filters depending of each observatory * add new file for distribution * update distribution according to the new filters * install requirements update * install requirements update * force update of fink-filters * update test data according to the new filters * test * save data * restore test data from main * add data for the distribution --------- Co-authored-by: FusRoman --- .github/workflows/run_test.yml | 6 +- fink_grb/__init__.py | 2 +- fink_grb/conf/distribute_for_test.conf | 2 +- fink_grb/conftest.py | 2 +- fink_grb/distribution/apply_filters.py | 90 ++++++++++++++++++ fink_grb/distribution/distribution.py | 71 ++------------ ...42a7-a2be-1f56fc5c6571.c000.snappy.parquet | Bin 0 -> 16833 bytes ...4e73-adc6-a480974420d3.c000.snappy.parquet | Bin 0 -> 16468 bytes ...4442-9cb1-9834fb0b6c1e.c000.snappy.parquet | Bin 0 -> 16522 bytes 9 files changed, 107 insertions(+), 66 deletions(-) create mode 100644 fink_grb/distribution/apply_filters.py create mode 100644 fink_grb/test/test_data/distribution_test_data/online/year=2019/month=09/day=03/part-00043-d0a99157-fe17-42a7-a2be-1f56fc5c6571.c000.snappy.parquet create mode 100644 fink_grb/test/test_data/distribution_test_data/online/year=2019/month=09/day=03/part-00064-73d02dc2-dbe0-4e73-adc6-a480974420d3.c000.snappy.parquet create mode 100644 fink_grb/test/test_data/distribution_test_data/online/year=2019/month=09/day=03/part-00069-60c7f1c4-fdac-4442-9cb1-9834fb0b6c1e.c000.snappy.parquet diff --git a/.github/workflows/run_test.yml b/.github/workflows/run_test.yml index a386cf316..a7cbbca14 100644 --- a/.github/workflows/run_test.yml +++ b/.github/workflows/run_test.yml @@ -103,7 +103,8 @@ jobs: run: | echo "PYTHONPATH="${SPARK_HOME}/python/test_coverage:$PYTHONPATH"" >> $GITHUB_ENV echo "COVERAGE_PROCESS_START="${ROOTPATH}/.coveragerc"" >> $GITHUB_ENV - python -m pip install . + python -m pip install . --upgrade + python -m pip install -U fink_filters - name: Run test suites run: | @@ -159,7 +160,8 @@ jobs: run: | echo "PYTHONPATH="${SPARK_HOME}/python/test_coverage:$PYTHONPATH"" >> $GITHUB_ENV echo "COVERAGE_PROCESS_START="${ROOTPATH}/.coveragerc"" >> $GITHUB_ENV - python -m pip install . + python -m pip install . --upgrade + python -m pip install -U fink_filters mkdir fink_grb/ci_gcn_test mkdir fink_grb/ci_join_test diff --git a/fink_grb/__init__.py b/fink_grb/__init__.py index 824f26e61..795374356 100644 --- a/fink_grb/__init__.py +++ b/fink_grb/__init__.py @@ -12,6 +12,6 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -__version__ = "0.9" +__version__ = "0.10" __distribution_schema_version__ = "1.2" __observatory_schema_version__ = "1.1" diff --git a/fink_grb/conf/distribute_for_test.conf b/fink_grb/conf/distribute_for_test.conf index 48c3d667d..b1f94331b 100644 --- a/fink_grb/conf/distribute_for_test.conf +++ b/fink_grb/conf/distribute_for_test.conf @@ -13,7 +13,7 @@ hdfs_gcn_storage=/user/roman.le-montagner/gcn_storage/raw # They can be in local FS (/path/ or files:///path/) or # in distributed FS (e.g. hdfs:///path/). # Be careful though to have enough disk space! -online_ztf_data_prefix=fink_grb/test/test_data/ztf_test/online +online_ztf_data_prefix=fink_grb/test/test_data/distribution_test_data/ # Prefix path on disk to save GRB join ZTF data (work for both online and offline). online_grb_data_prefix=fink_grb/test/test_data diff --git a/fink_grb/conftest.py b/fink_grb/conftest.py index b56abf0c5..fcee57958 100644 --- a/fink_grb/conftest.py +++ b/fink_grb/conftest.py @@ -219,7 +219,7 @@ def init_spark(doctest_namespace): doctest_namespace["AlertConsumer"] = AlertConsumer doctest_namespace["tabulate"] = tabulate - doctest_namespace["ztfxgcn_test"] = "fink_grb/test/test_data" + doctest_namespace["ztfxgcn_test"] = "fink_grb/test/test_data/distribution_test_data/" doctest_namespace["headers"] = headers doctest_namespace["maxtimeout"] = maxtimeout doctest_namespace["myconfig"] = myconfig diff --git a/fink_grb/distribution/apply_filters.py b/fink_grb/distribution/apply_filters.py new file mode 100644 index 000000000..47ee8fb21 --- /dev/null +++ b/fink_grb/distribution/apply_filters.py @@ -0,0 +1,90 @@ +from pyspark.sql import functions as F + +from fink_utils.broker.distributionUtils import write_to_kafka +from fink_filters.filter_mm_module.filter import ( + f_grb_bronze_events, + f_grb_silver_events, + f_grb_gold_events, + f_gw_bronze_events, +) + + +def apply_filters( + df_stream, + schema, + tinterval, + checkpointpath_grb, + kafka_broker_server, + username, + password, +): + df_grb_bronze = ( + df_stream.withColumn( + "f_bronze", + f_grb_bronze_events( + df_stream["fink_class"], df_stream["observatory"], df_stream["rb"] + ), + ) + .filter("f_bronze == True") + .drop("f_bronze") + ) + + df_grb_silver = ( + df_stream.withColumn( + "f_silver", + f_grb_silver_events( + df_stream["fink_class"], + df_stream["observatory"], + df_stream["rb"], + df_stream["grb_proba"], + ), + ) + .filter("f_silver == True") + .drop("f_silver") + ) + + df_grb_gold = ( + df_stream.withColumn( + "f_gold", + f_grb_gold_events( + df_stream["fink_class"], + df_stream["observatory"], + df_stream["rb"], + df_stream["grb_proba"], + df_stream["rate"], + ), + ) + .filter("f_gold == True") + .drop("f_gold") + ) + + df_gw_bronze = ( + df_stream.withColumn( + "f_bronze", + f_gw_bronze_events( + df_stream["fink_class"], df_stream["observatory"], df_stream["rb"] + ), + ) + .filter("f_bronze == True") + .drop("f_bronze") + ) + + for df_filter, topicname in [ + (df_grb_bronze, "fink_grb_bronze"), + (df_grb_silver, "fink_grb_silver"), + (df_grb_gold, "fink_grb_gold"), + (df_gw_bronze, "fink_gw_bronze"), + ]: + checkpointpath_topic = checkpointpath_grb + "/{}_checkpoint".format(topicname) + grb_stream_distribute = write_to_kafka( + df_filter, + F.lit(schema), + kafka_broker_server, + username, + password, + topicname, + checkpointpath_topic, + tinterval, + ) + + return grb_stream_distribute diff --git a/fink_grb/distribution/distribution.py b/fink_grb/distribution/distribution.py index 9fbefb457..b0e8c616c 100644 --- a/fink_grb/distribution/distribution.py +++ b/fink_grb/distribution/distribution.py @@ -4,15 +4,8 @@ import json from fink_utils.broker.sparkUtils import init_sparksession, connect_to_raw_database -from fink_utils.broker.distributionUtils import write_to_kafka from importlib_resources import files -from fink_filters.filter_on_axis_grb.filter import ( - f_bronze_events, - f_silver_events, - f_gold_events, -) - import fink_grb from fink_grb.utils.fun_utils import ( @@ -23,8 +16,7 @@ import fink_grb.utils.application as apps from fink_grb.init import get_config, init_logging, return_verbose_level from fink_grb.utils.fun_utils import build_spark_submit - -from pyspark.sql import functions as F +from fink_grb.distribution.apply_filters import apply_filters def grb_distribution( @@ -118,61 +110,18 @@ def grb_distribution( cnames[ cnames.index("triggerTimeUTC") ] = "cast(triggerTimeUTC as string) as triggerTimeUTC" + df_grb_stream = df_grb_stream.selectExpr(cnames) - df_bronze = ( - df_grb_stream.withColumn( - "f_bronze", - f_bronze_events(df_grb_stream["fink_class"], df_grb_stream["rb"]), - ) - .filter("f_bronze == True") - .drop("f_bronze") - ) - - df_silver = ( - df_grb_stream.withColumn( - "f_silver", - f_silver_events( - df_grb_stream["fink_class"], - df_grb_stream["rb"], - df_grb_stream["grb_proba"], - ), - ) - .filter("f_silver == True") - .drop("f_silver") - ) - - df_gold = ( - df_grb_stream.withColumn( - "f_gold", - f_gold_events( - df_grb_stream["fink_class"], - df_grb_stream["rb"], - df_grb_stream["grb_proba"], - df_grb_stream["rate"], - ), - ) - .filter("f_gold == True") - .drop("f_gold") + grb_stream_distribute = apply_filters( + df_grb_stream, + schema, + tinterval, + checkpointpath_grb, + kafka_broker_server, + username, + password, ) - for df_filter, topicname in [ - (df_bronze, "fink_grb_bronze"), - (df_silver, "fink_grb_silver"), - (df_gold, "fink_grb_gold"), - ]: - df_filter = df_filter.selectExpr(cnames) - checkpointpath_topic = checkpointpath_grb + "/{}_checkpoint".format(topicname) - grb_stream_distribute = write_to_kafka( - df_filter, - F.lit(schema), - kafka_broker_server, - username, - password, - topicname, - checkpointpath_topic, - tinterval, - ) - # Keep the Streaming running until something or someone ends it! if exit_after is not None: time.sleep(int(exit_after)) diff --git a/fink_grb/test/test_data/distribution_test_data/online/year=2019/month=09/day=03/part-00043-d0a99157-fe17-42a7-a2be-1f56fc5c6571.c000.snappy.parquet b/fink_grb/test/test_data/distribution_test_data/online/year=2019/month=09/day=03/part-00043-d0a99157-fe17-42a7-a2be-1f56fc5c6571.c000.snappy.parquet new file mode 100644 index 0000000000000000000000000000000000000000..939b58d540373350e5073d3ac24c250d8b92de6b GIT binary patch literal 16833 zcmc&cZERcDb&s-T)3$6Yp2<8VN~&n~v(eZVKO|Z5Y*>$!NRgIkitMkIrpCLd+xc1hA*&juWP$&&Ed+Ax!PRR0g8I^bQ?ucnaPlM zFv~5kELIx6cFI}JRq<=FQa{yBQI-C7YI`-0kmd56i*j|kx{kSMSG#M6Ye#24b+oOy z9`Lj}TC7xn_`cp2_@o`%S}*-m_d|Oq=J}RRS6fG4JGG;Ob}n#pgb}22v{4<}`I00I98!emO^StrBFVV4W4_1bbtbF+V)%}&BN8i7*@Fa|@Z?ydI-`tg<&pz{X z?Pu`)vr`{l`C4~n$gHk{p`8j1RM!l(QupOuovz)Ax~sL~bR|pt2Wc;J=zt=gfr^O5H#-|S%4j{WHBPak=LS^JBt|8?p2_cLp; z*f%~m@X(Ppv%1Pk+NsPzVa!k~b^otTRd*+HuV84W@`g;5cNJ<7|JmPBJ+NmkU#(>;wb@d&M!X=Y z@yiF8>koh$W?D8JEY&Exies*Q#0vx4e{jR$=(=?lWa;|C&);=8+n&j7|4)b0Ipp`Z zI-M_Hxzb8|hJN^ycX7tc2%dINi zQsIcMUvF`LxrMpuV*d3Y=xU~8L)}tWUBoG7+c1~C-=UFyPD+Q8r{1EO*XaB9)IQe% zWxwpCw}Jh_fpMeo{nXKI0H*N+W@tCWtDWj$w7Vo$Yn3ITAW8wX>VDS2+bh_FUIkw}OYTC0vf}#B}I&Yh9`YdltBAp|YH!ul@ml?|^Th&ubyqt|pRJUk&oT->u5Gt5zw^%oHkex$sqo`&9>X=P~B` zV;0a+t#)H%sYqG*xxZz;gs|7`wsU_2lM73=7mEL|7JQi)Yn(#bMJh?`A6d& zYmd6;#(yRLu7xTJhIT4*k(n84RONq;s;cZnRg|+&(&fD)%qvGg7c(In^6TpY5>V7L z%R^ru`S!0qdi*5|T@(ySs$?!QGeeEKyl_m_WiRT&FXW~RmC8be1ihruoli4wd>S+| zle1xBsZqb{l(HQjo_!Dm@p-(1$35`J-{t8&Z!7Ne_o#+zB#FiSyz@ zU3A0i#RWcF>AKjc)(S$GRN`lA1!K+tgYJ|U(QfM{Kub*)&r;0m{i?rq7fa&&bdJwf ztHf77b+~^4zUpIc^rElUC2gtIldeqjdejTthu-_BV-3EkzynamtX=)M)!9a!`TK_* zv}bMPv4;rCJ@Vih=%%~zN!txE(i<#UL+c_bJrn@iYKGkYE}hc2Fkj$n*=Zp=L$riz z!;3AzV_ys+~RkWA71LiWN_V82 zrF)}Dir2v6UnV1RsP+@%7QSxR4x~! zTq*6kx6d+vepYswo|FxP^|jdWI`o5}s>NP%ahP6)+XEy-;tolX_XEs#0-%V(U427E z%uwsP%)%Y!a!@fh+{{R*gVCQ+t)+z;+;KUqD$j?< z>zVvA2Xl~>=1!q7L$FlqYa!a&TMJifUIj|zld4HsKDi!~M*50IlEQHQYbmM6s4TF0 z^ysEFQ?vX&A(kUn}6wLw(2g* z{1&c*%vN02Hu>JfY{h+r+2kt%vn3@wvsEQFvqdE^vlS((`$tFIG~;w%K6((_y!4JT z{A`%5(tk@o)m;+vg_Y^t0>31P)$UVWPdwE<4={M^u4KiTLie$*ZZRtq03a^$Jd8-? z1w#*cUns+?*GJw9tT{y)3rt!`E@UerQAYZ5@AUw zH<}3P7hM!hDAcm~Y%L4ZpXx4%Fuf$s;M%Fl(D8xpr!~UprRY|E6-FvLS#+5Dk_Zj< z^f3@j*0Njb<(N6Da09=RSnKa4F3{=U1Z&hcF_W$rW@akfz^uqGWG&fg5bH67maR2M z8L`1E;htn>Cj$+vnl=;f-ISW20nO|TNCR7q?evT_h`SlJ!kIZrunlH3XJJVRw!w_l zU%2U_e_m&P!`+@xQWoFp~Z!9;>&s=7n2IR(aZvKE}=Bq<(ELW*nU}mYp4g56m z{~K)#xglZAadWZ_vTXj2W6fBH-e8UOKd_kjoACw#?sEg7`sBwDGix2n!1aC|tu~9y zeDyd3|5pCwILmAjYnOi_SuF_{r1=HXAgjoFy;*AvrDak7Z{8tyGQszrsC0l7=p#pfK$_OgK)HpH?sY;cCa0s$Sqj=+&7esp478p9`{Lhuomt+TKWz6lZdCX3)R z4Sxgmbh2+gor0yq9{32Hj0pTvCYeI_FG&2q^poU@cqu2jc@Y$@lD&*sDlfU$LXyqpV0`Xj?54SCE<zy*$q2#6*<~0v#~` zashAepr?=u5L`q|EM=3i2 z-j@TJ!G@)LvagIXfV`+b*x6ezBs4lIe1ZgD%cSD7 zK*N)6+HGZUEQq?OeA7PUV=tEuZ)I1All*YnBgcT3O9lG0GK0N5u{ledk~k@X-yx<{ zTZv^)h97Kz%$7cm=<#yg#Be2-c_}Z%7qY3C6gQS}Ww3>LzZ(4fiDYb~OLqQ?mPib5 zoh*y?ek)6PV(VmqScrLYVB-wzSGybqdwDfK8ufC?#FFe^tGM!V0zb#WUa`vwO0JXe zpU7`~9`dh!OeK6xxvgw=f4Sc}PeHED`4Wx15b|*TXrGZF9qB z$j9Y1NXLg8Sis= zSF*@4tKj!WJ~RmD8q|Au|H3=q*7+9hVHx2Q$Z0um>+Fd(OrWi5aL={5&zR;|#Us|< zP=iB0wY~e{T{S^`5u1y1P|xM1J!w^6dGqMkv_Hy|x}*g6YOLkMd!N|(k-1t)L zVssGu4PpF<)}dDsn1EgcjG;kserA|ugT2t6U}srqJ%n(~R&m^e?TQ#11pExPEpj*> z!7(g^u@BoG=a5vOj-)1V%+_H%oP!s35kE%p1DYuCAWt?nd9tTx0A5DU;cQ%r1N@Ml zdH4ztJK{ZsOCdB6?77g>11})km@GnRbnB5`z!-OAQhs^0rw7q9Y_Ci&z^)J! zY041!$Pl!KN4_V4O<}L0@3EhAt|Ie~u-pZi-o!X!WL3#44yArkSeH*ge|5cRSWozn!qu-k>Mg-5d<`d+YJd;Wpl z`cdA-LVqR(}$;&04h1mU@U+;;-^0!&J=TG?|`*tF!84GTY6?-|HA`= z03>aEjQNYC5~X(tcvxS@#0_z+5P?GgSAvm2Z;5^#4*(L(iV>7}Gt!6oB;J&sDGreo z_?av{L(yI3Y-Mq&P&=M0myc($N#7+m;k%^vJp3DG_^0I?@`w7~35vP^|9Qjz1A=4$ AW&i*H literal 0 HcmV?d00001 diff --git a/fink_grb/test/test_data/distribution_test_data/online/year=2019/month=09/day=03/part-00064-73d02dc2-dbe0-4e73-adc6-a480974420d3.c000.snappy.parquet b/fink_grb/test/test_data/distribution_test_data/online/year=2019/month=09/day=03/part-00064-73d02dc2-dbe0-4e73-adc6-a480974420d3.c000.snappy.parquet new file mode 100644 index 0000000000000000000000000000000000000000..cbc4b1f36ff037401a9dc62ecd8c6d89ee972b2e GIT binary patch literal 16468 zcmc&+eQaCR6}Q{8b=wrG4jZRrEM)Xijndf8N8=7Qo}JjSo1~4K`U|n_z2}cS$M41U zXB~vbrh)i_A;$K{geHa%LJT3nHpCEOTZIrp6O*P&WBX%&Z2ZBbjXw|+2_cYn&VBFt zeK$Z8^OQRG-E;5xosWCZJ@+{388#L44tPhrH_v-_cwL8Fu7kG^x?HaOax56;_)4o< zZ+KnaLtfu878eb`Q9ii|%3X;Lzkj*X|*|XPvJU zg=V7Q-QgSXz2Nq}>}KW|&t?pV5ZdM4;ewBY@XO^QqFhHM_6&=I4?od(W`JmR9NSD8a}{uD;bT>SQv4p; zLJ->4dSSAD@xO9FTShxehBfr;AoIz75Z}tff%E3#AN|kY{yy6zQT|FCxkrT9Ixg&?%8_3EXEKzhlilfI%4N3VIAN4+4um4yRiTj{Uy9-;wtkAEkC z$0G}7(@(jM?u2eXewf62WJrti`$W0Xthc11tSprc-;dnPtA6HTKZbgZgRLAM?;w?9 z08b*zZUh!CKN9pi?cp0V_+7;|MPBvQm^ThHj~_;0oMj6MryGUbJF*Kn$BK1Hv{8N+ zA^4SFCJ7$>x+cN?b-q!ox4Gtey-Swa18(2LZsw(7=EuV*iII=3OozQUQkf0}_5#+_ z^!UVdXnHbeBiG&@(XTa4@?H9dCg0v>T`ZT2b@e2E-0geX&AfXf^W2RnoyJ1X%~ndo z#$Eg29n2*-!%jc{^WuYV+MQ)|gx^EQ_+aT#ADhnUV-?TZEDaG%hV-d{SYtrPOM>I|LpeFQ8Q&f5D zX6B)rK@}?>2kx7z0s=rBZQuFi>`NbhcIz)}RG}0SP03gUv-X-)`QWIg$^lfRx-P5~ z>-F`zF0j6H3-jbHppccD1BR_aQ{IzwFWj(tJ;)IX1jDxl;Aea!FmW;vfS==mK)_~a zT<o{?BhO<@JaFR4u z-}~OL{`YH}bC6Q}9@;{y(ze!X(&^nX&2xLvbG7<9ug?zOKF&OS9K2@bn@rlQ}PR>Xf3n=9k2D8Gz+T>Owu1v&L^6$qcnE#H0 zg{_<%_;0R6D*KV^{^^hYWL1Jv{2tmu5ZcyyO-j5tttl~3=bA;G{7;==E>D5{RwfRt zHJDB_K07b0a956Ol1jKM% ze0Igb9Y|3Ok1c8~W3Ne(ORT2IU_mUEZ1i~TPUg`&RfiavIY4Z#$1B%f{>kG%-DSxl z{T@ouS_oq8HRTrCAw=_fMr?f0a{u;5MW-Dli72bSXRq z!)_(D*7xXBU46fnV$xnBpOU?)@~H`s@@C8-6e&z6uAY;2aH#@o$AV#4FZwDdB#6!1N_$XZn>=Gkr>VnSP|CzTe*D8(=)X4@R%UDyn~X4ZaoQ8va-P_l$~i zp?G0MSg*DuxiNZj40wBR*B+UqrFhDfuB z&EUOv!;8Xig)lFwY>iw>VzF9SG4?59tu(tegd1(iqOe|2Bxp&sZjYdR6^W8b#U@wa znj8$jXS688a8WMf+;hvZ+onhF)d^#iVw8TJiChe_7${dI5!&nCV<4ES<*4#yq!m%? zH{pqLb83Qkz@UGxTc^GSOu0>1!L)u8SgE?s*}~}vn*l;e)E-eKY%wHUi>z>JrwOWO zGx>{~Qup7^q@nNu_U#fwb?JteP2VCg3j?4lMhD ze6eJO>~(J~n(j*^E2fFKwaC4%hpe~;($=CXwE-)n)^Ebot^Y5zG1U5lJ>vF68-(nA zQDP5lAaAk8#&0mJ_*S|}fUDdBUh&BaYap3W zuU1_w)>{#2?-qk;m31qmk+#LahH9M^+d|%Aczp@mif*QEF?g@;Y(+MZOt6sV7Lks# zj1iMkrY7!^8+Rk!oKcXJyk5k=X&{+$Rpjejy^D!*t;@mpz8##^F4i03x{STS;Boj? zb`-MLQB@2*@%Z@h$y2;B6;%PC$7rV(i2V)bJ^>RW^uJTf$nh-nXgfzLzXU^f6Rv~kt zF&||`VLY`e$V<@2cGUN_5KT-aW)h7=G`UfTr#r}JAsZ|pJr@c$3YlOl8&a_I*&b#q zmkCyJEH9@Ud=&BWwS4BnS~g^Z8;^nPv5o9_YEp=!OsrVQOxA$^TK+7-1{s5$Vmcfs zW+Q}eLJEtVl&%z_gx<_&Q>(xukUIp4sB%+>qsaoaZsS+>6njp^<*rJ`2YSr9$Dw$XCRevm&3Uk$t zIwr_VRg{6+JIH4vlX@K~j^dACwGfYiPgAQsyUhg}sEZD_f_~?mifc`?RZ&h!+41D6qJ!DTtZ1Oi5mjpJz&E)rSwD9EGvR{%CjaH73+-I?i`fg- z(wdV@J({hZsx8e1R#^;w&7=dlOlq|dpY84M(%DWKVnJ_TCZyV$vfoqdTy{~lt<`^= zl!AR9>lWgxHDO#?H+JjFWDA@9YVz~vkx@>HPPP)H1lgN?lLhSMBnv0?*)9<4!FOGV ziB;IIPB9Ah@?t=MJr1!@@vq&wlG80Nv)Il<+{IXD6BE=}r^bxW<2NZC>S0>qKE%^( zz9+Yx|E?yMdj?Viw)Zm+XI4BmGhx= z_X_8ZWlX>9mmg1xSBmony4^aHO#9z@_L$=B=Qjh48wX# zwgr)#GZL3cH}bKtE)EyExoQ~Bz{L`*&n`dS=eQ=Px?H9p1NJ!%;tAwQGfr`?Fqf?5 zLxrkPg}f@0PN!!V9Aofy>|te%MY^z;9rF##nEtIr@a<$B^6y42ySU+GLpbAGTrgbW zA>W1E6Z7hR#g{e6zXQ2+xS7icPVvt ~W(^(HOS2=@h^6f6fZ6%NHqdx^H+{})r zJ3>6&b@V6fZ9W|i3UiUFy2tJOX^L&u{LN%j<9){AOjGl>QbEG}tcrINTu;^*19?C$ zn+#;f^Kiezy{p){Z@!gMwY-$yTu%FOeX}RAA>_hvckMX&+cL&>J!0;i4q!Z0_}KUx z?y7KSMPGy#W6hQj3@G`RU4E5@dtfwK5u`N9OYnXOxqO4mBw=49C-FTYM<$^S=ZCVl z3sSl(gsN>`#QSRE!ot~!&U}Rx@g4wU={+n2aesr$gjX=w&Qh2>3pW?=TVGLs<4W zg{&eSWVM9vYzN+F1Zc45@na4@pojtwVzP_NC&tF6p)q?3N8?m{!4KtGgI^J1Yupcf z7JDPnu`^?1&;Z$%r`7p0WD-han2-D-aHr10J+`+$$e==dV{8n`Wp+a47hx|D5-G|M z`N%714Uc6%Vm;?nUByqZ{27(s(gIRcRY?W4MtgE1xqG!iS#D7sX)|(%*_a82mNG~r=V;oGU&n*G| z*)yR3L~p)>|8cEH2tL#qDp|%wY^-k!-`{VRsPc^ zd?*gWqg%hEo~P)Ky3d}tRk2rsojn6fAJ^Z~Is5g3y>q?%EtyBo>; zraKWkk9eVqia)E^+afQntwAG$ohc%XajP{Amol@ZW-}TY*0MK zKM|s;&OTWBjGRZ^5@1thKhCUCJas>k1+QC=WS2vY0PI|WaCgnfglA)7=Kt+k!)RkzvB8KwKBY`bY{+~q<719rB$VVrSck1*K!y1o4BW7; zDHAvZa3&ZT^j7$H@Bko-nHqtvH#2`2Pu3f)tKuuN0zZ?fcewl`HLiZXRcziS)M~eJ gSfn3Oi|`{#c^>{`8UB)dOMYB09e26T!2c)ke+-z4>;M1& literal 0 HcmV?d00001 diff --git a/fink_grb/test/test_data/distribution_test_data/online/year=2019/month=09/day=03/part-00069-60c7f1c4-fdac-4442-9cb1-9834fb0b6c1e.c000.snappy.parquet b/fink_grb/test/test_data/distribution_test_data/online/year=2019/month=09/day=03/part-00069-60c7f1c4-fdac-4442-9cb1-9834fb0b6c1e.c000.snappy.parquet new file mode 100644 index 0000000000000000000000000000000000000000..a64643316a4b9bc4bd4cc29334307d6e6957aaa7 GIT binary patch literal 16522 zcmc&cYiwKBd6%*+Q;ysiuE|^_YNjg5tqVUyQV*M6viMRGDKe65(K25}C)@`~k>b6g z`7{*->x!)iiZ1Zlq9}$f3-V*=hOH}#A}ER?G13+Y3bzP01TBIf2#T&l*PtkxpfHMT z*!P`tFYnumYl*$EFV8vOIp6zy=R4mue3?&&0|x^a1EFR3VU9D*#N(q3!xYxz;iw>X zuD83*xns=1PCLK^jt7FL1OC9Vz;IxAe2zIeYM#ZKdYv|)7?~J?x8LVGaD=<%8xM?* z%^qWh$5{WiSS?Deq#PIu`h&0gg5URX>zsc#gkz_X!N3p$uVe7TFwma}JgW13?D1$X z_sQmsXOEuMe(fByetib(*XIP8lP;q+?q54N9yq9RKP(BVT%o+b<^%S*Zy(@p4~^di z^4%r*Mr!&K(FYnAU;V*<{pZ`k#zm(j`V7{u&p}C?qc%z2Iy@dYqD!*VDs40bVg)M6 ztzqs9!yt(hjR*4GB{{tOmE`xPK7RA%fB5AON8g{g>6Ao=uzr0m5_68)B>Bw|Lz2TN ziCmOSqP#Q0y)XizIPrL3cNOJW;KXuZh-?f0IJz7@P7!nn>(^UIL~k3Tu(-b|{0#TI zqlQ}@C}}?RZC~)FkNd?a_tGe^@5JGOd3W|t{`VU{nRzwFJK5JEtY2>-5xs4U+Sq^p zm|=nYs&ca>G+LXLW{X&Xit+RNxo_PMVmNVmP;eC^6o@QC$id*m9{kZQpKsyaCxMrn zUwY$b_{+R`=;u#@R3`$g5uI57k8a`f?vt!vpNlk|qX0efDC*ho8#*8}+>0j+v4$EV zk>*Wb@Esp_TVPr!us_V64BenC@^mJsplUsabKlj zirV72Aoq3vxObxPKM*_$79dVdF0K^lq6}v~xF((heFpDo6dvA>y=fIJQ+H5u2 zN>QaQvlINRk9&5Edy~bZz`)?jhd(f?^KlT*E61(`6by?1vv%Hm$RK~H*wHC(2J74x zAL9P!K_rZ;TuH*|r;{9+I0TsE#fDOG5&jSXu=K~q!P_4*IJj?HY!(|Gp|#!U5jXps zFZi5~`|QWK*T<0)D;` t=FJ^K>*c0<`ANPtTl>oSzN5@HNs0vPRR!-}4g&e}%It&Nh7p z>(}Q5!S$V^X4Vgka$kPLaQ)$OL$sc4e>KT{|Ks5PPBb3KcbDWZ-uY|)msZAZJ{+vB zyr(|mB8d)R{rX%a<{Y(2@-@zohA2l-l-jnmQEW7}8>Ykh>nFKie*y$@V)LNkD$rcu zg1#3X+I#@`h=jt?Ga>jhJrSC@5DLMcv!PJPWoJC#F_rZKn5Bf1{PSmwUGoub*Q{5R z;??zeBIa*>!QaE7eu}$s3b&3IR&R2y0$Wa!=H|P9{*S-KwJn z^v#fAxe>Hny|FFQY>^%)FHUj4JquQIV)DT6Dnu~wpl-GMP96pxuBuz=c2~8_0cty} zfmpp^kBbFZHR zIh;s5P`k=uaeJT+&u?7ZUWX91kvW>ONZS~NZr>QNm0x(=;QvUev8`;h>-8dyR@Cor z{3-X&1!@f&Dh~p?OQGjvPAljTBX2_@&QY5b|N5jM#Zi)(Z*(9%hmb_fK?V8yKjYqg z0t9hl^PsW2AixH5^ItYR%z+T9cx>s~vX0sWc{yeXGAdU}B^Noq&U62FQ8S1YnFop8 z<@nD1@Bhv7e|v~=8bpV%e!Yc6^tLf-ljFyW#$9g&?Epqm>Lz2nyudxT075u1dEj>w zV(f#lS%w{MN}EMR2!xox=wm}hz2z}xc^udZ9E3rwjKKtsV24%$Xh*+H(|LF=rQ_x+ z9vy+(k=Cff93*8*>2+c_OjB+1h&lB&j~gkcK1%qb*__6omJrdNHK(9l*iK+GCgUK} z7&eXx%d~zJrc_8EqfJvx&^WMcFFI!NRH{hB=#QOP{=)I`(_jgP3Hryz$39^9r#{B* zd)%-6 z;p>^uWWUQkJy}uZ;`I$_yVh3J=H!KmPdz=k1!K^fYzS((IC*+vQWcaU45;l|4SFQg zf^LYkx9iZHAU#Xl`N?NY%viaY)IWbB77H&H$USj{`g-Ox zFs4a4Nk3XTDH`K8dKI-bH$yC7k-tA|65m0L-X@&HjBy*WQf*ssC1(QcMi5G*?i4k^ z9+QM?k&~P@Xd~6vnR@rCG<{ZclCuGAWKFuW5;g(uq|}D!q)5m1m{8S4lXPs42^rs; zoRV2#Ho|uo4jlV|d}wl#><{lPHPc5WCrul0Zz=ab7dh!#KzmD-)&`s;jd2@2lmGug z8$+u_Hir5euJxrAAJpsN0 zcX3sx*l1R^RU8e6&%#&QNyuI&H8#w^v(sm1C!b;1Q9hl%l)XR;-R!NiQ}|5tX`biT z;4eN;@%#)QEAw%lFZ1(dG8T(j;9VSC?8BGVR%jpI1Pg&DdA`fTI`}0>;Fm0dcOL%6 zy7}zvRz3$y7rO9@ol7dUb|I0O$wgK>;!>t1sw6Vkq?53E#igVg>_Ih)9p07yL=}k#pgE79V^*hphaxS^B)=bW)cH~5+i+EOYVHx51 zNVF+u!|hyz(k|rsG~4-XxQ25@HPaO5kzS!*$X?&dMO^46;=p@+CpVp*l@iDkUy-x3 zb-=$>xRONLz+<>u%tS-QT#VpND$$CdWUBH!fwu~|^d{g4<*xw0OU+6lJ6jba>AI*$ z1YahTPAspND!wYjPqEHcfB#^|lmhqAVu5 zRkXX{&6fo(x#$&qA+%bha)E7H(o)ULb6r%|BCBpXP0WVH#P#U%Jo=iUtD>4#a?`0z zDudm|e8mD+W8i<_V_~x$3r#T+zY|Z2n22u53;EDp=xI()6iToks(0iIY?JR1|8cXQ zjV{PdvjpC zfWEwV5tISz1!6tet_Lx(2K&`3M!{ZQ4N0)aAr?~my7`rwX$#raP66UB#yXdnpv5{Z zW_%dCDVa#0rX%h{Jk1sQe7o82o?^d8oPt=pQ^=(v5MMizy5*&_PF`KDNooQ1PPA8~ zXOCNK*pJvAl9ZZSh-7+qICmU#2Bp8kbgFWvIB$Ww`I%zd|IV|=7H>b)d8kTC>W*=q z@b$-t^Qf-~oDbnB_*1GQRmeG`2-!@t5RaPTu-wbnqHqSTmcT!I;`D&ynwstjSy=_z zR|SYCkS8s8#W{H?RWC&3npA_lDx1k<7FRrD@P6!J>SM7U>}Ahd|L=dt0LsPkb7coJE-`w1^IUXjoc`)wDhCW=~sebLMY0om%fR9%q`Czm;SK z^RpV>P0*j5F$VI0d@dErO&8#PhkKXmd0@6x()EH;*qu-Jc>~iY(IMo*aChx`+1oM4 z)gLkU&V(?YQamp9hPx`zK z!54kD;3r0Ojr&2&;b?4r>hjbSG=R7D^V<3avIx1ctVet?xKkJ49^2m^WRl$3nVLdy znV-?{#rW$4M2a$mKhg!M;c@Iotmiz{(C8<5@v??*Z3Q8ksHBNqS}#fz;Cu_0u@5Rzdk8;EDU73JB-WYM_+Loc_$T@{@t?v4a9?i1U9Ti&<0|%7jrD!` zV_5A%)52p~4|OkCZ$8^VZ*$0RZ#`^INH`|IQG_Cr^R z_4DEk#=%VH>Kf3$av9{G>96;&Kl*x%(1$uhH3#`O#Qh!&^3lHh^D`Phy8W>TXO^FM zZ9muS@|CHM#{VT7eJBpXWAa~8&!h6A?2G44QGF%(#mnIO=znWh-Tehem-_izs(_gJ z1zdk#qaX9I*D}tu-HG^Rq!+1b^cShV9sHti#cB{=+A<=EufzOPYrD`R{sH;X4&;zw z$Q{YrGSWM@qtm1Ii4jpv`oYp?>>A3J1fA0TxUi+u)Al2Ac#}VpT@Ez@&}Rb{Vf;w) z3lI~iK0Wpy>N;Vqi||Xv`pft_%HDvQty(;-T`fuG4ST`mp*0K|L{O320