From 58a730ba874c2941a17a6d4cf96d986665149793 Mon Sep 17 00:00:00 2001 From: Jonathan Rosenberg <96974219+Jonathan-Rosenberg@users.noreply.github.com> Date: Sun, 24 Dec 2023 14:11:05 +0200 Subject: [PATCH] Unity catalog export implementation (#7167) * add databricks client for lua * add databricks client * remove redundant comment * register databricks client * open services for lua * linter * Unity export implementation * remove unnecessary imports * PR changes * sort imports * createOrGetSchema * remove redundant code, extract full name creation procedure * fullName -> tableFullName * remove unnecessary assignment * change structure of databricks lua * update unity exporter example * linter * linter * add validation to databricks client * linter * handle error if databricks client couldn't be initialized * PR changes * update unity exporter to use the updated databricks method * update delta exporter * change unity exporter to use table names instead of logical path * fix delta exporter tests * give descriptive regex name. fix error returning format. extract "alreadyExists" function that. some reformatings * change databricks client lua client functions to be a part of the Go client * use table descriptor table name. pass table definition file names instead of table names * validate table name at the correct place. fix delta export test * PR fixes * pass "get_schema_if_exists" to create_or_get_schema * rename function * use new "create schema" name * change OptString -> ToBoolean * delete unused sentinel error * add names to errors * validate commit id * Lua: Insert format to all `lua.Errorf` calls (#7189) * Unity catalog exporter: tests (#7176) * add tests for unity catalog exporter * Beautify comment * remove false test * Revert "remove false test" This reverts commit ad13f3c12e44b3b350fd9d2fbed300504a423534. * add name to mock table descriptor * update tests to new databricks client * PR fix * fix returning error * Unity Catalog Export: Documentation (#7183) * add unity_exporter.register_tables doc entry * add docs to unity catalog * add catalog permissions config * fix lua docs * add to unity_catalog docs * document unity catalog exporter steps * rephrase Unity Catalog exporter docs * fix `unity_exporter.register_tables` docs * rephrase the unity catalog exporter section in `catalog_exports` * remove unnecessary numbering * add types and fix docs * PR fixes * Note that only AWS S3 is supported with Unity Catalog export * Change the supported exporter section to a table --- .../img/unity_export_hook_result_log.png | Bin 0 -> 29965 bytes .../img/unity_exported_table_columns.png | Bin 0 -> 25799 bytes docs/howto/catalog_exports.md | 11 +- docs/howto/hooks/lua.md | 175 ++++++++++++++- docs/integrations/unity_catalog.md | 207 ++++++++++++++++++ examples/hooks/delta_lake_S3_export.lua | 4 +- examples/hooks/unity_table_export.lua | 24 ++ pkg/actions/lua/databricks/client.go | 1 + .../lakefs/catalogexport/delta_exporter.lua | 43 ++-- .../lua/lakefs/catalogexport/internal.lua | 2 - .../lakefs/catalogexport/unity_exporter.lua | 61 ++++++ pkg/actions/lua_test.go | 9 +- .../testdata/lua/catalogexport_delta.lua | 37 ++-- .../testdata/lua/catalogexport_unity.lua | 170 ++++++++++++++ 14 files changed, 699 insertions(+), 45 deletions(-) create mode 100644 docs/assets/img/unity_export_hook_result_log.png create mode 100644 docs/assets/img/unity_exported_table_columns.png create mode 100644 docs/integrations/unity_catalog.md create mode 100644 examples/hooks/unity_table_export.lua create mode 100644 pkg/actions/lua/lakefs/catalogexport/unity_exporter.lua create mode 100644 pkg/actions/testdata/lua/catalogexport_unity.lua diff --git a/docs/assets/img/unity_export_hook_result_log.png b/docs/assets/img/unity_export_hook_result_log.png new file mode 100644 index 0000000000000000000000000000000000000000..c954195ffc24a809bdb830dff9303caf949e483d GIT binary patch literal 29965 zcmeFZXH-*L7dA>4MMXi1B7&fRbVTU{PSv3Im4B2tcsPdulmyG9dC z@NqM+Lt!6X**FNuyGt?6^E@hrPe@~n1`~3YvE@N2o+w)XA@`mI@B(J*T z?-0ojD4q7Gn~{7BpMO=B^G4v>r*|OI(V-za{|^IoEsRlDv!s&Ybl60x^FiI$IlF%> zaeViI3g|xCrpb-etq~`zxShDDXvFpn3ChR6=h~r?<0F+W-&&+2zE9u5ck$!1+d?R*6_Yo4eRa6(Ck7bv?D@I(@cZcwfnLg*OHI4efE#Z6-e) zP3_%ldf;4h-*;CyUMuvu<_cQ=yolfor|9|S({(~)|E3@)
    qRT0b){dp~Dyf}v4 zI3<>GnR8OoyDBK;jdY5zwCV_x`P`B;6D@xvgRRf^7ZJLl);pos%#&rLh4rdLNKaX9 zM=84d1S8g2q}p1je8Ob+I>}mFZj5EVxYX|~Og0`j%e8HCyDyfKmPRhR4@t>Wm4@~7y2vYGg6p(==USY1c@2=qK2}7jI_G59mI}Z!ERE6eM5*=d zajN$b0db?}Er`#5Cza45HOg*{Kf(|&fUD+ z*>e5oP1nF0I};k`yUy&+3`JMHZ;ifMZ;P7W@p;m4eTJOn<;uM9w!&fBVcVg{;pAc9 zi4sY2hDHg;;T20twnxODQeEG;Cb*^u$})0tU%B)~vz?UzzKioo%njv4Ww0{(`|~pX@z{npymyN48pOcf7svShfODx{(|yZneO^Iz zTwYvO9PF#?tCXX2>GReQ;|mH;+RA8X?D4x}?Pw0$+nuos-G#CF?;YOP^I?tBxm;M0 z3FaJd9&`R66MXJT35j;AR;zAEeu>d1)t?`}lD^Vb_?Q`Q8DsfnUa{lxTLZdt@!P7e zRcBQFa~|b1sxrhM+fi!3mQ@x8Mg|TB_y>;H#Oki5JqBkc_QU!m*VwyMyQ;dbc8zyE z>9YOc$4J>0)A}r1P((OdXiFG`97t_TXHP1BrrDM{>G4f(=*iR*GVTuUC~mACDF6PG z#wT%4s=tWjgPyeIC4b@5>DHMn^ipwpnpU(`(5EY)FO!Q`CLIwRrW|}|?z`>%UnhbeL^-pn$|f@X1k)3Xv?c(7o!P|qPoo1jwmgFntVx8!r0eWrc|}v0Qda8x=yQ^D?>Z|JG^~v=M^<4@z3VlbOjCecciGV~L9E}`j#y*Z;aP*$^njF}mA1kbOD961( zTL_f8l(4w>d%y9_^YUrQEDtqgGBmhZ8htC;;U-uz(7nvN%T;1|4^dw0QHHs_(t4zb zA;(~Q(H;YyKkJ5S_G<2%TA2nxHr-He3w6VF(%z#UEA?n^7K@aUxUul@VrgFKJGVHc zze%M@8Th7SA2s+lzHO9fjBjXbK%IGz|uszG`;q>yTk98Xon=Tp{i@yYI)#}&$XGAAnN4aaM#^e5eH zQ<-xLPa@4;K7FuU3g~xYfATJZRv11hQ0-H*J5KZSdUwIe4fc z?}_$k)^O&l0Y0I_q&G>YLRpcG3@+wV&F#)A1MhKaQ3*BzjdkiyhFxXC-Ajoci7c>= zZ;Ib;y*>SUX@10AX+g|w*o|EZJac)5r%AH;oy;BAs~;{<%$v@uEu5n@p{cri@owGS z1kTG;(y!vbimC*v_`U!3KI}d99dNdM?t@$!9V=aM#0U8wnFn-6JY@!uh~9RE`Sb^f zicNd=_e4MN5Uv1ws2T+`bF{N^RL-o5pRlQL{(Tu6h3Cvj{T_Pj+3!NV<1xZ9>v1@g zYLDtnSCv#+!%4%;lZ_`2^FIx{4{aP;{NMjrKTsQ)!y}J=dr9>#B<~d%PSeL(Zi7k!#73} z&>fq-Q#ecW_NHj~Q<>pSgGuu;!aDIlNP5`qbCc^gfBm{_e{;fwdQ@#a{IYyL_xRzI{|^`Lu8$WobE8G6@(O|f;Kw)0`M!<_b^6(pIntRqkf8AI&!g%7=HAFX@Q%{DsM&r^B(lkVwovWtY8-^zy zHm`3uRZlW(`ApaPCwgFy8((4Tw!&x8fhCJ_V9{p3?PpzWqOkicQ8H#YGFcs}=sFhDh7VOTk*=af+mx%w$FvPG#Izj3gHWG#i_kZo&XmV0U zh(8=YbRA%M8>H7P5*RQK#>%+7laJbKj%`TxsozQaRZog`nyrqafPru^$Itu=4zqrB zhQXi0B|u7nw-}Mfp#+_s;YEk_>R=ZzhYlx4yUBU)7uRI9UM(yT`}h$Ny?8)WVN4W5 z4@M5@l4ua9li0T5KG3an8b6|6)5Met4=IOD9f$>Gr!VAPQiQin5j$G3qmqxjU+BZH zy=o&GsxVfN_m%2xgM5d)tazy~mO-@hh02mI0iANdTDzn+qAW|01MO)PzuP)_%;iVEVme%|}POfKJh@^ZZfJ-N9$TL;KT=2l*Jrl{*jWEa0&$RLd#a(u`q;(YnpK2fkYA7ubb*zXRm%Ojjf9T8;(xLO z|4FmiK_IRY0s`LN-u&Le{4Vac0z%^A;sS#A1@7PH15)sL_&P(L`S3Y=u>V=eU**VK zdsw;KyF%<;oLSGxeP-$636W-FJFDpLzdze)?PLE>P0k+w5epDd;OvQj5Wk?n-(>?? zrOxh3XxaN%I~vN{I|1DT)BzF}5|#QR{r}_1KQ;cA^XWf1g#-m3{+{)>NB_?H#KYSC zv5ON>Cj|7*(EKO!?+^czQA*%U^xwMT&wl>nF3{1S3sM4q4;tu#Um_kDd|4tDc{x2F z;?3!ENp}>+TXrVurQQ1s@4pFFjmdUp3#GYsML~<0G%v$~l}DV5O-@elQ2q{2t9&12 zp7Iqf;vXty%9@EycQDt?1uw{PA2?f&AhGPu(-#g#S(*OhU1_MfzU_0O|Gg zS~40lnZ8;6yL6yv(Eoi!_w$61w3*JmN;Fd`Za6lAdQ2TYupi9+*(AYUbRsP+ZC@wU zXuq6nqh5}bz`QY*F;M=et3&MX^~aHmD~h9Z#u^aGE$c1~$?1Ohjt9BbqP-H`>h{NL z-|mpe>sFG9bu{cO*X#{;7w5VybRue(llXcCPa(tH**V((j9o@1(BpK|g7o>7Eel|2 zhjk~1Tvd&op>l#6Er+@?Rj&*`ikjMT#1v@@ zx4cl>v=;THr|Q2eLBA280F|-2dRoD)y*D96p-j0^Y^BpT31yeKIDeCdl~&NsTl8P5 zsr9#vYQHz_Or$U`?7wf3b8}ntn+;sDD*a97KS~6OX8C_#G$@{FdD)NSvPAd>;u|Ue z23AEjevlSw+$0$_@z9XiZXrdTlR0cX5uH1dc1y>Y4G&Ix&!!#B_@N<~UQvTWyLc`n zQXn-(fBdNT1hx{ULR3*jZDI_A7_q$%Y6w#BpoN?T(K@K$v#WE z@a<`Lbi2HT>__|K4+j%2kh58EK5ah!mA~G{h}yo5G|rLSs$t)(Kxv-F6;=$Y zuBn+H>k!tx;3nSFo@+ejN2;Ks4hHj7dSl1ZlOrXW;{hqVS$|CLWbX@kFoU!iWskUx z%F#{|b6iEu?0fR9e7+PNw>m{uzf>`hDHa*A)~JTfh82oSH12-OhICoeqTCGC4&NCr zO&~g}g8Xj<>YEz|2K6syoU%}BsN!tmNtA+eM59b=(FJvz=adlRwwi`HrZ`_w?q)R4 z-%IX0-VkCrTr?B7$)aHs-;@!6sZ>Jrj;8y!8Xo*Y^}+Wa6**?cJ6G~*aX#hj=#b&_ ziDk1}X{2oy=a?zN9WKmjd?!69ETRZ^o6GN*T8bcXsLqj#K@%YDqy73-5#}=qj2I=OfqpB+)$>=t7j}F=om- z(R47e`DAHmQgZIf7NH>}{TffdWn(0rom6q!YwTN8yZ=HtBM#}UsopIx5f`{q_@i^T zbM!{qw=KMl1mQS>nS3+WC0~ieiv+9qdAM}=m&dM??MKCKY{1{UUPv|ALKb^yDMUHZ z>^f9aL9q2ZN48l#BYH1@u(7k&%E_5CF#d44K9Mp0^6AOy1N255J6gCnJ*KbG*%Hc? zVl&|DHz+r`pBoS;1akW5c{;OX6~K@ydg5Ry@vGtd(S!1yq%wMhwmlk*U7l`_T=J=h zkD6^T)PF@odTJWh`OqDZ!n&VDxJ^+Mq)~cpzU^DK(8C>iXR`gy&8ZZ9^CE#4%_j*F zYwtErC2u8tSpM;D)U=c{z#;#KrnwZ<#JPFQUwW-4pCenejQ#{>8_zr2mwPm0xPa;q z-G9PgP(PTr*C=pc)2+FdGpaK&-{NK{I;FkVir~nR!B@I`q%KJ6#YPO+htwWdEv>uc zLiT(YP)pUCJy4?TlhaKR#$lEUc@X@xKM%4n9k;8m&V92fe#s4K6*maRi`_mIpX_s- z-7nj$@8tAD^$33`SLps_)*G;1NOE8qQQMiewOjFBN%VMrMQSCQ)~KDuZ=A`}=*9~9Pa)vcAZT~_nxeP|%D z*Z&wL5OmbN*Fs9Yi0P*b3?ZjiHZb$Go^WYC9n48I`>1c~IlPC0;~x-CmJ-c1m2UYy z#~)o1or)PT4>~<@!A^UFQDFGPtF{rl&AVMZzU!xL)H11>GN*fcJ;KO4MjJ5cgE9YF zb?kR`ST;HkTOZ;76#vnoD7k6|x?ZtfRG+JzmD{BFxnR>^;{v$jM*NEbrq}y#uJLX7 zYtH-d-Rc%%#wNJzPP-imL1ewQcsO=_K$4e5e9(*g2sIqJsHQp$1gvE>KH!P1wk!-1 z?)ywfDnnj2?3V&cv zZ7nB+sGMuPQ>avEh^I$`?t}QGyulNmn1Qt!f_j&Lzd6*&KX5>zBXstgs9hA#41+Q6 zg~DGZCvlk%1&rn+Q5_ftbB(nnUW*yW-t>DUUb&xg3dbf65E}Ao&aCda z(p$iT+S(zrv(8)Qc;sM)s-O0 z+f=Dai1UWA*H5;RFr3+I<-Juqzp~1l;s)b`;7Kki9tTWu=difb7ykH}37&VVdKX+_ zXzD=W+XBef`O&|S;cqF7F37xJdYoukB}@leD0k9S=#4aN*$Hoc9xn#g;P*~yNP`U+ z-GXV;n~gRxz^@k2uF}?sUYMwbr3I%!4u6Itc#3M+8utd()*3KhXOA?Y`@;sP zw>(8j;7&5Yt}s6acHWN!>HQ(yHNRP$8NvSv^wFqKT}xahP1H(<{=OxHZ|L>}Zi=CY zyMQuR;5&DjzNwAn@%I;5I%wldbCsz3u2Vk}HNU%f{50v@DPK`pKq0-?HFz4V^gy5v z!_mDo7RG$bR~uf8(0-}iy*zo=MyS3S$xd=vOB(;-&A8&`AL3;QLQNAMKE89*q`g+8 z#E}yuEp?B1XMJZayBdmdy*2$%5~ky;iZ8Zi7HWbAXntHN^jiu?)rttz5dJaNa<3#! zJ%7-lq}#MJ%bHgi*bW!Jv#e&GFWk@%<>%N%!(f{bXzrWLzo z9Hoia6z4#7vA0yPl9cNQOhg;gdOD*|^L_7J?-|iUkz>eKrj-etg^h-^**P$ly z?W`XABrbI=v`L~Drjo;bnQ~@NmPL=0y7}uJjRKFBP$H9#am^rL#BsK@`y<9UQm-vT zPR6FWP^w*|{kTm`tyQ!*2X(F$hfyPo|8}d7_}tD1UClM>F&JQ~H0(>&?ZN>g?zHe! zh?W=!EV&=Bj8ENv+{IL9O%*W3U%aFE8Q~~HI6|U0QXJo>`^-n2oPIg!CfzT1CC!cG zG9cTynKCs&Q=HXvQKu8PA8nf}{sJ-6@ZOD3F!-WydZx#?agF>j^1ZU2`qh#k*7a8R z3A2EeI*%qv4jD6G7TvmA?<6E4BO(U!!T&7C2{^?*uWt3vKv#6Lqmd%IixS(h3&#Z; zhg04uGyd0kBUR<1`Dn0X!k0R)Hzo&`Sh<+3{qT!@Jy|Q#wVCzzDYWHzNK=02kKz=N zA5ur!uzG#KzDe9~u(@3SR;a9zoeEM4KB!Ab`7m4DE4|&}+&>xQGiNyyTRHKb@!eWk z0Z!CZruwO^-f>s1%R~DmcyvQhvGsJXim>{qgdL}B%aYZ2V?~9f((x2UfFi-CqpDz)n!(xlp>mG_g)86Mj(lD%AfNf@^&Twzxahbx z{-^GGv?|Ox{Lh*zoY7-glKZt^o9HMG9_A6=L znip-9$lEVVZo~w&&-p@1;QLePG%G`@SwZ`s5M9CpbMG`_U=2KVqHc9i^K1TsqvV>)YW|uJucM*o_2MD0N}8(rmC-4=rw|%n-Xj z)>&(9caRJnYy*Q;$J3~F(^ezeHx*(>dZP_!3b(R@Y&!2@XBc=S#dFt^X+IkyD@*Yp z?!RnTcDan=wTj+{x8G#rII-#@YGB(CN&EC4qbEn}^9pxd96xx9H{u2gmPG0eJkY?v zMri+PuHS6IJ;YEe_RH{>s7zh%=a!B+1wDuu*|KB{MnrA~ypm!Zs1tERLxrp_b?RnM zG}tB;p%E)v$iVl}kbzl_+%O~~+@SrMZLVK{Q18cm3NDTVS{lwFGW9?^Gb!!tRK2DqIHpN;8!COgv$oZ8lIrY7XPYon@AYyHIwrgp&^bF z;>X#zZi@Z|kMct%8rmc`NI|Lva$hl+)ex5EBL!v%skn{Nr&duX>E5B9BjGYM*abK+ zBYjI|WgfpuZ?}lOb`RRQ-5_T7dmBqrTo;RfbFxG0gfJBnN`rR1te8B%Fu32T zo;a)c^`yDx`6IkR$|$X22fZA2C|lf?SX!Ngkfy?wx&9LeMIK2+piT2M_en^I*Qydujv2g zaBGeJczDAb=F%Kw87Lze_G#1g`7F7cTV3p4s%$XM+NH1lGq&;t!zlXGM!KibXAJ$e z?=k4vRIp{H1MjpsLqLA@qn&b1xbUjJP^`cas!l0mcUpJmhBxb^K96;nW3Pw@`1pI% zxGZ#B;(?kTuVBka@j|_cK>b&W3(Qb!-SsN{m&GL}(pTgWixuPW^5^!Fq)eR1Nit0%# znDlM}LFRM~ytYw}>_ekweHNlHyB&>to6F*S2rF4-9f&t9!86n|)H{$k6sx>*9W&(M zWZO&ti)gl&jTcJjS;nrG?ndz`t?*6hJq4wBMwyGP_dc2*fqC1~MOWJQ8Rq&Idwu*I zpuux&3cD=z`ub_QYIO}F%^I>$>-EAZ&HOLCBk#Sec~G~AnmmI&i3mT5{lj#Rgb&pI z^ySDT(?$Mt&$irG2XA?k;7O)n8pzgBP=H*3x{^g#%kzud?Obb9u%iLTE5am-f#nUq z?$u4ucVC}&jx2jr%2)irf{Jp%Y_=!pdE5M!{&- zl=i%RA=Vk(47TZ{(F?!hw+&1|@A+H)b$xKp5yQ2Yw3@5%{iBuHUpgp4-z4_sLUcx| zxzihYi#`^#N5QxC`4xZqPR9l$yD8H9@MjI=z?k$qos8dF7vog*ekE`R7c&CgD?Mm=e3k3^_imSl-Q|3A-tj=_AApzhqan z)Vi?iytjjwWLQ@LgBXg-2KvfzrAME$9SB0)1aa}L@aFs7;Vm(5gTLggdNC2pl5LWa z#xhR5)w_6YzI=Y+w&dP&b7!EVh`S`V>%QQy!}#|BTT^MFW`+c~L!{q;ci+~%6*Hqi ze8g;oO^jG*Kx&f93)=WQd(LCfjl2SD=^S1DZtja^MO#eGE)XxUvw4Qw?p<2cMJwoF z$^vH&Z5iCvfiqx@JqzVI{v8WH!XNI{Qb%>|Zv4RHo**_r`%BJ$G|#$;GK+j(^ojtAUa02X`VhG5qA@wX3AVa?5DvUC+)JLb`u-}-){o2 znp{vd)aDKDXQZ!S26N$TW#O7SVH_DK(u_@&+sJ#DG00lCR@|`Lbp);xGL?+jI-arx zvssv1$3NK(+41QGH}8~tJW&~_)2zlCZ>7Z$rkJerFAbgVPv(eTQd}~ZtcRzivz@S! zU~`N2`?QacT=*>{?QU?Yxd!WMZhVUQ6V{rbhmwH?rmu|y#iBVEn=?u_jTaslG?{=8 z94?O|pW~sAKs3?|p$(GxLu_duo^UmYJL`a)6|#08_ytvOB#Jus1oDMWzB^>s9_T!J zHfW4k`sr2VRMixi`_~c+$(QZUXdFw&6@vF{3fmV3(F7?Wf|cH)fa3!R76ox57-P-a zU>l;l-WSCen7qLmWv5^+%=AipaHag}!u5*ikps$O`&z;?kry6xx^6FjQI0Ld&atMW zCcG`ZZuhNQ%9@*S@_zvz+qg(9zB?b0u;nm&Lj_JRMG=Uzq~BJYUzxQV=eQ!aQD}U{ z?x*B^5AMq+B40QzZ~ZXCUK2G*c((a~H%$8!?VpXdo$}FO{i=))pyQN2%G(W369+MW zJdLg4oBvvEbnMIA{YGhqeMb@nVFF34KWssE#pYc+3_OtBd1tB>FW;}p;6vgax(yBS zDJ}5ttLV`cpsW9iVOr~`=zm0&Gi+ck&2lB0e>eQ5yE2~EwSJ>aO#+ztI8sNF=YfGy zax2QhNGXp%mZR2lEUDEWo3|g$euwQCuR*Li`D*4ukI?19EjRg=$Tp?~np-KD7F(sJ zHzOkkcG!nOh|IYu52VKI;WtqlY0^XXfZaH_`T!5>a^Y929maRN*TTG_JH-?u;Yuwm z=$-Gr@61gPmyzGF@RfNF?jc6+FtXW!aZnR@MDq2FU~0EA%Xgz@>fY#2%Eq=b!#-^? zoH?^5lDB168f`z%H0T6d!Ghs%t><&~_y@k_>TW&XI3hPYGU9AEggIEQ?78?x26b`E zDIwBeOo{t%*Sg(m(j0Ki{+WMi1XsU6f83=P4@%HB1f?i=!!EQd2rOpwGE#R}SlPv< zI_dtdJK0-E;tJj_P^91xqFqCvP5<-5Ff>9BC?kKN(h3iWhO+>b~6>>Is zK6Z@zii(Qgs_}Tv=hvYgcQXYvo9bDUYz@2CBDH7 zE{`h1jT=vQn-!aE@VGoLjHnE|I+9WqhrQ*$7g5okN#%84t<65N)hDYZZN#UpL$nUr z_-xt9zvZ^FsF&hvxP2}~Ga-IS_%;)9jD>V)m7<|==;;DXCY{ldKlE4;RRx8slGL@u z8MvSqEh>in3qb-NxYB~vUwu1_a?qP$`5PxTppR165(b3B6UDkz56N@5)A!>}GH#&! zG=b>Y+A<7d2&A!HK*=>a%blMF3x{>%YYOcI*LICLmz#f;JU&>ziRqbP*uo!F^Q-0Pn{%SKN^neP$(Q5i)`Ou zXsR@=vmef^46Mnvljh`l6P#8?%O&5^2iDlsTNRmgoURFm`B|lCblO^$2swBQcIhR3$e%>3~qvASZ){z5`CCdny=2WKuu z(WmJOz+=4m0yf4syd%r$jq21;T}>8V#Y@V{6(gSRJLZ+1xz+*eZQt&0Nzd+{k_v3) z>yIcd-K))Qvp>fmrUrrYzf@(o#QfzB-%2 z+|_MeeSM(YKFYD@YTQ3Zx9?ZDr0rO)^A;%#ljnN&8|?41Y%O`&{Mvt4VR3kVPrYk!o2Tlo{=X$#q#BbBTy(J-)%rX6A4x2d zE&vjsF4>m;)6su)l44V^9Paz;z!tGX{A;A8Tv+ny}7yYFi|I?zs*PH*}Of=>t5$N7(1prtgO07D*9>8h; zwZ%ZfWnH`rY#xrfhckOApD`~VAK2?avGucQGXNm-f__NnU#n09ABpL}Abx^m0Y#WO zRfJl$84Qw6HYO2vQn53AkV&WNoCsRZTml~3d`gT%fc8H_MoqnQfb#`rlh-IGQ5vAB zmlZ{)A-Xx;_CNp2jtKco$GF3{Nw&NniW#L(iaA^;7k7b!2q{KAnk&sq3C4rVZ97zxF5x%I;@`9&X{n@%ohrGT)t$Q-_^F}7aOS^qzn_WCyApL+l!3c@5KvjLz{Y#IPb)@b2}@_&H%k`q`t|x@#ck zLC16XpiLf7aFi79K^{z`R<(ut-H+z#_4|4Iv%8X$nMKg&Vr0JIy$746PLy! zsq9DQ@n`%pbvsym>BC+lI-pq0_!ckiAuqtm9pM11-JXqg*)1zdmVt!d&-!IY5tdj(k>Ko1DdaZ3O^sKb5@nIOcen4bCtP^OJc7 z)J>0`*l3AgdH2f17GoaM;5G)7Sfo2_Hyeb236l+|n3Oc&2ek$$*nXayY@-VBo7REqLR)P>meiuA<6roDtDdDh-mIj!9cK2jSZ4K?) zXHaIvYW_SAmVe^0Wi)cD(%7lEXn^I106r$me8~_$muq(h*KU(Ak=*4zcjlqBx0(gx zFseVVOc@gy5gZ9=yCiyT*7-P<^`m|ei5vHAO+dS)RZZ){L7*@2!8nO@;wXw9Lt~AlDzeLTn}3^I36!@HhZ^$$9#d zI%xHIRhNbujGSAOa^_shm}V4ByrdrEB@XS_N91HU4L!S9BncI7Ow(SDz!EHofSbjr zhBJFl2{kS-qmwL!K14}TeosCaw}toec)yK*b_T69@qLEOKYeC?0&qei3TAuw3{k4? z+s$4yofoVT?ycE>?=u-Klehb8WDX~_KfX^GcW%+_KE4Q!>Z8a@nNmGEKA|0a2To{v z6gdkl5DCR}eEVmf7&9-Ly14U)Om#`f>ex}0=^1Ik5;U!wfFkbi{Qw8Lx8N$g;sOB> zrafDOj)k>z7cWIsxG};tX!4`tSZY zl69@5I_@%PdS3)^D<_-4vH==^c9rG%WK5m_v#Pg&lUVLB9JKcdKMt1KX_p$aH44=- z(qRYu@&fGRG(;j2kJznAgNPexl3o0wQ06wZX%qbR}>jcT;(ih zpE2JGJp4&KUdkuNkw^K~dJ~v;M};(d-;x}PYTd(x`^4BqZEZ;W4h)wfG z^-iE%5*#BPIoY>G4Yh~Wnp*l@O0T}A`)wuLdDfO>Jew3r%_>$@f0EQkEfbTd*i~&k_ zu4dpa+>?i5<6Wz)_4)2km5oVUZD-zY#5fY1%)*_RTL4|(@IF%vhWzb>%g7zlMYahU zSZ=a28v1qDDEa&^wz+=QwCuFQKw=TElKH>g*@|=#b)%MwV11pi4~>u&G@=6ozEBOGaDC&@7Dar; z@(4#2XGBYU1>jGx44Xojr92g_5-2?GkZ%zKcz9P`?JE(u$@63E{n=_VjimbK0sALr zz9e1^H31=A(ZJl24!~pJEG5llmjP=8?O{xDIXhnjyGP54nhb7O4UnCeCV6I5S33PH zoOyTTry76lowpxk2D^jxy|}5rqe)o~WoIzKa@4iODf(6v;!<8HYaR~n;KO{4 zijlaa#3M$sn4;5)Xb-0IR$SL;TZ8tr<^qhi&zIR-!{mcC&8*5Vut2G`?zL=Qi?wq@ zq<*Nax(l~w9E(w9bY5aD1_vFZeDj0IHubn&r(CBL?}8V9^F9bBK^bM-$5znJxv_A! z`Z~=^gCbc=4lw$~D>YWf6^ozCt*2*{u!IJtR;W|U3^xl1;kmP^O27sY5(S|7g;iZF zlAAY>>Q__UsKtnFcBsvbTzv&-i0KN^cS7mwMn~0zx_owo>vZBQAqRH1@2(bV4B#>t zGln&FmmNZpj3zMuwGmBO22#ZJS8j%1pNp*u}q%FP#!RL z2D8G}&KG^=dLwTw579u3k`|xw&WAOG*h#u=_OTpm={hG=e1jQVhBZr$XVMIrdgpG| z*OwWI`->hl*a#0E#rg;*#zu*}J&RMzo=~q{)(noT&xN;{<8SE64oGNrsFb%qc$Xua zk9#<3B5a{El013s&`PG`woXNXN2E&Q%X=6hi(Ac8+rUB#ZJCP@GJDh*&|ntWg!pHt zkxeFvx3`DUA27;tn<=$uCjRm%rjvE*KF07g$Ic>}=Bxvn*Qa?cY-gYVhdIb?q&`I@ z`7kV)4#N#99=!qP;WO>4#U_lnhEkFU8r_@JATz44q5UwZQTK$cB-1)!3r&WuU=$v^ zIh_!BFEF?0@RrPVrMI|(TA7dOB#;5yqmFi5HnecNW33As?Na(AQ8mnx!7+iB_nVve3R1R}H}!L9TbZ3TW?dUalyn;Dp^ODY%37$Qzz zw!q7DVs$3DHJ8Zc93&$q@3WO^E>5Z_=x>Nhee>Yv8*Bdc#HhW@bxwr1OO*jD7>yLa zyecg_zj6U^7=|t5Xf!L{d#K5c_2YuueTW=AzP-1QPKutnwZTBP^-fLp?P+#RA%^oQ~ShDTw4eUtd^pr2Y{dMu3&;uJ{zn*R@r#iro z2&DwtjrGr{nkO>Y-$ECqRmY89JsfyaZ*~k=zaP9;qHtfcnEV>;*BiZooeoP7k>mr+ zu%H0Y^^lA4BxE8}?COz{aN3hA75(HXTf-OU6ua#<=Ugk;(3PWRr`UR0vN-C`W4r3E zeIj?Y-eyF>Dp53jah+ap|`kzA;MMk=Gk3p2~2GP;RXiUXe5X zkRPX1#WG#G_6T`8-N27O@LBA@%O*39MHg*A6jnrkoEN-jOLknrkNT+IEhHlO9Qz%E z3D~>xOi;Yw(7plKSNW*zJ_lRM*nmPUDYBTp$k!U$Rxx#6wa&l=z3Y|rRhg1Rf@*uJLDs!*So9k3Te$;6z+);yEP_2Ilmx6 z1G4o~Dkd>Y`Rx?Ps2b8mzFFW;7Fmg$eD#xalCnv%(Y8^$rO(~IGP@Gi5=ZoAE`t6s z|H<{w^=95nc1!tuqQpe^Zbia{eVExDP-Pm4a}y(STVcz$o`*_>)+ly!EgRxeyUul~ z5oyTM@by3pXYCPbyBV*XWCfKX?Z`wS3@f&Ej2d|3*<*dn?A+u zaS}`E|M|`*w(z<5)|I@mPdcxhRJbYf0DX3vkqnXbbrf-Bn*0z@Z#e4GHOf0Y2}{u+ zbjsj`W(_PIDRyn8ULEceJ6FE3E{n3fNJjC7hXd-a6-ZIKC>*F0Bdzz?-!L$yEw?4+ET2oHEuVfIb%EIS!b>U7(Hj^4a8{v`iIDj=oW#x#Of|MYr`0M z+K5ep@R;XqSDOcZTOJ6VF!!r5s0++;+FADpZh*T3E9gZ(7LDT|rW|wGlir;6r5*HY zlIR8xys(2CGN1>Xu7L4@D{WCD=&De%l}YDXmmiOxo>tr`=M8Q50vs*E6BeK-OIa$n z9bxXoG4ss9?57knw#J@=lApwat|LpcGhdGEu1;t+38g?V~7b%l7QlP#c$Wb)Dw-0i-W-^*m z1?*7^9dp)GDM7%Ah(YEBV~OxpVz1=Sal4}Rh`O1^{SLW>7v1VHQe$&fBE_vu-^$W$ z5v@?(pckWvvAdw%Dc813n728r<5$qQ-^dDp!g{xw!Q4*y{rr?UJjkCwqE9qMowlwzbGlKNeg+56W*{PuwK%t?x9&V;`L$cy@PkCp z{n7z~gA;f`#BVGKg@9mYeJ6E#4%DNlX8!F_un8m|1o$TI`hCgQQVpa^jjgL?Ne=1% z{q(_=AzMxLVO)A9qj3_;G5WXf)1NF_hF-}1umkO!gr1(?CjWn)Jph=UL9X84BJcj- zghJc_KBjyBf#`27`y;a|5P2pLHTdq|C2<)59H9x=L-$`yTE_n>_={xwUl;v{)g>nX zpBDW!fBwImXsdjDh-&ey>$a&v0SipC$^evaT8BSg#|-Wr9ORyz8_y6F&9hBW>8{Bc zB)ike0&hB~EIx~GEY0#&>S|{^nI)W>6Mk(p;kMQW)dV(!q;=q_UK935Ujnu~;7lX= zPvZl80Dw>(aHh91dht1D5QCI~p9LE|TY_%4oX zjztFbSaF+M05FQm8Ca3NldTdr*u_(9f`3F2%f4iZYX+e8?iJ8#M%W;L9{#L2J(xrQ zsJ^HvfGHo9)(q<#Ut+^ndnW*H|D1e4u$AxaTfU326+bvq8XC!=4r>5d<}%nGL5o5F zKCXk^!ebss6ikjW(NeqLE7~qgW>2~_H}1JNnqJpGk~{_?vDE!n3L6h5To_>8X^H^g zSRy;&yOM?ipwDyw|0XQCb+F*K!ylZn2wyMoWCOx4L2zSCC<5DPX? z0L#XyRi3$(!vZ#})|k;gN@pQ60PvhPBtuQhSN5Il$oF{%ok35c2LM`PJ8#2}dTa+R zBl$+)22v4YwgU(HkEmR9ddo-}j2&Tpz$Co_EUO z02DnP05q!}0A9jFukZkH8GmNKOA+d5^%i?01)9Wq0kX&*2+Bd%$YW=Nf+V(T76t*m zNrS($0hAyb!x$~>I>_|!#dez3Ys*IL`b8;e zf-o7o)k`8G&aSg}0X&?1>2YBt(Pj-W)Z9J-)ecrS?H5XEIg4ysfU-#30hF*0rm(!y zojz(JKg1Djz%5v-%$jk=J-y)E@l+vUF!OnKj1D5CoQuAyobX})+Bl?)S^#j7i&&7M z0bbidHEVyy0M#skm@)dBI}FW-3V{R109+tWaAtBmnh+27dM}zwwuxe^lt>hv5UmMJ zw$Du-RK{&NA26F9cx^)bF0WYS&(v+5p?Oyh)3FVlfLb{0bJ!o5UH7dp`Tih1*khHP z!6XI>g}R?>Ld)#N!WKv0o}u3}jB*V-$RK>V43RkyDhqgUy&VJsG6680qr^@-Er#~O!#C6KzeO`kKFOE3hEtTBtQ-bOJ>eN&?m6SK zFct|T>AGO)%HG;8xTBFvoUW$$U4aL3xXHA`uXnZE2*7-3n(hD`o|+e;C24}xcuVDj z*3lJ4KkXZ-!TIiqY3RF&Chu)kA`gH_M5)z{N%8Q@5}5b4j-LGJs(`F*OvqPu7g z)E&uCB^unAh3<9Vi(cc_y-F^3FP?>u$=|&mF^Am(UP=%A5pK@9Br; z)X-sPAV7gM7twJFgoj+;Hrr8<^u30_9t}}lY!Y|-l_E79?!o_Q@5}$8Z2z}gL?Ua9Eh=it8ikqcvSv#Z zO?HuGEMpnQ64J(m+}UCnQFcO>!I&sxWFLFR)>wuZLJUpxy?UO{eLtUX|H1Qm-M`K2 z{N=pne4X#*I^M@|Ty;+z_rBWfQBJ-ee|XmZRI%|{3Hg!tzyM{-V#I6ONf&nXyjJa- zAj>y+_?^b27oB4j<=f6_S9?_F?`fRx0}?{;)T#yE&_|&@?YhDCaW8+08KO*IwH+8b zU2^ONFt9=F5#TMAY+QqDk=jlmZs)l*zi2~z`zW#R zAB|!e&7K>C!y4D$P3DEKV)wrn{@nTvKkk3dESFK<>h5cJNJs4N(!hy|BlV$pU;KLe zu3{VEBawr|r*ZJi-ftYC>1eyRy!GOUg}07WBdyBf%cp9HirNyaw;?(5#cay%1vaL) zVh;e}$5mgyU*K#%wYM4q0(*vu`I8^A-Ji1%pZL+TTk~S*>%@@sTjaJgp^*BsopGqS zt3W@``z+}qC4t_jc;{y#yw30E99ZiUL#~c32j|ksU?V|`v+3i#VF<;z!)|jCJ%m<8 zBQIK0RrYMEZ*@deHJ6Qj@XcRzMS4n-xCp|xEIVhd!tvLS(OpV;?>v1)^lr^kLDm9O z$*sJ3q1gU@^JwVhCUH;X$^Kr{o9xKg0A*&P@4cpxJN4s?ai}VJFJtz&#eR`>F})`m z#Dp>>MjsJ}RwjlLK^pIE&yo0;5T-L7g~QPIwkQE?U18N;bl1-{ZQmFoxWw0|lF+ur zL8+hAlL3%CU-mI$mT$avffP}>okHyWF!Ae~(B>1sP~&J$S`IKmXiyh#1oGWLb(egW zU|-{2NM>h8{-7Ij^Six3&7ksg?HDvnUIntm*R9{ume_JokHJmgEGE9C3z~FYVqp7+ z=RSe4-Pp~{=w!VR>jjyqtDQusjygzhK+Tw?Dv0>A%W?lN$Be-KO$Vt5 zutSQC!~U&Zkq$vy`CSNbT>OEBFR$BGsCkclw5cYE(Z^inLw644mU<~x=6!gG_G#=T zXQzs^xe$--^1QC3H)t2zXl~55^;GZbI%m_cQTN?@tV$zetNLY1iL%Fr=f=96@ePXJtI0cBCH#O(a9iaW*NiTic$Uq;&75I0cn$EqQn2m3w&k z#N=quvPr#S9S2AQZ!;lZj)$cdX8R2EB3d+cT1cAR=_f$E*)=7Bme=G|7!MJxEr{k{ z`%Lb!sLXN>O8jhgWMbQ1F@8aTFXljdP_X1WMyn3-+I_hb`jEa4G*fd7jrV{O>zV@o zn{F6Ta5iTX)sI#bwccy;H>ctjY^7Pd#Y%&C6WS1lpvlDn!)OcdL!@4IFy=Wf@q zjA#1K4KG$@2U_K$X7_F}fvDm@Vs77)#idtgoY!I=Yc4dh&im#7j`Qh6rP13Nw50pr z^CNCYJoyVo7mcv+hi{(zxc){zg-5xca)fbk?>Ds+o)+Mb2&^)1@QEvCWT zdw@2eieHHH%+#pHxQ|NqR2&|MzO8xZcb(uqO{T*~-IqJ=(J@)Sh4T37q^&%43^yc1 z;C;PE^4_AMRwi!xrE6m3U*S2D%4({$v&7&FOfeGosGs$9gm~hNLfYU;)o|m0(FX=w zDJwcC=twCTU;NUm72WI+EBTANhvuh5 z`R1Jn;{hAO;9VE$!;(#=Qn@iwCjgaBlBcR~YibgLg%ZE62L-QNIombw? z9xACTwJ4oGp#}C|(8cQYU%m2s_J!77MLZL^Q{c)YK9bFU7VISt-+dryN$CtH>hb8h z94G|ll4Et44x@-T0U+A)A@t?9?;YyHt%QGfE2fpYF4K8}>HKQS6!=GKu6x8mk`Hgt zud_~`s(Uo8U!NGz2Y2>yMi&;$G{0NFDzlq-UNrLxd}RKqEBMzb=4gkErr-CmUKLLU zB)9hAwUc~u2|-#4tfdK^JTVe94jA5uGr;)G>17cnB7YZ!UG*ncct;eE-ml^}S~qMl zF!3%VMaW{OAaKwFrG4C~H_xm~^kg~#%0;$~(?#xsP7_HW)aCFwY8slWH`Q)287K5i zZ)W8D0qP+|)?MeYFi;yRbws?k*3KHi$v+fmX>!2O1!WE4@&7WkifD>5S z?z?Pe`km}7q4Sl)jX+{yH)yCNKA#ek3L`s-vC}FSnuXLAH2rAF*VG>HhV}3~dkz7S z7&p|ItgX_NSO;fo!GVpo-IzdcMdk4dqNj+)Z!kWaZBSMc_73v?OB=LlIK)+rBx8Tg z7;|{h<=WeWhf*6zwH>p?7CmhIHL%X7+J7=$-&U= zP%5*BZ(55}7H0arOy(A=e~(~-jw$I1iE$O7nM1O@$|(qJqam^TEWRWR2%W8ht>#va zAh&*TMqBGPKqaHGtW~6?rT9YU;zq{LRsNw>`^W+3?Yx-vX{roObs_y)q#l)gRZK?`-0$o8~Mpd%i4;!3JRDfv(Y{%4G!6D)-;NH2A|{+SnBDODf4b(8oZRRMb%zW zRn&+9wK^*9#78z{IKr=rNaMEABcE^43W|>UF6bX&6DWy@>d(G%qsnyMA)kaCsU_pr zHEGwSIU#KN^Cz*EWUE`!`54Qzaf9U0_t}Pxt()Fr){Xr_?LIn|FDHGu*|uNgSz>fWEXM6$Xf5j;8@&Lc zp8DJ4N;n+4P2b2Av}Q(^EcY1tY9(s1EE1PFhO}DwiOE<#D{oZvIVs$+$axlxMMt}X zAG$b1#PFPQZZ%G4;+M|!7+<*GSOSS`rXdt1l$T8T`)1KB6ec(g)no+!t{6TT3u^PYYq9M<6`8P)yRl{-NUG+sy1l}+f#l1Wuo@BY<@o?f##9aKE?UUS6lC*!%LlYv%Zefs&- zK>T8=fr5ebNW-Mm{UvanezVeW_}k$|AeHTik2YG5`%{EcA5UB{a5X6NMvR;1jQVaC z%iRNa`6|mz{Y2!}I4kx0tC~b7b5uTb4&E zSirA-`SM$(jm4mslaRfq?Bla06}BHia?Zq={IY~@;|AnRa!1xY`GJv{m|X9WwZ6;^ z)4`Qw<_jZY%zXS{;Pt@M-OVNBgbeL!2kKn_xqIeWWIgjx*G2)BZDjdTbg<3UZccvx zmdqWz1&637{FW%FT!EZ@ZX`YJsf!7p9ETfN+FE11EmiwA;=z0hT;hXQ3}*y$t(Uy zlzL$>)_1Gw7f~rwW8g^I@?n_o(t0~x)|xXQ%doJROdxw|ZWVAR8H`HO?bwF8LSW6K z9!+&kcz4>l4JT1DJ!9sjTW}^UJ1gEkQ*?e0AQ~!Z(Udg#X%2*syZ1eBzPz60SQ_m1 z<0u3=;uQ`ekDDC1ec7)6(8(At#OH!4ZU&f9hl`LqeDfh1SL0f2$( zm;dQ2L1NTo{WMKjx;UTBqD`1gK`&893X3kgMhN)6aAFSMnz^J=ANsAC2)T1rpyr)} zUHUtTqrf@R2a+V8k&lFD}kT^MNCe4{NP1JwrDlkF_3=YPGWa-nQ&A zW2_kg4XkPetbgWXONWti1!plLR)puvCow&px-*|Zrx{I(A7sJ8tn9P*PxIM7Djo*g z7N{vR%mR|F6cElWk7TrLV%R~cuW_o+!kM42%+s@TR$1I&fPUAQ>LMDW3K~y|txo2a zyT^H%lr91W4**?Cjsn(4n*%n%ux)x)j52dO3`UuUVpbOa>O?wUzrvT65|Gcd)NEYp{cetB?t$nN|1CtPC|J#HF39a-W9w&UL^Lt^E9u2U`)G4j~!sb zZY1^KY~pcF4_=o>7wXOz6_B)NsQBb){+U9EPbm$G;><&9K_(rRJ}ESl=tj=eKIK3u>9iG>Y9DZdlA8*L8!ljrd!iBm^vA5f9m;J| zXWTq>+(*9P%iW3h#x5t9QuV!g>Xokx>v`Rq?f}DQ9e>~R)J+YVSC~4Yj2Lp6l7n`G zecbWQ1BpyW1%c!W45kz4VsAOa45(_ab|&q*-wy{8B%xaUjlB&Iy@d$r@d9T6P^9Xa z0n;EtQ#q-;!f?ESDJOi|k$whtnq780U13Et^H_ujvu47I*l7YXg0~ zQn{H&-DdD@yK#}tD*Zrfq_%ub9Kj(vJ&1YNu&2m(P;Io>jvL|Afw-ff+$CAj_X!&a zot7gG`9jf~m&1BdFMq}__j`Z-kw@6 zxeclIt5juk%-hc)xQeGt>RW(gj=v`~`W50I*%$2n48iM0#gm}89g-3{ZaD!khEq`S zQi7aK6=M0yTd2qVvg3&xXtSufIX7GVjDkcL@1h(y%WLjKFY(b;PIbLvp|T4GNLqA> zS*^YG;`r4BM=37teH=prmc09nLkHHtZfzV9M>^n!U&v9H7h&IbsT7*>Q>8yYF4&$6 z6&pfm2R@KO^z%a^0Ak{r1ZTqGq_n;9;Je>A9~||%yPJRWam;(6QxDOs3(nu5VmP+v znZIECX`l4D5A!&8DS7zJo{7~)hcE9IEAg#;5kU3(Jmu$6uC|Nz-=MbL4ku^zP0Tsc z!vw%?_G5zK^jaH-S{Re!%l`zKizstveis-gWGI))^cyU@h~g{4w|3oFKPZ&Y}+0xPEp1dCqyYaHXAq0 z{@s5n1`p2Y6D z@(jA%UNR#hNF4+4sMXIBSvO1S0)3?y23Jt|kr}zi4)+qtoAJlstEzIB5UUG?PayBx z8D}V`9&`imM(4Sihc*pLBBwl>+P|v@++X;R5*@>FH>~x#itJITog6e}pyPe0qw1#W z58t_Ww@5OnTGl7Z0Q`4XX{>OJ7Qd{c*~6zJ!F03sw^}ELxN;V??bBcR*1g z&rlQ;XHwXxWO>Em_%8IuH=#9BxMd;`U243fbW|Oh{o%~X*<3?4h^t5-s;Yi6o1FUK|w)2$88wgF0dL?>| zmv8zYh6&|1j6FTtz%n1<_q@o?^=@Rvv?Ow75AtLK6VFkBUPMad*BZ)bVqbV%o)NP% z{l1?!aAh0&P1gLXBuuLA!r%?h4u%1}rPI_Vyz2rJg=j=>Je}XQP1Lrih+?D&G^9YL?b+pLsS{%H6#Ct>po8s8DV~eD$Jdf3Lcm z#qZ0+@F2{^3+QI2JeqHlyJUyq+|F3O`?7YhuBoDSQ^3*C;zX};E#ny}XyuOG;?T$x ziH0asGH0PBn)89USRk_b!z#|xd5q(7yFtVg)II$@;r1%3Q{QYe<=PNK3rLWpwr;&gI( zq77%!vx0yHIwZ~2{>j&>;Vq-m<>*(^v>|zL;xN#EY%5gs00u+BeNJ^zrGZwZTe$)8 zBU3K=A*CoUEx&vA-n8P#9%u$d&!a;K?gHmd91f#-Y{=$gV&(31lb~thbr{gtLal4r z#goh<^?STCN@n?MDiv0qKl4Bi&8<^;_t>X35FFMiA4y-%w$wV@_S9pgyM5t2Cn0aU zm}i#T%l>jz(ej06#?Qx0b&N-G!W*Wby+4j;0gZ)ihJi~wYYEmT{K*F1yyZ%=*j4i_ zSWm%C;clCQM>=`WvxdyiLEPktxf+eC=Frr&lnWP6P-?q3GSmMr!KsI(6A)lqGj`lP zu5Jk{uLTW)wO^O|sj{tPSW%2rzvJ9uTCup%;PZ!(ojN93TEndx*rAAO`}0o#Z|TR_ z1FmqrkRi{TVX%G=jae~w5s{czV-;85os-x9o1+8wR2Utc_?t`cAEHU4U2b>UwxzxA z($_x(f#p}Lxo_4bhA|I#G>Ccc))0lqABy6e)P?V2sXoJOk>O8-kM z^M6_?AD~zB@{=(6FO|~obzlQlx;=i~KKOrgvtzsf_a2*YrtOtK*5w5+daO|s)=K{9 z2CeS^8;?^+1AqL5pen$L2ILoEWB)gE`9ECoabV-&Da|zVKi1IzSG*~7w(091P2w=H y@!Rd~s-QpC?PY=HktjNS^AAI*qPmt}hc|MsB>e?BBlP=IO~(4>dKJ1(5B>+BrwKR! literal 0 HcmV?d00001 diff --git a/docs/assets/img/unity_exported_table_columns.png b/docs/assets/img/unity_exported_table_columns.png new file mode 100644 index 0000000000000000000000000000000000000000..c31356695c0586520dd91d1d149bcf8200c1ea9d GIT binary patch literal 25799 zcmc$`byStx7dA?FcXx@TDAL{CAtf!+y=jncq&o!+8YPrYkq&9;Mq0Y=+UNX^`rYr} zJH~Yk2Aj3t74KYY&H2n{&S|8YiYz7?85#@>45qxClm-k8Tn-EjtPlzkXnEy_wgm%& zR&FCHsU|NeNu%cCXk}w(2?HY+nUaRArMZn4c<Uwn zni)z66H!7pp&>dQT|URw`-oSzuomSJW(EWgClU1#lHIFmSXmF?!mGX zXgnDnxX-)JhRUj;q8ipVc<`EY>goD#)01FEX;qAb5 zYk&APOy)<=0cO|R>+5^gC|_{_Y#0`jbv}OqPQrEkw=ElyjHEC$*J`cCI$@B+%6z4? ze9Xcdi$weqFN?(05|ZGIC{*+#DSbp3xX}I-1{kqX+1mjnW4O%lmC%~v2u^~67!mlX zi3vjAg7yB>s3@Dz*WSELfJrt|swKQGA=tcMA zs}Um2t|BFR;}bqqWg#6=rqk4wB;0A(XJ=&N(lC(6vh?+MlPOGk$f<Y zz=JT8w8(O3NIUcz1s{(m-R@T^YB~k(uo=hr?AR<)V4tAit$fpN?&oNnV6I(!cH}mP zV>aF4D~LHvwg$4;_IjDBD0OpO3QYEwt|Wy=rG80MsLx?hzrqWsAwCnsCV_YT$m6qw zF+Uo2D~^K#Kl+?>DM%^-je!PzEy!05L7|PPic;&9@N-T+l(<0ie9~+r;xam%J-Tpzf}xl_`E)0vb6H?T^Gd0t(H6`MV&veIK_LnaSZ zZnH9tu3DeIolcrooc5Xa|A|b2)!Rn8OyT6;@WK$!fzg59fw%(KlWHn-w=;I-2%_CY zx`0R>vbn-@D1DK0(Rtx|F?ZpABMX<2uUy4+fn$b3CjncK?G)jZ>XgMLM#9XBgBhXH zMMIaDE;+9>FKfrRz;K4&D<8!`pNRe-CRyqmvvvZRT$-G!+`(6i8jhLQO%ZGim5jOx z@rjiQK7DyC3Iv)_%$BIq3NuptV*ILJa$d5<8kq0)CrDl*x#NF|^L~95bEO`~WKG-q zdbPj&^@l{;#4qgU&vIEDX@*lxnDQQ)a0D3MmE2Upskf`OYX*I&dRCyYoZJB)sxF;b zm|~V-_I^dCTQW+Q5IN;gAxvRG!M9kVxLJWXgeR?*(k^8)i&Q}+#E^O z!nBe25y5TxZwlY)zTtkG`KJBNI@yN=r8A-3^bHp;PaO9?kI3+7c4sbqdabESXZDN7~KWN!s=Ac|V9~cfQSd&#uw0F<0&(Z?BhAv0pZ%$*Ch+ zawi8r$@Lv&RPSB2?rK=i+mh6h$`U=-(hccBQ}(dbFFJ`j8I$_uwDy^GAL`V;C|zft zDV97+8dNgX;hM{!7a$Yx7tqKd%TdX3HlqLG^aJjP%#TuU2H{lWCgTj_tc8e{x;waA zpl=wpRImj;=@ms zuj%7RGAOBfmt#|Cld`SMGd-}N^rEi6%$CEWV%2bTezp;N;K7F7iPDMpi7|SZbiQFrx%?a&h`=!hY;>r2of$OOI^5=<$Uk#6p%#8xv_MFF^S3iIMEbKYu zy7}e6llpm9Rnm0$Or;!Luw+$98*M`^MJq_PgyG-DC4(E%xYjPg1!1 zqxu&dOmoIEPhmdY@8yiZT?B-G;F8Z=*Fnt^UHz5 zbWvle#qcI|sHn%(#pL7fWRs}}-Ft)c*JdVNEpd5zqTly)=S*tucVS0^a)W7+=XNP} zcCc(HW(~2Y6nDe1IQe~dBc@)bBAtX~%E*}asXJ+T|5_ie7F`r2aw^;7+l6$Z2IGHc zwqcIW$}h<-dA-rKk(;`eTA_JYI{PfgK=sb-s6S|2p!CBKOWurcs$1RR&#uOK*P_kt z&2UZz!!28bg|iE@iS93Z(X)b_h=!kCW;Tzzu{?SkA9Cobn7DuT-XYt4cH(}$tNiq* zL-x)0{i&p!vmE88ln!jNECnKYQ0 zx(rc0UlNF^g|4o`LVRFgUOtBTXaJKyq&hsI38#FIolbX{2l3v|#q)vL`2mX}{RHK^ zku5Bj*blU~m@;{t^RRa2^y3+qo-cLs2tqqyCO#TSOL++mcDe-zeEmgnpVo436U4T= z__Gz}z67yqkR#v{;8z_>J$Wl-Wf)fQ83hJ5+y({#e1Zi(WZ(w|20k$i1{wUu13yyv zaQ}S^my-|w-)C4M=!4>#lJfH4x2Cy^rKN-G3rDxvrbag~)TE7;o|~Ssl7P9RJ%_1< zqnRZK#NG+o1x5%W06yAVx|z~I?Cl&}1t7w7|2!c8K0}*1>1h6W#LZTiPET2lM$*y6 zl7^Rqi-U_!1dWDK9Il1}y`8l~B zaXxy)4xV6l^>T1Cg|Iug(*HZif8$75x|+M#IJwz4I?zDJH8pc|cN3oVMR+Ne#PbtXjd%2!gZ8v}5I>0TvHStbUTlKONTNGn6$J1EP zuDZLU>12rfZt?y)z;x`5yy>Vkarut(iPg}PO$yUazqx0IHJiEvEMLkQ`=<QN%(yTYt=(>5>g|8a&p|VAwth$Hn z$dlGIhx5(Z%h``Mrgbw`Sp^?YJwyl8+K4JhME!k9o;nz&yAJUiG;IFrvtb?rs~5Yk zT1!+^X|3+!oJ}=wY<}W)upm6ZGc3fYF>%MbvqJ9Jgxhkp^Fc38;f#Dnq(FFTeTcm= zrZm_4<;AYaJ((coSrM(^da`Cme8A1B;f&D5Qt)!~`2u?D<)EWWab>*}N(K<05O{gP z&_ec)BNC=yFR)LBMFUKYY}wWL-)_y>)hb70$p^d5+LS8SW5;&6^s^It%scmP3$$$G zS^AIQ82RjdG6mAb2Jh)@r8viV>?FmUi!s_bp`jQ`eY+*wNZ$FH9$?#B8FKq%MLs%= zEu5qmVg#n}$XH6Hqa}-mt^x^V?z@xh7*s-hW%Uc*GvBohIIJGmBERB+dt>~wIL#RC z{%WeVBY{>@B2UB*t-CPi@_O-}>TKS1Sg#kSI9XU#_Tp#laG0)%-;(s8 zT=}|4Fml$ziUzu-ihjKZ2!aHs>x@z>ks^EZEE}^oQ)h*PoFIQ7zqrP*~@kv zLIpeHesMV*V2qR9db7^i*@4bU!TZB)+^WF+7sC01k0Zg4>-{f|KPg0j{13ra(9e6! z*_*cBp4oaraH6$X$lL`)a$A^Gb14KHHw5YqTCO?N)FJQ7pN#f}Zhiltd$EiiKqBl7 zDL&dC52E(oDuAv2p6z$kO8|BEQ+Mgnv|?cYN!wS?>tZ>)Q~Bu+ z0{zqdA76GhbsoqFk+@5BcJ?vHFKd_hALpm&K3?mqCmMH~xGtETdUd@~)rs|C+Vo}P zaKmyKc}Kwg^&&}kv1*rrlGgdvZe@I2l>hCOb__fz;nez>mWgZh5cFkDGb~dhbB72t zO7RyUtFfJX^^3+hIZqdTT^oMZ^mo`+HWTSQvnh5uY)1|sgmfVtM7tJFX`j>=r&rDEaItzSG;pcWAd`-nRXG1SyID`dT2rohp65>H-E;aduX?+8J~ zN0Z%BxyVTil<%iF1;qG&qbVdO1Q zyo1bRNAQ080^YkvC8mnHz#@kECBklH!0qmZ)(@>q^n_=oum@JG6@@MW)KOp%i^el#=SP|V5x(n&POYa^qYqa!K-Z+*l6jneJ)d3>(g&;i(# zxN~IeQm!s6{eHsQfJl`fm^*7CcGKl>phY$?o4@#w|Z5RD=>IkEK*Bt zY(2|4Ytf`%TyenhH^*s%J-dE@Ewz+W*(6EVs~LbBN~?;gChJpVipl397=qqKc*1o@cX2y*ckM znCIeSNO_pNTSsAk(PJQXkbQ|iyu+n5avgkvPL|(zoJ8GOcXw9b+AWf0-WJxRa7vnO zVlX9o<+x8cE4TwML#}KhMtzfaf5Uv%_|b794`|XaZE-F`!sqj_dy@KF0Va^;P`-~* zw_Y@bdk^dEjARrEr*wB1%4&NR?AMpWsf{Dv5^m0swYp+5NePWqjYBM?$*A&}^L!uO z0QC{K8O0nQhtLjZho;gQm%zenxV_d#5@8YXs7KH&#@hV4aWj{k69tnxwv)=|CoU(# zVq42rVYHdI0O^>C?@{-1elS)PaS9HXDm6*|_Q$WQTSf74lr;V2Hd;DOuR zX?!TFQ4qo;%igW{!V$&8N(OiO?O|n%SttTzcq`(vq68=jJ+(i#)@@7mSk4^-x+4+v!PPkJC%<{2uGX!g2@sg*dv{fJn* zosXyn^vO2KSq}rud+lRS;gK*R%$QPQciS%bn$izn@dJfJ=F#U9O|rR+VjF`zw={Gr z(M3RBn9~y_6mT`3*E1Az{fha%?xUM;4})vGkxL(=y6;b*qSTv4rWgFqe;g5vUr1Dl z!lhhg@2iuk@zE)5wc!XvDDR^S+%hahlO82VQVJZl!JGH02nLIFWzP0At59}aYKbx!Us#Fa$97Y4XfWx4y<@62CrU5tdDWVxf0Y6z|Dowq7^+u?&*(^-TttWjnStC;JM z>%0Q3kKtvd4z9jsc=k>MZ(jvyP_Bz?HhEpN!6HzXy}ORsn|GX`Eo(gPmBI|*BbxVW zScN50kL2R~ZapD!t9Z(Gy84bn=*>v$xK7(^WK4)t9z!Q>a)szQsR{xJ z5T{I^-G`W;)G;2|Fk4S66g#@D30o63@@6%^$eNl}m^ifb(!z~nZDv-6SphkANs7Yk zaSH^22prF^#MN=m<}OJZA_8ztjU*C}LxZ<08eYFx!nuIZkwgr}4l;1Y_#R!9SV{?D zGE>Et9T9{*pY^}qTTLR)WgAMuqW8L~*;ZsPb$*9}$c9NOM;K?@9*ns8O_LS_B39ts z+t8Z%FhFSCl%_u36Um;3LnL!eHXI~UG#3U>m z_u9rd-@E3e)ez7w-M!++M(WF)O&vRo6dytl-sVeSuv1+@z?D~S3Db@U2ouADGvtr|A7KF2>#__XmYZ_FiN#MAsE z?XQ5?Q_@dr_Zk7c!7kO+alAaAmb>R_TXC~|xpu5xikfD-pIM%pW?YvwwAADx%TnQu zQbG<#k^4wH&gyCm3zM*tWeTf%M011)lRiG?AgNu%!7IztS6q287$@V#Xl`^An`1aC zPcvTd93zUUDekSAD8kVJ02gm>H2uHdl@u%^(agV+4N-{TY7cI!f^pzL5O_+S5{U(? zNX3|1%Y(M8l&dAj=B#>M&9SR4*Z(fDN?8Axs}u)mxA91^SoxYDfZ23-oS?^jGcsKb zUxt6oJ?3H5-O{N;#f|irkF%;nl-zl_R!3@1EdT;jmkNe=J^!3?e?6)}^#D6gUUpbh zI9HO4q!8QqP41L+wBZK72W4< zXH?6%Lkq(*w>YmtVj8UFd*L~-lS4j$lQ0;3{2YT&uIJGTosZ#hmxhm&Q8&)SJaHd= zyCjl(wnx>c*zin-3w~a>_^&b#6tOF%q9G%l4}-;|AgRY8Z_a4@AACRg7`sYfVj{pF z*WMgoAB7zIZ0kaSyRSlVD$AWuW9-!7{eo+&nyeNsMvp zAKM>PDg;;$Cn1~cJhnH}JXb#omUdlN-ORQeu6}D0tIs~ss>SRicPN(oWO6i@u~gl| z;OLPbyE>&Ck-hb2$C^F0BN64>4chFl59V?6CXO^D>odO0 z+z1#c0{2)otd69f5P=M`CXH)tb4qO6i1%iS=h(*Ck7d5_DHjLOSEYZIqwG)&St?fH ziF^0Xe&(djC-Swd2dT5#yp7elV!s>Hgl>ls1||(b#K`rJ1OK$dkF!q~66}81e-rGG z=gtnQBdzYdRlW|kW{PcPP=24cebw;%X;h|VwPJSef%(GuV zw3=nt@I{!((nwae2Dcsb$LB|`EwA#?Qw*Z z1Tw_>-E+ti3wDJs0uwfs+gCS%5P6O8&gvHs(~fZ0_cE8YmUexQFySp%?-tYa>b!NuDz`9c5yvdKg$O8i`V|$x`v+&`-JEf_mx~&=9}}SvHIGW z;sS|2ak1ut@If2E0lfeFhW|hlV{F;-PK*qp~2B2lpj)1BppWO`XFZMYrV!ll{xs`7`cGFE0-=G@=7rSQCtxx_ZbHq%2) zcuiOl3N(5`@Elg>x9s>zdt#81u{+G(g&omHY(HiSCC^o%DCj5Q%dOeU-zwvhLM^~$ zr8!6IEt9z<7d+pXJb~%@v|=nSMPEW%nHu(J>4!B8-vTXDZd+g&b8J8;_Y}6s8}nC2 znnIZZ)E&D*LR?3f;?jJXpC>c1m6da-=NfV$H|f4T4W?wO`+F~Ab2dJYaP!1vn?Bk$ zjY6I;sUu7azV=@XL$;x+!huKesxa7Li5e%q-|#PW<=-u#)WcWD~`cpizrYkjcmetB}7Q(UA$3d_g2q+@#HA zD*W{40GSf*Ow$;eDvraAMlWlMXz<-~QkYn>LXxMEw~Ou0^qT^4d7a4P(xzvtSqUiE zsKph!?~QE?sHkFdcwXW2QhZ3oJm6Y)*vZsAzRCJ*k~OR2i?u%QnVeypb!tesX%gMq zr~)y_{2VaeoHdkp7JZdj@0ZWHlozqo*Ey=jIjPs|7=w)+#|ML>5Hhh14=Ki32}V9E z`?S_FxId-SKlnp7TUYVhq9K3Ob~Uk zSnNK0&AZ^%@$iiI8)>u6KFPYn@ZR-PPE z^kO~)4vDJvDK3;}Z*;T`(sn(^-JDK5eyUrtB2tvPLv1V_bGG&-NhPy@ zCH4pP15^GzwYt{!ECm$ma`l(S$^yo=0mI3qwVzbSRzGFKs0zgpcl)VM%OnAgbz~Od z&|&$L`Lw*PbnUY*G3@+QxXi}vVZ<2ieO*#vylw=$TkDR4yGk#p;ve2)l%(!+OIzD6 zAW|v*k6wmFbyI`8f0|Gc3|BS%7xDX@8C*>f^Z4H+E`op%th9A_DRMb(o-m>=j?_u+GP`0+qQXVdQ{br1V1eg-~$Mt*H<2kp= zH;?sse>2}|=m@^5*6fCe!+(aK;D_zpb@vZq&;3m=zeNPB^|J=F+~7Y=uAoVvwT(#i z_qrzFz~cX}v81hYOrA$G|Ho4!ybJ>5HBrT@%|EZWlMR}NIIFs$(*C#klNfkwblrB> zKPYfg#MP7z@~7dd{SwMx;b+QY7$fUyYZ6sAA@fH z^co3585nz~xwTMkyY;=+XKV74A~I{EbV?vXCjE%bv*0IbDD9UU$mSh|=)X`-+z*JX zGKw`_zcd*ABSQh2pm&<3p~Fl{X!N3#)K_dGtsS-S3-CnxaIv^w83h*awrWSrhbm0( z8aM6OsHYVFl>-_=PfQj`k7_nWs=h@_hX65%A;c-uOr@(x>_GUQ{JVBBD!5;yzbG?- ze1oKD(zYxxN@-6+1|0wS_C*|wU)bt>R?(UVsv8?zqm*^8gqaQFpABMF2BNnq-1xJL z3|#QA(wi7^hP1yomO;%190>*dC!)W_MH57dvgXf!gW)sO6$L2fhx33!Fa@-f^T9&1 zPq<~OJ`-S{Axti=#DDcUlgQVWVm3Pg6rQccs}Uu$6DX5LM?m|zo5+AgWUk+6)l1>` z86d)Y`+19k*G`Wh6-7B@y{4Z-@W%tD1LlsT$K7+KiiW>d0Yi?~eXnLSsc(&G^u+^{ z+W*27O8IvJpPalWmYkSLIg^0M7_wYQF?+TNNL2h`e8$jLEDwb4Z%&rGiA~si7XTsGa1Bs3WrFT&81EHab0@-+~SVrgO|E>vMf*2yImbTj?D;~$M6b16ngo+>{{hy%F9vZMPEeIvb z%*~hDYfK(bJT14U0XMH?kK?@U=nAqc0^(fMb2h7!ybJ37gUuDX6}Perec<_ z?Mm!?0FmFhnHNBXa&W%${UKZ9T4E@}+U<5}UKo@q54*npwT(Yci1(OHDM#Rm(CLJx zX zRECI+XpbV@aa4K0{hfL`t&6U{Wl967=^5A%gt{Yfp_co*YiDw=7_uj<17KrK%Rfz` z)eiC+IHv}21(d4f6h1d}?k4Qq8Z9cyXIzGousER)E1O*O%wwMvHu#=?*R}o{Oc>E# zM$0vh6XZq}u?3jnFUyOGLwLD>^yO1=2gGWD&-p$rCc9W09m|a8Qd*I@#lNWgL^F&cnj7Lytcf?qh;YdKdX zlL9S;Yd5)%nHX{rn0Ha!lc;xVudB-W`*V@k8ItqL8s^^&R!Cwn)5f$}{rriqDunyl zeV0h`KpFMa4Wsq<@oyWzd;5N?%e9YWP$(%_F1VCq_)XDDhgG-sQQd}rEob3tE*{#Q zRI*oiucO+OHpL-XixCpXRz7%zD54$nWxaEHc6N#@i67YsOS&&pt_Jz(C;5k~&3lnL zZp&QFv>!aH_|LB7;~~5kDB-y+r46xmO&xpVm#JtnNZJ)VhHszWQaNfx{@@B2&uWjxSnRmk&{H(%Rjb+5#jyL_L`LcS#4hTxudaDJEC$61r?{YV&@V2 zgBQGebkS@Y>Gneq879JJ5PDfY>ZOkF!wGm;Rwdz)#*~)9@Cy*eUDcT=k9w3{Fk+fA z*C@ZR5e!VauML2<0B4vLTzgo`!$$=)VZG2kV@AiUq-UZaVS6NymmvO+7S7dr}B!Bk4E`xJIK5cME#_#lHY z`)6Lxbmu|YYtN&jnz`Eka#}uq<9DkBdFHOIwVQi2zt!H07_ko#LvDu~Ar`K4nj-$c zFBUB7M^3nMR)Ldzb;w461=(-yG17vf<$mUUEwMcF+jhXBYDEL&@yudS_cw@<%ZDNl zvE{!5|DRCS@N#M&4>aWpe42UsPF}egrH;;qKC%(L<8neCs z8rB4{B!n%VEx~`3Sfm)o=*$P}bFSm@Bh!I~NN*v0{(mpQb0jzj-}b}5o~*c+lUn&d zP{!areo=Dk;(ss$N29kha_Q&a03*S`jW_M}-0y1zMXn@4hRdft9v*)gj6j5AfQSAM z+0X>l0Pxe_u{renyn+J)noiUn#r~Fgf|x!?DzIF3wf-$snjj(28N>-x(3o8%-tf+r6um7hb4K`?^QYd--8wBF9K(~j@cc1M~$As{}LLO|! z@!yCK7ByKK>===VXX>AhUnRvxGr|*Xej~m>gl}~DTWsigf`1*AHQM~G0m;6{f4#LO zu9{nF=y1+oQvCl@Tx)b(-oLgCx0M)x1YqFmNGW$k)Z9GEcJ2SbGqEhNpLM7n*sQ;$ zBmk9e@8^Y7o!{d9hzgpro}cIce%X9;Trl#A|Ly72avyUZMvj0x+K2MGXIlW8?MoVh zpqlB=>FVXDt(VICeo8FhW{JW9_~MZ+;Qq1?-vnqdNe~@YaBwXK2dx@)#Np<=_5Pof zkq;H0UqS=uy+~t23MT5KUOVqf!&>i7D^0&8KfsmT9Lni@P@0YB518T2j^m|Z>@X0n zR~^lWPK{(3@khN`LKa0a?}>S!ANWtK8)S=cG!CYKJJ!p{Wj9@Qc#?N76hkfA0%IC4 ze1`HOp2O%f!Q`jj(7q(*aFd%AV$*Tur038bt#^b&{F~WTr;`S~ zS#+pFCU?harog1d(YWTkjsynzGqV$j?DvL)%-Eh#cx)jqJCG<~TG@OSwZ1=B@AxBa z5f11LcKSo0hC?K=_!-z*E>?oxZfPj86OK>^8QQ8`;K%uG)~@Rh_aV*1-)p{ydWV14*#B}pVdu_$%81xw#xkvHOyObZ?b#e>6F^d-zanNju>{$$ zj%J)XF*`}Ev(=|JfTA3^Loo!ALRd!{>@x7y&U)V4#hul?ROBFN{UOj;uG2kpj6@!J z9t>&<5K*js?{{4jO82k%;=zRumw@6WE>X@Rf;|$0P7)5XVmQlczOh!vdha)G76xO+ zN?wPmDNSm9Ns2R42=)aa7O|H?BD1c-xSXyKU-uje+g{9n@z{Iz{Cl}UJzvYoaKN3b zVR+k`d?>d;X`a7#x_#Zu3Zfs#HbK9B6Uy0o^Jt@wWYmJ%_fVXoxBWmqiK%-}Xq`r8 zyQ~647Xv>~PCZ_nk03U3Q=h*D!i(N+G{qL!%CDL>iw+%E5oC|Ul)t=Y2RA}EXg$qD z#Diw0=VsEtF*bJ;8ud{-hh|z-K@%vh21Y1CHbG#VSP-MBBwq9m==nEE9RSJWLs&!r z4>sH4lpBP=z@>=*{EyHoXcB`eZ!m!5kA;4XNSdbVTScx^KZt$p1|yLauJQ3JG_7G; z1gFlL3e8xH%B~t@amPLf!*fwsp!2W{gp35-UR;0#i%fgZv1&z| z1Q5v<-00*6xm1HrAaY-;m!|Xj@JjHY34fVpEwhH)p@H^uyZTK`>$F|%AROfYd;L5f z5*EeuL`AHFAXv`o1*a6aj0>dDEQKoI1@!!y*K=$->T$cKdwleDOurgiBAuB8n8UZ2 za7q$u?YISUlES?dt|8&n{(ND}GpRoV;jtnp#jge%IOr(dM1;PsRLfBXgESpZu%;*D z#j4krK&%rYux>K%hhuZ>jbqyl@e3b99+S#?b>Yf}DD%!RcnjPE1t9e!Aug1QQTPn_ zIS6s$b0M0T!yDjIkZcyN*?W?W)4U)?VC(P>r}TJH7P05=CoRQH7dI3rFQz;L{6R8A z6S70W&1_ehc=}!INg$(^hFy@$MAcebyRM&Rxc?xaC(TTrW>x(=&<(S6heg!4EL=CC zSb-TorG%o8r|@-DW!|Fu?3ULT)+en)qk+z^VEaw{PPKn2;NljedMd+n30tuA1$n;1AUv%=;sUrA4+0Q6W>f-Z7PJn8KzVAzi#4 z1i{GISiO{Px#OcERM2`Zka~o1W~~x9;F2SSnpC5?1bPBZ z7RsfkqmM{qKn77D7kizE-&L2kl8jjsQUOno<_oBzrU|T1l8?|ea{rEDtb)STkf$~U zO^YIV8iyQ&8lt4}=EU?EC-}}L7=nmzE(l19d_TPBWlXgb8Pk$kZRz$4+kShWrh1lL zLh20)xmlG9<~@)rw}CTE9gfGH{nohLtkAWey*?&Zs4x-sR%y)(Sg}uc+fx2X;l7Y_3ZVY

    jy~7xAC?g`MOY#;qPQ$svMV(Q+ zLEb~;_URZQpQOySG9?=KhRSM($l`>Kzmm|(oJ@Rhf9bUDfA0{8mYSeOG{aHzl`eMU z>T)wL91sTeqkA^+Ifw_9$D+v$bpDF@9;h)?!UAj{DUgs~JZ$BV41Xa?+&U>nUfgHD z(YgWTZmrtNKzwA(_d2^G@y(~%1t$g9SL?x=&Je6h2J?pkppJc}%LHODh?4=)W{?JqHg9qR;s$)J{uM<3F3$=ooiu14vI5X{K7+*F=%P z8UF(|)Aiw6DEzJZ8uNZHEkVoyE^NxR=uyF+CgA3(xec-LpZ}Qx?<*kIGRRX<|AL!V zsey&A|JPtqL-TK;80}b?OzDj1(2E>GL{m*AEF7$V9ejj)_2+ei?~b3 z0s#6cEB;?z&;T3Y@&*!J|0eF{5nOH$PUjml)P76yBVJIcYz!tbmX9!o;{}H$y!SZ< zeCi%4)6p%Wc;f2 zA*(QMXT&n3&^zUpuRf{=X%~2 zIMVEMeh6i*uthGOmnfX=e5ef5RN=QQIdf-*svi8Wn)l2~={Qgd?-1C}PX`^|XW(c@ zqcY&~TR1I}A}}v|)LCOMqngUYIu(L;N$az@Z=~lBVqAn8I=o zxg!SD_VDezE$Vprp`1vRImb=o;%v^blhSh@H;!*5LhK@@DGZaGXzcF(W>d72%(e`d zchUF63GjUcZ%5_#0L&>cY4!hyAp>?%#dHoJ8>%?by9+M2t&s-5(j1SdP0tZ=5c@yLyG@4RZm>!|x7rD)yM+KP-m%mkjLuxU|9LkI zBnC6u)$}n{$I@s#viz3Z;URHEZdWC)XBtY_p>Lf-IF}AgabwjFM6fNlIBn*AYx!l% zKa|XcW`8VbV^;^ws8?m2b>m`v$0Z5nGu=PLGqBcJX+|FU7l)nLolrVzA5gRZ0D7=tkL?=QQgC>kQoy}N>0~(j7?r97x;~voW=+|EWEC10E4&HlMR1!8Uo`mPe zXR`u1|>j2 zGXA)mmvamh&QqBKw-2qKtyT#F(wB<@yA?Cb92Z;E16Qh{sBHCRQy!3ja4ka__Vd*mc<=SG7JM1ovljj zW(lzwxJeyh7&+0UgkaEkx6rZ`N@UAa#>HsO7+qo`CS(=xe|JC0{=4IPZ& z2#?%0_$`@;8QbNrB(a4b|2kL{jp_k3tf2SP+U2ShAt)kc;Tyj~BsQ#Cde`>lGjLaW zZ?-HBHB#qOT<`E>Y?vjr6gV0Iz-!uFel1HCnS)PBkOz2a4}o|EQ(0)> zB!g*W?8;=`KUsk56_0+M9|^yUg`UN0>9LZFW{~x%ay~~5^ujf5U~f&eytp0tBS?>@ z3!XnGVXC8OaYa0j{gYAs81!L8(>>r)@rQM0@O7n^r; zZIG#~)CwvMdbo2t!?CfqgPBv03jaMJDev#Sh9cx;RYfi>XdmDb^}Zx_Y5VL6K#EB&|zsVk*si*J;Bk@R;$~n}3ZV zs=lZ+ZiZVx?a@2?$$$HNdBvRjatmBFR0!|ko0SZUY!-a9XD^jxXtY$*{Q#E2E|QIp zTb0egk?t)9KsJR|?cG)0rX7UkhgFt6T&D+#eF%gw6yP()7*GL(VOcN}yO&KwBT|SXV$DU2OzK4N5Y3=t1IoVA` zwG$-6WbDt3@^nhWj$#(EU)U9y;q7#+>hJjZK)6YR``#38#C%L*QhuL~Y(C%MY+BK{ zk?BBT(O-2P@?q8vW*HumGS8c4rTrMqAzgdPo11PH0GhoSXta#@Q);Z|AYuFW5kOH3K> zMQxK0t*oStbJ_w z7p?aH=}~QJ@WCInqb3i~)I<64rvKgvSQrI>rc%3B z6#jY#Xp>H2iP!G{G2jLn0P7do4alS40!aZ)WemzEe~E}i9zZ?3=!bv7Fji=jeC7t@ zU#mz27?FWU_b(A?L7NgE5B`bt%jGiwam%w+B>$ce?=3@+;k{3DBJyL-->X$ag;R|u zA?Kht^Cnq1H2Q}0Yo8Q}ZH|Yc$+A@=%V_Ewz<1>s^PT^_i<(Ff{7?W})F2DnJu^qN zE9D(g;jh?+XUqCZrKbV9N^ZISSKCm&3Jp|pIqjx(K=Bi0RNM7ID>Vn8s+>RtN0`2q zbFy0a=L;eoI7|eNhFzb7jqY#^GK~g7vhQo~^fSVNC!m7r3ry=W04t8iKXpMo&Z(PF zHw18ck28j1!F;#4ZRYZF%~}mYaR+wO#&!_NneaX6BBQq6j86mYEz#o5e1EH-d!v?{lMrFc%S_w zMJa%mY?wlE5m3~6Kb`zRt&%KR$aj?X+h3)kn4C?A5(H7mlx}Krl$Zr~Krsq{N~HkQ zb5MEI0LUU}Az25sm?9Ul63OMV_O&5r09wkz#~af`4eF1$z=OTMAHsiRGXgpp>U%E( zzaYl_b$B0%{1`FV(6S42L9risLS8O3XvZMf8v>4>pk4FnWOXhWu@^C7 zt@|(=R6+43oP)9!oH-0UXt~TXV0f?qOKT-WWiBQ!=)BU^amN8Xs=chXz=e}xwZLWi z66#}JHZAMvc0z4twdzQz8J=$2*BG4ia0IV`A+YTUyGv3PL0#>Mji!6Fo5P^R`#+>b zzelA55O@^;KwXO{15ge#Y6498)IyjL^5Dp!1rX0ike^lHi*UwNJ-U6t7bE#qpynt9 zngnsz!=vyFV;D78zl+NQtY7wkrLZ1Em7Q|}pvMPS>`=WGCr*+5la_CTQsXcv=qdo^ znd6_}KcWKU>Xwug26=1vJP2_GnUe)O3-EZRzjBYoqBaRSlcHFl4Az-^AQwOFSsAVf z^|X)LFQ=N8iI4(<)ZzV2mtDNKQKk5*>)`%TU!5ONlluwfg27`M=|6@N;{}DQDS-%5 zlrnRoUCZT0bk5Kl`J|QiP{+g$5P#5Y0``#EF=x3E02n^n64d_q?Edg(f4I+RJ*aQ!hoLoj>;8;Qn?<4xyqZ-o|~hVY^1_`e2c!em`-ayyYIP+`?jGpsSgG? zUCM3af!43ELb;Pka=U?pD+;X;b)MXY((<6jHTM;Ak#42&^68WbXH35x8azJkq0XZ- z{OexPJKRL%kMC|3?CE`n-!KZ%k>t)8dCV}T7HN4;0ZXxt``nk&ycxBKCnF+!bgl0V zWj;q7rJ&!1omKo-NiO{4IN`Htz2S2}C2kke@MKk9gHHCc0Kc8rjxI*fp8m8=9J5VI zSL#A`o}IQkuX`xoE9BBGoCDzk%BmsIGbw?@{1I*%2_?n?QWNQ$YfzzRE+9Y|BQF&* zZh^s)`Pzf~e23jEd+%5ya9j#m#_ndkuE-9pH zdBJKB{ON^X&X@Y7k+FL~=~^Rw~Dw&q_^7W>l2a$eCo{k9U!^)_(W3*LAIH z@BPRA)9;!Yzr)P)eDBZqd*9N$@|L(rCOMi37aeWK=5KaV8Dxx1o|C<35!&RS8WuOG zt>&mkK0k07Z=#IPh<|LK`K&cqBR_A~vS#sZfh}=zMZga#-o=&&R((|1f3te)n(D@T z@R0k&{uFnT)VG4m99&QH?3MsZt!0VgNcegF)~VZ`j|_=FI0((n>^pIk3mPmRBGXX{j~WuSF7RO4#KeH4>x_qi-IuzQXIt#kC{|%>rX+jp0VTV?ur|4WO3WZ(<^~LH>+~B?2zAn4 zSUW|lWo!LCUAv>aQ7i!!ny9NdrRU~8JF(1H1w(7^nMC0Y#HM@yVcV3o!1b4F)UJ*Y z&rCmd%x?K*YVmwepYQLUj9%VMICQbot!8^D$Jo!Y<+9^3eICh^+5Jb^th$93Ayg{; z4C>fxkA84P1wjuMlazGa`OY2qg|;pKtq-?Y{7d?9w=s#xsJmwv)ZU)+x>dpJbsm8Q zI4;pQtq3+T9VJe)i9!%8UQeruxnR_>Lez5jVFJLB8R7XCG>zPBXJK6tsi1)RaPw+b z-3dI?W{2(cPk=3|?&^12OM~^O@s)7@t%E0(N$SFncDeGZZL}>S2KqJB6J#LgAfr$$d z6P)fh$8v?i2cxj2omgKU1nk0mt*(tFc(j|ME-th9L1gGcTW&>oY`w<`p!}W^bFE$Z<()j!!9l=S+Fb+tVpEm7;R)%`b z+#3x>cppr5DNJ256^n$42j{^Sl~`D*>`^(6xG8-1BnS|(6G77<9s*~x9cH1HccXiG zrNI9bb-Cqv!0RsCSM=AMj?#03S_())3gzM$7k-U#=Bh#N&13LfmPD91{_*atebyfz z>a7?rI*=6X=Z?_{9IunMYOzSr%g-yCzBw3;#g_`Astmn(IEq;xtHc= z-qB#=EvJkCAOwaawQ|dPYZcxr= zusQ#i4atlNqT9(w)h1v`)G&A1Id1^CTN{r+ZX@Ly{VX2j&KoRu>4(7A1=~gwOpT6U z^lP89MrMWK!yXg^Nu~NTdsnL%#^K8OxS6lj=pC?A=OG$~{pmgQMN%}5vOablK0}|0 z0=17lzsq+(1L?iY>XYTfyGYI1t*yij)X}e2en&u_TV%E4H8kgd0IkdXki+fvIPw!> z#2-}L;b8{ZVjT3e`iT~s@I{ihxInvFGCOB7)0{VRW#Ly z#PNNQd-ad0g6yhu8wG9;Y(DRDkSnRJyCg8P zjoM0noMnX3^{*VXMb&#-(=ChRODJoeV9XUuwXeV8WV8r~S)8m9y>4!V262?akLUd^ zn?FF)?G#N`m%xx5bR=joj^3XGu=MfmJ}%ONH%3nNAq##B!By9l7%O90cmrPDca5u~ zPOxn1=>-W9(!z8VEt$qRBTjP}Q_-e%c-a2S@P$v310GU>4RM>x)U0Axq6rez-FpwL zk0^Ny*#_dCC%oRMIOy0`;nuIdiFZZtwP=~oXu?FoyAZ16_ic{tprhHm^D%4I9X3CO zPtfXm`pD{_?bySeJaUO#1Og_b^}N}!@?S;uxVfRT#otsg)1J%g7X7X>{yL3@@E zP8fXi5R=xB;*?Ub>(kJ^MV8el72rn=9HFAmoKP%|*Ob1s<+X4AbE(i=7dGok{>>w2 z&rhtNk$`lq+onjfm`@@G9wk0a(e6HTNSR0eYPimuswQs>7YrSrEqW+SFW0#%@#&L| z;n(VrFFPsPd6>f_Cl1-(9bZLDDR`EXp7z_STyL7JEI~VZ*O?JDqZr^l^{5r1E{$XS zTSZL*G;8~Q|Gx9A*h+Rmtq`AVD9XPN`Jz^ckvIFbX62-nHpt2Kr&r>{ejh4=Lqq3| zuZf%eml-l(3jQ~2TY!NL&VxP8URUR43GqCt0;4B2{z>;qJLdim13X}Qsz71`j(Sv|FA6spwc@A!n5?y}JNrZ$TiZ>eww zFXgjJ3At33@cT43{C5VS_^v@&iT$wJedd{L^W>u`1JG;8k3a_A4Z{L-ViKGhu`x3` zv}e|P&CuoHOPJcXINo~SFG~lhYvfV;OD($Wqd_Fi#I`AL?`fSVNL7aD$X4w0nh>j} zjO+_H3{rT%(Ebx&G5<8yqentNloN;wr~CXf{y7shX9N zjjx1|HpE^|H$?gVS!OsXF?b2SnDOG6d#H(zR<}$;3tT=C10QGBtfi1u3xXMAk(o@H6tKzACch zaxs7R_G15eWqE`$_BiVO`1)+(8v$HHLA2CSAU-IVt^=kfatf!H$1L#l$Votc)}Dvf z5zTtoW7uBEzj6ar9z@LVhs{U?g?!#V1GTuI;TZDaEH_PhC{4kF7Z|SFq314(wxBD; za33X86;M30^K^?TcDj!wDzov$#~EHoK#6_b4hnJBag{;#IxEKhgh0I8T~pN2;PgSo zev1G3Nf_}3e(N57QsS9CZrUec>HZrBqjJfcVfYRUe|K5hRed(b%xnF!c=xFK+K*oV zUi6rE)Y+sb;1a^qn220kGT@)cIpZEeLg^C1THh(4nCOrsTasy8{ z8HR%efs!`#EeOx+04%x;&3c-i^c5tve?`bU7zY(y-wXlC2Y!cM3eqn@HtO>nir51> zUO4y3j~x_vI%aL%C@TuxErxCHXV784|2JgFI`43Nq5lQrr5%O6uj|?2MLO?y__fHH z7|rCHiF#xw!UaUIhIbyqe;*^QDCUR}Z`lJeOmFXr+@oY=-E69o#H%Vny<4HP)W;?7 z+Fc+5TgAL!j8_mOem033rgjx~(UcZPf;)?l8KTK6@cn*B@uBHtf{?-uF}m^|GGFe+ zf{$v|3j={Ws-f{(1kjGOG5V4lqi~vrs2^uKz)BapWsz8OtJn(-WI}PL8=wsP_sQz^ zN|xw9o>7$YNnMvoKWbWTASJ4~X$q*Pwm6>C&qZ~=$v|9JGt(;5us~PD-x!-w3I6#+ zn2b)AjF$8EEV}UyPCy5^a+-1-trr(z}C8rd;5{qHdjQBSiW3-vfwimyZV{p!aLMi^iFE8)pg)Na#nt+EB zbmHCPAW##!D0myx#|%PEOi=i$6^+CB4cH`c>MS6;|fAf7qD+zVZ z#9+SDD2}Eqs#L`nxsYfi;5G8O6^k4DKo5F{3TY1HfXm+_)N3A(OUk<0s@!->s1YM? zVaR_Hmy0aG>s$=86ag?WPRXv?a??*U{-s6=hB+QVU9VqJxU_PeB`km5yqrVqW?UZO zt;_AWFVBq@lS8&|b(cD~>5ga2Cx;OHR$L-wXms&8?bdM{K6OWGw}0LS$hNtfuTxyz zK&gXuOD)9c$&#>2qvcvUp{Dbs<*fsih#UIXY2Eg7#}S4BO>ROtH3(Z5@mI>9a}#Rs zvSvwBcJ{j)!t|f{jTqvVez#lI3s5O7L3;qfS+*_Xn=WH%OH-U9%rR;`J^i* z8&M32ttdguIQ`it{`rvCxPkqzP`)-u>|)x;Z=`wEJ9eWlS`jZu{Uz!`QzO52;u1`) zl>Zc|WVZFi?K|7sPu^~yirZV)YPVXCv_Lo)Pe^HkptdY2M%&g05-QZ?_|5zKAbDpM01`H?ymHy{1wzuIy$bZnJodPQ(}+~I za)=Rc07w61OlQ!Kwq_}%b@{wUBS5*vm42jX^?Uy)nYZ-dg7=DsnjOub%~;2}#Lpg$ z{rO{3pH_oYFear{XtEC&^ST5nrknI69uN0g`MSY4(9Lxjq1suw4O@s3PA~{h=4o;k zQ3@9jBh$wuo?2ZI>oT&_D{9=uNGx$B!w%g$2A7Br1y?Yn-nTo`F#Y(NF-%m$SBf%L znjVWxi{diXT)S4Zs#zvaHMK7c(g<_JTb~w`6v(yB+#z7`!zsmTWiKH8)ByUR+u8@< z;S^GK4<=}}Mla&?4><(E{tE_MDqp?!T1?uWeilMh&-0US6V)ubpbO^EHXd`)qjfu8 zc2~6M+?N;Wh9OPe9EdTiS%`mlX&xqELP7G_N6gB-ZFndvHof3M=!Pz)4@xn*tteo6 zrGFAPTdheBcN@$ZK8aIONAAGOCPI-l(UL=0xI}+CZfp_nD7-HmmP?xZsdnr?S2Ath)VQ zd6mmg(nr+>S5U^PZI5^SR*`s~F@8|S17xk}*>k>(m*Ex4y~f>mV3svPvkx@1eCl#u zHG3I~{#vNn$FDnQR^6iygJz$9{48kpDL%b-&Ku9eJsH!Bn{yh;LZE`RIv?Gca}tJd z=zN0tT;qHJ$Y&YO+ugsCurrently, only AWS S3 storage is supported | #### Running an Exporter @@ -139,4 +142,4 @@ sequenceDiagram Exporter->>Catalog: register object store location Query Engine-->Catalog: Query Query Engine-->Object Store: Query -``` \ No newline at end of file +``` diff --git a/docs/howto/hooks/lua.md b/docs/howto/hooks/lua.md index a79f1784795..03a175c0283 100644 --- a/docs/howto/hooks/lua.md +++ b/docs/howto/hooks/lua.md @@ -220,6 +220,57 @@ Returns the MD5 digest (string) of the given data Returns the SHA256 digest (string) of the given data +### `databricks/client(databricks_host, databricks_service_principal_token)` + +Returns a table representing a Databricks client with the `register_external_table` and `create_or_get_schema` methods. + +### `databricks/client.create_schema(schema_name, catalog_name, get_if_exists)` + +Creates a schema, or retrieves it if exists, in the configured Databricks host's Unity catalog. +If a schema doesn't exist, a new schema with the given `schema_name` will be created under the given `catalog_name`. +Returns the created/fetched schema name. + +Parameters: + +- `schema_name(string)`: The required schema name +- `catalog_name(string)`: The catalog name under which the schema will be created (or from which it will be fetched) +- `get_if_exists(boolean)`: In case of failure due to an existing schema with the given `schema_name` in the given +`catalog_name`, return the schema. + +Example: + +```lua +local databricks = require("databricks") +local client = databricks.client("https://my-host.cloud.databricks.com", "my-service-principal-token") +local schema_name = client.create_schema("main", "mycatalog", true) +``` + +### `databricks/client.register_external_table(table_name, physical_path, warehouse_id, catalog_name, schema_name)` + +Registers an external table under the provided warehouse ID, catalog name, and schema name. +In order for this method call to succeed, an external location should be configured in the catalog, with the +`physical_path`'s root storage URI (for example: `s3://mybucket`). +Returns the table's creation status. + +Parameters: + +- `table_name(string)`: Table name. +- `physical_path(string)`: A location to which the external table will refer, e.g. `s3://mybucket/the/path/to/mytable`. +- `warehouse_id(string)`: The SQL warehouse ID used in Databricks to run the `CREATE TABLE` query (fetched from the SQL warehouse +`Connection Details`, or by running `databricks warehouses get`, choosing your SQL warehouse and fetching its ID). +- `catalog_name(string)`: The name of the catalog under which a schema will be created (or fetched from). +- `schema_name(string)`: The name of the schema under which the table will be created. + +Example: + +```lua +local databricks = require("databricks") +local client = databricks.client("https://my-host.cloud.databricks.com", "my-service-principal-token") +local status = client.register_external_table("mytable", "s3://mybucket/the/path/to/mytable", "examwarehouseple", "my-catalog-name", "myschema") +``` + +- For the Databricks permissions needed to run this method, check out the [Unity Catalog Exporter]({% link integrations/unity_catalog.md %}) docs. + ### `encoding/base64/encode(data)` Encodes the given data to a base64 string @@ -348,6 +399,76 @@ Returns an object-wise diff of uncommitted changes on `branch_id`. Returns a stat object for the given path under the given reference and repository. +### `lakefs/catalogexport/glue_exporter.get_full_table_name(descriptor, action_info)` + +Generate glue table name. + +Parameters: + +- `descriptor(Table)`: Object from (e.g. _lakefs_tables/my_table.yaml). +- `action_info(Table)`: The global action object. + +### `lakefs/catalogexport/delta_exporter` + +A package used to export Delta Lake tables from lakeFS to an external cloud storage. + +### `lakefs/catalogexport/delta_exporter.export_delta_log(action, table_names, writer, delta_client, table_descriptors_path)` + +The function used to export Delta Lake tables. +The return value is a table with mapping of table names to external table location (from which it is possible to query the data). + +Parameters: + +- `action`: The global action object +- `table_names`: Delta tables name list (e.g. `{"table1", "table2"}`) +- `writer`: A writer function with `function(bucket, key, data)` signature, used to write the exported Delta Log (e.g. `aws/s3.s3_client.put_object`) +- `delta_client`: A Delta Lake client that implements `get_table: function(repo, ref, prefix)` +- `table_descriptors_path`: The path under which the table descriptors of the provided `table_names` reside + +Example: + +```yaml +--- +name: delta_exporter +on: + post-commit: null +hooks: + - id: delta_export + type: lua + properties: + script: | + local aws = require("aws") + local formats = require("formats") + local delta_exporter = require("lakefs/catalogexport/delta_exporter") + + local table_descriptors_path = "_lakefs_tables" + local sc = aws.s3_client(args.aws.access_key_id, args.aws.secret_access_key, args.aws.region) + local delta_client = formats.delta_client(args.lakefs.access_key_id, args.lakefs.secret_access_key, args.aws.region) + local delta_table_locations = delta_exporter.export_delta_log(action, args.table_names, sc.put_object, delta_client, table_descriptors_path) + + for t, loc in pairs(delta_table_locations) do + print("Delta Lake exported table \"" .. t .. "\"'s location: " .. loc .. "\n") + end + args: + aws: + access_key_id: + secret_access_key: + region: us-east-1 + lakefs: + access_key_id: + secret_access_key: + table_names: + - mytable +``` + +For the table descriptor under the `_lakefs_tables/mytable.yaml`: +```yaml +--- +name: myTableActualName +type: delta +path: a/path/to/my/delta/table +``` + ### `lakefs/catalogexport/table_extractor` Utility package to parse `_lakefs_tables/` descriptors. @@ -506,7 +627,7 @@ Example: ```yaml --- -name: test_delta_exporter +name: delta_exporter on: post-commit: null hooks: @@ -538,13 +659,61 @@ hooks: - my/delta/table/path ``` -For the table descriptor under the `_lakefs_tables/my/delta/table/path.yaml`: +### `lakefs/catalogexport/unity_exporter` + +A package used to register exported Delta Lake tables to Databricks' Unity catalog. + +### `lakefs/catalogexport/unity_exporter.register_tables(action, table_descriptors_path, delta_table_paths, databricks_client, warehouse_id)` + +The function used to register exported Delta Lake tables in Databricks' Unity Catalog. +The registration will use the following paths to register the table: +`..` where the branch name will be used as the schema name. +The return value is a table with mapping of table names to registration request status. + +Parameters: + +- `action(table)`: The global action table +- `table_descriptors_path(string)`: The path under which the table descriptors of the provided `table_paths` reside. +- `delta_table_paths(table)`: Table names to physical paths mapping (e.g. `{ table1 = "s3://mybucket/mytable1", table2 = "s3://mybucket/mytable2" }`) +- `databricks_client(table)`: A Databricks client that implements `create_or_get_schema: function(id, catalog_name)` and `register_external_table: function(table_name, physical_path, warehouse_id, catalog_name, schema_name)` +- `warehouse_id(string)`: Databricks warehouse ID. + +Example: +The following registers an exported Delta Lake table to Unity Catalog. + +```lua +local databricks = require("databricks") +local unity_export = require("lakefs/catalogexport/unity_exporter") + +local delta_table_locations = { + ["table1"] = "s3://mybucket/mytable1", +} +-- Register the exported table in Unity Catalog: +local action_details = { + repository_id = "my-repo", + commit_id = "commit_id", + branch_id = "main", +} +local databricks_client = databricks.client("", "") +local registration_statuses = unity_export.register_tables(action_details, "_lakefs_tables", delta_table_locations, databricks_client, "") + +for t, status in pairs(registration_statuses) do + print("Unity catalog registration for table \"" .. t .. "\" completed with status: " .. status .. "\n") +end +``` + +For the table descriptor under the `_lakefs_tables/delta-table-descriptor.yaml`: ```yaml --- -name: myTableActualName +name: my_table_name type: delta +path: path/to/delta/table/data +catalog: my-catalog ``` +For detailed step-by-step guide on how to use `unity_exporter.register_tables` as a part of a lakeFS action refer to +the [Unity Catalog docs]({% link integrations/unity_catalog.md %}). + ### `path/parse(path_string)` Returns a table for the given path string with the following structure: diff --git a/docs/integrations/unity_catalog.md b/docs/integrations/unity_catalog.md new file mode 100644 index 00000000000..3ec205c207d --- /dev/null +++ b/docs/integrations/unity_catalog.md @@ -0,0 +1,207 @@ +--- +title: Unity Catalog +description: Accessing lakeFS-exported Delta Lake tables from Unity Catalog. +parent: Integrations +redirect_from: /using/unity_catalog.html +--- + +# Using lakeFS with the Unity Catalog + +{% include toc_2-3.html %} + +## Overview + +Databricks Unity Catalog serves as a centralized data governance platform for your data lakes. +Through the Unity Catalog, you can search for and locate data assets across workspaces via a unified catalog. +Leveraging the external tables feature within Unity Catalog, you can register a Delta Lake table exported from lakeFS and +access it through the unified catalog. +The subsequent step-by-step guide will lead you through the process of configuring a [Lua hook]({% link howto/hooks/lua.md %}) +that exports Delta Lake tables from lakeFS, and subsequently registers them in Unity Catalog. + +{: .note} +> Currently, Unity Catalog export feature exclusively supports AWS S3 as the underlying storage solution. It's planned to [support other cloud providers soon](https://github.com/treeverse/lakeFS/issues/7199). + +## Prerequisites + +Before starting, ensure you have the following: + +1. Access to Unity Catalog +2. An active lakeFS installation with S3 as the backing storage, and a repository in this installation. +3. A Databricks SQL warehouse. +4. AWS Credentials with S3 access. +5. lakeFS credentials with access to your Delta Tables. + +### Databricks authentication + +Given that the hook will ultimately register a table in Unity Catalog, authentication with Databricks is imperative. +Make sure that: + +1. You have a Databricks [Service Principal](https://docs.databricks.com/en/dev-tools/service-principals.html). +2. The Service principal has [token usage permissions](https://docs.databricks.com/en/dev-tools/service-principals.html#step-3-assign-workspace-level-permissions-to-the-databricks-service-principal), + and an associated [token](https://docs.databricks.com/en/dev-tools/service-principals.html#step-4-generate-a-databricks-personal-access-token-for-the-databricks-service-principal) + configured. +3. The service principal has the `Service principal: Manager` privilege over itself (Workspace: Admin console -> Service principals -> `` -> Permissions -> Grant access (``: + `Service principal: Manager`), with `Workspace access` and `Databricks SQL access` checked (Admin console -> Service principals -> `` -> Configurations). +4. Your SQL warehouse allows the service principal to use it (SQL Warehouses -> `` -> Permissions -> ``: `Can use`). +5. The catalog grants the `USE CATALOG`, `USE SCHEMA`, `CREATE SCHEMA` permissions to the service principal(Catalog -> `` -> Permissions -> Grant -> ``: `USE CATALOG`, `USE SCHEMA`, `CREATE SCHEMA`). +6. You have an _External Location_ configured, and the service principal has the `CREATE EXTERNAL TABLE` permission over it (Catalog -> External Data -> External Locations -> Create location). + +## Guide + +### Table descriptor definition + +To guide the Unity Catalog exporter in configuring the table in the catalog, define its properties in the Delta Lake table descriptor. +The table descriptor should include (at minimum) the following fields: +1. `name`: The table name. +2. `type`: Should be `delta`. +3. `catalog`: The name of the catalog in which the table will be created. +4. `path`: The path in lakeFS (starting from the root of the branch) in which the Delta Lake table's data is found. + +Let's define the table descriptor and upload it to lakeFS: + +Save the following as `famous-people-td.yaml`: + +```yaml +--- +name: famous_people +type: delta +catalog: my-catalog-name +path: tables/famous-people +``` + +{: .note} +> It's recommended to create a Unity catalog with the same name as your repository + +Upload the table descriptor to `_lakefs_tables/famous-people-td.yaml` and commit: + +```bash +lakectl fs upload lakefs://repo/main/_lakefs_tables/famous-people-td.yaml -s ./famous-people-td.yaml && \ +lakectl commit lakefs://repo/main -m "add famous people table descriptor" +``` + +### Write some data + +Insert data into the table path, using your preferred method (e.g. [Spark]({% link integrations/spark.md %})), and commit upon completion. + +We shall use Spark and lakeFS's S3 gateway to write some data as a Delta table: +```bash +pyspark --packages "io.delta:delta-spark_2.12:3.0.0,org.apache.hadoop:hadoop-aws:3.3.4,com.amazonaws:aws-java-sdk-bundle:1.12.262" \ + --conf spark.sql.extensions=io.delta.sql.DeltaSparkSessionExtension \ + --conf spark.sql.catalog.spark_catalog=org.apache.spark.sql.delta.catalog.DeltaCatalog \ + --conf spark.hadoop.fs.s3a.aws.credentials.provider='org.apache.hadoop.fs.s3a.SimpleAWSCredentialsProvider' \ + --conf spark.hadoop.fs.s3a.endpoint='' \ + --conf spark.hadoop.fs.s3a.access.key='' \ + --conf spark.hadoop.fs.s3a.secret.key='' \ + --conf spark.hadoop.fs.s3a.path.style.access=true +``` + +```python +data = [ + ('James','Bond','England','intelligence'), + ('Robbie','Williams','England','music'), + ('Hulk','Hogan','USA','entertainment'), + ('Mister','T','USA','entertainment'), + ('Rafael','Nadal','Spain','professional athlete'), + ('Paul','Haver','Belgium','music'), +] +columns = ["firstname","lastname","country","category"] +df = spark.createDataFrame(data=data, schema = columns) +df.write.format("delta").mode("overwrite").partitionBy("category", "country").save("s3a://repo/main/tables/famous-people") +``` + +### The Unity Catalog exporter script + +{: .note} +> For code references check [delta_exporter]({% link howto/hooks/lua.md %}#lakefscatalogexportdelta_exporter) and +[unity_exporter]({% link howto/hooks/lua.md %}#lakefscatalogexportunity_exporter) docs. + +Create `unity_exporter.lua`: + +```lua +local aws = require("aws") +local formats = require("formats") +local databricks = require("databricks") +local delta_export = require("lakefs/catalogexport/delta_exporter") +local unity_export = require("lakefs/catalogexport/unity_exporter") + +local sc = aws.s3_client(args.aws.access_key_id, args.aws.secret_access_key, args.aws.region) + +-- Export Delta Lake tables export: +local delta_client = formats.delta_client(args.lakefs.access_key_id, args.lakefs.secret_access_key, args.aws.region) +local delta_table_locations = delta_export.export_delta_log(action, args.table_defs, sc.put_object, delta_client, "_lakefs_tables") + +-- Register the exported table in Unity Catalog: +local databricks_client = databricks.client(args.databricks_host, args.databricks_token) +local registration_statuses = unity_export.register_tables(action, "_lakefs_tables", delta_table_locations, databricks_client, args.warehouse_id) + +for t, status in pairs(registration_statuses) do + print("Unity catalog registration for table \"" .. t .. "\" completed with commit schema status : " .. status .. "\n") +end +``` + +Upload the lua script to the `main` branch under `scripts/unity_exporter.lua` and commit: + +```bash +lakectl fs upload lakefs://repo/main/scripts/unity_exporter.lua -s ./unity_exporter.lua && \ +lakectl commit lakefs://repo/main -m "upload unity exporter script" +``` + +### Action configuration + +Define an action configuration that will run the above script after a commit is completed (`post-commit`) over the `main` branch. + +Create `unity_exports_action.yaml`: + +```yaml +--- +name: unity_exports +on: + post-commit: + branches: ["main"] +hooks: + - id: unity_export + type: lua + properties: + script_path: scripts/unity_exporter.lua + args: + aws: + access_key_id: + secret_access_key: + region: + lakefs: # provide credentials of a user that has access to the script and Delta Table + access_key_id: + secret_access_key: + table_defs: # an array of table descriptors used to be defined in Unity Catalog + - famous-people-td + databricks_host: + databricks_token: + warehouse_id: +``` + +Upload the action configurations to `_lakefs_actions/unity_exports_action.yaml` and commit: + +{: .note} +> Once the commit will finish its run, the action will start running since we've configured it to run on `post-commit` +events on the `main` branch. + +```bash +lakectl fs upload lakefs://repo/main/_lakefs_actions/unity_exports_action.yaml -s ./unity_exports_action.yaml && \ +lakectl commit lakefs://repo/main -m "upload action and run it" +``` + +The action has run and exported the `famous_people` Delta Lake table to the repo's storage namespace, and has register +the table as an external table in Unity Catalog under the catalog `my-catalog-name`, schema `main` (as the branch's name) and +table name `famous_people`: `my-catalog-name.main.famous_people`. + +![Hooks log result in lakeFS UI]({{ site.baseurl }}/assets/img/unity_export_hook_result_log.png) + +### Databricks Integration + +After registering the table in Unity, you can leverage your preferred method to [query the data](https://docs.databricks.com/en/query/index.html) +from the exported table under `my-catalog-name.main.famous_people`, and view it in the Databricks's Catalog Explorer, or +retrieve it using the Databricks CLI with the following command: +```bash +databricks tables get my-catalog-name.main.famous_people +``` + +![Unity Catalog Explorer view]({{ site.baseurl }}/assets/img/unity_exported_table_columns.png) diff --git a/examples/hooks/delta_lake_S3_export.lua b/examples/hooks/delta_lake_S3_export.lua index 88b01030f5b..8b92366b8a5 100644 --- a/examples/hooks/delta_lake_S3_export.lua +++ b/examples/hooks/delta_lake_S3_export.lua @@ -1,6 +1,6 @@ --[[ args: - - table_paths (e.g. ["path/to/table1", "path/to/table2", ...]) + - table_defs (e.g. ["table1.yaml", "table2", ...]) - lakefs.access_key_id - lakefs.secret_access_key - aws.access_key_id @@ -16,7 +16,7 @@ local table_descriptors_path = "_lakefs_tables" local sc = aws.s3_client(args.aws.access_key_id, args.aws.secret_access_key, args.aws.region) local delta_client = formats.delta_client(args.lakefs.access_key_id, args.lakefs.secret_access_key, args.aws.region) -local delta_table_locations = delta_export.export_delta_log(action, args.table_paths, sc.put_object, delta_client, table_descriptors_path) +local delta_table_locations = delta_export.export_delta_log(action, args.table_defs, sc.put_object, delta_client, table_descriptors_path) for t, loc in pairs(delta_table_locations) do print("Delta Lake exported table \"" .. t .. "\"'s location: " .. loc .. "\n") end diff --git a/examples/hooks/unity_table_export.lua b/examples/hooks/unity_table_export.lua new file mode 100644 index 00000000000..e5d7a29c58e --- /dev/null +++ b/examples/hooks/unity_table_export.lua @@ -0,0 +1,24 @@ +--[[ + As an exhaustive example, it will first start off with a Delta Lake tables export, then continue to register the table + with Unity Catalog +]] + +local aws = require("aws") +local formats = require("formats") +local databricks = require("databricks") +local delta_export = require("lakefs/catalogexport/delta_exporter") +local unity_export = require("lakefs/catalogexport/unity_exporter") + +local sc = aws.s3_client(args.aws.access_key_id, args.aws.secret_access_key, args.aws.region) + +-- Export Delta Lake tables export: +local delta_client = formats.delta_client(args.lakefs.access_key_id, args.lakefs.secret_access_key, args.aws.region) +local delta_table_locations = delta_export.export_delta_log(action, args.table_defs, sc.put_object, delta_client, "_lakefs_tables") + +-- Register the exported table in Unity Catalog: +local databricks_client = databricks.client(args.databricks_host, args.databricks_token) +local registration_statuses = unity_export.register_tables(action, "_lakefs_tables", delta_table_locations, databricks_client, args.warehouse_id) + +for t, status in pairs(registration_statuses) do + print("Unity catalog registration for table \"" .. t .. "\" completed with status: " .. status .. "\n") +end diff --git a/pkg/actions/lua/databricks/client.go b/pkg/actions/lua/databricks/client.go index 6a5e99c455e..d42c6fc1509 100644 --- a/pkg/actions/lua/databricks/client.go +++ b/pkg/actions/lua/databricks/client.go @@ -112,6 +112,7 @@ func newDatabricksClient(l *lua.State) (*databricks.WorkspaceClient, error) { func (client *Client) RegisterExternalTable(l *lua.State) int { tableName := lua.CheckString(l, 1) + tableName = strings.ReplaceAll(tableName, "-", "_") location := lua.CheckString(l, 2) warehouseID := lua.CheckString(l, 3) catalogName := lua.CheckString(l, 4) diff --git a/pkg/actions/lua/lakefs/catalogexport/delta_exporter.lua b/pkg/actions/lua/lakefs/catalogexport/delta_exporter.lua index 6f6da379dea..f872b75ead0 100644 --- a/pkg/actions/lua/lakefs/catalogexport/delta_exporter.lua +++ b/pkg/actions/lua/lakefs/catalogexport/delta_exporter.lua @@ -3,7 +3,7 @@ local pathlib = require("path") local json = require("encoding/json") local utils = require("lakefs/catalogexport/internal") local extractor = require("lakefs/catalogexport/table_extractor") - +local strings = require("strings") --[[ delta_log_entry_key_generator returns a closure that returns a Delta Lake version key according to the Delta Lake protocol: https://github.com/delta-io/delta/blob/master/PROTOCOL.md#delta-log-entries @@ -33,7 +33,7 @@ end - repository_id - commit_id - table_paths: ["path/to/table1", "path/to/table2", ...] + table_def_names: ["table1.yaml", "table2", ...] write_object: function(bucket, key, data) @@ -41,17 +41,37 @@ end - get_table: function(repo, ref, prefix) ]] -local function export_delta_log(action, table_paths, write_object, delta_client, table_descriptors_path) +local function export_delta_log(action, table_def_names, write_object, delta_client, table_descriptors_path) local repo = action.repository_id local commit_id = action.commit_id - + if not commit_id then + error("missing commit id") + end local ns = action.storage_namespace if ns == nil then error("failed getting storage namespace for repo " .. repo) end local response = {} - for _, path in ipairs(table_paths) do - local t = delta_client.get_table(repo, commit_id, path) + for _, table_name_yaml in ipairs(table_def_names) do + + -- Get the table descriptor + local tny = table_name_yaml + if not strings.has_suffix(tny, ".yaml") then + tny = tny .. ".yaml" + end + local table_src_path = pathlib.join("/", table_descriptors_path, tny) + local table_descriptor = extractor.get_table_descriptor(lakefs, repo, commit_id, table_src_path) + local table_path = table_descriptor.path + if not table_path then + error("table path is required to proceed with Delta catalog export") + end + local table_name = table_descriptor.name + if not table_name then + error("table name is required to proceed with Delta catalog export") + end + + -- Get Delta table + local t = delta_client.get_table(repo, commit_id, table_path) local sortedKeys = utils.sortedKeys(t) --[[ Pairs of (version, map of json content): (1, @@ -82,7 +102,7 @@ local function export_delta_log(action, table_paths, write_object, delta_client, p = entry.remove.path end if p ~= "" then - local code, obj = lakefs.stat_object(repo, commit_id, pathlib.join("/",path, p)) + local code, obj = lakefs.stat_object(repo, commit_id, pathlib.join("/", table_path, p)) if code == 200 then local obj_stat = json.unmarshal(obj) local physical_path = obj_stat["physical_address"] @@ -99,13 +119,6 @@ local function export_delta_log(action, table_paths, write_object, delta_client, table_log[keyGenerator()] = entry_log end - -- Get the table delta log physical location - local table_src_path = pathlib.join("/", table_descriptors_path, path .. ".yaml") - local table_descriptor = extractor.get_table_descriptor(lakefs, repo, commit_id, table_src_path) - local table_name = table_descriptor.name - if not table_name then - error("table name is required to proceed with Delta catalog export") - end local table_export_prefix = utils.get_storage_uri_prefix(ns, commit_id, action) local table_physical_path = pathlib.join("/", table_export_prefix, table_name) local table_log_physical_path = pathlib.join("/", table_physical_path, "_delta_log") @@ -131,7 +144,7 @@ local function export_delta_log(action, table_paths, write_object, delta_client, local version_key = storage_props.key .. "/" .. entry_version write_object(storage_props.bucket, version_key, table_entry_string) end - response[path] = table_physical_path + response[table_name_yaml] = table_physical_path end return response end diff --git a/pkg/actions/lua/lakefs/catalogexport/internal.lua b/pkg/actions/lua/lakefs/catalogexport/internal.lua index 7125cab9b56..1db29e312c7 100644 --- a/pkg/actions/lua/lakefs/catalogexport/internal.lua +++ b/pkg/actions/lua/lakefs/catalogexport/internal.lua @@ -1,7 +1,5 @@ local url = require("net/url") local pathlib = require("path") -local lakefs = require("lakefs") -local json = require("encoding/json") local DEFAULT_SHORT_DIGEST_LEN=6 local function deepcopy(orig) diff --git a/pkg/actions/lua/lakefs/catalogexport/unity_exporter.lua b/pkg/actions/lua/lakefs/catalogexport/unity_exporter.lua new file mode 100644 index 00000000000..1d961ce23f1 --- /dev/null +++ b/pkg/actions/lua/lakefs/catalogexport/unity_exporter.lua @@ -0,0 +1,61 @@ +--[[ TABLE SPECIFICATION: _lakefs_tables/ +name:
    +type: delta +catalog: +]] +local strings = require("strings") +local pathlib = require("path") +local lakefs = require("lakefs") +local extractor = require("lakefs/catalogexport/table_extractor") +--[[ + - table_descriptors_path: the path under which the table descriptors reside (e.g. "_lakefs_tables"). + It's necessary that every
    in the provided `table_paths` will have a complementary + `/
    .yaml` file describing the used Delta Table. + - delta_table_paths: a mapping of Delta Lake table descriptors yaml name (with or without ".yaml" extension) to their locations in the object storage + { : } + - databricks_client: a client to interact with databricks. + - warehouse_id: Databricks warehouse ID + + Returns a "
    : status" map for registration of provided tables. +]] +local function register_tables(action, table_descriptors_path, delta_table_paths, databricks_client, warehouse_id) + local repo = action.repository_id + local commit_id = action.commit_id + if not commit_id then + error("missing commit id") + end + local branch_id = action.branch_id + local response = {} + for table_name_yaml, physical_path in pairs(delta_table_paths) do + local tny = table_name_yaml + if not strings.has_suffix(tny, ".yaml") then + tny = tny .. ".yaml" + end + local table_src_path = pathlib.join("/", table_descriptors_path, tny) + local table_descriptor = extractor.get_table_descriptor(lakefs, repo, commit_id, table_src_path) + local table_name = table_descriptor.name + if not table_name then + error("table name is required to proceed with unity catalog export") + end + if table_descriptor.type ~= "delta" then + error("unity exporter supports only table descriptors of type 'delta'. registration failed for table " .. table_name) + end + local catalog = table_descriptor.catalog + if not catalog then + error("catalog name is required to proceed with unity catalog export") + end + local get_schema_if_exists = true + local schema_name = databricks_client.create_schema(branch_id, catalog, get_schema_if_exists) + if not schema_name then + error("failed creating/getting catalog's schema: " .. catalog .. "." .. branch_id) + end + local status = databricks_client.register_external_table(table_name, physical_path, warehouse_id, catalog, schema_name) + response[table_name_yaml] = status + end + return response +end + + +return { + register_tables = register_tables, +} diff --git a/pkg/actions/lua_test.go b/pkg/actions/lua_test.go index 976632e911f..78449c6d6dd 100644 --- a/pkg/actions/lua_test.go +++ b/pkg/actions/lua_test.go @@ -367,9 +367,12 @@ func TestLuaRunTable(t *testing.T) { Output: "testdata/lua/catalogexport_hive_partition_pager.output", }, { - Name: "catalogexport_delta", - Input: "testdata/lua/catalogexport_delta.lua", - Output: "", + Name: "catalogexport_delta", + Input: "testdata/lua/catalogexport_delta.lua", + }, + { + Name: "catalogexport_unity", + Input: "testdata/lua/catalogexport_unity.lua", }, } diff --git a/pkg/actions/testdata/lua/catalogexport_delta.lua b/pkg/actions/testdata/lua/catalogexport_delta.lua index a73592cc689..82852f8d779 100644 --- a/pkg/actions/testdata/lua/catalogexport_delta.lua +++ b/pkg/actions/testdata/lua/catalogexport_delta.lua @@ -1,6 +1,7 @@ local pathlib = require("path") local json = require("encoding/json") local utils = require("lakefs/catalogexport/internal") +local strings = require("strings") local test_data = { @@ -44,10 +45,12 @@ end package.loaded["lakefs/catalogexport/table_extractor"] = { get_table_descriptor = function(_, _, _, table_src_path) local t_name_yaml = pathlib.parse(table_src_path) - assert(t_name_yaml["base_name"] == ".yaml") - local t_name = pathlib.parse(t_name_yaml["parent"]) + local t_name_yaml_base = t_name_yaml["base_name"] + assert(strings.has_suffix(t_name_yaml_base, ".yaml")) + local t_name = strings.split(t_name_yaml_base, ".")[1] return { - name = t_name["base_name"] + name = t_name, + path = t_name } end } @@ -56,6 +59,9 @@ package.loaded.lakefs = { stat_object = function(_, _, path) local parsed_path = pathlib.parse(path) local table_path_base = parsed_path["parent"] + if strings.has_suffix(table_path_base, "/") then + table_path_base = strings.split(table_path_base, "/")[1] + end if not test_data.table_to_objects[table_path_base] then test_data.table_to_objects[table_path_base] = {} end @@ -102,8 +108,8 @@ local function assert_physical_address(delta_table_locations, table_paths) end end -local function assert_lakefs_stats(table_paths, content_paths) - for _, table_path in ipairs(table_paths) do +local function assert_lakefs_stats(table_names, content_paths) + for _, table_path in ipairs(table_names) do local table = test_data.table_to_objects[table_path] if not table then error("missing lakeFS stat_object call for table path: " .. table_path .. "\n") @@ -143,11 +149,10 @@ end -- Test data local data_paths = { "part-c000.snappy.parquet", "part-c001.snappy.parquet", "part-c002.snappy.parquet", "part-c003.snappy.parquet" } -local test_table_paths = {"path/to/table1/", "path/to/table2/"} +local test_table_names = { "table1", "table2"} -for _, table_path in ipairs(test_table_paths) do - local table_name = pathlib.parse(table_path)["base_name"] - test_data.table_logs_content[table_path] = { +for _, table_name in ipairs(test_table_names) do + test_data.table_logs_content[table_name] = { ["_delta_log/00000000000000000000.json"] = { "{\"commitInfo\":\"some info\"}", "{\"add\": {\"path\":\"part-c000.snappy.parquet\"}}", @@ -163,14 +168,14 @@ for _, table_path in ipairs(test_table_paths) do test_data.table_expected_log[table_name] = { ["_delta_log/00000000000000000000.json"] = { "{\"commitInfo\":\"some info\"}", - "{\"add\":{\"path\":\"" .. generate_physical_address(table_path .. "part-c000.snappy.parquet") .. "\"}}", - "{\"remove\":{\"path\":\"" .. generate_physical_address(table_path .. "part-c001.snappy.parquet") .. "\"}}", + "{\"add\":{\"path\":\"" .. generate_physical_address(table_name .. "/part-c000.snappy.parquet") .. "\"}}", + "{\"remove\":{\"path\":\"" .. generate_physical_address(table_name .. "/part-c001.snappy.parquet") .. "\"}}", "{\"protocol\":\"the protocol\"}", }, ["_delta_log/00000000000000000001.json"] = { "{\"metaData\":\"some metadata\"}", - "{\"add\":{\"path\":\"" .. generate_physical_address(table_path .. "part-c002.snappy.parquet") .. "\"}}", - "{\"remove\":{\"path\":\"" .. generate_physical_address(table_path .. "part-c003.snappy.parquet") .. "\"}}", + "{\"add\":{\"path\":\"" .. generate_physical_address(table_name .. "/part-c002.snappy.parquet") .. "\"}}", + "{\"remove\":{\"path\":\"" .. generate_physical_address(table_name .. "/part-c003.snappy.parquet") .. "\"}}", } } end @@ -179,13 +184,13 @@ end -- Run Delta export test local delta_table_locations = delta_export.export_delta_log( action, - test_table_paths, + test_table_names, mock_object_writer, mock_delta_client(test_data.table_logs_content), "some_path" ) -- Test results -assert_lakefs_stats(test_table_paths, data_paths) -assert_physical_address(delta_table_locations, test_table_paths) +assert_lakefs_stats(test_table_names, data_paths) +assert_physical_address(delta_table_locations, test_table_names) assert_delta_log_content(delta_table_locations, test_data.table_expected_log) diff --git a/pkg/actions/testdata/lua/catalogexport_unity.lua b/pkg/actions/testdata/lua/catalogexport_unity.lua new file mode 100644 index 00000000000..ebf6330bdbd --- /dev/null +++ b/pkg/actions/testdata/lua/catalogexport_unity.lua @@ -0,0 +1,170 @@ +local action = { + repository_id = "myRepo", + commit_id = "myCommit", + branch_id = "myBranch", +} + +-- table names must be unique +local test_cases = { + { + name = "failed_not_delta_type", + tables = { + ["my_table_not_delta"] = { + td = { + type = "notDelta", + catalog = "ok", + name = "notDelta", + }, + }, + }, + error = "registration failed", + }, + { + name = "failed_no_catalog_name", + tables = { + ["my_table_no_catalog"] = { + td = { + type = "delta", + name = "noCatalog", + }, + }, + }, + error = "catalog name is required", + }, + { + name = "failed_no_name", + tables = { + ["my_table_no_name"] = { + td = { + type = "delta", + catalog = "ok", + }, + }, + }, + error = "table name is required", + }, + { + name = "failed_schema_creation", + tables = { + ["my_table_schema_failure"] = { + td = { + type = "delta", + catalog = "ok", + name = "schemaFailure", + }, + }, + }, + schema_failure = true, + error = "failed creating/getting catalog's schema", + }, + { + name = "success_all_tables", + tables = { + ["my_table_success"] = { + status = "SUCCEEDED", + }, + ["my_table2_success"] = { + status = "SUCCEEDED", + }, + ["my_table3_success"] = { + status = "SUCCEEDED", + }, + }, + }, + { + name = "mixed_statuses", + tables = { + ["my_table_failure"] = { + status = "FAILED", + }, + ["my_table2_success_2"] = { + status = "SUCCEEDED", + }, + ["my_table3_failure"] = { + status = "FAILED", + }, + }, + }, +} + +-- Loads a mock table (descriptor) extractor +local function load_table_descriptor(tables) + package.loaded["lakefs/catalogexport/table_extractor"] = { + get_table_descriptor = function(_, _, _, table_src_path) + local examined_tables = {} + for name, t in pairs(tables) do + table.insert(examined_tables, name) + if string.find(table_src_path, name) then + if not t.td then + return { + type = "delta", + catalog = "ok", + name = name + } + end + return t.td + end + end + error("test was configured incorrectly. expected to find a table descriptor for table \"" .. table_src_path .. "\" but no such was found." ) + end + } +end + +-- Generates a mock databricks client +local function db_client(schema_failure, tables) + return { + create_schema = function(branch_id, catalog, _) + if schema_failure then + return nil + end + return catalog .. "." .. branch_id + end, + register_external_table = function(table_name, _, _, _, _) + for name, t in pairs(tables) do + if name == table_name then + return t.status + end + end + end + } +end + +--------------------------------- +---------- Begin tests ---------- +--------------------------------- +for _, test in ipairs(test_cases) do + package.loaded["lakefs/catalogexport/unity_exporter"] = nil + load_table_descriptor(test.tables) + local unity_export = require("lakefs/catalogexport/unity_exporter") + local err = test.error + local schema_failure = test.schema_failure + local test_tables = test.tables + local table_paths = {} + for name, _ in pairs(test_tables) do + table_paths[name] = "s3://physical/" .. name + end + + local db = db_client(schema_failure, test_tables) + -- Run test: + local s, resp = pcall(unity_export.register_tables, action, "_lakefs_tables", table_paths, db, "id") + if err ~= nil then + if s ~= false then -- the status is true which means no error was returned + local str_resp = "" + for k, v in pairs(resp) do + str_resp = str_resp .. k .. " = " .. v .. "\n" + end + error("test " .. test.name .. " expected an error:\n" .. err .. "\nbut returned status: \"" .. tostring(s) .. "\"\nresponse:\n" .. str_resp) + end + -- status is false as expected -> error returned + if string.find(resp, err) == nil then + error("test " .. test.name .. " returned incorrect error.\nexpected:\n" .. err .. "\nactual:\n" .. resp) + end + else + for table_name, status in pairs(resp) do + local expected_status = test.tables[table_name].status + if expected_status ~= status then + error("test " .. test.name .. " returned incorrect status for table \"" .. table_name .."\"\nexpected: \"" .. expected_status .. "\"\nactual:\n\"" .. status .. "\"") + end + end + end +end