From 70d201de3e9488232b5efe3077a7d2d5f3d14d3c Mon Sep 17 00:00:00 2001 From: Kaushik Iska Date: Mon, 3 Jul 2023 17:00:46 -0400 Subject: [PATCH 1/7] update banner (#186) --- README.md | 3 +-- images/banner.jpg | Bin 0 -> 40895 bytes images/logo-light-transparent_copy_2.png | Bin 39604 -> 0 bytes 3 files changed, 1 insertion(+), 2 deletions(-) create mode 100755 images/banner.jpg delete mode 100644 images/logo-light-transparent_copy_2.png diff --git a/README.md b/README.md index 9a00b594c7..fc88ed4b53 100644 --- a/README.md +++ b/README.md @@ -1,7 +1,6 @@
-img-verification -

PeerDB

+img-verification

Modern ETL in minutes, with SQL

diff --git a/images/banner.jpg b/images/banner.jpg new file mode 100755 index 0000000000000000000000000000000000000000..6037c9e7bb81086a681c780adb3c6e045e1b0be8 GIT binary patch literal 40895 zcmeFZ2UHW^);Bx|NL55Zq)3n=2vQW0PDBk=L=cFe5|Jh%MLGhBf=VaSL{Nx=G$|3J z2qFPNs)F<;y-H04LLkZH9lhnb_y51&x1Q&DpKpEdTJN33Br}t9&dk|opM8FN|MtKb zWK012jPwlk0464Y={VDI0ANf49f$xICjc-p0b~IHUP{r65x2p&CbWeOYD-D zhnTX0vVtn~JIfQ6-)-(<1s2p%o>7=rTlB$N%NwE`Z z8cL_21E@b7t+#x9JT(**-Mtm;96WA1D%g9tDF)biDxOeKQUtW%0iJgDu8uxpHyxc_ z+_lBmJ`lvkTpYB;kt!xiCZ6XUZ@C!U^K!g=@1lkMJy&~m2XVNLm{x#BfSaeAqmP|f zfZH8+Z;b$L@!ySWK*zr>D~gN#F5=^=Eq=x1lGr&9FGsP{3Q7t};!tfb2PX~l^Ll@+ z3!Q0;|MemL{rwgEl@&a^oE1-~tE($2om4z|QXVQH?;YswV;3Or?k(}R7S20*+k3fq z`nY(wi~VZR?xu&YkG42;=f6CXo9Ew~{g-|JRnSwz1bSKrJ0H9Ac0P{U;=g1lucRuk zbkYKn;Rz+^PyUxAmHtxroR{OTv9j>+xT7upug2~liaGwP7JrY*Ut3g!B%%0MQT#gm zyG(V&{=NNM0{@o4za{W*3H)0E|CYf2eG>Q&ROaXoK{0;_a{-JMz~Ry@4<8TjTOOVe zQd9!e&KjDq{DPMNa9o7R24EJJ(iefC=PqEE$Sxg#{TG-~QdK%BvI`DyolsLcrF0U4 z9!g5T>q-5tx37{tI3s9}i?RKKdNs-snFEYEfR`NrS(%ub#DQJBOw7DYj79(kK~`3# z-`nqM&|$7D~boK{s+*En|`p{H+PXmr`a z(h7OS+Q#0&(aHIi%WZETUqAnVzS`|OiF&5@+>tiGwXGBPHtZQo6>h> zXRVPR%r`_(U|UH-p1&db7ja~7xc2JpDB$1we_sZSZ~qqM|BwnXy8bRoxoT3}b0 zQHay5yhL?XXHtUkC--5FhSLJt?>hPCHzm#6Sqikc&AvUhx*W>Bpu73f+$1Nlmmb9c z!cWqi7{IGa1Oup*W&l$%f4(HQ(MjG-FWw*w+fP2UF^y2*54L}GKK}Nll7@@XkFF*i zZeReSI!iM*c2!Zw-ugQC<0I?W_9h1q@)XBjqatIOiVC0D?^!K%?28 zFCjN95u*LN^ztasaqGYGWLQ@*aQHLkBsHG4$N(Br(aM-_Bbf|f`~U+W@nQaOfuGKA zreFR9<93sf8qxy#VkW;31PKdnU6{2<20)2||KS{r%F7yVfmf8EzSPcOGzi2wRLuCs z=>@`8BLkozV1GErAS}#Q9AG0?@N+0Hcsh#Wt=QzTBkY5sPGLZC>_49pGT+FMOLsdH(toTX&+q*}zVF@Z3z(rqVPT@Ztu<;i`E&Wl86H=Ep?1C~%dR>{ zM{QlU&^^Htq)L#Zo~lH1h`QSDCK6z85gZ?w_2~XV?J5TbFekea!X&$1?R4N zma#QmwDd02OIR76cHsFmVR@Smlqh)^kk#;HHVv%jN#)$TUf(X(ER|Hv)b|LD>5t4U zSv4=>ESbw$4ULk?xy@&W0RA6$WG`BP9!0o~5(@BY|6D6TbX=KB&-;{euqR8u!r--_ zm7u4meqPZU0H}+Kpt{k*=c!voA_HSsY##nCX7WA*xC7tWx5fb0@(7e! z7_kmh0i(0~Gk~pL6wNETcxtB)Hjlceqmx-%&j21X)88_Hg>wYRj6oA~ItkOj1P)?0 zj*LUI)C@yvQ}__xpT__+s9mH@eliFB5AC040B=oK9vo^9;4-6Wy`yDB7f)`!fp^qy z1cxI+%)joe`H%Wgjsat&-C#so&eTDpXC4$nZZ6M4ROg#IU*HPS*6I+%oJB zKkL(!U@PGa;Ot=rK;IN#03w-7RdAau+R~bSb1!lb{)d6W{-Hi%!1I4&;5N&~--fRQ zv5gZC=3quGJZ74zuA7o|>G@?Z=y+%t4(-j62-eYm_@;{dX^hu7hZn*SE!lM{dWlS>*p2laIiiJ|H1?4IcBEb(E|6gBW!mDs_26@*N4YP^n>hWO}O6(O2&rC z9b3>9*%yMif}ir3WB|>IF*r^%DqK{icjLG~nfG=X3s{~~s--@|@3Cs4GVcpIZ+<{vq%g#v9wS%5aL7p22`hs7S)iy1 z)fMw?6*%>Pf;fDNiJfHXLE8`huAPfLdIM=Qb2W9Kkr z*?>EQZBZ;GP<5pUf<7M@z{H*u1|ZS3{0wtr0K5glXD~KN$e&0Cz;l#dPFV2A(3naY zz|1}P@%^2pEns0OIa;=<))D z;}X%s;~Sa;5E#Pz)D~?G{~zxQ!HO`Ef+_24<{9R(B&An2?eQ&t*q!;-^-@3a+-zzL zWa)dGQak#VGG3Vb3?r2q9r^Qmd?ou*@Y^uFg&D#rP#1ww887!(nWkO|R;Ooa*-d!t z4k%wPD^fN~;JU?{o)s~DNl8`fc=Mv6%KH4Ddbwn2nDOPL(a)%agnV4Ev)(4Jm>3cv=??(*l;u z;AilEQZURPmBXQdx|^`BPlD89r#F+3K=`w^{7E~s6M6orGZ*g=CUQ^}<*y%%H;Mbg z8jj)|bZO@Suzb~)6M{GO=&pMr`LLL>BG~o&_IH0~^xJRvc!h!VG~3pAE$sesD-3*H ze0X1(K+#3VAuVIMbaCr4gFM?%GU#p)fPf) z;fi9t$K8{(R1bC^FEPliu1;Xz+WqQtQRnFjp(26X@g;Bc0#vrOsrC$@{$wye$&U*k z0b`yDJ44bDGCbbPzKnmK-y(uI22!@K&?h*K!>`Y$TXK=+33Yj=lDODEmatN_mlYU#Xb=hUtfHWP>34wCrc44SBxi*_MftdHYH`G z^HTTx5VrEYh7g0o*<(l&ZUpeeBE-{Q90|2tMFn>`e3ouo?dIHjW3g%nbD~^HGR2>NNU~ zm#&qL;xeY-(QxGUBR(_poBxQ+SavD7&f}v{rP$>)QrhSg<-w#TS1Jzv!%UwX^wJcg zA)kO|jo9Shkmu`B;Fn?x_@KPKLw}8#h}QrgagRyUpL>j3Jz#R`R$;TOf^9b4k%gDO zzl@!-M@0{J#1>HeE#H4Dr&`~^yGh>44fQ$EDiY;snfz9lEl-8sgM=Vk^Rn_p=WtNN zWeWDpV|mr4{&-9#iYMOPZ|{-!&12?jNNUU7m|}}1q4Z9F<`=Ch+q+Ly$`*KoWRp~O~JD$Uc%DTWZG)M)|;v~!51in%Qw`!7(erUl4tT1S1>LpQGg(WCKR(s~y%a=(s*Ra&SO>bN$tjG^ic5BOB$C z!kG|Ob3K(}LK+9zS2%4!fIfn!TewW`5P6l?0a00iwYN&kZJpx%W8 z_}063<|&hjZ!%Iv3hcSrr^;gWd&!=ueFi!-|B~oI3zPYMZybrmR+2nq@xW@e{mgwfHz}q&J!xM4541& zc!lbPt%E03ZwN*k9}IfY!sdt{ye*rQHu&RJbP;0gMH;Pkn(~}NAYPvGsKXv?oxn%g zSX4VM#&$hzK%5G!3wV94Ab*oBtiGNFnLI$wB8^N)H(-aQ>-OP zU*jYrTQsluFE>195kdaEN;N`vjH%F5(bA(d4N~ZK33hC7>`ijq>JYYFe6xOeDY5*4 zpajbr$M>C@4=3Y}c(ofUHh&{>%48O9{-Yzn!|3@ZkR+2?jpd|BG-YViBwi%yVS@_4 zpUrTHzcSM5xtPj)_ex)cpqK+G@yv5$E3XYCgxMN2(cj=3;uHw0v!=jl?pTVm$1ofC z5Va4238nK4AhyfLv^;!WqFDwH8JEX?^p}_g1c?XaQov9~VZ@H{G9Eg6?GZ?{j-3oZ zICDX}5S}WkjtGL$lq*-5As0X~0zg05WkEu8D0it2}+;;q|j~RmX z5qb*hA%D8wN77!Mxp5PC;x$Yx`Y>`C3cc`#SzbZQ1oWJjejcm9QAd94MxdJKV7Rns zeJCz?${cpVDN3gJ@?_`q0C~>Jba=kGp0FTygjR=7a$Omx$c#no2&_DxzH-RU{b+gB z<;9~Fk)4YMG`T6+rMMP48|RuHPb*p%!FO2|8vgr#+IQs0XnH=VjQ(n~(5EAOJEp=* z%{Be>^hxXQ7=oytu!zUmV(q)1gO?C27kOV?b1KFzHIR+zWuPF~m&Rct!T=lt;<}Z# zsxNd1Aw-xZO`1~=UQ`7*GY)ab(Bn}Jn9v&6DSWu?u@X{cs7H2Bxrv6E{KrOV?il!4 z_Q|9fCCPod_$6kt486DvHvDd~d1RHv4}G-zEaJ`3vFW?icX~WJ7SGfUTRyn_Vprfi z*KKU#m!F+>)HV#;EQC#|X9;5PiM;WW4LEPGgyIU<1Q&{eTkxE>h&u6gG{!zvNuw2* z*S-)avSNn9Z?sCp-5M55(oj2+6cfsG332{=0pk zg&di6ZkB?vW;5$9*)kX880Vpv2ImQOe0{x%k6OytTnt%}Ey-t9TZW8D&aHo35C3+I zaUhg*($aDp#;E|Lp$o52h67)5f4X$M(s}+pcB-=vv_tS&JbT1Xs{scHcGR+pA8LN z7cn=>kfdQk#r(?ck2+$mu$UPJVANGfuYtQRp*C%`d$H767&N&q>z6uWZdiei6x`q* zep>3nIMJLia!x<~DtLg|C+y8NYn%7*%QKH@&ya&SHfTspDIG#9b@j38hat}0Kcy@G+8^EH+T_&xt}~@V0+PO0Xne5!6*8}`%Z0N z>~3xNECwXS?-dkraJ<4C%5#=uqx#EFM*)F$DJMvJyWj%wi9%fIrvO|7FW8;2rxG?S zq#-HPCN}4KVM_h|V%LFktocLHNK~gK#@qH7u^&6F$i1k;OJX@2w8wn-s&Pao32`#X zjuPrh$)*$$8Gt$EGD*>v;#QEpC3JX$d}sRp)X^i?f|MF~?63AD_hAcKNEdxZk9i#a z2%iweAo#3utN%8P|C8V7c}zVY2X0k1FOT4V9_k6{1Mo48TWnI8{QTDQYWvL5-um)<^txT z4{{b;=RuL(sv|^JdftNVU&$sqWuIU#y=EZ&)=3+V-j&ozKXoRH+P8ys^>qz#JG_z< zeM(s-VCJl#^%aMG`YgQPqQ2%-YPC^rC!ypG1~Z~$Uf=M99{>d@s?$6Vd|}}_I%QYC zDqfCgmK7B=we*31TJUTejlj>ClC$a0X-KLf&8nIbIJMv1?OI6I#gAO&?rv9}6vcgE zmt7hPzZ;4veM7;KiS%d71Q%2&(of?KQ;p%P!j$Us_k&l&V`>D!<+jt)ImFi+MMpF> zMT?PfKC)h|R6Isec^j6Qu|QIfJ7bwN5mwJ~S?(oODKkojd1DLYfR;Duo0w&;K*-fa zpn66zN;sk+&$cEM*uQbLlg=p%W-M`jz`-&wHMA!XSdX86#J^34M1GQli4NwW%AuA( z&OU{WppD++@$Pq=g91a}P249X-w~bG8d$%}0QRU-4KZD3Ns8c}VghBwMaUnieeEFf zI0Akv+wBgD zudvLWvF(TIwH>TvIc6<&k7&4wZ2r81P+ON%nXRIt5GvN!&%I(bXpwk=(0HeHA0Gwx zfDNU!lana0K>OS}SmQk*!;seoS@SNIUc`{2W|fCkA4zA2;1Q>mS6z$>^S@ry%%T|j z+#1XZEzD&RWfy;rvIEB;G*DE_2ljyq-%ia8IR|=GC%-NWo^#nyKTS;$pE5J#QWe^w))d2mp_t1X8Z!!_Kji)s|89?d6TqG|`>|!pr4U!UQg*%nad!yTiY86~gT4 zq?-ZWl6NrfKm02szQ4SRqrXC>S+2m};TlyaW~A{?sC{!7g*}QEl}{%LKfb@U zCaAS@nj@{$Nfz{M!G>yQG7cejET)aDq`1lE8IW}LM@T#QLUns*~@k)R%x)?a4mR`smX;owRrZ6(jd_IqiuF2fBp}1}hkHMm zN|Re0zF6>b(R-nKwyeVcX>pec%YuD^>gCYFLMC&bOM0$4glt8Mtl~26Rq%O=y+3K~ z)6;04NG;WUPdBr!34dUj+?p;pOVB8pzu`DP+q{1DsvZ)llfoJuCyvwT+SIjOr>X=8 zlBAc%NkPUG+~ic};Sbx$in@tw!v^}#c~=s4TV%xXk@34T^M1VGo0^u&f>&WVC|T!7 zgvX~Cz#)p0Z*@<(almR(wk?+LjiBkwJ-^~m0%S!ep5TO>Na6LP$O#aCEC|PWjl<1f z6x|$P9U3?9@M?WQ5P_~iN+zdcAAG*#bGz;N0{RvNyf1^C`Xo)V+w-R3;3J!+yW7c> zL7r#BwfiU0&;433OX(Pea(ZV#V#AJew3a4j(#;)yy8Uv+)9y*0Uohe>B@}_-6;&Zt zyqfS3ygkOzN0?sH|CrI{K3-GlEsQR>YJTs^j>uzzin@7-3bo^AluVInFr+ho%H{xD zG4{A^71MA6nF;cqy3W=n;-AC+c&XUhy|eQ!>@%)__9;px{{$RsI+6ljoQK>N^1+mC z&TXUGB6<~OW;GbY0PI{aPw+_vMR(AT9)sN40y7X?QNHnOl)_@t~}mWkY8_YHa)5JH1>n2&W{_IxAGEJJ-+m#1J}E9u0GcsG!KwEmz;2n z@8nyZhx-s1E`GW@Bt;D^%fP9%!SVW(w6L|bZsF+tA^W=n#beOCZ=#ysEC$bogkz7p z%9EW(e9RCxgTEnvwn9tTWuke2F}bF<+#pFdAk6e#@BPEdWg{gtn_Z?=_(gZZi*d^F zru5n1yTsxm?BA~{jv4x0fpumVzXI@a4#IBGC{A(Y&>cwuiM^BSI4wuPwt346N)&X*=mVR21Bo=}C z-Sq2(FW{l#vad#sPUNvGnDMiOFP1{!N(;-44VBrk;}jUtx%sY^?MY%ze14=3sj4(e zZ|AYxdzh!U@$mKmQ8CnQ+%)9hfJblP7kQ=#wEcCS>5}KSpODH3Yn!sX02Wet%dHgB7;)<6vMUDPjfEXgg$XJy>jGG zp!T!nb3^N>G8hM8BAEe{t~InsyeD1HZjQzeu(anHnC}~3Zs4_SewV6KzSl+6Px@3t zS8&lNS^wvV&83B%kBxc(@=iX7t<54J4|e7GCmaVT5U~GLbiJ)`Z-cIajoK1sC9 z%Yl~`K`4o@Df{H8WefmDVIw9jkV*`htPbl=^vF?P^`F|q)_9r$I1DTot!09q-1nSmkSH;5s?Xw@G6f#D=bk3SH_(F2 zP}MjQ_(Yu7$cRZH*jX#)GV0Zyvq<4?DBeE_2re@>u@!9Ln=la&O3)Dt7AKt_;S0Ie+C}nDIb4_}RQx6{-aF5T;(y z9X#rC5XXnPbsBfEt2o)|U3aF=^{-!?pRAu&=DVXIOIzOSRpY#6JlR>d7RCVBmlK*N zD<5yr58EFa_t+JAbu?S7Wq$NoTB?P?%xC9Q`G%jkQbAZC&ElHNaO`VW`S1JB$8K4F z%Y$Et%0`X8%QVQAWJ)6H&IoNUP3bQ`r>X^8S3Knrrf6Qsu)g=bfxV)t6Q91f6~1Ti z^_y*xw^QunjeHR|H~g=Z(;nbw)S1GOR*|0{VW@Jx}x z)IWC_IcrMWA5DH(MZpnmrziCza!xL0a3AA4J})ozaOh|C=Fg}y^n8-0ub^{k5qA*olvk$6d%?L^OWJ4H&fl$rTyHrcP4BZmM2;gExRT^R_Hvhi z3~tiw`uy@w+rg$6cR%ho|4tWlCHGh5e4GnbaAf7a(CbH-&x1TQHVzuZU)I~7q7)Ha zuI~rMn5u}gwKU)3DXJ3E#5$VXPbY#YgjEDO+lK;Jv~Z z`HOW({viQ!og=P`P+(@A^bjaNuW?{~5~2SNYv1|Q?2$VS7Y%>KLkZz&Anfc{fzddl zAsQeHgqTJDCDo9{83F#kY2$g+_u8Q&cDMkFDx;yUHkTvvOVY zT&XCK^@tPAwO=lM?Gd-r;zw?EN-peEW!m&epwq2)23YU!zxMBkVXiZ$2$ z4@Ur#k7`gKQ^QM!OK{Hk*~N<2WmOwPd$=wqBL__1L&Mj*1*!B)o@E}gFr^kJL~v5% zqT~byk5D!5=8QYc)^<5Zw0ihoD#g?-E!lwH5J3hLohD)$M~x?(J}4P09>^WO<1fNm zTWM?IM_)cezi^w)28=04yZwc7t(|AtwM|>JVD3f$zClAZHuyvy1>Wcxxk0yr;$hD^ zNuU23538X}5}(^rgh{At6y%hWNmIO6*YSrpOgOePlY$H$T?x618sSHug|ym|;AmJY zNfA}VW^h!Tbkz}hn3E;q`V?o%QUPj|u{>n}o|Ri?axYr>X_Vfv$wm&fyP!jO=66Hk z>agWq|B!FQbugOwdSj(NufU}lziaC|$SpIMJ>?$@t;j|NoZ^Lc=!Hv40J+%+Jg~I$ z1)A+>uzN>wg_Wpyv1hf|f|AnUiLlSFYu<{TjVrTfi+5e;52H&D z``fm*2dSlyz2JGO<`H#p$9KHJl-FjFayr2!#YJZ|!#Xm|c|;IDwTqCiNI8aGQOpV! zrU-`+r;V$uNYc?OV;zSqXe8^_pPkBkU-Eys6(AL-=U0xAFaIc&-C5A$Vst87_1(Q- zX!wduedrRes%3!22mjPXF(x;W%9A)MGz*@2q9qzPDEX}sr)hCrHrZt7hbx*u}G z8P$fP@gHb<8_+-|^cTrHtl#{$;&$qrzDY}lE0)65Qb!H5G5=0JX@EL_hMbo8?fKsx z2NTK?Y=f1eungcfK?GwL;3Irk^h?2Qo2;CLqHi0PA|AvcGS4-xh26d&(k8We&0S!-(rTfOGz zca=l79kfR~ZR>7u1FYxg?c2Vr^At2cg$dKNzToaNfOi9*)`r^aODM{d85tEn?z_9c zZn!pT0FSd);)3aZ!Xp4=r5VB-c38d5t?lBnOy;{y6O_@n>rIrXP>5O6cOO zZnCnZ@gY-(gIy@<7(X8>?BGJr#l75j$Ll(AFM|iN3*_<~)Y+aOSk=m_4?elC$0OF= zr7Jz1yV3xKY`%R9#yAI(kmR5;?8Sj;tF$G<$g@|+?GtV??^4IU+!kQ~zLC<~!!_?d z^sLUlB}7hSu-%%8L&z>HdRji1*%u)8a}UOkp8_Q=v6|E4iadOV7=R13Fj&B0G1+!U3PYnDk7}FNwNb$Pd%2otdWmK>OY8Ws<%XIE1{?l(aZfR(p@l>=T7dc zkE9nFfayA~rPrf!F&b}kQ_|lk9>_yOb!UAb^AQy-e^7Rn)y=5RgZrfBV4;HSdsoJ_^@Iq zDD9FT$iz~wali6lH*EUxh5fNTy{dXFcWqk{%nxS|W(zoBXL!gT=vbORRVJX-XXQ$| z{uzRkhF@M}xfFAvNZ6Bvyv3?|k$efY?Y)GG&KMX+upp^OwG26!NmroB_GyAWG32njGoYxDM{9UAivEU3Xfu?EE`Lv&8b! z+$XL%2WsaB9RtJrC0gOJXdUO+E(E`GAwQJ?Kso}&q~i6P$2Yg_AeY4$b^l*pHoH9O zocaP&Z92Q1MV^CJ3;U^1!V3?|(avRiw$FIMucYxQ%e<6^dnzMEj|vNtKGww0=h)T| zyo(yjq!uf-9ra@Sdh{ORc*8j7l>q5arwg2Oe!MN*cRrmTS*6{oe%w?t!Sv{xlcj!| zN|Y+IT=RWnRs-#qb0HLd@pnv~1c4q2QWzEf31SX}bJ`<)1+Q9tyyO}D%v(fkC){g( z=@5Q?rxs7I)KS8GTBYX%n@)3C2aoD-h2;+J^CCEv(T7W^2ImYJ05d*G?u~3)^XTk` zcAThE&??=U3Xvr&ml?p5ltz!8*=&sF8~P%?9z_wy(H_=HU;G_;j8-zLs8zoMTuXwy1Ha-^D(`a})QLU5P>3SD8X22l^twO&( zK;j}6G_FnHqC|yBX@_$S`;MP$@LEUxdU&#&r^* z-i&5IQJ=yw)!aN+;DYqaWA9v(j1A>o6=n&F0*}g3-{YrH)0ps^#PBApK)^-TA<@(# z)Ntm`aph{BZ&tk-^WC11Hx$XE7Z}24b`@rT&VW#cBz$Jsl>Wd2z9Cbv6NH~T@Mpfn z4(UC;-noTKfM$NFmiIbDAXGAdqYv^Jz@QNn71Klh>_ZK}R_Mf_DYj}AI5oe2puEQ=0V!Y5faVIvqSSJQwJzw?{9xi zLCA9P&0R2_l1*Bhik)x5$3Unv`yQ0_cW2!2l)|KuvkWD=l^!qjmtxFdX+tIh@&rXxkM>7ahT$JbAd_ zboL6&78wI-nyr{YX$T&dKqLi|N{i@14fxYEr^%|2r*p6q1fiUk=3l-~FuI8WY=}b( z-Z->VRz&;Q$eTQ~nDA{}OKb?#Fv~i{k^ykpliw1*#(qINK8-%oukgNw!}&2*;=S^r zUaAoI2=mXkw#_GIksT=qh?-W_186>q*SKCyO?XS%l{9@P?2!uE@}M9!jgnl7BOp`dGzo=qcaJ}=(Ju;sH(H4vC(f+6 z=DDQsj$XZh3zUisljC`{i%`lllrr5EM%AYYccY~jKJRyZ_!D2$JHNaYe_skaAh?~P zl_VQ9K#D{`7O@_Vadp8sp`vg@W75fl$^9fQNuj8ph!1fRRc2t5=Y;Rs07;p=eBrFN zub#GJDW}jD(s zR_?)S@OkjQZHOkkLDeq?!~L;ZXD^or$GBdIEiz3u((Da=6-%JOzR+4TqvS4+{+*Lr zLkmG1(^@@5NnH(Cnykn@o%qIPM)F(6n|oRLZPg-Pg`62v?`389GSlAKz$ytm@QHUB zQYAh1rIf6R78T(nK&3z3%Pq{lD(d_bhpmPO3#`tx3((WG2lQC=NcMmSC`RM@=}oM& z^!g8uoxYLgDlC5Azi1=S$IQ%HMVoQ-!g!e8&zWGVh=Bz^drhXgbR=9`!Ef({SnX~`Rj9H8Vp_~CM=TZ#<+R168+^Ar= ziK45Qdhq4Pq$=M(9!*o<_aOAOFaNqf?S#h)grCk+q4-vv6_mjNHZ2rcS_S#6x4n&u zit6bF)qBXt=!LJqtV9$oAsuM! zoE^MXkuds2-@N3QCbI7t?TuyvD~4}Olk~WN#MU%Sz^XJn|9YDo^rPgPp=QdB-!T{} z^t7xu>4k2-&y(n>T5SGv=Ul`)@y3GNHWu|!BND|8>n)n6yZWvmCs%wf$0m0cw|d5< zOIuxS{|SkK&+Iy(0nUP=8Bro%1#5TzlLjJR6zwGqSz!vHs?&nHYHn@u zh+@6PhZeiSetvw>xcjE&vkpm=Z}VAK>K+y)?p(1Am9J_;IKkA3TD(;k#ddr~-WlSv zpW|JS8lvxZUAG=PPc-EY^D-2-XPtOBg`wDdq-3Ivp9++FWxO=b$c7LhtFSWk3f__akw9Cby{aLr643|%} zf%tJ8MZ)w`gnW(W6h1bk3k|ONbghWQoi}5BdgtC_r@@C4^ofn_SZ?Nw_-H-&gzw%CciM&qcpHi!!ONt!u(it4^==1e2_?Zt3&;_@w1ma| z3fHmtr!o7SS1TBRO`hKJN}Zjk_?vK(@mDkOvNamtZCw0l6hNp(*vL!2YhGRK>vf53og@t$`Aj8LX z`|c9@Q0@C3yxp8;w6%>Oa$!Li_5eQ&VR$^q8Ce{AyJr%!qU=+7y7KrF8`r{D4w0>O zR|{{P6lHsU-{R*sODQN~b}lL?b;>hosrhN<_NM-pJNpg6i@MqPFP#)AC|fv)6h0wG zk#0=c<6RhV`uw{lzzB|+fgkB!<<9n#C47!Xuk+h|__rZE&gw$P zO+fkIRFBwuhu^zh{p7xXHYvEQt^=(=GyVqd4L&)bBTa!gnM}xxUMrp7t;fu9 z2tF3ymPw62Gc9m5v(h;Z>>jUx)f0w1E=1CTe5E=He4elPd{7ddv+PiPVqkG0#{H&p zSlNj~_*T;cn0n`0iSH=o0L}V)ZrK%R%a0jXBiXl(%{;*6if(Qzlk}%h(P$+K)q)Z~ z9UbPKfBW63Zc)FNaNm6*lf0g9Gy%E$#SuNm-b=(G6?Q}r;;Y0KZnBbg(5u6Ba2b@R zsJwjee2;Io=^Q`ww#OxC9l~HZsjAbcl@G+YEwV95|1;sC5KUm~?7*^BSN@6b_5Oj> zysgGT-(!i=5-dl=;!$na1#_3XEKlCiOi1#1xBZ)7yYkJNeQ$nPezd+kbu!-Id~uUm2L>%fUPn zJ@jHTW5k_>pQYTq7mU=y4r{l!^ox(&=`B4&B zgc>x#NWzF_(U?+jSX2$$9|RFDwJS$FmTbDeZ+Wp%e<&*U!45$FdDW)#k2TuYzm_$=w#{rvtqa4iN9_>0e!lW6`se*9Z%c$~|t zibtyoBKjgX-YJU!Y?2B0n9PwGv`ZB032aQp-YTL%qo&@+?qZ_jBd3JJjXv9%j_y2a z9Wo+rzF$kW1F24!dea)|31K-xIQ-q`FE$+?#mmeWyW*wNQ?(B8&N@Buli0d_@2OZ3 zM$;xEH{bPM(oR#!U8%m&-q_QIF#n-yD3ox-;$__lu9U_{V5DW8hEzqg)@z5$r%psWXkV@qom->x;z4 zp7FWZce9-bJV^#@7I*7c_p;Xm14(AOC!-SXCUoJ3nwO`k1~hegP7O0pYz>m0F>n}uUWtn+j-PHs6+k;n4Z1UMHo%0EBGPAMniryMHx1*?-3Qt%&NTBZ9ewG_wbu9v-! zeeUjgH{-p^!l{^S0Rg8%{_O(jIWJS7hB(*AH{KJoe^FvreMMiq6j7=)9s4qV_KgI5 zvzuU4u$=~>ugYK>2Jmo%98p2>wO63R*{zRXcsUbNytcnhpT^vFdBLKu2kHDO`4air zvvEf}>vo@{alJa$saIFeP8LMb=9c( zwF3hvu;1coLz|yw0B4eE5(6up%L34o%7_YuwoedN1OmS1<_!9*xZ5WJ3nkYQTVs5z zO^*Q^J8IW_U?zQ@eT*Qv=#ND#1dbF~-?>DaC4&`ClcQ5QhMODSeinyO0 zh^K-U8XEiB+8V?{)x_d-`L%BbcTs*&A@E**ELduKs5sbzs%6L%zG=;6sF18+^~q^5 zIFm`~1h~o_0+z9LLIsrceTc*raCPc&w0D+m%!+jnhZyX$aLnsyu=|AQCjC89 zZd#E!KzZuuBgMzT{@FH5Lc!zr#(qkrX_3`#c*_u9$yvok!6xG;&2+LU z3QX2fz(8*k`%!V9Dj8F{7xSZ-%?_0TLhNj)9=fMiIDUrs za0eMNDL5$h{`5NU>bo~$aKk;f-ykwpL`L>pUb#A8N`XagJ0n7zi_xEO<@4dMPmEL8 zES}(LRPj-sj)R`AQHf+TdhWDa5pk)J@Hp8EljMukzu9OWWD>GK6q1A7-pOQ)%i4~# z?NLZXCIir`KUtt4k_-g|GkvE#Xndm>Qnf!K=r-m{?M8qU{~CtaMkw2&AHW~LOyn&6 zdanqYH3d{tOqT{p736d$P)4D*f$|x^NovdGeNAH-4-LwNQb5rbJ+njTBuZ^#JAV8{{tG1a*{7|(G9(`9}WeKwi zoc~OSI0)mc6@bvYUN%bPm)awD)@rZK&f(J>R03?oJFtX$?w-%y4&Up zG!w4i@>S^Kn0nhb@R~R_Uy_ruJ_RwJTx3fM z!b5gZM>@>qQ}LL zl*`1MtB|H1GT>GfFq@q0@osSVDRlRn^bsD#mSKzr+yx6|t~SA9Dzgf7KoAsE1S~*QL`1|03ZV#5Q4tVOkPcZ20@9@` zC{YkWilKK2y^8c+L+CZ3gg}zzd2#RWclPZZ8Cs>BdFJsF!d(TlVRcKI~aOo<7!yo?L!Anes~y$ z%Vq&0dzU`_#6$n$`L_3N$Y%I}%m%n2Pb}@2vk>(H`9@rQo>sqN#~l(|>A?Isfb>`# zSbI3&%QAFFnr1xS2j-=jRVJUv6V#5(5N*8_z?4eO$X@SzFE$8c$djJ{5_`E+j;TUb z=tM7T#%17b@QGv3)Mg4%NT;L@aJ- z+PL=WD@KVlb)bx&K++ND)mwmo&S3^YQ(K&XoNo=munVC%B+XfY<7^j!!3Fg0q{yUP zzd5i({um1#OeczQuAQm?5^!w8F#7NSsvT_~={5GSyJFk7W2;xi z!osO-#An8_|&60gRRT!!ACy04zZ)mVi^RMFZmW?o5l#fpS>!~ zShh|fBCq3(ThjqUE|T~q{R&e$ltY!ZMm|wDrYh%w0g2o4?C7<40$;`^U(GV!>&%r@GC^*X!9b4S`qlXu){ilH zO7pjf-G-+ZU!R+`o|2V&P>5-`Pg=oeVmYC5fOj7wwp%{CNVzX#yWir~%Ohr5TDhXh z6e+ckD}j^r%5+K1x3}w3t4)u~Z`_S9;i0-RKbP*uwyScgo}^hk|8el1Nau&Lo%U9T z#*V$-wR_*10eAY(Jr`MrW);&5NjJD(y?>C_g8qrjiJmJY=TmWHduQ9)yiaoXuF&@+ z?%g`krZ$M8974LvFXd;JBQ$g_ISNZ(9C(0JWRanfD> zi+k+b10TX_P#XY7S$}9`4MIxv^sbBAT7G0wb3J~D6k|mWx@)A zPNHStKKy;Fb@gv=uT9aP6br2S-)SB-2Ep=eC-4`LLT@JP{#QPY`3xCiI=Tic)$V@i zlg7lbW&X|H{-m|U4;2R)n!G!UEt{v2@Xq7pSa`2(Z-p&lSUqdGdJX^dKp-NkZ$j!N z*e7=yxccyIMBafahuNkvdz5!3IJ2r9$Zmb|F)JXDJLBL5RACLO@LNTp_`(7f@=KfP zjMV6Si#Qjk6BU(ZDf}ly8gMuhDB5QooCoQqpW3=(@N)-#EVhg>#xg~zxoVfuT zSy^gn;nd%>iM4rVBF`S!4ixAbfkGB3|adLWsvbZm4w_J8F5?N7&raUu2 zykguN#c#E0aZV_A4f+|G-1olP-bzgUFjL^_8I{^gXB4={7Y~D#w&q6YEA=pvNNOR$cYp=OSMi*s`oT2lsqNF1#i1U@|aZdwRmv+f0g4-~Ym0 z=kZ$B$gW*q=n6xN7kOh1DKoLY-|vpJE}sc3{f-De=VX*H?J5=jdGuewqH-!Jx~drI zMPO%WQdHM$P8;w1CZ<2N%vJ?fH|v`eSb=eCc-Yi3y4}+CRggk;RnW1lla1#K8+jHq zn;>-z*}ygcNe*@wZ3VSfS^**4L$R^m?6Ap`j2gzq$8^XsxN+bY zWrlmBkO#n&qgSu#Vlu^cA5UO*NPE22f8(Zk!=`S!DV45#I=#3ZfQlE`cdhC!U~jU0 zEofVT_la4n3c_q3zML6HiFj{&@~E`eU_XaSlRbj(Kwg_M?ngI+j};)}W6gaDx~<0& zhs}sr&D!pllbWuVOp3;9NI?U9>%B%r3;AE2qwkbZM01bEEZ>aNdN-zGYG{ysawAOp zA=*e^A9=B|?e^E2m?*LpltkXSVj&M*Uj`jxr<(Dd zSp-=D1iK2WcA}BXO7>QIKYUXli#yu@XQuXY1O2L1J=BUz z)0%Bf%*>FohcJ2`m*?!WF$&u*4qR)?pQwtd1=;DZ#&`egJRQK%4kriD%|{POPA)w! z;j-=)fFo(vm}~%Pu5fBJ&_0p0C6Ti$1xIR`cG0$QvkuZb4r_^jg=+*(MDhUIpCl92 z`6$ivXX!TX$CwccA7n1m0=~?wg<0*bSLRAm+x>dmbv`eFN3Xr=rnq=F!=Bqox_927 z&3ih~W8DnZ^HjiLd_?#o$^7lVSi+_Nbg#w!z^$-SG~lg$(2@ySyjTR&TYy`qHg_4x zELp(_5&9{pg{vAt+hqYX^UtA565$bQV+$<-JXmYwx@@qfBiGyjia|G@J9u${ezSCc(xbPG%PGz5@{-|`{# zM^>(B=lFx1(g(yEMc$vfQGbA~=St%1n^y(Ski_jVk%T{gi7)X(H7id$Er)w+s7S-d z_J{v3>#G1o2Vwcq(@qRtd1?a< zLpM?*iqq`xr5Gx{=&m8ruSuB=$d&{ZD}PP=shsehlqcW6d?nvAt7D@i>-{wG)6P@{ zsL30(O>uXRu>JL7+gEb7GKtv+$u%MJPj9MLd4az!cqazmG&zql zvwqKvuB3)~$SYKWe0eAHGe)cjM|P;Q{-+OvL)idS%8YsErs;K0mMz9ba;_SpRA z;-b5umlFp=E~K2&+~9ykOFTwhl&wkWRyFp!Rq>F$YuU!Y)t}IcG+czVM*5v(9A!qk zn~#2{OM0lxrGIy)4_tWV8Yx#i#R-Y%ZED<;>WokYzuI)IIUyV37xoHnIsk|xsbu03q3UA&;8;YeL0)LA0QRi zeADy%TZ!ML!Uk?YS3|TTWMF!3p^Ljfc_5Ny$TaL{jFB-28tevR#2Jy>Wu#NTF27?Q z=;PC;BMiX?@pAbh2C(q+LFAf#BKy1o39)s$<@K=nS)Sw-UaHb0UcRqrXm8+yRU zs0B0C^mJr9!Lrz@`ljme$KIc)uZBkgz-OgfsGw{QCwDrD24OYEzXOdU&ghRx3H^(u z_MaIynH3zo(*!_Tez|!ytI>d-{2to1E^YOCfm`RXGkhC9_6I?wn!|b~rVSJXaw`#) z7>97#os(4oQ+b25qSen=#J2SrPDs@<8Ov8!x82t66?L@2^xpft@m&>AXY4agy!y!3 zx$|-rrHE3BYW_i=M(zBlnNhXY!`@nj3|^i)wN(-xToa2h$27$p=0WyxG!oMTi|3Wa z9wg7~xlmkEl?9ER+)F*!Juc#e;VE%Salc6m_~x$OWl#NJv*Fe)@N@WVBb#W$4xKp} z?{5XEGKi1&9@C{yCiOy<0QX;?;sC;JE*f-)Vw!J7BybvOLCKh_8U6ibiDO|`&TxQf ze45p<^X=*)Vh1RT3BicGSE^XtJ!Gj*&x_597rLo&(>x;0MKsorito4W6@qu(8yVha z*hibB2KgViH@+*bEqQ(}9d%QpmZxBWYoS(*vl2bW85tQD3rZT3F2A|R@hbRC$Kk_x z7Aawt^IHtB;myjbCqse65kCpi2n>?57QADevZ&{6ql_8l6uLU-HD1#;c$IK-9}iMP zVQN2utd18rCX}x1_FxHem@`2yb7wMTHnzqm^j%#VviG-{!FVF0fr&VRtcmYSsMxFN zLdoeHzF)oU{ozz=I76qPvSxVx5c^O}%EpJ=@-u#Y zX*MM}Q1<||F9$t?n8$-`35lXKx6W2uh4Wt0PN!YGG2K}+o5})BeqrI-&pgdyMtec8 zU0TZ1R?%5`ev)@qEiSeQtigAgE0{`LABh@C(-C%rw^QPhtZnxX-`Idh9#awaA!FM$wMn@I;?-S1w@hzjX^#8IBTIrM^<1&Dkp_?0{=W zf3f^vQUTWJaCU-4H+s?$VJnT`w zu-j$LMmEQvkZ+O#BObkJ9lR}trUmj+%hdZ`)%mSYN+}K-uyUzfZnX03S>5z^hOWC)xQ?_naw8{?2Fz^1clpD>J~bJ^ParlEqNTJ&tmd*cisquL4`hplOdlh;MsMaHrlxeyh0J#ISUo3D1*cuOEj-opW z`Era8%^?6f9G6?-TZ(i@4{6C5F87UR{B20Yw*e;KwggH6IceV7{8L@O8HN9Y%U%0+ zJE=lU?THbOXF~*q)J|1)_bcJV{WS)5d0Zmk;6OyN zx?l2!?9Cipl72vaMVG^nZTs>4MeC{5BAU?tc`h=$Db1GKD_L`Y-}4DcA=`uN@nb7{ z>;m0n1e4FpuSI4zfNv0k zC!Bhgf}NHmyX*5PX&#e|>9=j&R|Un~zGh+LLMv&A61C((nbt08t3w*9fMWJka^KyN zuQ}4pJ$vqUP^*n{Y(e+b*2_H`h4^TOF?iSyXz6B!C7dKq8C}+o-+hcvL5m?dyW!K) zp-19uy`%f`Q_M*Ud{pxq+a4im6pe@I)+|gL*33I6;nS>29a`G2zI@;?Be3yM;|bQC z9o;n!n!{#A9}I5aX$iBclcBRQz1?k84`S<8w|5KV*KdEH$8G*?cDt30M_nMIR)OJx zUeN$|VNc7nN@K`TxXK%At0(+ipBVL|qU?nJJ(+}1M^)pyZ~<@!;Cq_NMX z^XlB0tlMj1tnOkhkyth;oTB5Ke9sG0nIEitG})P_CWhdta4#TTlMzM;}YjJao?12d8GOMU)}suB!&N`Cn-7J)&A3eE4)Y3BV(bDZr& z?Mc*0wQoj89L5bqxzz56=nM)R`4;yQLjbX~I&PiU-2-skQHYIgxwI{RbCFM#-=*+R zT>iQVm}|*rZTp3cZZQ<~S%0w{Jc^(3Cq#k_pe-Zr1|SDa9T6KhBGZpITOIzz;*1Sh z)|hnIL}hpWKxO~;LWQJSp@v!BYld%Mun369RTSB>J0of^dvgm)ltxUXXVHYWFEWfS zwqDNY*HtIjQYjW`xEMIr$Sk6jEn5yqnyuxG|}M%cE$GK)-x!k zT_B6bh4IX;(qWJ&v00d`)K8TTcaSLNkV-@FIpfsGXrElJ{j?aO_49ezA+6MRvGMBM zQD(F)?cWb`TlXUTfJrA;&`}Yb56m*H>$!BY%YlXK5c;|P4EA$f6+&HsCU4qIzr=XZ z2_xOjy0J2|9A@Vd%THXo;U4Q;cGYa9;Ui0EXO>%cAo{jmWJpOFkr}k3eYSg3E}W3Y zQ~^W%Ke_~9en|dR@Ko__vUU=;h!h9o`@nf}Vv@mU3v06iCL<#<1P;WHrx3nkXwt)s znC*ZMxhI2Q3Sb)bp6m8(Fansp8-z_G`4;M>f0iz@A65<)3*>)v29y>mFcaV=)h#77 z6EbT#yrtt!t<#rj{nmj`Guc_&Bd~i6(4#m8PbIu_eH7a3oOFXWPsvBQaSGjVC~QAX zkB~V&*7(D&NSM$2hJc#=A=V(A+lEk@bi;<&xP8eTOHEy`@A<301H8TX33-@fHmy7b z+nHfd)gPTE(w=pjAxX zbOyc8sK4f1N%o1$hyY;3VorNFqAf@Ct^CeK>+WlP!d(4ko z2KOw8PLARFaHwwl{Y@5Ayz@cb`t`=wBLb`3XP&`C{ETNt%4+7)`l{Q{)6v>kGXAG6 zNZAAGT%VBD1iySJ+qlTrEE*}w_2balF#DC0pVVL)zBL90TG|o(Chm}*I@bMSiMS;9sp&djZ+T=3wczcy08B9$3?)FqX;L>!PPSm$%^5sS)ZPOyafcM+ZDl88Nv z$_DevR!aUxwVL^NCExB2n!9oN8Ed&6*V9k04`*H55Su*O9;qpH(Ik6TP-gwnW(y)V zZDtC?2|7fs`D9Bp-Y)sD6u0&K{CtsNSyp4qT9ouf_Fhz#N%Gyr0o`{0po~-$f!*cRWf; zL24)esXFIf2knfwQCE)Dp>%WUoQ4Ith2x_qI;FIt;xpNra&zZ%qMSn0G8z&^gtzPK zA2NRY+|PeZh8FOOO8DDi$zC) zgv((spXVVBS8&=BO^bMCzuGizOOqcPNNU>cMW`F$2~H6El=3RAX^CN0*{MsHgJ$bs z>Ev{6d-V>6Qo5A4kC)P9Uu2x1C3)OzYb%W|DJsruU9eCe)+flRB== z`S!9&1Grp2wF#FW&%zi#P=C(X%Q#2Al1h=O?USpV-~DLO=(}fwSfIDXds)(4VLg?v z#Us{2CVK;97CrUd#X&~YfqtTVv~emA<6JbOebEp20pp5ZVqYEIjBuDTX$TIQttQ#l z=k(%^$0yhI&*^Q(49uR77hWS-*lf|_9|oJ7GZj7tuSE9VBB~QJ*vTtgm4F4mft|oqJ8Rl4~ zSi$=cFZ!G+W(6DvowHU1Zi{LsM)e|=BdS*g@DoMRl z>wlQDs7;y&yrQH%3AF#&rN!`ePsHfM1B4}@fdwZ*4xtX+px2d$yi^Ho7YzWbH~t+k zVEKzn{uU;Hi{)}yq@lYZGcMhoHF?0MTloNg%>%P#g(T#oXK>xcaXRlBh$JvIgt-Lz zB`ah$vgU*QsJ74glZ$l4#3#N?kN2Nl{LgVVzL*YEbr2vaFkjGQYOn`renc(ccywy% zJX7wLw|QQi{8Ew9%ns!f$^fqKyR#~0Zcnmz7fR4;0wV9pP7xJe9m6=ybxFD#xTSNo zwYwTa0+DQ8=OMlMFWcTokpOOX6aq6%*H9%PJow1u2p= zJJN4RS=w@Xv5$k zxnS>x7Y6jqBk7$cdp6!N&%3>GT2_?=a~um-YtiLs*C0OU0d=Lo+`3MiYkM2bZ7X*@ zzxSppgHvqJE|%?=e30=SItKYpj_%{G?Wsz-w)(s}ru*m`?hx60uAf{)A4LX=>$ZLk zZsdR(L zdg8|a<}2_=-9Qs191(k!oklT!LkgX$2c2t11}$B`R}KtZb+xf?)p)4mtC|| zHx=gMRVmGDrtQwWR9e9Fg_Vs#Yl-VnyJP^`-Z7eK1lYHjAa&pJ>@fMk;Qd##SWE5aL zeK&k7F|YZDE!rJ;`}%K)2X#k=DVFz5J-rRHAAI9+#Q)04@BrNaU~vi@FhY8er2TX4 zDJGUaZ6>xA{$u5-{YS=ctFTqbUcY{h8+*z1d(EaHEz#hC=njSDq(I~YRBuRa zbe1JnimVu_@`EsYBzW*%Xmcva@T!hYN(67J=p=w01lQ2+R;Y8h~ZZ zN%FH+Rv$fYn1kO|e$Xv;n)g$Yo^rMAd;a4sb*i_=#xgjdPXrEZ^>9{_CmyZSqP+97 zA8A~46p_m(_DS@aJK|ZB$C=lm2ZK-W-M+h}jOFGGb@vUoru~8^vlo8M^QgDEpG`1w zcDtCbr6qJgcW7uwO-Ha)nVz1=3zjWL&vguN=X%h56tUVN5g>k7F&>o*kk~uDp^_GT z>*N{ngG987VKASD<3Y3)<|hD}C~8qAg<6Q1TO==#wG$sNyA)NmBEsCNi1)U6jQfTS z2F6Y69T@!JEG$!{chlHm1u0o7Mq9i?!;uEfMkPTdD>c{Y_`txJ%(bd{#9Z<^DD~kO zodS;hC7@FB|7o?zU!x-Ct?s4FO~e6AwCcN|*`I)P(rn93ES!IUS>$SkA2SvQY4RYMZ+o zlx(1ppmnU}g9_m_QVmsGw00&C2Y`7|Ds;Ns3t)XTzIW9im{ZLGq5jMg|3mq>_JMH739q*#3`7sEe96Ov~HYd)!_ zr=ld9IyVa@u2m0d3Xi{gyQB#$FT}e#h@CjK@bUS2baIX{^zItamxLs3@;yit^ruRY z?&Y0;1bOqP))^V8g_QM4dh=$J5kt|l#Nixd8HxjL!*H6@Oi1Tbi_Z&8`Z$Kwhd$)o zIlv+tT%ZCQF<`KfC;|ok%1P%C@H0RpOA6RRo!g(gYPE$QTI!4^$^7lGL6xzPIcDQP z2{>P&bZA|Wa3?vbS$2ZZXMkU)HP;cr4O%nbFE2a$R^iq*x`7Es$^h{7MD%{x^QhbX zIy`KV%L*o~)oP6Y+3^8nP9|oVf{X661t}8h&5Wnrx>nI2MZ--Gvct?OpHdv?PqH8T z1G9bcerE|MQRvL#r}JS8vmOIe!$ar~??0_+=dSYczQ@^1zin97j~M(8;=r$GJ^f{b zs#FiPFeE;i&|<5)pBYS3_nltb&vW7`DQLUaNfj2}t>{*S1U!VvVFO(b>E~xQnj&i> z(`R?|*+lR%MqN65-nhDG7B06x)fqr99mPC^`-gJ@=()u(4!@H%%;kGJh+3px{M0=j zYw_OER83_2j$VXyRQz*GzLJC;q)KuDdI1kJN^)H=d73@bSQhW zEr?7}F_poU*8XzARTqi7DG4)IdABeTQ28M&&!$dXNSF4>gCFR`G5dJ-0iE~{i}i+w z8-eIq6_>>dTVH_OLVV7#S6a(#=j5*0b)wA7?ea%?oYRxcHH9g?dPyhyWrm5Zm66_g z0?kauq-$lE^RZK$Sl7e7z0+h(00%b-f=9>;I{{c6#fPlQ`4hNT7ySfF)ob!SR7gk9 z_DI<2v1iHhx^ciU$dRO9EFiQqgd6(ef_{g9+MATJ>g{(fKUH>Lt%HX77-{+)QPHZt zhdw>NmO7g>0G@}X8_-u|66e`(Q*IEtos6SUUk!K{Ym`dndWL%m`C4>???<{2+vQ)Ozf0 z_7zqk#%B(IN7zCNzb*VjFXo>H@@D&~uP$ZYY zR?8+y78l*d#4~Ojd!u2S<{}usg*p4SJOV!DiwtrX!=w_(2CrTsEjbUx)0>E?y1?DW zvZ1lA(VdMa{=Ua0K_Lw7Aifg=dJ=(Z5pBX_EBx!V zR@-c{clfoyD;j+Z%e;W$ST=Cw<6GCmbl@4>jhj-lkY3mJsu})P{P?)$KpgMJqjT;0h;2%n$N$sBt0Loh`M7H%*HW?kxvW(FWwX$;H z>XCb}G#hUcybM4IYPASRW->r2Nf9F*GsGUviFpPUsllYw!lF0MSLIL6D&(Sn`LYHXNFVDkv5yBe2Xj1!~Hyi zG5m+G)@Hraz9(%B*bNzpbnT(&5&~kjg_#mx>PY6Ax2B(FXsfGoQocn#6AMg}zi~VX zNo}wWgGjk6e7T*ubGz)#4>t-0rheM}*==97nzrcm>4rUR`m2)h{&*)Bi+fMjqL)7J ztvS^!#-=CwIgah{E?)ExQzSE+@Q~n>p(W>6Vw;$g;Z?NjtoPWuNJ)WG2J$9SngN>e zg54%S(^~d4s(1ND#`?aC3Ps69MD1xhy`@w^i`ZIbz4PIgzlICyGb!oHIue4Y0n9-M1 z71fs0Mm8UW`2D*MyG#bPscvy9t7o|@#p2vuh43>~q2WgjOE456{lYGVqwq?%tij9P z+a4V{y|>v!yr1!v91L9*z#8sGmaIlYn-s!78f~3v8vf)*YzSicYp%J(JwmJM{7nbM$If9Z^+Yg;|XsOy(E|;n$k|(eS+pT zDi|X8xb;Pf?AXbnsswuef%~LB_4}^dw-QWXesI#uT(y-FRe=jfvNj68A>KyAbHT@DicBm<*F>SKF|}vrZ8!ywO}CK++KJj zcuHn z4n{BSI~d;^%(taNHB>B5Q31t+CajTkq=(gUP&pPYO7T;Tq*cc2&g-17sFTHRg=XSr!F~wV^sl|6vLC!~P1x4M|^G8g(@9^yH(A2y$B>-%zY;!p7P7QV`wvzJ;rxmGjWJ;-+dYH2E)AQum;H))8Ot zBPRz+Cp-o*4YdqDB^V_MvR?S}TujMp%)-CEhrElrPQ#CI1wlEbfWjk2g6%af)ZW@$ z7%h7H&BEhPdiV15G!7gLM#ptNZ%5tgu7A3E&!n(C(Y(kiMfKG0q~q5AdB6vq#*noZ z`)-XS&CPp|3`h56Q{>!iG{LQu7OQp5A)nghAdh|eZ zaWeiDM4UioBA9!I*R=_oaUb$e|2~Vk`^Sbq*nzG<{^=k={ORAq_zVv%Fg&W`Yw`;- zG9xY^ZcIXEUdGHYOhGoZILQ>au(8>>=7Y=<^+NkgMOU9hC}f>@e-=llkGD3^IqyE> zf=vj8xX%%2lCM8uBpb>8L=A9oTIi0q&*^vG^c_Xsw)lzUswkDCfenDokr`iNvO7<8 zOJ;kC4C~mj%umHdH=+LMH7F#4Q{je&wsHa=GTZjn#xcrp6>2Nv1esAPP4i7Pn(A{m zF)w;^Vbvj#Het(#TebghfFiqt!7lms;%GdwLLHIWT=O%#MHb-kB z_m80FIo$0jXkEAgs5Y_|tbsSq5yM|BZig-#fsm{7C~yfm`~_WA93_|BJH zEjT`A`;c7(CEUHNNww|*t;VjauB?USG>2R;crP`#on`(`q_v=hD9nB|>Ed$z5XN)m z_i^g^&+TY)Tl#air9X0)GAoks&Y(>JrPDn6^GtN8OOz_VzxMqn!OC?RZ+36^eYoRx z6yIzKzX5&{!#fGY{WQr`2D&uBy4GLLJ^4UzUewtO8eRGkh&A==gQ1fXk<2JTZJQN3 zSpA4^D>{ZnKk-S zNFN3kr@J40Ao!3yagZwhgn4+MS^E)!cxo$F*8Nfk_88-4H!NXU$YbTyJ~%IVp8tvj z&s2c7Xhq3P(P&y4_T{UDx#Sw$LjeCa!SzR(6)SkBJvkcYN6%~B8{Gv2qRuyVDroLR z9DRHL%yy5G}8Xu9nA>@WhNnGoc4&l;Wl;f zBcS6&@zX(3k>hO4Dw zxJ%hhv}6OqVhZC7er`Wigr*Kee8$v!q6D!wp8ke07W-tU!*5w>c|YShSyq1yjqxju4fKE!x`lnW4~cjOJF4H}112_tmaH5L(M%8+)iaN>G(arVs)gY!S|l`jnqV{fiC( z;9IGb)DIM!^-KDX&u)tMJ`$(h&gmreZb}d2@8Uav1m}O7R$OWz04ID2G)O<9aoN10 z$4dhqt_6Ji+;Je(hku%$jGnrb;{_LO>UnLfF}=h@>meK#6u4~|tMgU4Yf-wQ$zUbt zQ+NR77GlC(@ zbR|;y*_R@*e453G33}A2N`;3w^cvZgTikOj;S8vU^B(Ys5XRlZ^eg$4s3baTRK`Hw zr==LNBB%Ofw47_n)f}%b(xmtp!%(@HDmPSNPQNh3&;lyOq*qVgk?x;j8=SZg#CwPQ zP?=i^i48TWUA0Z-x$7x4VN-WT+D0uQ*sAq6e1*keV5M;rg^%vX?2%B*9sZ z&S7HLKNsZ2Ori#bo)vw(1UHE~)fTGKYUY*$5p^8W9`A2Bj~f`&YpUr@03kBCq3o95 z$c7nDGXv|%t_{oYey&wj_%Vi4-Xh*m&0Gk2Vl6k8tUKlrxHQJEIzia~7$~79RmFB) zE4r3aWNN#DbV3qW0Y1JC_zSQa2OLJB92A4kSdTVIwl|`oKGT%wTUL|X1mvzHQo}OY z^&NInA7_8Rc**JMPJG>l^zFm;l`<6-*K~YRn?NT;py@IH2=*G5=cfBQpMTg@?fNJm z-vbOLdU^-dRH||B#y#St2IMseR`Ww!?#!QQR>S10!MiZQ?3GAH+Ao$h5$wKJU2u!C zV(eFnZ=~7UDsEyfupQ34`^7SD{)?qxwgR$1(WGtCDQhDW>EvilGQ- zgmRoKK8VV$UZc)SI$d&}ARKx8`4g|%Zv3vDN3RItl{UNC>czd-OsJ#hCtndpGS8tr zOOAh`od?7D2SZ0~{;k7+mA|v#tIi!glT_IJxb>T;H(l9aprP=5(bg08n243z z$?6BxHXqxw|NhuqHniy?0BK!#j_Z{qjxT!7D|0amosx%g$E&A1us;l}Z(r`ETAbK> zRBU3VrZf+PS>3-_I>AQT&$Q2yo=VS);{U6?~buP~e} zPlS2W##k3rYvk9vi4)(`XG|r!<=i#WuRSX1kPMq};X+60ANzs*9Bz2}W_(mlfxC#{ qeyUC`6S}!cjK293gZ8g%+W*&nX8l{==07qS|Nra%$!7ek_rC!fW&}t8 literal 0 HcmV?d00001 diff --git a/images/logo-light-transparent_copy_2.png b/images/logo-light-transparent_copy_2.png deleted file mode 100644 index 81b2021dcafe6c6111719379c9bb167d82f070a9..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 39604 zcmZ^K19W9U*6xjM+ji2it&VNmHabbi=-9UHj@7Yk+xAP(%>U24S?`^@&Z>QO?XPz2 zs$H%dCNC=v2aOF4007`5B}5bf03d;{mlhKIi&Ix*8Sxc>nhDAX0syr!FmDE6U*CjA z5{fbafG0Ts;2#74ynM0zj{pD{MgZVM9{}J=0RS-UGujk*z9NE5)Fn-2WB}A(G$a5F z2pItKMFD;N06^G)zo~su04X4x|D_dyDE}n`3;={!06_mGqxlv7eSUqte=+|IL2`ip zonj8~|40J~1>@y{+Y=Cq$6VDWaMaJ?`&abOZYck14BC(XI^6Bzl#3% z`DZ?zElmHd$=2y#W_=mR@Hd8miJp<+{{=H~xA_0S{>Jq(6k8cmzMTs%YU?ZrMN;%nz0++?_^lt3{RC^sYyQyzJGSbqAzpbiD9h`OC zV5Tx|Xhu<|fK9LysHSF7YiJkUeU4zr1(LUQ+J9sc<~EMNRd@(HX<53o7q2S%`lbRu zqf&zC^RbR}xzT_H?OIuS63MF-`%lJ(Y`&SVUWG%~_rij%+g!UJus?Rz4PzIZ?YE~! zO+v)1F)uXH>gdoMj)KXJf_NaXp52|59& zW=LZ76Q4Hk%v@x7C4=&{5zH#OBy^w%;5{4Ib9F5g>r9=?>8jc-#{w%16KsUNmXjnq z__=NQl9yn!`B@n@)JeMN>iCOg$h5I8OI|{euPqhW1aM{=@Gvx_QLYNsFh3v7(6%wN z_V6Re^t+pjnI#7@H*LI>Wva8{ogc6<(JZ43AM0W*>yV2gAs!(nbVz<8a1dTqKI>}q z0*BkopUsy$RC`fnbi8d_E|JgI8?#hk2%#U$0DQ>#kw$vJMeGX_8%xUJ~ApfN<0VbgS5dlID5hL^>E?3SBX%c+# zyu`o;=e)jY*jvd(*UNf(yld3OGT|RKtor52_d$FzBiRtLe>HNPzwBqSj*lNKuSPSS zoX8>pc57g*jvH>BXFDe5Gn{&3p?Q~UL6{b!Np^z;S!zElQ@@`~+WWxhtUlZE80aoE z2yhvhmI+0F*tNkomvCky1m~r#MI5aHQ4Z#zfw*v|Z);3^I&S)KN9vEBH})Z)RCPIsL_K@)PTnn z@7Tl5%c3*!N~)oEY5cv`sfH~fB`w$7Wg3%Fip{96TJ>f>xx2mWhn$|8vaG^-7o+I* z-i?(J?nCkkKjl*UNj$wfkP|E0RIISuTJj+OtP+OLkbin$)Mho7uEJ6=PmzClps2T5 zI23H3%H((1@3>a6jJ3gjg79&ReN9n8qZRGr zn83gJ^Ka>3?A*0iZ#`A0j^E+M7k)0p$(We~HAm}FbZ-nOhUuhbju-XY10e?2Za`mE zvcvksW=c#pS7F|lwaH;ge#`IFtoK1oyuI{o@pP&;{lnTxGaiLdfyHdEe@89MUIh!< zXQaKj5O@6mTixu^7b!x$#ew30L-@wz&uJ%ner5LNMmfrKDNgDqzI9(s=lAf}VQT!M zUu`?8dam8aaw|QvMAfo8oOi)E$^qUNoJ6oVdGUcb(vTNFjZ~lu>Qc`x-KPR=sXIWo zR;dusrlwV`GU=JxJIwh%1R}}2FT7_LIW=L}#3Fx)(GC7Oor1_RO#v4;I>vR85x3!( z`V(4k)>3XWWb1uBfT7x;-q_1DQ>5}lq3ZTRaUsq!lAOfxRa@#iAaAzo@u3loA(nt_ zjBwNS`l1<5QdN*9Z7Xo#;_!A|p@Q!aTP%_44URdE#s{hV6k_?;)pT924$={3-bYkM zL6~~M$l_}zF5pv1HXH~PCk#CcKG3`csb@=M6Z4s>X2(u!nI947CsFOLF0`#agDI7c zwsK5b6Nxb&GGDFuq3;?)EeX4_NNMctC9Vrk!yt&lDwrE@eT9-ceSx{u9lcKs9Win% zIv`~=wp+)50pk8Bzu6r}@T5s9$48f@Dofbr&AIlSosQA(FMH5q*g45s-I{ zefKN=icIwCJ=-_=Y`A(q5tKV29lW+R%q6c16Y75l_3Y@udoIg5HA7 z-!Z5;woZZo6#83PByHy|bKcfWRB`=TH5f3XwQ~Q~#kY!z9erHLA?iR(+y-m5`XA~M zukqJ=d8K_R)b30T1_%KQFGc44dbFUqj-87jviaqLtrKzCkuBdt<<_dM3H= znsaCpfv1z^-R^RnuNlq5e>QH042sdtvuv&_p6h9S(Bl1dOy@zz=By{&?CMRBhsb4V z@@y;6F{>KXj2KZhX?Fz-dv8xft1_zE)+QVNm|T|)t<EAIQ0^$=B+Tdc}&KFX~ z_9MPXSKs_)-rFs!pV){K{0a9iRg)1J2NRKgdcxRuvhuO9*p>EmBO#u9?_DDlE(IL>9h=)WosR3IJawSbhg>Q8zdsVL*tq42ynasq4VzOd7+E%H&_H4hPG*v z#~#_coDbP9bj+r4+^~O)ir}u*iJsPnNj_R~+HPDqbo&MGaK)xj1@Xl{9R&*%`PGvV~&mV50bGq;BhB{e3*N4YbmbK5yvKnt9p`lnF6g!b78| zjEy;&;#U3 zDwsXJKK2TAx_un$JUg9yA>@209rkkkGop{oNDqc}iFv;FpdOK={u;q{*4dVG`RKNa%FUS39}jkgjT&tl>k z2d|XRJ=D0d6`y}-7P5IZ?wlJpj+ZoyBdk=OJF!pxlZnuy+U2-*@&%4^2=_JNs6E>R zK>Jaf-vpl4_Iod14H=y+PpB&>BWJNjg5Mxb@aE{8G`D1KX-eS|ZqDIxOYPi(BcTU7b-ArU?Va=Y@AUYt{|#6CcH zpL}ek94qQ|W94SY$Zg#of0&Sm*T7fInp2JM1^jPs-M`ylcC;Xa(B&?{h~3&d*I16U zIw&y<#P_z=P+k{v0btzQ62&ghSlKIIfikHZSz{yOE9JmgSv_GQ(eC43Z?&j^^h zRW(`3XFL#cRNNINd0$w(CLeMown!&;0mGsV+3PUKMOxJ88|6=4m!DuKio>0*c1%-n zqij*LATRgGs*vpXNrtkh?n5tC;$V<0p`(C(IUStQ>wcrcvP@>7`^KZRIJ`OSp)b?Z zNiv+Yb=#9dqqIZk{We|?d!`>%h;*s?219^2sOU*|RFdRb!^_}E4MBXeZQn2%e|P!z z{>groLEl>KiX8pO+@A!=^)IHdW<4!$pF>@^X+l7Wy1`^k1Gn6Bu|&^DM&}Q zq(I`ZKNVR%iaJksYx2Id(#dtT+K_Kl1;vcb8Wks^ln3wD&1-Ya%H&*Va=f?h>NHiM1k@F$GH-LxZ0?^>(xr@LiIg(d6l?73FN0^C%7ZfV=8ECtx&un=*c zqw+kDqVrEIqgq@$AV`8Su2isQS=>g!Dk`n$pU@qCU!xO*c_%x?YI<^GCKg*1SHLwx zSii)=9vVV%a82yrQCIp4x}G+iSa4cp<;w7qk(}P~DP1QKHfk*xrQ_ zv+Xzdb-h{Facys}bH?c`t2+NCI9n$vDw}q>?4H0OE-NcX$fXmNAGbet*N;{)I#B>t zmNUSe`|2y-;o|GzSlsE*&L}7eH7>AE?7Pd=DuCN>|AC#5m(b5+R(81^t(1# zr9$k6so_@84lUERLb*^;+HLJprEfD#zizzlM0CA|>CMZs^>rpB*D^$@VF3{A8xl%{ z$_iQY6UpREP(B7p(u0G0t1xN>9f+8$e`!*|&9qYXr0#);FwvuYh+_@4y4j91eV%ve z`cIB%(SAI4z8nJy(6CCh(H1p-`?N8ZkI9N*eck-p4A^2Y`=xcFa-;CxpXR*gRl=8jb@l8A^aBwcVImx&E zA_32_>XdUwQ3<;Yz$BP%d|^S{-oRMw_ggWHLdr)RmeBF98K6UB^-m59_T4A`-r0IHAj~X#N8RSB z!idQwIL|2AAT0!=7CwOzOp&x3Rh7^GOV$delhdOgJy&|!kyiOA>uZpBl@jkBv9w{O z!nuxmblhGACaM46uYxsJ>t_Tkur%l z{M_m79PaI#R(ilnG#-{wogS?Io2m+bR^?rH$?#~N^OCT@-lIgkoMY(Wmuzg?XlizN zak7uqNtiV@0K9hkIm1=2D86AGq^9a@UGYw*+P>7la6#efU3nSRJ|J1$E`$>mUX+IQ z4Q5k7u7&Z`^l+2w+C8yf^+3VbW42Q^S*W4Qi5Pun-)B3M6|^a4miuZ(q zyf{%Tv-LqzLtzm;*W*JDx5ZPfP0ejHYkF3NXxYw=In!G%Xw4s&R1a7)$c3F$Vr?=i4D&ZQ-^m)ZWlo=T8d&Ja{VT&2bw5+YrP#8#%ak*-n zvF&Wui0OU1U59o5WNvI;pxR`iHRr~n$Cp{Wl=~6IommGs5v%YBDJy9GhZB%LNhnIb z6UK4r%p&ePhfSjiLttr-WuH{}E>@)FXFm{>I`hqQ2p;C~zuo%W&+a~*aQmzm5EMkd zkmjwzqMwKs?8HdD)!91%$5B2>4prl}e}eTr8W)kgugjb48TTfqPJOz@i`V7#2V?wKM%b7>>b=?nyWSckAPN!4z8kE}4XJH$- z^~KKFvjq$~PuUW`?2ohna&0JNNW%|fEMS5`SvZ$_opiM@VjUkPkxnF)Y zrKvx9ODiW;+V-RbynNb7nSyXpqlGPqg(N?5dZK<-q>yv|QjR@0=FS*{?sdJqfK^dy z6B@tc9D2kLFznhnpN0=@MXSB?hbqdIF++I@!t_v2k+7_?cjtIkMTY$0$dLX~e{qDs zef+(?`t$es^&$hw(f%~4tso%BU}^BUo+9e4QobiIE)xl6&heDhJvasl9YO)~&>(q9 zvyNE}mT^8{(lom`=G!7|VE|g~}Q5c?hh_Bb6;qaPyv4GAv~MA*+epXS_q>&Dke@7?E-FTx3_)=F-SnTR2%a-CW}n6roG zFyxUUfwk=YokMt#=()}`1)=Tusc)eGg}v0PR>;cCMb!9DYp3@g%t+tBVf9m>?aFaK zKfeXYSQ5qTu>B**Y0O_>Jd&RD!$_rf#UiL3V$pxVmV+L)DwV{n6F#))oqLXdJF^{J zV9onN`Qs0{p~o9!Am;}q)F<0=Ei$fKEgA5~5bGw(vnneSk6_i{3o1zDXRinx&AgWu zy#LZ;U)U^_COa+!cm%4q-((J5AjtiBe1eoErhc;f3&Bw6B$~CE=>%C zF8M;{8f#56B?a||0v}!YTWR;I(_P&Ml@UBy98jbrKHQB5l2(qVDG2ZS5WWqPScQn? zO_$ocsP+Ev6PU3xVU~302%t}iHem@zu{=DHCUyB-lO*||^hs{z_jK;irM|t_AF9^_ z%4t#R#&do#Sbqvs%Tp0%K^gORfWsZY+Gzs5qKb_ez%%HfdRbaNBVg@o8I3AdB^-$b zQbK227LfXLvOq??&j!}}Q~^csUV*9`fPN?tL6}>P-D#QC$gO&;(av(rig+wYVMN*+^!-FuPo{gb)oYh`ocXk55nDL27vwKfX{5;DLkAi=Eu#pN%E+9Ul^3 zv;W-0T?->Tm?P9u4vIC#HI=peaY7CKJS26ndWY=Qi?{wXHbvTI{N`en`X2YB+c)1A z040|Iu^|3A@{e|oZg0Td#XU4SUTU(~_dFYWs*JIeaYBnTN^q#fUpgAt31WCfbmV1{G5X_dzX- z#>;C|DJqf+x!-V79;7L(HeYw78JvSxJeJE|&DK-=*{7wh^xQuW(8tZEHMs_YweZ&& zsSz2Ah!kZ6WA1zt=shPW8dl{ZzC{sOc;;j{5u+3Xw*%XEP`|{L=2R}v6KfSFl3&WQR-m{WmRy}bT>Is}Y z_ZH>_lJ?0K+&GgMXJyH@va>f>d*_fO`k?BoMjrvc_;i?t`*ZjZDhL%%)PrcU6YcUb z9u`=jU^e4yf5;BGkf%spjKsC+MG^urpZO}n{N&>_y({(OC^a!s@-DA$sG49QeLjJP znHTe8N=YDvpv<%Fyf{1{!;yso8@CGmRxQe|m?-5c0nfLs3ERJcIAXMI>Dsf+6Mz7* zPufvt35>eq-p(F$E8+Q=yvLU-Lm$a-*iU1&`FzFM-Hao+5e4G?di#J1v=42yPjk41;K*vzhS=v% zRTa2HO?>T=iNrbllx|-UVZH9Ie-X58H5Y#@e5^daI-6sLwGeKR>~EsS6xIEyl;Icv zin~;~g{-iy3NDYQFCb|PdUjBZQW|2L{_}1TL~L|TmLWE1)XwzL@6PPQ z5Z!Lf0l#%Xea+e~mnm9jG&{FZSGYl(J5eLv2~991DCXT?tEA&x(x0b*G5Pf8BdzZ@ zb5_)VcRQNl@^bh0VNwV6^f$6IXPsa|C}0IbX9CF0Dc3&I)t3l{NoO86xdvbw;~SMv z2IJy@1qj}3J@)JfAcKV62+=c*z`s`U5%gumA4r_+5_doBIY|lT4_on8lQooNKay!d zw{ca*EM(v$rCvC$de%#BqOGVdj}{3$oNu?xr&2?4MpUN2Y6uS+jEaL~!$dJykys~{ zx5?fu5{?jlH{C}K-R5iXFgq8p`bbFD&z6_YsT2}4Mv*BOY+g|gGUcW zNrv5OUjTfA7NqzM9Pu74h8!F!bX)2gXKS&AQ%ksLSwDPSZ&BJPt$cOQUe?gkCyRvj zg5k4K2vHM(cIm-UXorBvN5)SS3&8!TNEUv>cti>Uz} zu=*AN!a$M{@7f?B-x$IfmDn7CJmg;3G$^YWHvvnk_>SwVN-e+LcEmsZe>hVw;_H z3ZB-*`X#|X6MB-mSd-N|;7y*ugjByE%Ib2LYgSCJn!Aytwi0k0&&240+MGymyRf}S zO9DrT47u){-Z7$Q#bD-6i8G7sY*rpbA%Hb>duJ8i82|^(xP31B{DAjrQgB{M8jGpw zFRRp^<#O9ElItb=gzyc@7F_><&01d1$u$)mDXyvC{fnfEq|SkGQ8zbhnC0cSaRJxN zy__TM{^%bctY(@0KwC;MdSnw{Wn4!)~RrM$HIqHmc6 zQc2ibVWLl6|0j4{u;2G_Dq~cnzF?9MuOixqr>aU34GDn7w;&hc8?+{a)0XUS=Mj}@ zMtVt9+YGLN*=ffYAWc_vMWSgjHbOqbmtQ;K& z=&mni>e({fW9O&e!AoM8xf@1oee#Wqm7TpxD5(&C_C=vblCF#P;YWG(+k+mxIkDS& zO&fkTd*>~V4-l%R$TYSgB=}shdY+=T(9OG1_U@VSCO1;3Cg}p3`4& z+TzOA>C~QKG)usU2DYRul9_E0yzulgYsXuP`X1_0Cn0i1WBRii`EpCMGGj9H>GCIK z?J?#=gFYpjkSH+eE_~Ie?t3gypb;{IrP;Y2N1D^)sonm|==TdyK9K^&^d+wc@j-P5 z{?;Ria@RS{s%Ng~`DPPVA~j(muvU(qQ72Zdf1q9er7K9jQGX z-zzky2H`>mzI+AR_mgPX1OR~f(Q!nB))NhRKG&6@&zi#N5BAtwv4R`()=!64ti(o8 zbbd{yE!4B)LT0 z?r(5Je2P@Jx$V;U{t0?NXmWJct#Y=@U4AlvzYecx|1CH;7K~bQgKc|!Xe19X*r&{y z^PxU2RCqoE^d>AlAZ{f~|79s;+GmZMvXThj&Xdvz2ccfc_E&K@lfW@A^d12@pIabR zeQE@sLxQhe@6zN2rF%Vyb!F{(N&9%C};n6d-0!Wxzqj=6PvDJRjQyvxPXv73a2BRn+r}eT5LwI?VXEs3XDI6tC-&&;2k z`Zf2V9V79#Go_rHp0;{GIx#0!Awx&@)6Yu<@8G_k|3B)E{ zt**xW0q)}++S`9?hsXbRIWw^wu^3e(RBV{KZwG3y{UUWkNe3|2?+~uI*RgI>?=aPK zOt2SXvD2A$#ZeWILT*zsR0)hWy&9mxE@F%V<~axc34$0?%%V{hMBLlY*Q6?y;QPP=TJZHqCa0%QoKg2L|W&(Ml~m7e^a zo6a|l9?6Mp7J^S(TRNjkJSFd`*(oJFk5!kXKTH$KJq3^^$xG%iV;>e5@afh}yU7T> zm!6CBW0V?P8LzMxObKevSP}=`5_3CIRch6R#IqO7Vig?&d(x5C&=mwL>$F`e(A6%v zqiKZ2qZr$4SRWt$L``Uy2FXs=A5CD4%q)K-gM?{DA>8FBna2sZcHcqa?s1_nWXAki zaKdD^h24Oi&=Zua7Z#mbDO>>n{K~tD?{CineROnPWO5L9QDYZe(a+DP9AY+dhCFIaNFQkHcSbU*q<} zTQajLRV?99mL9+!d`6%};GLFeQLgbcwx~gV#ljUqC(FCaQoKUk^}4kVkBw+}l&X;(w3~fbG#&l-*_^ zL{}6NFT=VHI&l09E(E}B2_0R@CY9L&3mFCeI0_V_b{Ne_lDS<`N-YG8mETxjAed*V zxOS0;0kTD`_xcsd(#l@2J-4tIUohI+vYouu!U9rJNzNJdU7xPjDx_eW0-~hfTs7>PJXMw==_s1h@1d|S5h4Fh&R@c3=4c#-;A~S% zm+xD3I&%eQL*jJACZ3-kmEYH{04GM~v)epFp}8`K%s^U+;m}Ovz+1@u+3xY#Aw7!_ zmDha(ZZbcbL&t+ksUoT}PFX^>!GG6YWB4%L`nM~wvYXXl{n-ioB6sBB4%M01XSA_+6R%wCiV-1@I`nnn8}s#-v5lJUw`s>tiDEDXvtt<2H|cv ze(p}%b!*N0tgL?I5Kd)LT;01*)VH&UCyj;tIX6*<5x#8Ij_0J~+P$O3ZBTMgE#1034-$=Xx)VLZfT4J}dtSmE>bm}fWD~Xu zPsU?aaerTcGu!JQoA_uiQT+V~!?!X+=Y4HGZhQHgD~3gTknPUN`<_)E7Ll|Pd2XPp zWt4=WZo5PJ^*f)zJE`ArMPlF`m-eL_!PKIPa;e)A1L}&^bQ6&dpKDXUV-01O(X2D7 zZ*(a7)m152e{+)fG20ywo^Z8!fMnd7VD%4pNX$j2%6!AgbUGg6!mK zG?M5gX(bv9WhQ3uZ+%>{H2{-nXXB6@zjoq&71Ha>^P3eHr;zMHt4PE3Asstw^()cH z`S93Jiz7E#AxiRC(vQ5fbjb$0K6&dccOHG7j1_m{yAC2xtLtq!wQF&h=vb-TU+0K zS-K4-D21mJ_4%#c!eB}2Qs@EhBRBparw=@fj@<*!QT8gK`D<&o(+>je)pBlJUS5BQ zqhK87zFMTBi!AWQiQP2I0~y5Yl`yj=3J#1ydix+oih}sF+a_D~*5SLl7oNUg)u@uQ zSMCc0Nt=EoAif+qv5`5KLK(gGrg@x}cMsKAJ*_$bLOcn>|fd^aRH>l>u0jD)hKJeGuPRkALXp1z^l(lOv%_Fej z@Uc3Y+LT-XK4*4%Vz!^j)Jv^Oh8A98&6!?$KBqDa-(o_-_OF(EY-E*Uao7rWFI_1w z+HwVf@NmY5QU6rrLR{F}p?!OCBQ)%~VKg%dD9z}7kbcS)ep0^#c>EIW@S-#aWssJS#B>2rBObP#r6=QYE&{(%tpi_z{(lISq6bc z(Idp_swiJE7mW-iz(y@kfoOby+aJ5zVl5XbA3a=$!bi!^(WM(&a&3ZJUN+qx6iOQ# z7I1Axfs&*S$HEBeU>5pYH(+e`8s6|OL7aDc1ZbhNEH_=muLYUR#poSh%5u5mG`!nE zW5tg#WD4FVbr^VDpq^_@bdU(W7Yx`P*BW8*`r2r^xU?kQ))|d6#g*WwLseESnL;!a zwZ@+^>nFv$_Z@J{TRBrf=BV7Sn(q+1V1}jm0u2M3TpV+|(Li|Qu{LI;ekS0=oRm+! z34kccm%8uNjXdd%1R31dOPjCf$_kY8^S#$5X>akbughEO{>VjvifMSi-5}v&KY1tY z?mzfyo~juX0HDs8jTMG_tPBCygo6Q+WL((Ku~RMnpbLwOk#9(TpOA-YlJp+DuFX)$ zDnXqPXY)|f=&|g;JTDi&dbWz`h3k_u zn@KDVW)P7r8r{7zS!t}MV$P8w^^m%bFt(GsLImK%2Da*&_vPkH4U!O19*>@VH?`r4 zbzd4YO1fVj3fRf(GfZZ_$tQYCzNn!cA zdL->?>x8M^3}ph<_&~v2y9-Q;jG=UsQbcK$kzayU7U(Md( zpC;N$N-3?Um*p6aq7ijLMG>%GcUOoKXnJJ@JN&IeoSELfcW=imQy5E$!^l}v?wovsX)&j`UGd4eZO#DeG0AJvhP=9Q(YI&cZzDc_**s7Sow)kH8=aF#geFy_A}w zK(x?&p;mLK5&eD>_kHlteclO?mb;QdYIR(%gp>oI;sQ zU;SPU=#q#1z=88*!h24nVb#D!FY?zZKpy_Hl!4=(ig zg_%b@hu-X(U-S~>0TF=S<{(;~oy-84Y|#@X)co|B3s_Kv==nUYEZ*xB{;wml!!ih5 zm#6yBX0uS;Usmezw&)QK{=_O>GF;1(ht5;s@H>Sy44r}NlDJ7qwiuTa#ppxzQiuQr z^=|!NE3}n%t3`lcH;BAK=W7M3c>!l5pC#G^E!)2z`Dbe?D)B|{AHQxT4Md8qN67;S z_}`w1H-81YPmH#6=RQb)cs=E!rz$Jo^W4cca*94zBsIx(Pxy+utxbmHC6tJ;OW7oW zp7NXjDpg^QK{P#r^6Wh@F8r|X75OHMTyKJ?;lRu_!>!Hxp<=ZhJdjFTyR;jy4MB!9A*;^5-e65C9{86ykvRx>{;N5wjN zyu)g+NZ-c`U$q*WDGna61o_pHK@rmuuqYezL4o9hCD>?5Zj)@NC(7_IjT5eJL}Ty_ z83nH^vo4D=#Ah^zWC~I^LB+m}U(h=Y^1m$1P8@F2qI#IU~k51kzd-KtHk}0_=#gnU|bFbHPr#8b_=t*u=*=#x%iGz;)#hivpA1m#iF=n zaWPQgkUmT7CTT_%wT5ht;N1W92nZXO~DA5iB|YXKFgPjJ(- z=;^X0a!nTzzpDpF!&Xg1emvH21pE`uxy~{>G>=o#NJ!iQ1v8c&LWjzqm|zK z57NP5?gIu$Pj{-l`_7Iat}iq%)tdGyqcBv`3m^+pf>J-K&DM zS-vpaJL-!re~~JtFk3`;i8^GQHcm!OPJM8VwBj7Iccz(bo}*Xk3y8P=r5VeE3~wje zh|j&wYtEK&r*{K(c~3c}G>2cOY)`-M1nQ{n9*Als4C(+cAw+Q0)Aa&@S0nX>LPQ)J8K zlQgY88UdQy;v!^Jcki$t?$up{J+6MQ3NlgeNi=swPMJzX*L1STdb z0;D*PJUvdi1fl0aJ^OEEsH9k>)f7tyAQ246GLO8 zx|h9uYt7`7dD^yHvQN`y-w62Gpi@G7N%hc1@0AJnF*fcK9;{!zm?-B;111FxKr?w@ zws~+Y(+X>84tdVVdDJ1&tZ7#5c;p1KHCfh`7LxCl!VUeOGb@&1VP>eVj$GuyL(VH| z4>|$#q7o6e?(6q8)<}W{J;yt_clI!_#TeoB^gwz5q`-|51sNZ2SCgIl!e)DUrOIOz z_i+o14L9pL?9WLua5i!z_I}xj7D|WNVk3R7%o%jO^x3*v3pFqAW4Gd{=aAo8&Xv?O zPBKv?gx>Dg=_+&C1B_ZPs6Ft8-=q-$2mzSu9;X*5=8s)%x4i53w3~S__7O*g9N#!9 zVu<5>kOq1O>~+u#mk2dzv57H+>B8EHgRTpaZQ0`AshCQe59^Lg$e}36cPiECD}5{r zWXmWDdvoYxNyF}$?-~?GlULQ$r5Cbe7Zh2b@=^ulMShaT=I{b?FAR-2W0zJ|X;>dm zkDv%7$m?O4q$-ma%Y4IbJ%60EN#MZmUF~gywAQ~~P=OO(bZ(rzOPT=IxOL!Pto2fP zhy&n7LCG4E0NK>PofXf;`9UBh4`L_vb6H8-3XK-E9PZK>4M3&BMA?xxgJX9b0QC+J zLHUKQ&}}pf!&W5v@gOrGc}CG*)jL-P-rd>2gpsAk>^~uBpPK9p@5`c2@#F;GZ^`6Q$k+bheTshfah^u9 z@Z+vQSWE`8CkSoOOAft2h=t+Bv{qc?>omhIdG|DR)BBENrKuUoWQ8!_v7|Cui(ZM7 z69R%f;0VWwFLCE@KvGL14d@JvXa4X)DhrFnm~P6Z&~T0TQNUyM78*744M|75dCJOA zmGwL!G9mmdnnW#o*&int?l|9J#q27oppCw#@z;sUqZzB%{PY5adii$|rrQ6pwUt zdvQir!+R4S#?|Io9Cc^zJi_ow@gu>Ms5RvgaUvfx!*F*x&nZ0f-557%mEIp!78Kii zCHo&mU;B?nG{X{bBapDT2tynjmdjTeVrMSiYL-7)--C40!0a&;csHvmZ~Q6e-r1SV zUc*tH7&93_wtD347#^WdhWm-`lH(F@(%Ivo`LlmJzjdNKeEn@uMB7|#wl_?q5|{Y+ zv-D@2U^PUvGB~5ADgeE3O;45@~b+scy75xMoqP>AFR)?0YSF4Y3%ghO__sMJWOc8AidZ} zb~-HfQSpd7-O^uo(7p=wKtZW=4D4Q8Enu~ycy~!u6?A|6k^=nvV6pRbSA?BHopzeq zDXGDym+8J5Fny@F$4#4SX}BaXSm>xudSraAxAo+geS45Dr{Pj8af0pwc~lBZr>;_& zp%wY5L&W#~<~60!{AVw-{@Z7G)0@;A)a>w{$`eW+MFmeB>NVzf*WBGXJdS>&(%Kn; zZTLcpv3{1#+?r=AV)WP23h3$V^^IZm#A`eD@Vg~~LKy!V?VlWCF%R+xZzS1i5dJ$$ z;Sv=*?_TXP_)i{Ee$bpg5HhceLcS#{(UpqsR1$-R$kv0O!K{y_<)-~oT~WxvjV7b? zy^ytgp0ZOPY6G`n?$$_Xy)i?ns#dZS*;iIibiNpbzfr5~xSlcORN+_g+VyQWrLaxy z5le?{r`DrKP1)^QX3BUUB+-hFo~a%@KS#uV=~A?M8XN=bM?X`;YIu~FyE*v3N#=N6 z3Nx=M2nz?w_h#06-y)DCg4mjGzA%KeA;p#a$}=Ta_9W^$5p(wl>L$<;8`&<#ERuY{D%p35*;iwt+Dh zMNyHI%&(=D&#xnan7~$^|7aHeX?h>l=|4p_+&7}I)pWK{ga4Fo3?x5TnF|5=pc|8r z*5?Q}M!+R-kE4j)*+P!x>h+8`jmVz-e*mFCUcW3luX6&S&<(q`{mZ6F!VZ#+KVV|% z6|VI1`kG_J*G@%IAS;VIOdn1%07qzf?f1bPV3a;RE7TrQ^4j`QAQUnWXGz|W>NK|}Je0Cc(;zTlRNk9?YiAJW#2&gQba^hVQq zg_eUF$&*Z|}EOd#8o%%N=SjHH=>7s&LdJF+jB^4ysCa z$TJ3fh8v0!GeJhHS%9D9q?1g;IiEYf$tCtl=~EoNt$M=y1nfCxjX!!W8LEGcW4Ewg zYPn;_-gg%O06+jqL_t&~{+N8%o12upPmW^Z0KFFF{w*$8-mPif@keO^U{P`H`CF(M zg?qPg{@bcg9#N^A!GVrcpF}T-9?&3sjpP9%9}@Qu$w}886RA%;LkuPY_RX#WK{NqN zq*5R7C!XshXU!ZZ{iv6Os}QR<`?MPe*sE`Thh7##8{VO5-Fdn+05B&$AlZ>AI7gm* z{OL}L?c=jLcXZV@#80z#jUYZ5v<@i^MWikaDN8y`X6meM1R7K~R_|kHO6;4Wde)T{kL3h&kTz1Pc%t2OV{+5`T+41fnrx_!J+ ziNFPzO*zrKsqV1;nxTCA93o&53x21r81uKya zI11Q=UVC*J0DFtn1;_=H)AI>yk&rMjj36+B-o!3Mv7O!`^?|%cR=cX=%;(zH)_$@0 z--%9{bW-)d0K33DHL1tWmjSTHNSk}GZ=o(kC|q9OMBT-#)pwQ{yYjDP3ha#n z{w`p5dbIBCs=j@$1g*e*dq8@u^#CAc$;G@mtjFHb*B@NM3%#!oO;5GI%mMZkW%qb) z0VLqvS|DyPN)PDRC(9RGC`-B91^xVwGBNt+1SoL;Z0Mh(y4PNluO6pgMB$Ii*OEC! z>VfR_(tdK1OxM1Y*H3y}?&h8pDB%TATzmQ+_QpNOp~wFgqaK-0Sn>LYt>qg}kQa^~ zUEi+(Ab@L-fAniY_(aJ6gdoMWg7O>JeZQA104kM#_tEK~pHyfO*zov6 zwemsq2es`yqj;Gb{geVF1%L|YJh%Shf zYhbaQC<&CDw%=<=~=;gf<5PkAT`NP(_(TEIqh>#O}mp0}ZP>+|>XVSaOrLdse$ zd9cdI+bAGLkgk*YELhjsg8b%Bnu5H%f4#JH7l!K=N8ez-R62Uc?JP3hTT=`*fLh=% zpHZ>>;ssq^TJJ#oJS2RFE6S%!kpjg565UBzX0np_4Be}BQQ$=n_y9Vxyz(XT?(Fb8 zJ<9{TE3J(Y{5=LZp65%hE$?(s`FxoI{fh#n2Y@nua9xIOD2}UGc6P@N^lnpfZ>u^k z8zKvIn5rO(F8ynI?@w=gcX;-vW#wyojsm3#fY3|Tl`5Nz(3wNz zFH6|vUe+BK*K8Fk$WdS<_8QTn{}w0R52DNBZ6d=4)1~;FhK53qq6a!gAkeF`$$wci zjq*?GC3A>-IL44{q9a|d3nKLJbOS;fwbpf)_;C)!GW!IS6aDshyYNtIq$oK6R0AYD zkEV5c`4@W7M8x8!cb8P|ZD}JJb?nR_a^HE2oSh*Y;2(|_{_Bs&&}|8|vJF!<2H7&v z7{d)30dw%oMmH$>xPBo0^~a%qTc1l40O=+kaJ>sSZzFACPgn~!g{1{%pz;JW5IPLb z&(=r-69|T5>7&Lqi_Evf-%k_|;;hN-(>lJBf>9&M~*ai~`f zF8|Zr6zEI{+1(E=`15*MiCxhC(pRe$tXr))EN&(G&sx&<32^TlBpEp;8Q%%^=>buB zrIK+w9H|(4%}mBQ^g2skVF12&iXRdVVg4f_AMtUM`!$m_s#Pl>w;L6!Hj2gnp9oeASmZhsulUXeQpR0Wh;QM?a5YkxSaKbP+uA33=Ptq=> zlkIRjmH#iBc7AWQw5>!xc;6gdUuEUORT_C_-ORDDMtz%$&Idlxb_%VI7%<>7W+3>r zKqz!pD&FoWRf+Cq!GBTJ8fXgiNJYgI&5d6MZ5~B&aJ!}Jf8usASZ zm$FM?7Hk;+yPfW>-V^m6j74T4Hn(U-O^rrMPhBm>In5i-LY03b0C!;^9M<4wuzjSZ z3KDLJW-wA&g>v7(*DbB36CGMQ-jReY496-fYtyM@t<}8omztYxY<=<7m36Yf>ktDG zkG0he4#kuD4o66TYg$^28m&gFg&P*&vlgyDOph9uvJp%*gQ2l5VS&kxT`Bg3P#_r6 z;RFo{70e{#Vh+g)?YNCSV$jrdqWvlx=30FcUYEdygu{=vkB0v%~`pv zWgE_8A%6k5JvW4XBhkLYL7|`G)hzB{C={vCQi=BVOggnvH_SKSa85OY!81akSR|R) zrNwt`Pr&*FBNYQb$#)8RS`3r7eCy6PU%F%3hqu?%G@Boscsg`R&Ct!jglMdC2*4Of zC*zs;u5DR3y(l-I=h)79aD=~MgSlOc>py0bP{t;uirDf5s)#y;JIkkk%wPm8rrpRDjyTS4Y)h> zSIXxRE^a_nK-bQue~&=7AnrJezg;_Whs?8P3;}BoZPn|mmDu(NPF;8Finebf{?3F0 zygXJlWH|UK&r8D=x#$m%ja3Ys19)6t-@IW#I-7l%cKl7ACAu~FY(WGs44n}{MaSCE zYIt0!<$U$Kf2xQLtGoyg$2tJ~6M&tTY~Sg`+qb2Fy)b&eFq^@eOe*hl_3_tk&8E|n5tXOFDSp7tW`b-Ib~3&z3mb2SB9*7ZF)qvK ziM0*O+U|{S-@54Ab*CsgD{i^E+Ofpq?q?+TUsL`pItr9502WGzV%0P8EvCL-| zjU&Eh$#}=%GWxH-^ul|J!X8+8R!jIUM}TkO`wySWQ*Q`GA}5NMZYr@W%?HmmBUp6< z*>sA!zv-WH&Ae5A-SpGeoVe<7CSz@%Yolp7Fx2#0#_5~+^JeR@jo-yOJdJy2AnuJ8 z9hofn?9OD$G((3+V^#lY#M{4!b@UM}ooU3<++{h2E3C&l+9SF!8kc`4BSg~+fg`tI z^zq6cINw;@c6i8er(xTf5{yJo6t@<1oG^HCBN7`B;GVE;Xa3u@^(J?hySzT<9wdgI z!Gp&-wY7kF4yb&(Lzh>0?>BLoxig(ME@r^$5TQQ?S8b+}9kNLz*)$A*a5Sgxj_QNI z#wPJFnuq(Waih0Ln=NslX?4wgwHymu-&2M%Y0tqNg$m<5m^Sb0y(J5Pa9^JMSq~|! zeI-XXT$fXd0|*8H#nR$G#8KC^Rh;RO$B*^ewU<|k9!on*Giu@ZH3r4KBq(O%du7*nt^+ztx4Q{LXoOLp-eJ?7BBlF z*LIt%OGiFM6V)N@+y?N~)q2Jw|Ga$eg)X84eMEj59((muZYFz!X$H;-RaQi>APW#m zNFhTJ8IJB?wxR3S);G5;Lw|TL0R1B0Da&yKn}l%j{QZu{o*T9RV$mGrmMfaJbaO*W z764(v3Oa}20CyWIz;N%tm$O=qIrLEFr^ZbV7TF^V-JQWL*My=~?}wWK_;!e4Ll1<) z7&c+o)k*y9g>!TZ3=A{8IGh4Rzz*emDS zspOYg-JfA`M|r=CQ6L4d!}tuGfW>qI2AF4cQ@^u*!ZAPb!Ot_8AOaq|FQHDmz&uaP zRnNyVJ$vy>pC+z)I^zE&=sJV=IHKR%mW19gd_y7_FxnAwm$^pp0c-qGzmYBr=h)EN zil5|a$Jz%MmM*%83gRqO5aY7-7a#Qt9`%bk%Qt;Eol4cQztvO>7%~J96kTgNgVNMA z0*6K`2Ydkw^K`3a!{dzCz0m8|+^4|LvM?rT`$ii!H2PiyhwS2xo68T=S_BR1-X@eT z02ayo@rR4JQ~%e_PvvGJfQJQ9^oF-Qx^`5+3}27>Xd+zA;UaD&c5THt;fBJI%CL^H z1Krem7Wn;c!hWcJ%E*7o!+iZ-j>~W?Ik_K44yGUwa1_gNJ8jyYfcpwzKsg((^lSk8H~11Z8AkF^Yup&+XnZgRlRfTr;M!t` z4;(bXG>xkfLqCo!Ls`4yJGUlrYl+0F21GLH#6L61 z{v%}^@7nIZzV!C-XNc&fo&!gr`6BaPU%Y=OT>(GgnCj5AHQ@}+f)}xRlC|{ZlQNl1 z9gBMkdcpx>bxw3_6UVtQme$kJbDo|_YB$bay8eE=iI>o~H|5o`*!R^;m9ahIT1Uvh__k(j(~lUbUN9_&q739bdyIbYjvk+uZwZ$-9^uR zC6Cu8j4xf5@RzxF7W&igqkp{;l?!7x;6t1Xz?Ej>JGW&3HL)Cjit+oD)jatRx<2*p zj-9V9=UKbtz4EMWRcfbZcDTFA47Tg2SnhXInaDGG<0z;o)x#YuVE`sp<6Tm$ypdg z)O{=*cH>T<@SodVpE+(Mqs_$OI%n}kEc`3a|Ki-GFW&*DGX+j;dL(wp`$YugM_G=M z%|959MepSAepw-vcyDC*#J>TZ6xt+kbWWy=& zKhvS#gh;gVeMGKcM8a_yGQ#hL&`kyqJ73IX(s#Q~Y@tOJMAqqAxG z;;g5yrOl#0y#p}2+ur`_MAvbyMqhiT83+Urg&lqXLS;Gr2zt~9E2@UxJbT%O`GDp_ z+^Y;GLLY+w9r#ddR?@6T+7$MAQL`;mrXHLV7Ce4p6MuBi7HVe@D z0(!z`IJhR-EUq`52jZptt}Nz!qW4@Aa_sQu$VM;&PWZE-ScR!L;azVkE)zLdaYQ(4 z{{~O*JKSB`kLsr#t$gr=E?b_!GMwWa<-t9D_-X^;mYZJC0(GY-qsk)QyU1Gk$~kT( z^96h~&w*3L;aY7H(pD_Tv13EQ&^J@M_7!Ne2nx)zCLa3|t~Mi4E5K0aFwh0e3wX$$ zEwA_HuN#uz3sEXS-rJK$L5bXx9V_&+$tb6kI%*a%jT}Mfn{)hJa4xNk$_oZm%z!ODW2e{Jo!;$=YaJF@+WIINIwkL5SZcKJ$fBl{5Cnyn6 z#p-6(4cAQbBAzu36(g2iP3{*?7(JsFB&C0hXKA>y{L*#w`|4|ki@H=DB0`FIT}Zn4 z+_D&wRFn=X1nZJvKcj&^o3rBOQ!!M132ym9^re+-2(n2euvnXn|N9~pRbS@Zbgb&X z#&Ue7lO)#y-Y|eFEMxGETsI5<;c6v|6D8ZRg3lUJYRmSC9#0|>`1OKtI5-J^T*SGZ z;=||iJNDIB6pQXZaA#>k3-oaPcrRk3OZ3zn-@;n(KGA?qMSho)wp8D1ys`PYg@ZEh zpFl^iVPT(vD2R(rM>ate1$hQgg$=wAk^VwNz4eIx|74M;41Lo;H)tLUcQ*(_i-@^m zxfbVY3@ihpxc6Z6Nj>4}fgjY5KWdfVe*wOF&-ZY@4IBP5y8F}T2vKLKIBiKbJA3@m z;#hr!wer>D0lsUo9A6fz9CR3Tk(g`@CR<&AunKPZTmeV6Okun{fzxr7SU+?h{6@RG zFZs`t4ZM_7%as8zf3&;zcdU4z9J5D0anq; zd>p#4<2-GbBhK4m@In8j+o%OF8dPa;NSSmx{S(7A@0?vbY_)nnglxnxBSsn#b*;TM zsu<4YsSRW-MQp6SL>H^{ji?R21rT1%WVi;-_9y{DoE055L;-!AMFst^+OQd5*!hce z=Y?YEQwdu^$^h8S6U;){ZxN*c_p$^2PS!coM>pY!WE$xhG*alxo^y3`$(zZ4wA^*Y z={uzf!qsT2TM;ci14%bGtH&PKDFV{(*PBSx06@*U_~^%Z;p5iwmoLJf@w#9z@@er_ ziQ`L1Y%SJp;f_VyC0YXbUekmL8UXOcQvl*JIHf!4CyWqx6D|`UKFzGhmqh2;J=T#P zc(6Z~c|a&?gWKe%Vfp5%s0)6eo5m2PhEPI?VUxeYp2u3UkmV zG?y5BO7tiLV0RB}tMcE1Gtz#E68wA~4=3lU)boKrP$xj;rF6=^_SVZsDCgxnpK`;0 zedPrQkuIx`8>i-5cQ>gwo+qHsd-xGokycX{zcs-F!nrLQtI>tsNOqd@Lg9*t2zp${ zk6@sNs7#P;I<;IgfY0N}o=YM@={IG;v$L2-)@=(lwkwCL7{(HOXG{hm31Hee3w6H_E$w zYjZdHHp*iQawx980k<5>bIZ9I_a5T;ZiJ(ngicgXv~L&Vm~0wZwu4aB;KL2jVo<8b+2`)F77T$rzT-ra8Sbe2%FSr59L&iFQLT3=8P6%+0I$Fckh>tduAgjZG0lm;yP;#Tb(Lkw~`BMf9I7l*yHI z?bGAG`v@23Uu6L7fpHO+ksJvAojh}AoK$<-bzoT?;>{SbjuPT#$J=+L*UO$BCO~~HKihGRX$~FsneleB2L}8 z?9JWs(gHk$jW3tFOzX_NBfy8h%U2L{iCeA!__nKr4Jbe2-Q1_v{_mrzRM+xP8vmq0S6dm zyvbD<-$)YL#iU1TvMxFHWod#pN630pL>>S+&fTL>E-}NHni`et_h86YZ1*5ie8=9y%a-~d*M$$~c0EU&{f=|@aonj>nfJHZGD*FqI zMicAkmYHL1GOeuP&l_+1QQLo!PU$LIJPilgQ5Z^8I9XbYI}%q5Tlh^Zre8(0dN`XU zOitIug~q9e3y}n$ecy6SZ;Jfzj5IYWXQMK3ELu0f37Iuy=(&-6iDRVXsMHu{Scq_u znC_~8I5gw#@+^r|?{(L1{K>4mEZt{IHfkKYi@e*ad7g`6Uufz6x3Po)aQ}(XTX%?c z)X%iCc5SPE^M%L!lXrjX#^zV<7@SE>0tlyLEj&XK@d`IV=94TU)exd2Zq(xK*P?TM!f_!j^DeGYjE(UHxJL}DXs3E}J5A!9%! ziy4Vj4vI-i{%j_*T(ixl&PWz6P;N2Rs4%GN*VGe@?bc`J>G{NXRiZ~301Gv+h`D~I z6>*1Xk}$8@VWNG0P<|R(UOCs%(=#y^UWB=j9CJBEx~@lvC!I_f)ntJ1=R};`_v@El z7lVmsY4yXeil}m1p-6n7@0IarK)Dd(xqx7vvPq8BvB0Jzp~=yGeabE+cdE=CQDV*GLJ=P(^eAQ4^j;QBu>#DnCn_ ztFC$xg~B-wB{Yz+6){(F>HF1?f?h23yfS@e z{TvLbN&x@`_S9e*Og5EHjT3C!eQwuM1{15x8CqLi?aZ##B_952G7sN|<(T|?`c#sq z3}exWdj)CbG>ZAgP6GJ3O23|WYhh_tp~B`l>C#eH}jpBKKT z^r@J2XE(Qf8vWi(T7EH3>n7YDDY~|bTN1!m84;$4jp@(C@!sR6vP)-OF;c~Ji~pze z7Vh5h3pmN&ed!2>&^W8mJipaB`%FE5TmHGy_$&ng>>3F_b8Xj|KKWaYBUB1qc_!BV{1v-H52`7HJNLqpgPHF-;$)P)cL3iW7Hrw zq5UhisXJU}_>=XuUMAuH0+DiG7p84J^VZS= zz@DK|L7(a`D3vA)r4&H8jRxIKgvq>Z!PwK@Ch8NJ;&XGcCY|1_bs}<752ZHHw=us`W0||GlG%GzllJR zBAq=ty(F1xu;>nnvBB6ls{e!>HdMhIvC40@Uw(9NWj6h5`poC(8t}lv4a+$x5R42W zYf+MoAVPK?Pnzm89ohGWmM#FIeBO656#tdJlWe}zEDsUm=i=4Ktr&0-T+2lO;r|eYH{W33X;tl1(>6ofrwy)8lI(jvAf0D`2%Q`*EeO1P_8Pfzg|)GFR+|z5Nn5!q_oqp z0o_9QU5a7jUvPjAkjZSp?D50atNU`X%)ZEc0L>|?TYU9Ou0LXg3ut3D&Ld3olK{UB z_K~s20RT+y7V+Os{xSf*ohE(ejEJ;NWLr@pZJuMbwd8N#rMBf%z-$IF z#>a@>imc31RZYWP<2Vc^w;~=(<+evy>GQ42bji@-h;O3!sv%}8{bQdY#NHYT$2Sw# zB1^^S(l-uk*-K}VoAw5N{X7=j5R2R|KdLK>t{EgLYjQn5_rVoCT3yEF6spLceWa1lbX zo9s@W;TLvycK6+P{_o6xp8JEy0_3DmpypGuUkx+{*!}|1($hq;t^K|Sr;lnTX%RRS zJo)3ekJy+1kAS0)UntJTN@2-th0b`wb{x@@^jEui_mzre-GZRqFCefwOa&OIBOOQq zQ5}OTpz>89)ed$-yY9d~(O}#;URI)!*$WUCJR4-bA|&Uz74vkZH2a<<|1%^!v}ig^ zo3}%yyNC*!#AAmMA(TkM6cCL>OT(zff`;?vOQ==5^iTF5pWj$nDM`+4vsv4xZ8md- zD-j6&y`2COy>(G^bJ=9^6COb;hdtT{$frlg$nv18--M6M`bU+VZ4*}d&kn@(RF+Ldbn3rxMjK`2&rm+9}St-r!DQr4_1w8CL1bj$i7p3 zy+vgL^pNo3V@9HZW{1?S+D2`A**<5{v~&J|L;Zu%u=BfSGC3P+@!V)h(MZ_6Tn{Li z0*-+5!G=GAx_oaRriX;nQz^ zOb-Wc!u)0#1Va;i^A6{+!;KnyATJ<$-f0=G<8JpNN* z2A?u!o&65Ysly&@WOIi3H5LjX(Q*_oO=HIbWc5N+&~3Cf{XHBCFxU zw`rNs6WAxovC;A=%;{G28JP;zO?Ok(sQ9RZsGx924~EFGu^)YxpF(r9#}-Z9x`X5DZ@o>6c6ZPC7-!uG9!_60$iJf*W5) z=;+e-O{?M2S>t3ji)Z`vg){6VtLj?{K-}NvDP%7#{MbiH!2_6REWpBwW1?_ z^$M<$B{h{RpT!Ew8yKx)**@WT=7i&%KEB)KTLMHFv~TSe**FV5B9!VeRW^pogh1g2 z;)q2Xc25z>Nwgg%BRTc+)etu~1sBtlvI{5v*;wDaRK(S__^vGQ*aL#hvl!!mC|W5^9@X7w*BEc>XsOpc8d78?;h(J-V|GT=j9Pdyq-Dx1E0 z-Dt|3$&fTUv=wx!n!ECzNke++aXjSsx_9lH6GAX&;H;m6q7A3Zq8a4afT5F20O-3M z0TCuqt!5LPK6N#fqyEDk&Bdms9DSEv2a>mYKs@@B0Fld$h4*dxPBza>ob+^~fO?GH z7Ii4e1&7Jv=`z)8i_x(6p;RJ%ld9=AAdq?}75GddPJ~*TBdq41u>WyWjeVV*^u(hKfq4nK()$ z+P}5T_%cTsvw7jQN9O4EbS`u}d9!6CVXPTBHKa*{T5Se&bpgVdX2OIK&Qz z!UPaNPM5!P{ zD>!tOrRhr_m^ND0!jldL0fT%-Jv-TqIt}A*57}8>Up)KD2!hECvbyrIF=5K7=QI6C8 ztGEGNE%8UaNSyPiPw1%d_ylotf|{FbhP`ivlM+Xi>7pR$jKoJbzX84ZmH;6U{HkW! zSSG+;oZc=W3F%R!VT={K!b+)H0qGijOw>}x@C zcOkCvf9ce5uJw=&dmpe+fJ>1IjTdFQQm+Azy4c~whOR-~qFh#^?h}WVyXyJ>KNDb% z*2tB9KpIBhct~?}7;;#;5I=)>!mf$`4{Uz>Gmto{5lV3lB#;2b6IymEMjl2)p(td7 zLQwT@aAE$)n11ey-uyfXlU{d~GU>HTe9GN$WLy`HlmtngqlzR5;DtUH2*WW$>Rh{` zX;0f}=TW0_bQ67##*`Jcp}y1WLf@$I(h#KHdDVNq+VJ~Gv32?lXp4ZxOM!oOgSuNS z01QvP7Nr|P3Nd{kzn5*HJ-&GRlh5PClSD^B-LJH4^ob;#{gw3<0oFhT);m;S3o39w zNr1Q@Jqd(7YkpBTx=4f;PDLqU8Jk)@if*3s96LQdh^zu4Eu>TtX$25!p^>_5rBaa4 zlzP1Jf-&=g_q(3P2D+-*;Zc)NaOq|YITd!S0QE~oZ)9X9K(P&kLs03Ua<~3NDEe~L zX<2Dh9$B2gH~7k!#4ZvyOb^1bW&^vqz~ut6!Gh?>1dxLC9SIjf;zqR^*Z`ifQueY3 zW{f++v2pYbrYr9J+(p0-1gJm)0N*IiqV+qr1_D7Oy!WRKkjii=j_e`fgyLR_bXj== z4Ui0@9OO~>>KQpI%F4pUCVh+M+CQt?jq(G9$99+CXbV`HT8s7tP zZ5!NEh(SaEvtyr$_b0+lSq7&H@^cRmtH$vkdTfLc#}|PeEB+Q%cO~@SmEWE^W-pz# zUYwezGSj|--AHXmkh-Wr-gPFP@}4zsjSK7gjSzfqMt8a4Bu%4bDsCzQEn-4J=(-48 zZr_g(?^T)=d}3kM2{{ud9=Zn2uOD2r?yc`wnnNDaztaaL5f0nZ#jbP2*9+!_=h%=jfHGHDlGFE%=l498 zN~W$6cKSBNrA$JTaM?ADZcy+@jzC`S2&k~~I2{xyB%*{`E84+yxNV^bb4i45Xb8~# zGbEm6huaUW{Ne16QBJcg|5ZrsBu}=OGLXDhpfD|*J_eZpn$O3nOVw)NbuTI=wYL4x ze&lNlN@YQakL%a~A^tR(R9AlcdiRDvCJIV<>{|KeWWUYty9oG<05jx=V*~vK`*V?z zK$!>@4T|96UJP&CSKy&uCp7i3%4y19XkK*ER)`7#VkBElzmRn{@bSV+Tw<4{`(XzZ z>*x;!7^c7=e-~_b*Ff@vBO{T7GeZZ_bsR6r?s#c~EF*M*BXIW*kaHgXvpss##%XSv znlcC@CUmS{)^Ijbb1{7c6A(0 zQ^;n6w6Zv)9_s0P8=lXBo$@)9n0)Mk%JY6lF>bU;DQYV`bOr1%@l;^|kNSk$WO3D+ zOrch)SPe&IeJEUfo|$Mvh>Oy3-hutnz$O6Vdo!Ij?=hy2ea1z_mH4PgD^x=n7US(N z#784jgHOiXK|`QpKw;X?!}fU#@VE*A<|;@J{YPYlo=Bv4P?++waRd1k?4C=Erm-&| z&y8-Df{Hfm-HVn69o}Ku%Ue$n=r|F3^5s1LPKT7yFY`oTEL!G?^Z6s85I_e5y}K7wV896Ya{W z$`<3_-<@Qo6Tg97?;=ziv2dS)6h!1#s;83XD|-&?zXo1Uk{-L_fN^>}k-Yd~hlkn? zUbZV5wg)e?LRYGqa=WGnKaIFms?lge)s4Yi0FQ17Wo#r6_46}G$B!A)M*rSB5VuU1 zu&Ui6uikEc>;(dTCcs|6l7+8!=Aqb7?E<*ib#xQMxqT3JN+p1LbYIYI>kBA5^&d#> zdZP86cQ>IQaGb7pKS+d-YMg2!LP&%ZKWJQg)_aQ^{x$;;ysjz=O07HSVn@xy6aP;% z#jmV4E-?*|Lrq0l3Q~KjzSjh85xTz2yl zX=n&x)a8pNVr}oET;Xcm8=k1Ca6^3DaVD$OB9;0@XW`7`A-W*oTLPrMHeHUh@19O} ze@a1H9ifpR=LB%EzmGs+r>LZK7*2XB!kI>|4(h*wP?3^P;v)H^I`YhKpwPe*LP@TM znk1XhxOO#ojtg4VL59v8ksz<}qRMmrWHhuq6D}&AhNewv1oFZ|l323PxX8TY#T_Bz z3*(bU->p#6V&t&B;Z6# zqP^{>s7|%=*r)+GNMO>=$z*v@;adVkdQ`uvk+{K!$jeF9brH<_hhRjzJ`s<73m)nj zh~FA%TWKoYlt%vr^70|`qXp5Z0( zh6M4q5iXEvxG`vs1=&~COgf-Xww;%Lv>=bnoiwcxJ@YE5+_$D>w5B^ZLJ^o7DlQpC z_Pj(p)!-8v92;TR18!|YEOVXnUi&)ZHn&rsTcxMGW1Iqpq#&m5TLSDi2@v^4w83~? zss(iTV(8}W1n<2{Rwem5)V%Mc(%KD(^Oyts*GKRHJrQrS5F>|59WfQ^(Aj9w_#jeT zzh-P~USV1?gyzpQCdGMCYr+atTj?+74$Y%Cn-U1^cVHj&qLq-&AdLl0!y5ZXV(-nt zAcn7E<3{8LN^d^J-U$O$S%e;#3!!ZuDzFTO<4eR2A8=cgdX5b<`Lg9GOBYv;hQr5= z3H3bPg)Qy(2F(8Xt`7-NMpYn7$SLZC)D2GVe4YDemSYbhk2Gyifu%H>QS(T>eHb>O z$3_l4W18aF)o9c+8FoUA)MI!wG|5ghzg#6#Q;he-p}!0ov*vS^hs_IhAVk*{H*PPC*|L^5z?B|MF2s)BiUJ zt&%#-LE58^b8(nHP*Zkr7{*`tev+F|ly%9X>7zSxq{&fK?GsOT{;%AzGxQ+=dIw1= zN#7Hn_v?@Vy2K&6O!&y>&ohU@7Dnn|CePE2muqH_47qxV%IbQTL9U^Y|}hMm{C?qCw6)J zj6Bi1ED59_Qi^KzHz zysUdo=zCwNg+~|p%Z2Xo$=l^R`NVuofUatBatQkF6ScJn6gJ#6CC|H;=|%A}X7RGI zdH277=;q=An-wh`g1#0~B21y;3VC$kFe$wPAvIT^4E(E*b07Kp!2_El5#j=JjqgaO z%5#Yjo*^(G%mQe((X3EbSG9K-LDz%`5X!ChT)mb{}hGf%uYC zgAY#KNHP&`G2`(kO`)xPsPdeSEFe0@j0uB1x_PkR{v|+H=;}9!5#ahE8wolJ?0E(m z2W2k`*ZzYhLsOO4vG4T{YMhqEz z|DuLn^{Sa(VN4mH>E}TGd_jo$>N$2S3JoFMSLSdF-7Q>7bI!M|OLmLLzC;!w$# zA?@7`lDm}V3madG3f3TuB>+OyKfQV4`WSAS11oDgEf&&pAhUZv?=Qv!3u0KCcIPG`C zPPjtYfqG-gs6BM(ooQ0}l_Hfi&0*n46kH)gJ5^~|hq0iBWqW~%-rz`VPx zPJT{x5MgK96!qFz2G8-6j#Yd$O1|R)jD8Xh6A^mN?$`lkkXT8D!D;eYMn8C%x}FFCgKy6+Q`J`L6WzR{v<$L zYPiNam0-H`I$!1OVt5lX+_Fe0iHAa;+k}n`0E(5{M1NFZz)6{Rq7?l#lt7R~{XMFd zoHk7dO^k-2;q@J`!ru&ua5E}GEX7!zsZ?GLX{K1F_j`T$=*DN}(cK9cQ^&axvMP() zVCepamJ5)oLXM3pNZvzHlG6$i&x| zXH@Tg%bx^Djz*HAj9M7m=tW^}nOHwGx9;qRTQhac797o%#hZ3tY^KcHQD<*PxVRMF zMz;_t!f42I96?o)S&$BAp#l1{a5~mgRc6nCUud4=T&HlZuWB1QQxeDEe??XDjn!0bkl<=mpu(`2e0ND!XmLu7}-bz6Fk zZCTe)&M{su+ui=xf3Kf&NkaDXMMzH9UL42#un%{1_2_z`M2prnQ*nZM@JU^bkRv6I zJ{z&t4^q;x0jCp!+Fwtl&1LOviDMia8EIXH#P|>{-yf2X39x^4UIh4Ne*!2fS}D-* z2U|zZ>CCZIU+^9@#D3ICsy7QonG-1)JUT^HgwrWYhR`@dgG3{9j>m{mO=x5J$U#Tn zh|=;h=h#iZ>RZoo_Q5&vwMrh{SI}p}D;ip1peoCRUaZG5h-gGLTKKhX%I6^;#xyR(q0b~?)Q%s_F1n-Nd+QPdzr>Hap8yz0@S!8#mEaU|nB zYo(nf56l=%QnTDhU8XjhT#>~?#2=4LfH_*n4-Es%^6zyJA?$>EhHulpdCjm^{KSuq zUmjXz9o~=nd*P2!{XGiVAgK$4Xqjd_4JXP-NQDnULPUOY^HT^)Spt%Hi>4>lM6{}u zRg3%4VM251$tHu$1xiSD_SvwFb#IIjYUn1Yxi=s;?HnpMA8R|9M2$w2;5&`7Z{7z^ zPlGR)dl?%Y>T-FxRqtkh^kh2xojgW>2{4Zr=TxTV*$K%TS^L7?h;hZB58w&@8Z4p6 zGJ!*gs^hpR$!Ind)=;M3O1JJ$YASM$BgJ>2tz|Vn|1--G4;xj=|HW_1ST@R4osV4A zj6>#Brqt)3?dpk@^EZT8xNgVAP;ci$BAx;J-XMez!LjjysRzTbj}8g}hZdx&Er)Xh zJvPer(xlyfycH-zCgU?D4z!;){~_RW0<6J`8U976M15LokXbWsuUQE>ts1F5+(#SD&(e?#@Zs+K6r8sB(( zeb=1h?fdpQ{cb1Bc*JR)Kz!sFN>Sx$J7Q0TKVt!!hfe~@%N*T!+krIlsCD!!4xlRc zFD%EYTU`0{^PoB@-I0B71ohaEMNA8N<$3a_hyWAdDFRk#ABVADrrFWFW#uWy(JWg@ zetYJ5q~`w4Shs5olJ>8Ik~|wEaX!`RL$-q~QVJX4B$Te2R9-gf8|52zE|0ZQ&hZ4Y zF~SR-N!xR?CTLfmZ8FmWD!P|jD>F6%q3NhEvmk_2HDsH}p2U#y0+HfU4djlDnPdaH zv^@4e#hCx}ChGRssI|QAY~Fr;95@It0p@6}k?WdGEw9J3JUSeQ<`qOP9Sul5IE>1% zyYciPNKZdX7n@U|I?vNlpAhAosDg1C-?vb$`Xi`W_!FVD-L+^XU3FNK-`j^VWPsA0 zqee*Q2pL@>NQ&eT5NQOzGAT!mPKA+^loF5-5EMr%AxaAZju0tn8T~%|-u<)ddCqg4 zbMAA0?mVSEIN&<^jOt(grvVv_=$C&M!s6)&NyJ6b*iMZ_IgS`>3rT~_puqBb+aiHa z4q>k6Ly@^5*T|z>zQVd})7ar~B+O+yf;d^~vlUNK-S_!7VN@^Sv4~ajt+u8}vJe;0 z(D#!i{t-qhwG$G}TVkGvE;kw7?w;Cj*UPwjgW4y*@xH5%w)kajbt9A=7kL-NPX9To zj3a2NSXU^~YUP{5-PBjj;vvd${;J7Bx9T2&1e+&|1+mq(qZclUM_BAf5)zgwLmf@a z({ZZSHs<@^SjkJ>p|hapzBkl$9ngJRPCDkb;_5~zH>OqMiIF0RPhU=Hk*BaqU4^a^l`Z{j8c-)hr)Ocn z76VywC&P`*yjqVMrW7(%N^k7=&lx=l#!td!O~!#mIlvZrQp!}}Q=Uw$sW78TK%v!{ zqLJBsM3A^wj@f^Ispq%pc6}CNITG{h37WvsADU>&9eCk^@4b>Ul?KUgOfPCrUF!Dd zlV8pH-R2+aA$b}f>(yB|zyGcA5;dLFG7>`*Mtx#mICkG2ngk=29Q$RU_rCUE0#bER z;Byn9yn{RBk~Pk_)*({%eomV{Qo!6u!w=i)imZRx6#8_sopr;1b#!$MbjiMvD@8Ah zn9@v5UhH&TNNnLY;(tvWF&IVP%HvMX3YARa2O1mc=}r85?*iyXzGA(xyG6F0t@-C&x=#S#z^Vjw|LkA=?xVJs61d<;eb1hcS#CiEZ#|HpyiS$ClQrDU z{kVGO_Yf60%8lu*_WMB*V`K@5xN?nV=oc7V5vL)!;OX+4e4Ei|8TzHLmuK!e-I>29 zpnz~3j>6|nQ*+lJ*3$iGJ8OEzQ1r5O-K^rBt>%4HeZmDD&%Uk|eM`4RsB+iC(&cM@ z-deQauQZrMUJYOgm)GQ0C|L0WxQ!!PtZBly99n)}Juw)pibbf)^V%J!`t=`VZ-DVIJFx*86y8RN0lp*&N_HhR4j)FCqLXJ zs_P1>#7$E>Wy&rUZ+#2TNfEv&DLiAgp%9~0yZE?A^rA1%#-V@_M!H+kj8NM2i;cauwW*`=Mf)G-Z^G|X8Zec8=FCYkenPGO|PAq*=!$% zxpIjd&;afHquXI7eJI(pSu|FX?ztmD_S~^gTm5Z7Dov@r(;gQrC)9ziNQ;KiLs(vM zD<>^ZNYYIH)R;LbE^@*e%61-T$4^|Rb$>Nsg81_=9MVgT{`*~!csof$i5{poqiuW` zxGSUI(~~aL3uKm+E?F3f&(JZ zw2-uMW51HOaTl4bDyxyvA<9_EkPjX-h5ad(s$6n0-$Q@n;a0}^(-C;b*f21oF z*OL#Orfd=F8h8ruG;`S*A1Bf96dSLK{PMyBsh%Uv0oOvgfcMBq8}v@S2N@X-_!`tz z&4~VNq@7rgS`*NHR-t=Q`>jbN0%;6b>9*&H8U(At<#o1--%+K!*OY zbZVWd%OQ(u*hZ9MudF(qJx?w@61>L==%zDAUL3ao9Ih48`3qAB5lix8b`9r#;Xnv&OF>DJCy+St zkhl5I;S_J9=P(Hf_zoaC2dBCFRHtDv$Hd?Wp!_3A>ZbvSG$vgz;Hu-6a|z>oQHaFF#)7NnXkde3OC@jX;~M;M5DVN{manfOjq8 zxjg(Vm;STVm6?3U^DI(4SWa`!Ekf$v;Ls9_BIsE_4vS*kv0BKZMI=J;cx9hs_vZ;M zFtQijxaTqyA;U_DN;cvn6;|!fQ`8x&6D?PhA_W2|{eSGl zQ;7<$U7eH}sd{az>SW>&Y#5AWkJJtUGJ_Q_1lra3Z@v8WuuKATdLi zjjha#94tZB`}(dhwk!wg;Kzr8a>>&6Jrq!qN%&o$LN#^C4_E*V*GK@%``*^oyz@Tg zSLU`WJTPD30kl*f1!b;%Rg>2LHU8gSOhQaTMgMc`kt~Re`ikzFmL(_^jJJ8ItZSwF z1Bxa|g9nh3Y>LFRF;OnR7-dGu^0fV5F;Vzk$<(f#TO$Xh5H11C0QV4Dm^=j`4F&MU zMK?G^@de;-zc%1dp(=q z`NDG5DtCEGtDpKkcuQ2b>vKI6L-xJv|2>(*$Kt4nqU6bCj_TMM7Ced>o? zX982bB1yFoo!m`9M>$D@lJJ!DJM5&m9h>7j9SGo>I|vt`0DjT?6VJ%2@#;=U^$!-h zK))>M_d8dAP%K92E^B|L0;$6^7=k<-K^Y6z2+jC#J2Q10j{lbnXj4=${1V6C^dzAs zBHZbrdWSC`@OIhj=Pfcdu0QnV^)4G(OK(|z)(l1=mV>}&z(n)pnmH?G0lPh(VtS%c zaj(3F7&vEogXThB02=y?&mgDY>@HQ4p5wlKE>92{IDQF{zgJ-;!Ib><{Pk-+Q?=#W z5%g?n5Ml<2!z5?g#*gGE42g)hA$pD|{XCj}M*$RLisqHF~qf%joHHy33+;WGaSE;xWeoH433fg?RA6->J+2DVF?d3PibK|9wmVA>z5APqcg_i2w(&IoXYFP%D=6e4J zM8VQC-1(7SP}j(d)23XRS3_~VO)h^%GlgT}SDvDlY4 z_ct6l>56#!fb)jr&ctduPBO;3FuzlOh3|}1x^jg>doI^VoqtTWKX|_3+f`P{FCaNa zJk__#SL6@4=~MQcbG&#;lhF8r5~7?!>;!V*M^;Y7YAf8rrJ4>ev9lO0 z#|O{71$X1Wys&Z^c-bVQYdB`c&Y6pFvHO=gp`p2_PRXb~ z8_`ARSmcC*)QJvxeirXxCK3Hqq)kQC_7jT)hrIe^_*-tV$7WDETMzloHV#t!?c}wk z@HtI7o#~h`d8D2#!PG0g&-WD&j%JW-4c4i06hctO&G<6J8ElQVlM)t!>vO7*re1iW|h#R4M zN?!V&Ix|ab!t5`2H!Od%FdlW2{oH(c?*=9@vY9NrsrOL9%KekBxwK*8ad^HOLJv9x zAx4oz0FG6Lr$gD^LSKRdYkx0+f=M@zSyYU&f0ibaf=iB$$~2k-E<1j(vK|nX=Z~BFt@$LOr#1kwt%cn|&zd z=Cd$1?w!{QH#+aqo$3N}ZxVJDE#f%!hlxERC9tu_`+bn>uHaB@bW@y8q~Le$!Sv zV0BF%!`9Y+{IAEQaB?usY2@{+Gc=!H5((e0o1juMeyc3YE-JPD-VQR_h1R1=r49S9)3Bsc_a>XvK2Q?JNn`7F&y;f_M4O9#SDS~{3liJb;cUeURP27^oG^;2krSI>? z@V@W8;FM&{t&-?CDYSe@pa|Ekf)DLFq0*#j$x;lGVmAb5fXkdqk8k%|XZf8EI=%85 zs`#f<9OJ#c>9u)$bnP0bLqNl}IT=?9e)n`_$7d$Ux78o<6cM_V;BZ@CdAG8vSRC!( zol#YVTV9Ul7rLjx@S*sr%OEdO${t3;Bqw^8p*mGnRSH|9v_`h zg_W*>is8Q|7q0-TBM9mc)bR6>6BG9i=AIt~&)aD~?>or?wrmArUsHUdyRNTOyd|BU z#}3h#jM?2Z-nKdEt#-fHW+7B! z{orSD#eh3mr*uABtaG~>o4&OjiKj19Ga_BlVisBdCbgw}D(vz#WsSU)c}^ZCLY4sD z9Y@WKTp4mM-Uw1n^n8&v2ft|Qg}q0u-Zh<13D~;%_NP&)N`hVQx&m-`cv_t&I0o)-+wc_zHOacz%wV&-Zc=u8mv?O z^ZNEyzhT?$CSId*3s|OKipF7bX$EC_Qm<1$bQ`czBYRbCPsna1?jxgwshgt7!TuQ! z(5)v>jnOXje4LuPoNMt`dViSdxsNL}ql5BHk#>H(G5*QuL(o4(`V=NwmUtUZlt~DE z!oz|`!Kb=X;cttJG&uv1`6`bWbu>q@^%pg6osSF^xWv1R${ed|I4+ecJUT@z9X|LfXb$D++CCe~mic`&AGwm(GLy%O zkOjp=;?cqyK}B4>(j0E;_4b_w`f$=d6GyYk^08o;-D`>s+DE zCUD!(tei)z(A$iXEOykGt|EVWLHs1G^iSKKBZ&~?Z{pTG@m;bOqg~PHs$n+GaH7)x z2TN+66Q9`$%#4LCVo6(_l&sO^9#ySY)L`LC6c8N`AYH&#v2^wl#wcV=amS;w(G7;j z)#=^=X(YUqdbhVp4EYUzM1xTmtB=%N?hN*GI}BMwD+t)DwZ?6RR5&u~U*~Q3$KS>R z;mdx1?n8KVD5@K>UIeU-ko-hJaXqtcPtamYGdSmAhXA%ZS|2c zU_P@GBG@z;CEd+8Wb?f^jVi{iEZQx%z1}azW%dPfusy#6Yb4}2S+En9;yLG~I*BEG zkFgE0(Wvr{1+!y@%^La4U%!aI)cXr-y7VpcMr%h#v3pU4%G^Wz|H)rWX3}lk$xAz19`dnGBd`bNN zA9GuScbs(;x{`?~!wy?TWq(zR!m(Z6HoR4l7!8I8NENTpGPTIB|l}<>uu=VJQzvdPe zCY7EasVZLY0?|ENgvFI3J-(yzx7=<0{w?#73U#N4+Vqo+X{8|?AJw=_dphzcUldIV zYNli(zRR}q%XN1CFA60k4agpk`1xxB{x>-$E>&!TNU%9(Vepa! zmbe}^tK_?hKPr9=qmaSKhSefg`#uc=4m=PSpLry}&rsJyr&bFY F`G1 From 75c65246ba2cb8989ed8c84531b7326e25563aed Mon Sep 17 00:00:00 2001 From: Kaushik Iska Date: Tue, 4 Jul 2023 10:22:43 -0400 Subject: [PATCH 2/7] Refactor QRep options on the rust side to make it more readable (#187) --- README.md | 12 ++- nexus/Cargo.lock | 1 + nexus/analyzer/Cargo.toml | 1 + nexus/analyzer/src/lib.rs | 187 +------------------------------------ nexus/analyzer/src/qrep.rs | 167 +++++++++++++++++++++++++++++++++ nexus/flow-rs/src/lib.rs | 2 +- 6 files changed, 181 insertions(+), 189 deletions(-) create mode 100644 nexus/analyzer/src/qrep.rs diff --git a/README.md b/README.md index fc88ed4b53..135a4f8507 100644 --- a/README.md +++ b/README.md @@ -1,10 +1,12 @@
-img-verification -

Modern ETL in minutes, with SQL

- - - + +PeerDB Banner + +#### Modern ETL in minutes, with SQL + +[![Workflow Status](https://github.com/PEerDB-io/peerdb/actions/workflows/ci.yml/badge.svg)](https://github.com/Peerdb-io/peerdb/actions/workflows/ci.yml) [![ElV2 License](https://badgen.net/badge/License/Elv2/green?icon=github)](https://github.com/PeerDB-io/peerdb/blob/main/LICENSE.md) [![Slack Community](https://img.shields.io/badge/slack-peerdb-brightgreen.svg?logo=slack)](https://join.slack.com/t/peerdb-public/shared_invite/zt-1wo9jydev-EXInbMtCtpAKFFWdi7QvLQ) +

## PeerDB diff --git a/nexus/Cargo.lock b/nexus/Cargo.lock index 4f41a9e804..7e78f44185 100644 --- a/nexus/Cargo.lock +++ b/nexus/Cargo.lock @@ -34,6 +34,7 @@ dependencies = [ "async-trait", "catalog", "flow-rs", + "lazy_static", "pem 1.1.1", "pt", "serde_json", diff --git a/nexus/analyzer/Cargo.toml b/nexus/analyzer/Cargo.toml index 626753becc..1680d9dcb9 100644 --- a/nexus/analyzer/Cargo.toml +++ b/nexus/analyzer/Cargo.toml @@ -10,6 +10,7 @@ anyhow = "1.0" async-trait = "0.1" catalog = { path = "../catalog" } flow-rs = { path = "../flow-rs" } +lazy_static = "1.4" pem = "1.1.0" pt = { path = "../pt" } sqlparser = { path = "../sqlparser-rs", features = ["visitor"] } diff --git a/nexus/analyzer/src/lib.rs b/nexus/analyzer/src/lib.rs index 66b09cec71..c51bbcddde 100644 --- a/nexus/analyzer/src/lib.rs +++ b/nexus/analyzer/src/lib.rs @@ -11,6 +11,7 @@ use flow_rs::{FlowJob, FlowJobTableMapping, QRepFlowJob}; use pt::peers::{ peer::Config, BigqueryConfig, DbType, MongoConfig, Peer, PostgresConfig, SnowflakeConfig, }; +use qrep::process_options; use serde_json::Number; use sqlparser::ast::{visit_relations, visit_statements, FetchDirection, SqlOption, Statement}; use sqlparser::ast::{ @@ -18,6 +19,8 @@ use sqlparser::ast::{ Value, }; +mod qrep; + pub trait StatementAnalyzer { type Output; @@ -107,76 +110,6 @@ pub enum PeerDDL { }, } -impl PeerDDLAnalyzer { - fn parse_string_for_options( - raw_options: &HashMap<&str, &Value>, - processed_options: &mut HashMap, - key: &str, - is_required: bool, - accepted_values: Option<&[&str]>, - ) -> anyhow::Result<()> { - if raw_options.get(key).is_none() { - if is_required { - anyhow::bail!("{} is required", key); - } else { - Ok(()) - } - } else { - let raw_value = *raw_options.get(key).unwrap(); - match raw_value { - sqlparser::ast::Value::SingleQuotedString(str) => { - if accepted_values.is_some() { - let accepted_values = accepted_values.unwrap(); - if !accepted_values.contains(&str.as_str()) { - anyhow::bail!("{} must be one of {:?}", key, accepted_values); - } - } - processed_options - .insert(key.to_string(), serde_json::Value::String(str.clone())); - Ok(()) - } - _ => { - anyhow::bail!("invalid value for {}", key); - } - } - } - } - - fn parse_number_for_options( - raw_options: &HashMap<&str, &Value>, - processed_options: &mut HashMap, - key: &str, - min_value: u32, - default_value: u32, - ) -> anyhow::Result<()> { - if raw_options.get(key).is_none() { - processed_options.insert( - key.to_string(), - serde_json::Value::Number(Number::from(default_value)), - ); - Ok(()) - } else { - let raw_value = *raw_options.get(key).unwrap(); - match raw_value { - sqlparser::ast::Value::Number(str, _) => { - let value = str.parse::()?; - if value < min_value { - anyhow::bail!("{} must be greater than {}", key, min_value - 1); - } - processed_options.insert( - key.to_string(), - serde_json::Value::Number(Number::from(value)), - ); - Ok(()) - } - _ => { - anyhow::bail!("invalid value for {}", key); - } - } - } - } -} - impl StatementAnalyzer for PeerDDLAnalyzer { type Output = Option; @@ -234,119 +167,7 @@ impl StatementAnalyzer for PeerDDLAnalyzer { raw_options.insert(&option.name.value as &str, &option.value); } - let mut processed_options = HashMap::new(); - - // processing options that are REQUIRED and take a string value. - for key in [ - "destination_table_name", - "watermark_column", - "watermark_table_name", - ] { - PeerDDLAnalyzer::parse_string_for_options( - &raw_options, - &mut processed_options, - key, - true, - None, - )?; - } - PeerDDLAnalyzer::parse_string_for_options( - &raw_options, - &mut processed_options, - "mode", - true, - Some(&["append", "upsert"]), - )?; - // processing options that are OPTIONAL and take a string value. - PeerDDLAnalyzer::parse_string_for_options( - &raw_options, - &mut processed_options, - "unique_key_columns", - false, - None, - )?; - PeerDDLAnalyzer::parse_string_for_options( - &raw_options, - &mut processed_options, - "sync_data_format", - false, - Some(&["default", "avro"]), - )?; - PeerDDLAnalyzer::parse_string_for_options( - &raw_options, - &mut processed_options, - "staging_path", - false, - None, - )?; - // processing options that are OPTIONAL and take a number value which a minimum and default value. - PeerDDLAnalyzer::parse_number_for_options( - &raw_options, - &mut processed_options, - "parallelism", - 1, - 2, - )?; - PeerDDLAnalyzer::parse_number_for_options( - &raw_options, - &mut processed_options, - "refresh_interval", - 10, - 10, - )?; - PeerDDLAnalyzer::parse_number_for_options( - &raw_options, - &mut processed_options, - "batch_size_int", - 1, - 10000, - )?; - PeerDDLAnalyzer::parse_number_for_options( - &raw_options, - &mut processed_options, - "batch_duration_timestamp", - 1, - 60, - )?; - - if !processed_options.contains_key("sync_data_format") { - processed_options.insert( - "sync_data_format".to_string(), - serde_json::Value::String("default".to_string()), - ); - } - - // unique_key_columns should only be specified if mode is upsert - if processed_options.contains_key("unique_key_columns") - ^ (processed_options.get("mode").unwrap() == "upsert") - { - if processed_options.get("mode").unwrap() == "upsert" { - anyhow::bail!( - "unique_key_columns should be specified if mode is upsert" - ); - } else { - anyhow::bail!( - "mode should be upsert if unique_key_columns is specified" - ); - } - } - - if processed_options.contains_key("unique_key_columns") { - processed_options.insert( - "unique_key_columns".to_string(), - serde_json::Value::Array( - processed_options - .get("unique_key_columns") - .unwrap() - .as_str() - .unwrap() - .split(',') - .map(|s| serde_json::Value::String(s.to_string())) - .collect(), - ), - ); - } - + let processed_options = process_options(raw_options)?; let qrep_flow_job = QRepFlowJob { name: select.mirror_name.to_string().to_lowercase(), source_peer: select.source_peer.to_string().to_lowercase(), diff --git a/nexus/analyzer/src/qrep.rs b/nexus/analyzer/src/qrep.rs new file mode 100644 index 0000000000..46653cb9a0 --- /dev/null +++ b/nexus/analyzer/src/qrep.rs @@ -0,0 +1,167 @@ +use std::collections::HashMap; + +use serde_json::Value; +use sqlparser::ast::Value as SqlValue; + +enum QRepOptionType { + String { + name: &'static str, + default_val: Option<&'static str>, + required: bool, + accepted_values: Option>, + }, + Int { + name: &'static str, + min_value: Option, + default_value: u32, + required: bool, + }, + StringArray { + name: &'static str, + }, +} + +lazy_static::lazy_static! { + static ref QREP_OPTIONS: Vec = { + vec![ + QRepOptionType::String { + name: "destination_table_name", + default_val: None, + required: true, + accepted_values: None, + }, + QRepOptionType::String { + name: "watermark_column", + default_val: None, + required: true, + accepted_values: None, + }, + QRepOptionType::String { + name: "watermark_table_name", + default_val: None, + required: true, + accepted_values: None, + }, + QRepOptionType::String { + name: "mode", + default_val: Some("append"), + required: false, + accepted_values: Some(vec!["upsert", "append"]), + }, + QRepOptionType::StringArray { + name: "unique_key_columns", + }, + QRepOptionType::String { + name: "sync_data_format", + default_val: Some("default"), + required: false, + accepted_values: Some(vec!["default", "avro"]), + }, + QRepOptionType::String { + name: "staging_path", + default_val: None, + required: false, + accepted_values: None, + }, + QRepOptionType::Int { + name: "parallelism", + min_value: Some(1), + default_value: 2, + required: false, + }, + QRepOptionType::Int { + name: "refresh_interval", + min_value: Some(10), + default_value: 10, + required: false, + }, + QRepOptionType::Int { + name: "batch_size_int", + min_value: Some(1), + default_value: 1000, + required: false, + }, + QRepOptionType::Int { + name: "batch_duration_timestamp", + min_value: Some(1), + default_value: 60, + required: false, + }, + ] + }; +} + +pub fn process_options( + raw_opts: HashMap<&str, &SqlValue>, +) -> anyhow::Result> { + let mut opts: HashMap = HashMap::new(); + + for opt_type in &*QREP_OPTIONS { + match opt_type { + QRepOptionType::String { + name, + default_val, + required, + accepted_values, + } => { + if let Some(raw_value) = raw_opts.get(*name) { + if let SqlValue::SingleQuotedString(str) = raw_value { + if let Some(values) = accepted_values { + if !values.contains(&str.as_str()) { + anyhow::bail!("{} must be one of {:?}", name, values); + } + } + opts.insert((*name).to_string(), Value::String(str.clone())); + } else { + anyhow::bail!("Invalid value for {}", name); + } + } else if *required { + anyhow::bail!("{} is required", name); + } else if let Some(default) = default_val { + opts.insert((*name).to_string(), Value::String(default.to_string())); + } + } + QRepOptionType::Int { + name, + min_value, + default_value, + required, + } => { + if let Some(raw_value) = raw_opts.get(*name) { + if let SqlValue::Number(num_str, _) = raw_value { + let num = num_str.parse::()?; + if let Some(min) = min_value { + if num < *min { + anyhow::bail!("{} must be greater than {}", name, min); + } + } + opts.insert((*name).to_string(), Value::Number(num.into())); + } else { + anyhow::bail!("Invalid value for {}", name); + } + } else if *required { + anyhow::bail!("{} is required", name); + } else { + let v = *default_value; + opts.insert((*name).to_string(), Value::Number(v.into())); + } + } + QRepOptionType::StringArray { name } => { + // read it as a string and split on comma + if let Some(raw_value) = raw_opts.get(*name) { + if let SqlValue::SingleQuotedString(str) = raw_value { + let values: Vec = str + .split(',') + .map(|s| Value::String(s.trim().to_string())) + .collect(); + opts.insert((*name).to_string(), Value::Array(values)); + } else { + anyhow::bail!("Invalid value for {}", name); + } + } + } + } + } + + Ok(opts) +} diff --git a/nexus/flow-rs/src/lib.rs b/nexus/flow-rs/src/lib.rs index 4906d05beb..70bddd9cd9 100644 --- a/nexus/flow-rs/src/lib.rs +++ b/nexus/flow-rs/src/lib.rs @@ -31,7 +31,7 @@ pub struct QRepFlowJob { pub target_peer: String, pub query_string: String, pub flow_options: HashMap, - pub description: String + pub description: String, } impl FlowHandler { From d1bcc741d4b2469a773a9825506fefe7d11b174f Mon Sep 17 00:00:00 2001 From: Kevin K Biju <52661649+heavycrystal@users.noreply.github.com> Date: Wed, 5 Jul 2023 00:10:50 +0530 Subject: [PATCH 3/7] wiring CDC to QValue type system (#175) --- flow/connectors/bigquery/bigquery.go | 92 ++----- .../bigquery/qrecord_value_saver.go | 19 +- flow/connectors/bigquery/qrep_avro_sync.go | 6 +- flow/connectors/bigquery/qvalue_convert.go | 76 ++++++ flow/connectors/postgres/cdc.go | 45 ++-- flow/connectors/postgres/postgres.go | 99 +------ flow/connectors/postgres/postgres_cdc_test.go | 89 +++---- flow/connectors/postgres/qrep.go | 5 +- .../postgres/qrep_query_executor.go | 249 +----------------- flow/connectors/postgres/qvalue_convert.go | 195 ++++++++++++++ .../snowflake/avro_file_writer_test.go | 16 +- flow/connectors/snowflake/client.go | 71 +---- flow/connectors/snowflake/qvalue_convert.go | 84 ++++++ flow/connectors/snowflake/snowflake.go | 86 ++---- flow/e2e/bigquery_helper.go | 46 +--- flow/e2e/qrep_flow_test.go | 10 +- flow/model/column.go | 23 -- flow/model/model.go | 42 ++- flow/model/qrecord_batch.go | 17 +- flow/model/qvalue/avro_converter.go | 43 ++- flow/model/qvalue/etime.go | 72 ----- flow/model/qvalue/kind.go | 37 +-- flow/model/qvalue/qvalue.go | 36 ++- flow/workflows/setup_flow.go | 5 +- 24 files changed, 652 insertions(+), 811 deletions(-) create mode 100644 flow/connectors/bigquery/qvalue_convert.go create mode 100644 flow/connectors/postgres/qvalue_convert.go create mode 100644 flow/connectors/snowflake/qvalue_convert.go delete mode 100644 flow/model/column.go delete mode 100644 flow/model/qvalue/etime.go diff --git a/flow/connectors/bigquery/bigquery.go b/flow/connectors/bigquery/bigquery.go index 798d4a6e3a..d4c9f9df1c 100644 --- a/flow/connectors/bigquery/bigquery.go +++ b/flow/connectors/bigquery/bigquery.go @@ -15,6 +15,7 @@ import ( "github.com/PeerDB-io/peer-flow/connectors/utils" "github.com/PeerDB-io/peer-flow/generated/protos" "github.com/PeerDB-io/peer-flow/model" + "github.com/PeerDB-io/peer-flow/model/qvalue" "github.com/google/uuid" log "github.com/sirupsen/logrus" "google.golang.org/api/iterator" @@ -442,8 +443,8 @@ func (c *BigQueryConnector) SyncRecords(req *model.SyncRecordsRequest) (*model.S // 1. _peerdb_uid - uuid // 2. _peerdb_timestamp - current timestamp // 2. _peerdb_timestamp_nanos - current timestamp in nano seconds - // 3. _peerdb_data - json of `r.Items` - json, err := json.Marshal(r.Items) + // 3. _peerdb_data - itemsJSON of `r.Items` + itemsJSON, err := r.Items.ToJSON() if err != nil { return nil, fmt.Errorf("failed to create items to json: %v", err) } @@ -454,7 +455,7 @@ func (c *BigQueryConnector) SyncRecords(req *model.SyncRecordsRequest) (*model.S timestamp: time.Now(), timestampNanos: time.Now().UnixNano(), destinationTableName: r.DestinationTableName, - data: string(json), + data: itemsJSON, recordType: 0, matchData: "", batchID: syncBatchID, @@ -469,12 +470,12 @@ func (c *BigQueryConnector) SyncRecords(req *model.SyncRecordsRequest) (*model.S // 4. _peerdb_record_type - 1 // 5. _peerdb_match_data - json of `r.OldItems` - newItemsJSON, err := json.Marshal(r.NewItems) + newItemsJSON, err := r.NewItems.ToJSON() if err != nil { return nil, fmt.Errorf("failed to create new items to json: %v", err) } - oldItemsJSON, err := json.Marshal(r.OldItems) + oldItemsJSON, err := r.OldItems.ToJSON() if err != nil { return nil, fmt.Errorf("failed to create old items to json: %v", err) } @@ -485,9 +486,9 @@ func (c *BigQueryConnector) SyncRecords(req *model.SyncRecordsRequest) (*model.S timestamp: time.Now(), timestampNanos: time.Now().UnixNano(), destinationTableName: r.DestinationTableName, - data: string(newItemsJSON), + data: newItemsJSON, recordType: 1, - matchData: string(oldItemsJSON), + matchData: oldItemsJSON, batchID: syncBatchID, stagingBatchID: stagingBatchID, unchangedToastColumns: utils.KeysToString(r.UnchangedToastColumns), @@ -500,7 +501,7 @@ func (c *BigQueryConnector) SyncRecords(req *model.SyncRecordsRequest) (*model.S // 4. _peerdb_match_data - json of `r.Items` // json.Marshal converts bytes in Hex automatically to BASE64 string. - itemsJSON, err := json.Marshal(r.Items) + itemsJSON, err := r.Items.ToJSON() if err != nil { return nil, fmt.Errorf("failed to create items to json: %v", err) } @@ -511,9 +512,9 @@ func (c *BigQueryConnector) SyncRecords(req *model.SyncRecordsRequest) (*model.S timestamp: time.Now(), timestampNanos: time.Now().UnixNano(), destinationTableName: r.DestinationTableName, - data: string(itemsJSON), + data: itemsJSON, recordType: 2, - matchData: string(itemsJSON), + matchData: itemsJSON, batchID: syncBatchID, stagingBatchID: stagingBatchID, unchangedToastColumns: utils.KeysToString(r.UnchangedToastColumns), @@ -823,7 +824,7 @@ func (c *BigQueryConnector) SetupNormalizedTable( for colName, genericColType := range sourceSchema.Columns { columns[idx] = &bigquery.FieldSchema{ Name: colName, - Type: getBigQueryColumnTypeForGenericColType(genericColType), + Type: qValueKindToBigQueryType(genericColType), } idx++ } @@ -923,45 +924,6 @@ func (c *BigQueryConnector) truncateTable(tableIdentifier string) error { return nil } -func getBigQueryColumnTypeForGenericColType(colType string) bigquery.FieldType { - switch colType { - // boolean - case model.ColumnTypeBoolean: - return bigquery.BooleanFieldType - // integer types - case model.ColumnTypeInt16, model.ColumnTypeInt32, model.ColumnTypeInt64: - return bigquery.IntegerFieldType - // decimal types - case model.ColumnTypeFloat16, model.ColumnTypeFloat32, model.ColumnTypeFloat64: - return bigquery.FloatFieldType - case model.ColumnTypeNumeric: - return bigquery.NumericFieldType - // string related - case model.ColumnTypeString: - return bigquery.StringFieldType - // json also is stored as string for now - case model.ColumnTypeJSON: - return bigquery.StringFieldType - // time related - case model.ColumnTypeTimestamp, model.ColumnTypeTimeStampWithTimeZone: - return bigquery.TimestampFieldType - case model.ColumnTypeTime: - return bigquery.TimeFieldType - case model.ColumnTypeTimeWithTimeZone: - return bigquery.StringFieldType - case model.ColumnTypeDate: - return bigquery.TimestampFieldType - case model.ColumnTypeInterval: - return bigquery.IntervalFieldType - // bytes - case model.ColumnHexBytes, model.ColumnHexBit: - return bigquery.BytesFieldType - // rest will be strings - default: - return bigquery.StringFieldType - } -} - type MergeStmtGenerator struct { // dataset of all the tables Dataset string @@ -1003,40 +965,42 @@ func (m *MergeStmtGenerator) generateFlattenedCTE() string { // statement. flattenedProjs := make([]string, 0) for colName, colType := range m.NormalizedTableSchema.Columns { - bqType := getBigQueryColumnTypeForGenericColType(colType) + bqType := qValueKindToBigQueryType(colType) // CAST doesn't work for FLOAT, so rewrite it to FLOAT64. if bqType == bigquery.FloatFieldType { bqType = "FLOAT64" } var castStmt string - switch colType { - case model.ColumnTypeJSON: + switch qvalue.QValueKind(colType) { + case qvalue.QValueKindJSON: //if the type is JSON, then just extract JSON castStmt = fmt.Sprintf("CAST(JSON_EXTRACT(_peerdb_data, '$.%s') AS %s) AS %s", colName, bqType, colName) // expecting data in BASE64 format - case model.ColumnHexBytes: + case qvalue.QValueKindBytes: castStmt = fmt.Sprintf("FROM_BASE64(JSON_EXTRACT_SCALAR(_peerdb_data, '$.%s')) AS %s", colName, colName) // MAKE_INTERVAL(years INT64, months INT64, days INT64, hours INT64, minutes INT64, seconds INT64) // Expecting interval to be in the format of {"Microseconds":2000000,"Days":0,"Months":0,"Valid":true} // json.Marshal in SyncRecords for Postgres already does this - once new data-stores are added, // this needs to be handled again - case model.ColumnTypeInterval: - castStmt = fmt.Sprintf("MAKE_INTERVAL(0,CAST(JSON_EXTRACT_SCALAR(_peerdb_data, '$.%s.Months') AS INT64),"+ - "CAST(JSON_EXTRACT_SCALAR(_peerdb_data, '$.%s.Days') AS INT64),0,0,"+ - "CAST(CAST(JSON_EXTRACT_SCALAR(_peerdb_data, '$.%s.Microseconds') AS INT64)/1000000 AS INT64)) AS %s", - colName, colName, colName, colName) - case model.ColumnHexBit: + // TODO add interval types again + // case model.ColumnTypeInterval: + // castStmt = fmt.Sprintf("MAKE_INTERVAL(0,CAST(JSON_EXTRACT_SCALAR(_peerdb_data, '$.%s.Months') AS INT64),"+ + // "CAST(JSON_EXTRACT_SCALAR(_peerdb_data, '$.%s.Days') AS INT64),0,0,"+ + // "CAST(CAST(JSON_EXTRACT_SCALAR(_peerdb_data, '$.%s.Microseconds') AS INT64)/1000000 AS INT64)) AS %s", + // colName, colName, colName, colName) + case qvalue.QValueKindBit: // sample raw data for BIT {"a":{"Bytes":"oA==","Len":3,"Valid":true},"id":1} // need to check correctness TODO castStmt = fmt.Sprintf("FROM_BASE64(JSON_EXTRACT_SCALAR(_peerdb_data, '$.%s.Bytes')) AS %s", colName, colName) - case model.ColumnTypeTime: - castStmt = fmt.Sprintf("time(timestamp_micros(CAST(JSON_EXTRACT(_peerdb_data, '$.%s.Microseconds')"+ - " AS int64))) AS %s", - colName, colName) + // TODO add proper granularity for time types, then restore this + // case model.ColumnTypeTime: + // castStmt = fmt.Sprintf("time(timestamp_micros(CAST(JSON_EXTRACT(_peerdb_data, '$.%s.Microseconds')"+ + // " AS int64))) AS %s", + // colName, colName) default: castStmt = fmt.Sprintf("CAST(JSON_EXTRACT_SCALAR(_peerdb_data, '$.%s') AS %s) AS %s", colName, bqType, colName) diff --git a/flow/connectors/bigquery/qrecord_value_saver.go b/flow/connectors/bigquery/qrecord_value_saver.go index 7bb82d8f94..672f7ee0b3 100644 --- a/flow/connectors/bigquery/qrecord_value_saver.go +++ b/flow/connectors/bigquery/qrecord_value_saver.go @@ -63,10 +63,17 @@ func (q QRecordValueSaver) Save() (map[string]bigquery.Value, string, error) { } bqValues[k] = val + case qvalue.QValueKindInt16: + val, ok := v.Value.(int16) + if !ok { + return nil, "", fmt.Errorf("failed to convert %v to int16", v.Value) + } + bqValues[k] = val + case qvalue.QValueKindInt32: val, ok := v.Value.(int32) if !ok { - return nil, "", fmt.Errorf("failed to convert %v to int64", v.Value) + return nil, "", fmt.Errorf("failed to convert %v to int32", v.Value) } bqValues[k] = val @@ -91,12 +98,12 @@ func (q QRecordValueSaver) Save() (map[string]bigquery.Value, string, error) { } bqValues[k] = val - case qvalue.QValueKindETime: - val, ok := v.Value.(*qvalue.ExtendedTime) - if !ok { - return nil, "", fmt.Errorf("failed to convert %v to ExtendedTime", v.Value) + case qvalue.QValueKindTimestamp, qvalue.QValueKindDate, qvalue.QValueKindTime: + var err error + bqValues[k], err = v.GoTimeConvert() + if err != nil { + return nil, "", fmt.Errorf("failed to convert parse %v into time.Time", v) } - bqValues[k] = val.Time case qvalue.QValueKindNumeric: val, ok := v.Value.(*big.Rat) diff --git a/flow/connectors/bigquery/qrep_avro_sync.go b/flow/connectors/bigquery/qrep_avro_sync.go index 09703625fc..58d19fbc8b 100644 --- a/flow/connectors/bigquery/qrep_avro_sync.go +++ b/flow/connectors/bigquery/qrep_avro_sync.go @@ -216,13 +216,13 @@ func GetAvroType(bqField *bigquery.FieldSchema) (interface{}, error) { }, nil case bigquery.DateFieldType: return map[string]string{ - "type": "int", - "logicalType": "date", + "type": "long", + "logicalType": "timestamp-micros", }, nil case bigquery.TimeFieldType: return map[string]string{ "type": "long", - "logicalType": "time-micros", + "logicalType": "timestamp-micros", }, nil case bigquery.DateTimeFieldType: return map[string]interface{}{ diff --git a/flow/connectors/bigquery/qvalue_convert.go b/flow/connectors/bigquery/qvalue_convert.go new file mode 100644 index 0000000000..fabdfb97f2 --- /dev/null +++ b/flow/connectors/bigquery/qvalue_convert.go @@ -0,0 +1,76 @@ +package connbigquery + +import ( + "fmt" + + "cloud.google.com/go/bigquery" + "github.com/PeerDB-io/peer-flow/model/qvalue" +) + +func qValueKindToBigQueryType(colType string) bigquery.FieldType { + switch qvalue.QValueKind(colType) { + // boolean + case qvalue.QValueKindBoolean: + return bigquery.BooleanFieldType + // integer types + case qvalue.QValueKindInt16, qvalue.QValueKindInt32, qvalue.QValueKindInt64: + return bigquery.IntegerFieldType + // decimal types + case qvalue.QValueKindFloat32, qvalue.QValueKindFloat64: + return bigquery.FloatFieldType + case qvalue.QValueKindNumeric: + return bigquery.NumericFieldType + // string related + case qvalue.QValueKindString: + return bigquery.StringFieldType + // json also is stored as string for now + case qvalue.QValueKindJSON: + return bigquery.StringFieldType + // time related + case qvalue.QValueKindTimestamp, qvalue.QValueKindTimestampTZ: + return bigquery.TimestampFieldType + // TODO: https://github.com/PeerDB-io/peerdb/issues/189 - DATE support is incomplete + case qvalue.QValueKindDate: + return bigquery.DateFieldType + // TODO: https://github.com/PeerDB-io/peerdb/issues/189 - TIME/TIMETZ support is incomplete + case qvalue.QValueKindTime, qvalue.QValueKindTimeTZ: + return bigquery.TimeFieldType + // TODO: https://github.com/PeerDB-io/peerdb/issues/189 - handle INTERVAL types again, + // bytes + case qvalue.QValueKindBit, qvalue.QValueKindBytes: + return bigquery.BytesFieldType + // rest will be strings + default: + return bigquery.StringFieldType + } +} + +// bigqueryTypeToQValueKind converts a bigquery FieldType to a QValueKind. +func BigQueryTypeToQValueKind(fieldType bigquery.FieldType) (qvalue.QValueKind, error) { + switch fieldType { + case bigquery.StringFieldType: + return qvalue.QValueKindString, nil + case bigquery.BytesFieldType: + return qvalue.QValueKindBytes, nil + case bigquery.IntegerFieldType: + return qvalue.QValueKindInt64, nil + case bigquery.FloatFieldType: + return qvalue.QValueKindFloat64, nil + case bigquery.BooleanFieldType: + return qvalue.QValueKindBoolean, nil + case bigquery.TimestampFieldType: + return qvalue.QValueKindTimestamp, nil + case bigquery.DateFieldType: + return qvalue.QValueKindDate, nil + case bigquery.TimeFieldType: + return qvalue.QValueKindTime, nil + case bigquery.RecordFieldType: + return qvalue.QValueKindStruct, nil + case bigquery.NumericFieldType: + return qvalue.QValueKindNumeric, nil + case bigquery.GeographyFieldType: + return qvalue.QValueKindString, nil + default: + return "", fmt.Errorf("unsupported bigquery field type: %v", fieldType) + } +} diff --git a/flow/connectors/postgres/cdc.go b/flow/connectors/postgres/cdc.go index 411e9ee41b..a71e229887 100644 --- a/flow/connectors/postgres/cdc.go +++ b/flow/connectors/postgres/cdc.go @@ -7,6 +7,7 @@ import ( "time" "github.com/PeerDB-io/peer-flow/model" + "github.com/PeerDB-io/peer-flow/model/qvalue" "github.com/jackc/pglogrepl" "github.com/jackc/pgx/v5/pgconn" "github.com/jackc/pgx/v5/pgproto3" @@ -386,34 +387,34 @@ It takes a tuple and a relation message as input and returns func (p *PostgresCDCSource) convertTupleToMap( tuple *pglogrepl.TupleData, rel *pglogrepl.RelationMessage, -) (map[string]interface{}, map[string]bool, error) { +) (model.RecordItems, map[string]bool, error) { // if the tuple is nil, return an empty map if tuple == nil { - return make(map[string]interface{}), make(map[string]bool), nil + return make(model.RecordItems), make(map[string]bool), nil } // create empty map of string to interface{} - items := make(map[string]interface{}) + items := make(model.RecordItems) unchangedToastColumns := make(map[string]bool) for idx, col := range tuple.Columns { colName := rel.Columns[idx].Name switch col.DataType { case 'n': // null - items[colName] = nil + items[colName] = qvalue.QValue{Kind: qvalue.QValueKindInvalid, Value: nil} case 't': // text /* bytea also appears here as a hex */ - data, err := p.decodeTextColumnData(col.Data, rel.Columns[idx].DataType) + data, err := p.decodeColumnData(col.Data, rel.Columns[idx].DataType, pgtype.TextFormatCode) if err != nil { return nil, nil, fmt.Errorf("error decoding text column data: %w", err) } - items[colName] = data + items[colName] = *data case 'b': // binary - data, err := p.decodeBinaryColumnData(col.Data, rel.Columns[idx].DataType) + data, err := p.decodeColumnData(col.Data, rel.Columns[idx].DataType, pgtype.BinaryFormatCode) if err != nil { return nil, nil, fmt.Errorf("error decoding binary column data: %w", err) } - items[colName] = data + items[colName] = *data case 'u': // unchanged toast unchangedToastColumns[colName] = true default: @@ -423,21 +424,25 @@ func (p *PostgresCDCSource) convertTupleToMap( return items, unchangedToastColumns, nil } -func (p *PostgresCDCSource) decodeTextColumnData(data []byte, dataType uint32) (interface{}, error) { +func (p *PostgresCDCSource) decodeColumnData(data []byte, dataType uint32, formatCode int16) (*qvalue.QValue, error) { + var parsedData any + var err error if dt, ok := p.typeMap.TypeForOID(dataType); ok { if dt.Name == "uuid" { // below is required to decode uuid to string - return dt.Codec.DecodeDatabaseSQLValue(p.typeMap, dataType, pgtype.TextFormatCode, data) - } - return dt.Codec.DecodeValue(p.typeMap, dataType, pgtype.TextFormatCode, data) - } - return string(data), nil -} + parsedData, err = dt.Codec.DecodeDatabaseSQLValue(p.typeMap, dataType, pgtype.TextFormatCode, data) -// decodeBinaryColumnData decodes the binary data for a column -func (p *PostgresCDCSource) decodeBinaryColumnData(data []byte, dataType uint32) (interface{}, error) { - if dt, ok := p.typeMap.TypeForOID(dataType); ok { - return dt.Codec.DecodeValue(p.typeMap, dataType, pgtype.BinaryFormatCode, data) + } else { + parsedData, err = dt.Codec.DecodeValue(p.typeMap, dataType, formatCode, data) + } + if err != nil { + return nil, err + } + retVal, err := parseFieldFromPostgresOID(dataType, parsedData) + if err != nil { + return nil, err + } + return retVal, nil } - return string(data), nil + return &qvalue.QValue{Kind: qvalue.QValueKindString, Value: string(data)}, nil } diff --git a/flow/connectors/postgres/postgres.go b/flow/connectors/postgres/postgres.go index d9d228f8b8..909423a3ae 100644 --- a/flow/connectors/postgres/postgres.go +++ b/flow/connectors/postgres/postgres.go @@ -7,6 +7,7 @@ import ( "github.com/PeerDB-io/peer-flow/generated/protos" "github.com/PeerDB-io/peer-flow/model" + "github.com/PeerDB-io/peer-flow/model/qvalue" "github.com/jackc/pgx/v5" "github.com/jackc/pgx/v5/pgxpool" log "github.com/sirupsen/logrus" @@ -271,17 +272,9 @@ func (c *PostgresConnector) GetTableSchema(req *protos.GetTableSchemaInput) (*pr return nil, err } - relID, err := c.getRelIDForTable(schemaTable) - if err != nil { - return nil, err - } - // Get the column names and types rows, err := c.pool.Query(c.ctx, - `SELECT a.attname, t.typname FROM pg_attribute a - JOIN pg_type t ON t.oid = a.atttypid - WHERE a.attnum > 0 AND NOT a.attisdropped AND a.attrelid = $1`, - relID) + fmt.Sprintf(`SELECT * FROM %s LIMIT 0`, req.TableIdentifier)) if err != nil { return nil, fmt.Errorf("error getting table schema for table %s: %w", schemaTable, err) } @@ -298,20 +291,13 @@ func (c *PostgresConnector) GetTableSchema(req *protos.GetTableSchemaInput) (*pr PrimaryKeyColumn: pkey, } - for rows.Next() { - var colName string - var colType string - err = rows.Scan(&colName, &colType) - if err != nil { - return nil, fmt.Errorf("error scanning table schema: %w", err) - } - - colType, err = convertPostgresColumnTypeToGeneric(colType) - if err != nil { - return nil, fmt.Errorf("error converting postgres column type: %w", err) + for _, fieldDescription := range rows.FieldDescriptions() { + genericColType := getQValueKindForPostgresOID(fieldDescription.DataTypeOID) + if genericColType == qvalue.QValueKindInvalid { + return nil, fmt.Errorf("error converting Postgres OID to QValueKind") } - res.Columns[colName] = colType + res.Columns[fieldDescription.Name] = string(genericColType) } if err = rows.Err(); err != nil { @@ -450,77 +436,6 @@ func (c *PostgresConnector) getPrimaryKeyColumn(schemaTable *SchemaTable) (strin return pkCol, nil } -func convertPostgresColumnTypeToGeneric(colType string) (string, error) { - switch colType { - case "int2": - return model.ColumnTypeInt16, nil - case "int4": - return model.ColumnTypeInt32, nil - case "int8": - return model.ColumnTypeInt64, nil - case "float4": - return model.ColumnTypeFloat32, nil - case "float8": - return model.ColumnTypeFloat64, nil - case "bool": - return model.ColumnTypeBoolean, nil - case "text": - return model.ColumnTypeString, nil - case "date": - return model.ColumnTypeDate, nil - case "timestamp": - return model.ColumnTypeTimestamp, nil - case "timestamptz": - return model.ColumnTypeTimeStampWithTimeZone, nil - case "varchar": - return model.ColumnTypeString, nil - case "char": - return model.ColumnTypeString, nil - case "bpchar": - return model.ColumnTypeString, nil - case "numeric": - return model.ColumnTypeNumeric, nil - case "uuid": - return model.ColumnTypeString, nil - case "json": - return model.ColumnTypeJSON, nil - case "jsonb": - return model.ColumnTypeJSON, nil - case "xml": - return model.ColumnTypeString, nil - case "tsvector": - return model.ColumnTypeString, nil - case "tsquery": - return model.ColumnTypeString, nil - case "bytea": - return model.ColumnHexBytes, nil - case "bit": - return model.ColumnHexBit, nil - case "varbit": - return model.ColumnHexBit, nil - case "cidr": - return model.ColumnTypeString, nil - case "inet": - return model.ColumnTypeString, nil - case "interval": - return model.ColumnTypeInterval, nil - case "macaddr": - return model.ColumnTypeString, nil - case "money": - return model.ColumnTypeString, nil - case "oid": - return model.ColumnTypeInt64, nil - case "time": - return model.ColumnTypeTime, nil - case "timetz": - return model.ColumnTypeTimeWithTimeZone, nil - case "txid_snapshot": - return model.ColumnTypeString, nil - default: - return "", fmt.Errorf("unsupported column type: %s", colType) - } -} - func (c *PostgresConnector) tableExists(schemaTable *SchemaTable) (bool, error) { var exists bool err := c.pool.QueryRow(c.ctx, diff --git a/flow/connectors/postgres/postgres_cdc_test.go b/flow/connectors/postgres/postgres_cdc_test.go index e45d0f117f..af32f112fc 100644 --- a/flow/connectors/postgres/postgres_cdc_test.go +++ b/flow/connectors/postgres/postgres_cdc_test.go @@ -9,6 +9,7 @@ import ( "github.com/PeerDB-io/peer-flow/generated/protos" "github.com/PeerDB-io/peer-flow/model" + "github.com/PeerDB-io/peer-flow/model/qvalue" "github.com/stretchr/testify/suite" ) @@ -133,11 +134,11 @@ func (suite *PostgresCDCTestSuite) validateInsertedToastRecords(records []model. suite.Equal(dstTableName, insertRecord.DestinationTableName) suite.Equal(5, len(insertRecord.Items)) - suite.Equal(int32(idx+1), insertRecord.Items["id"].(int32)) - suite.Equal(32768, len(insertRecord.Items["n_t"].(string))) - suite.Equal(32768, len(insertRecord.Items["lz4_t"].(string))) - suite.Equal(32768, len(insertRecord.Items["n_b"].([]byte))) - suite.Equal(32768, len(insertRecord.Items["lz4_b"].([]byte))) + suite.Equal(int32(idx+1), insertRecord.Items["id"].Value.(int32)) + suite.Equal(32768, len(insertRecord.Items["n_t"].Value.(string))) + suite.Equal(32768, len(insertRecord.Items["lz4_t"].Value.(string))) + suite.Equal(32768, len(insertRecord.Items["n_b"].Value.([]byte))) + suite.Equal(32768, len(insertRecord.Items["lz4_b"].Value.([]byte))) } } @@ -175,8 +176,8 @@ func (suite *PostgresCDCTestSuite) validateMutatedToastRecords(records []model.R suite.Equal(srcTableName, updateRecord.SourceTableName) suite.Equal(dstTableName, updateRecord.DestinationTableName) suite.Equal(2, len(updateRecord.NewItems)) - suite.Equal(int32(1), updateRecord.NewItems["id"].(int32)) - suite.Equal(65536, len(updateRecord.NewItems["n_t"].(string))) + suite.Equal(int32(1), updateRecord.NewItems["id"].Value.(int32)) + suite.Equal(65536, len(updateRecord.NewItems["n_t"].Value.(string))) suite.Equal(3, len(updateRecord.UnchangedToastColumns)) suite.True(updateRecord.UnchangedToastColumns["lz4_t"]) suite.True(updateRecord.UnchangedToastColumns["n_b"]) @@ -187,8 +188,8 @@ func (suite *PostgresCDCTestSuite) validateMutatedToastRecords(records []model.R suite.Equal(srcTableName, updateRecord.SourceTableName) suite.Equal(dstTableName, updateRecord.DestinationTableName) suite.Equal(2, len(updateRecord.NewItems)) - suite.Equal(int32(2), updateRecord.NewItems["id"].(int32)) - suite.Equal(65536, len(updateRecord.NewItems["lz4_b"].([]byte))) + suite.Equal(int32(2), updateRecord.NewItems["id"].Value.(int32)) + suite.Equal(65536, len(updateRecord.NewItems["lz4_b"].Value.([]byte))) suite.Equal(3, len(updateRecord.UnchangedToastColumns)) suite.True(updateRecord.UnchangedToastColumns["lz4_t"]) suite.True(updateRecord.UnchangedToastColumns["n_b"]) @@ -199,8 +200,8 @@ func (suite *PostgresCDCTestSuite) validateMutatedToastRecords(records []model.R suite.Equal(srcTableName, updateRecord.SourceTableName) suite.Equal(dstTableName, updateRecord.DestinationTableName) suite.Equal(2, len(updateRecord.NewItems)) - suite.Equal(int32(3), updateRecord.NewItems["id"].(int32)) - suite.Equal(65536, len(updateRecord.NewItems["n_b"].([]byte))) + suite.Equal(int32(3), updateRecord.NewItems["id"].Value.(int32)) + suite.Equal(65536, len(updateRecord.NewItems["n_b"].Value.([]byte))) suite.Equal(3, len(updateRecord.UnchangedToastColumns)) suite.True(updateRecord.UnchangedToastColumns["lz4_t"]) suite.True(updateRecord.UnchangedToastColumns["n_t"]) @@ -211,7 +212,7 @@ func (suite *PostgresCDCTestSuite) validateMutatedToastRecords(records []model.R suite.Equal(srcTableName, deleteRecord.SourceTableName) suite.Equal(dstTableName, deleteRecord.DestinationTableName) suite.Equal(5, len(deleteRecord.Items)) - suite.Equal(int32(3), deleteRecord.Items["id"].(int32)) + suite.Equal(int32(3), deleteRecord.Items["id"].Value.(int32)) suite.Nil(deleteRecord.Items["n_t"]) suite.Nil(deleteRecord.Items["lz4_t"]) suite.Nil(deleteRecord.Items["n_b"]) @@ -397,8 +398,8 @@ func (suite *PostgresCDCTestSuite) TestErrorForTableNotExist() { tableNameSchemaMapping[nonExistentFlowDstTableName] = &protos.TableSchema{ TableIdentifier: nonExistentFlowSrcTableName, Columns: map[string]string{ - "id": model.ColumnTypeInt32, - "name": model.ColumnTypeString, + "id": string(qvalue.QValueKindInt32), + "name": string(qvalue.QValueKindString), }, PrimaryKeyColumn: "id", } @@ -494,8 +495,8 @@ func (suite *PostgresCDCTestSuite) TestSimpleHappyFlow() { suite.Equal(&protos.TableSchema{ TableIdentifier: simpleHappyFlowSrcTableName, Columns: map[string]string{ - "id": model.ColumnTypeInt32, - "name": model.ColumnTypeString, + "id": string(qvalue.QValueKindInt32), + "name": string(qvalue.QValueKindString), }, PrimaryKeyColumn: "id", }, tableNameSchema) @@ -603,36 +604,36 @@ func (suite *PostgresCDCTestSuite) TestAllTypesHappyFlow() { suite.Equal(&protos.TableSchema{ TableIdentifier: allTypesHappyFlowSrcTableName, Columns: map[string]string{ - "id": model.ColumnTypeInt64, - "c1": model.ColumnTypeInt64, + "id": string(qvalue.QValueKindInt64), + "c1": string(qvalue.QValueKindInt64), "c2": model.ColumnHexBit, "c3": model.ColumnHexBit, - "c4": model.ColumnTypeBoolean, - "c6": model.ColumnHexBytes, - "c7": model.ColumnTypeString, - "c8": model.ColumnTypeString, - "c9": model.ColumnTypeString, - "c11": model.ColumnTypeDate, - "c12": model.ColumnTypeFloat64, - "c13": model.ColumnTypeFloat64, + "c4": string(qvalue.QValueKindBoolean), + "c6": string(qvalue.QValueKindBytes), + "c7": string(qvalue.QValueKindString), + "c8": string(qvalue.QValueKindString), + "c9": string(qvalue.QValueKindString), + "c11": string(qvalue.QValueKindDate), + "c12": string(qvalue.QValueKindFloat64), + "c13": string(qvalue.QValueKindFloat64), "c14": model.ColumnTypeString, - "c15": model.ColumnTypeInt32, + "c15": string(qvalue.QValueKindInt32), "c16": model.ColumnTypeInterval, - "c17": model.ColumnTypeJSON, - "c18": model.ColumnTypeJSON, + "c17": string(qvalue.QValueKindJSON), + "c18": string(qvalue.QValueKindJSON), "c21": model.ColumnTypeString, "c22": model.ColumnTypeString, - "c23": model.ColumnTypeNumeric, - "c24": model.ColumnTypeInt64, - "c28": model.ColumnTypeFloat32, - "c29": model.ColumnTypeInt16, - "c30": model.ColumnTypeInt16, - "c31": model.ColumnTypeInt32, + "c23": string(qvalue.QValueKindNumeric), + "c24": string(qvalue.QValueKindInt64), + "c28": string(qvalue.QValueKindFloat32), + "c29": string(qvalue.QValueKindInt16), + "c30": string(qvalue.QValueKindInt16), + "c31": string(qvalue.QValueKindInt32), "c32": model.ColumnTypeString, - "c33": model.ColumnTypeTimestamp, - "c34": model.ColumnTypeTimeStampWithTimeZone, - "c35": model.ColumnTypeTime, - "c36": model.ColumnTypeTimeWithTimeZone, + "c33": string(qvalue.QValueKindTimestamp), + "c34": string(qvalue.QValueKindTimestampTZ), + "c35": string(qvalue.QValueKindTime), + "c36": string(qvalue.QValueKindTimeTZ), "c37": model.ColumnTypeString, "c38": model.ColumnTypeString, "c39": model.ColumnTypeString, @@ -714,11 +715,11 @@ func (suite *PostgresCDCTestSuite) TestToastHappyFlow() { suite.Equal(&protos.TableSchema{ TableIdentifier: toastHappyFlowSrcTableName, Columns: map[string]string{ - "id": model.ColumnTypeInt32, - "n_t": model.ColumnTypeString, - "lz4_t": model.ColumnTypeString, - "n_b": model.ColumnHexBytes, - "lz4_b": model.ColumnHexBytes, + "id": string(qvalue.QValueKindInt32), + "n_t": string(qvalue.QValueKindString), + "lz4_t": string(qvalue.QValueKindString), + "n_b": string(qvalue.QValueKindBytes), + "lz4_b": string(qvalue.QValueKindBytes), }, PrimaryKeyColumn: "id", }, tableNameSchema) diff --git a/flow/connectors/postgres/qrep.go b/flow/connectors/postgres/qrep.go index aa74bd9e21..10f1d4249b 100644 --- a/flow/connectors/postgres/qrep.go +++ b/flow/connectors/postgres/qrep.go @@ -28,6 +28,8 @@ func (c *PostgresConnector) GetQRepPartitions( switch v := minValue.(type) { case int32, int64: maxValue := maxValue.(int64) + 1 + fmt.Println("minValue", minValue) + fmt.Println("maxValue", maxValue) partitions, err = c.getIntPartitions(v.(int64), maxValue, config.BatchSizeInt) case time.Time: maxValue := maxValue.(time.Time).Add(time.Microsecond) @@ -286,7 +288,8 @@ func (c *PostgresConnector) getIntPartitions( for start <= end { partitionEnd := start + batchSize - if partitionEnd > end { + // safeguard against integer overflow + if partitionEnd > end || partitionEnd < start { partitionEnd = end } diff --git a/flow/connectors/postgres/qrep_query_executor.go b/flow/connectors/postgres/qrep_query_executor.go index 5dd4cc1ab4..8bf3c2373f 100644 --- a/flow/connectors/postgres/qrep_query_executor.go +++ b/flow/connectors/postgres/qrep_query_executor.go @@ -2,18 +2,11 @@ package connpostgres import ( "context" - "database/sql" - "errors" "fmt" - "math" - "math/big" - "time" "github.com/PeerDB-io/peer-flow/model" - "github.com/PeerDB-io/peer-flow/model/qvalue" "github.com/jackc/pgx/v5" "github.com/jackc/pgx/v5/pgconn" - "github.com/jackc/pgx/v5/pgtype" "github.com/jackc/pgx/v5/pgxpool" log "github.com/sirupsen/logrus" ) @@ -38,50 +31,12 @@ func (qe *QRepQueryExecutor) ExecuteQuery(query string, args ...interface{}) (pg return rows, nil } -func fieldDescriptionToQValueKind(fd pgconn.FieldDescription) qvalue.QValueKind { - switch fd.DataTypeOID { - case pgtype.BoolOID: - return qvalue.QValueKindBoolean - case pgtype.Int2OID: - return qvalue.QValueKindInt16 - case pgtype.Int4OID: - return qvalue.QValueKindInt32 - case pgtype.Int8OID: - return qvalue.QValueKindInt64 - case pgtype.Float4OID: - return qvalue.QValueKindFloat32 - case pgtype.Float8OID: - return qvalue.QValueKindFloat64 - case pgtype.TextOID, pgtype.VarcharOID: - return qvalue.QValueKindString - case pgtype.ByteaOID: - return qvalue.QValueKindBytes - case pgtype.JSONOID, pgtype.JSONBOID: - return qvalue.QValueKindJSON - case pgtype.UUIDOID: - return qvalue.QValueKindUUID - case pgtype.TimestampOID, pgtype.TimestamptzOID, pgtype.DateOID, pgtype.TimeOID: - return qvalue.QValueKindETime - case pgtype.NumericOID: - return qvalue.QValueKindNumeric - default: - typeName, ok := pgtype.NewMap().TypeForOID(fd.DataTypeOID) - if !ok { - log.Warnf("failed to get type name for oid: %v", fd.DataTypeOID) - return qvalue.QValueKindInvalid - } else { - log.Warnf("unsupported field type: %v - type name - %s", fd.DataTypeOID, typeName.Name) - return qvalue.QValueKindInvalid - } - } -} - // FieldDescriptionsToSchema converts a slice of pgconn.FieldDescription to a QRecordSchema. func fieldDescriptionsToSchema(fds []pgconn.FieldDescription) *model.QRecordSchema { qfields := make([]*model.QField, len(fds)) for i, fd := range fds { cname := fd.Name - ctype := fieldDescriptionToQValueKind(fd) + ctype := getQValueKindForPostgresOID(fd.DataTypeOID) // there isn't a way to know if a column is nullable or not // TODO fix this. cnullable := true @@ -146,216 +101,22 @@ func (qe *QRepQueryExecutor) ExecuteAndProcessQuery( return batch, nil } -func mapRowToQRecord(row pgx.Row, fds []pgconn.FieldDescription) (*model.QRecord, error) { +func mapRowToQRecord(row pgx.Rows, fds []pgconn.FieldDescription) (*model.QRecord, error) { // make vals an empty array of QValue of size len(fds) record := model.NewQRecord(len(fds)) - scanArgs := make([]interface{}, len(fds)) - for i := range scanArgs { - switch fds[i].DataTypeOID { - case pgtype.BoolOID: - scanArgs[i] = new(pgtype.Bool) - case pgtype.TimestampOID: - scanArgs[i] = new(pgtype.Timestamp) - case pgtype.TimestamptzOID: - scanArgs[i] = new(pgtype.Timestamptz) - case pgtype.Int4OID: - scanArgs[i] = new(pgtype.Int4) - case pgtype.Int8OID: - scanArgs[i] = new(pgtype.Int8) - case pgtype.Float4OID: - scanArgs[i] = new(pgtype.Float4) - case pgtype.Float8OID: - scanArgs[i] = new(pgtype.Float8) - case pgtype.TextOID: - scanArgs[i] = new(pgtype.Text) - case pgtype.VarcharOID: - scanArgs[i] = new(pgtype.Text) - case pgtype.NumericOID: - scanArgs[i] = new(pgtype.Numeric) - case pgtype.UUIDOID: - scanArgs[i] = new(pgtype.UUID) - case pgtype.ByteaOID: - scanArgs[i] = new(sql.RawBytes) - case pgtype.DateOID: - scanArgs[i] = new(pgtype.Date) - default: - scanArgs[i] = new(pgtype.Text) - } - } - - err := row.Scan(scanArgs...) + values, err := row.Values() if err != nil { return nil, fmt.Errorf("failed to scan row: %w", err) } for i, fd := range fds { - tmp, err := parseField(fd.DataTypeOID, scanArgs[i]) + tmp, err := parseFieldFromPostgresOID(fd.DataTypeOID, values[i]) if err != nil { return nil, fmt.Errorf("failed to parse field: %w", err) } - record.Set(i, tmp) + record.Set(i, *tmp) } return record, nil } - -func parseField(oid uint32, value interface{}) (qvalue.QValue, error) { - var val qvalue.QValue - - switch oid { - case pgtype.TimestampOID: - timestamp := value.(*pgtype.Timestamp) - var et *qvalue.ExtendedTime - if timestamp.Valid { - var err error - et, err = qvalue.NewExtendedTime(timestamp.Time, qvalue.DateTimeKindType, "") - if err != nil { - return qvalue.QValue{}, fmt.Errorf("failed to create ExtendedTime: %w", err) - } - } - val = qvalue.QValue{Kind: qvalue.QValueKindETime, Value: et} - case pgtype.TimestamptzOID: - timestamp := value.(*pgtype.Timestamptz) - var et *qvalue.ExtendedTime - if timestamp.Valid { - var err error - et, err = qvalue.NewExtendedTime(timestamp.Time, qvalue.DateTimeKindType, "") - if err != nil { - return qvalue.QValue{}, fmt.Errorf("failed to create ExtendedTime: %w", err) - } - } - val = qvalue.QValue{Kind: qvalue.QValueKindETime, Value: et} - case pgtype.DateOID: - date := value.(*pgtype.Date) - var et *qvalue.ExtendedTime - if date.Valid { - var err error - et, err = qvalue.NewExtendedTime(date.Time, qvalue.DateKindType, "") - if err != nil { - return qvalue.QValue{}, fmt.Errorf("failed to create ExtendedTime: %w", err) - } - } - val = qvalue.QValue{Kind: qvalue.QValueKindETime, Value: et} - case pgtype.TimeOID: - timeVal := value.(*pgtype.Text) - var et *qvalue.ExtendedTime - if timeVal.Valid { - t, err := time.Parse("15:04:05.999999", timeVal.String) - if err != nil { - return qvalue.QValue{}, fmt.Errorf("failed to parse time: %w", err) - } - et, err = qvalue.NewExtendedTime(t, qvalue.TimeKindType, "") - if err != nil { - return qvalue.QValue{}, fmt.Errorf("failed to create ExtendedTime: %w", err) - } - } - val = qvalue.QValue{Kind: qvalue.QValueKindETime, Value: et} - case pgtype.BoolOID: - boolVal := value.(*pgtype.Bool) - if boolVal.Valid { - val = qvalue.QValue{Kind: qvalue.QValueKindBoolean, Value: boolVal.Bool} - } else { - val = qvalue.QValue{Kind: qvalue.QValueKindBoolean, Value: nil} - } - case pgtype.JSONOID, pgtype.JSONBOID: - // TODO: improve JSON support - strVal := value.(*pgtype.Text) - if strVal != nil { - val = qvalue.QValue{Kind: qvalue.QValueKindJSON, Value: strVal.String} - } else { - val = qvalue.QValue{Kind: qvalue.QValueKindJSON, Value: nil} - } - case pgtype.Int2OID: - intVal := value.(*pgtype.Int2) - if intVal.Valid { - val = qvalue.QValue{Kind: qvalue.QValueKindInt16, Value: intVal.Int16} - } else { - val = qvalue.QValue{Kind: qvalue.QValueKindInt16, Value: nil} - } - case pgtype.Int4OID: - intVal := value.(*pgtype.Int4) - if intVal.Valid { - val = qvalue.QValue{Kind: qvalue.QValueKindInt32, Value: intVal.Int32} - } else { - val = qvalue.QValue{Kind: qvalue.QValueKindInt32, Value: nil} - } - case pgtype.Int8OID: - intVal := value.(*pgtype.Int8) - if intVal.Valid { - val = qvalue.QValue{Kind: qvalue.QValueKindInt64, Value: intVal.Int64} - } else { - val = qvalue.QValue{Kind: qvalue.QValueKindInt64, Value: nil} - } - case pgtype.Float4OID: - floatVal := value.(*pgtype.Float4) - if floatVal.Valid { - val = qvalue.QValue{Kind: qvalue.QValueKindFloat32, Value: floatVal.Float32} - } else { - val = qvalue.QValue{Kind: qvalue.QValueKindFloat32, Value: nil} - } - case pgtype.Float8OID: - floatVal := value.(*pgtype.Float8) - if floatVal.Valid { - val = qvalue.QValue{Kind: qvalue.QValueKindFloat64, Value: floatVal.Float64} - } else { - val = qvalue.QValue{Kind: qvalue.QValueKindFloat64, Value: nil} - } - case pgtype.TextOID, pgtype.VarcharOID: - textVal := value.(*pgtype.Text) - if textVal.Valid { - val = qvalue.QValue{Kind: qvalue.QValueKindString, Value: textVal.String} - } else { - val = qvalue.QValue{Kind: qvalue.QValueKindString, Value: nil} - } - case pgtype.UUIDOID: - uuidVal := value.(*pgtype.UUID) - if uuidVal.Valid { - val = qvalue.QValue{Kind: qvalue.QValueKindUUID, Value: uuidVal.Bytes} - } else { - val = qvalue.QValue{Kind: qvalue.QValueKindUUID, Value: nil} - } - case pgtype.ByteaOID: - rawBytes := value.(*sql.RawBytes) - val = qvalue.QValue{Kind: qvalue.QValueKindBytes, Value: []byte(*rawBytes)} - case pgtype.NumericOID: - numVal := value.(*pgtype.Numeric) - rat, err := numericToRat(numVal) - if err != nil { - log.Warnf("failed to convert numeric [%v] to rat: %v", value, err) - val = qvalue.QValue{Kind: qvalue.QValueKindNumeric, Value: nil} - } else { - val = qvalue.QValue{Kind: qvalue.QValueKindNumeric, Value: rat} - } - default: - typ, _ := pgtype.NewMap().TypeForOID(oid) - fmt.Printf("QValueKindInvalid => oid: %v, typename: %v\n", oid, typ) - val = qvalue.QValue{Kind: qvalue.QValueKindInvalid, Value: nil} - } - - return val, nil -} - -func numericToRat(numVal *pgtype.Numeric) (*big.Rat, error) { - if numVal.Valid { - if numVal.NaN { - return nil, errors.New("numeric value is NaN") - } - - switch numVal.InfinityModifier { - case pgtype.NegativeInfinity, pgtype.Infinity: - return nil, errors.New("numeric value is infinity") - } - - rat := new(big.Rat) - - rat.SetInt(numVal.Int) - divisor := new(big.Rat).SetFloat64(math.Pow10(int(-numVal.Exp))) - rat.Quo(rat, divisor) - - return rat, nil - } - - // handle invalid numeric - return nil, errors.New("invalid numeric") -} diff --git a/flow/connectors/postgres/qvalue_convert.go b/flow/connectors/postgres/qvalue_convert.go new file mode 100644 index 0000000000..d9b821f2a2 --- /dev/null +++ b/flow/connectors/postgres/qvalue_convert.go @@ -0,0 +1,195 @@ +package connpostgres + +import ( + "encoding/json" + "errors" + "fmt" + "math" + "math/big" + "strings" + "time" + + "github.com/PeerDB-io/peer-flow/model/qvalue" + "github.com/jackc/pgx/v5/pgtype" + log "github.com/sirupsen/logrus" +) + +func getQValueKindForPostgresOID(oid uint32) qvalue.QValueKind { + switch oid { + case pgtype.BoolOID: + return qvalue.QValueKindBoolean + case pgtype.Int2OID: + return qvalue.QValueKindInt16 + case pgtype.Int4OID: + return qvalue.QValueKindInt32 + case pgtype.Int8OID: + return qvalue.QValueKindInt64 + case pgtype.Float4OID: + return qvalue.QValueKindFloat32 + case pgtype.Float8OID: + return qvalue.QValueKindFloat64 + case pgtype.TextOID, pgtype.VarcharOID, pgtype.BPCharOID: + return qvalue.QValueKindString + case pgtype.ByteaOID: + return qvalue.QValueKindBytes + case pgtype.JSONOID, pgtype.JSONBOID: + return qvalue.QValueKindJSON + case pgtype.UUIDOID: + return qvalue.QValueKindUUID + case pgtype.TimeOID: + return qvalue.QValueKindTime + case pgtype.DateOID: + return qvalue.QValueKindDate + case pgtype.TimestampOID: + return qvalue.QValueKindTimestamp + case pgtype.TimestamptzOID: + return qvalue.QValueKindTimestampTZ + case pgtype.NumericOID: + return qvalue.QValueKindNumeric + default: + typeName, ok := pgtype.NewMap().TypeForOID(oid) + if !ok { + // workaround for TIMETZ not being defined by this pgtype + if oid == 1266 { + return qvalue.QValueKindTimeTZ + } + log.Warnf("failed to get type name for oid: %v", oid) + return qvalue.QValueKindInvalid + } else { + log.Warnf("unsupported field type: %v - type name - %s", oid, typeName.Name) + return qvalue.QValueKindInvalid + } + } +} + +func parseFieldFromQValueKind(qvalueKind qvalue.QValueKind, value interface{}) (*qvalue.QValue, error) { + var val *qvalue.QValue = nil + + switch qvalueKind { + case qvalue.QValueKindTimestamp: + timestamp := value.(time.Time) + val = &qvalue.QValue{Kind: qvalue.QValueKindTimestamp, Value: timestamp} + case qvalue.QValueKindTimestampTZ: + timestamp := value.(time.Time) + val = &qvalue.QValue{Kind: qvalue.QValueKindTimestampTZ, Value: timestamp} + case qvalue.QValueKindDate: + date := value.(time.Time) + val = &qvalue.QValue{Kind: qvalue.QValueKindDate, Value: date} + case qvalue.QValueKindTime: + timeVal := value.(pgtype.Time) + if timeVal.Valid { + var timeValStr any + timeValStr, err := timeVal.Value() + if err != nil { + return nil, fmt.Errorf("failed to parse time: %w", err) + } + // edge case, only Postgres supports this extreme value for time + timeValStr = strings.Replace(timeValStr.(string), "24:00:00.000000", "23:59:59.999999", 1) + t, err := time.Parse("15:04:05.999999", timeValStr.(string)) + t = t.AddDate(1970, 0, 0) + if err != nil { + return nil, fmt.Errorf("failed to parse time: %w", err) + } + val = &qvalue.QValue{Kind: qvalue.QValueKindTime, Value: t} + } + case qvalue.QValueKindTimeTZ: + timeVal := value.(string) + // edge case, Postgres supports this extreme value for time + timeVal = strings.Replace(timeVal, "24:00:00.000000", "23:59:59.999999", 1) + t, err := time.Parse("15:04:05.999999-0700", timeVal) + if err != nil { + return nil, fmt.Errorf("failed to parse time: %w", err) + } + t = t.AddDate(1970, 0, 0) + val = &qvalue.QValue{Kind: qvalue.QValueKindTime, Value: t} + + case qvalue.QValueKindBoolean: + boolVal := value.(bool) + val = &qvalue.QValue{Kind: qvalue.QValueKindBoolean, Value: boolVal} + case qvalue.QValueKindJSON: + // TODO: improve JSON support + jsonVal := value.(map[string]interface{}) + jsonValStr, err := json.Marshal(jsonVal) + if err != nil { + return nil, fmt.Errorf("failed to parse json: %w", err) + } + val = &qvalue.QValue{Kind: qvalue.QValueKindJSON, Value: string(jsonValStr)} + case qvalue.QValueKindInt16: + intVal := value.(int16) + val = &qvalue.QValue{Kind: qvalue.QValueKindInt16, Value: intVal} + case qvalue.QValueKindInt32: + intVal := value.(int32) + val = &qvalue.QValue{Kind: qvalue.QValueKindInt32, Value: intVal} + case qvalue.QValueKindInt64: + intVal := value.(int64) + val = &qvalue.QValue{Kind: qvalue.QValueKindInt64, Value: intVal} + case qvalue.QValueKindFloat32: + floatVal := value.(float32) + val = &qvalue.QValue{Kind: qvalue.QValueKindFloat32, Value: floatVal} + case qvalue.QValueKindFloat64: + floatVal := value.(float64) + val = &qvalue.QValue{Kind: qvalue.QValueKindFloat64, Value: floatVal} + case qvalue.QValueKindString: + textVal := value.(string) + val = &qvalue.QValue{Kind: qvalue.QValueKindString, Value: textVal} + case qvalue.QValueKindUUID: + switch value.(type) { + case string: + val = &qvalue.QValue{Kind: qvalue.QValueKindUUID, Value: value} + case [16]byte: + val = &qvalue.QValue{Kind: qvalue.QValueKindUUID, Value: value} + default: + return nil, fmt.Errorf("failed to parse UUID: %v", value) + } + case qvalue.QValueKindBytes: + rawBytes := value.([]byte) + val = &qvalue.QValue{Kind: qvalue.QValueKindBytes, Value: rawBytes} + // TODO: check for handling of QValueKindBit + case qvalue.QValueKindNumeric: + numVal := value.(pgtype.Numeric) + if numVal.Valid { + rat, err := numericToRat(&numVal) + if err != nil { + return nil, fmt.Errorf("failed to convert numeric [%v] to rat: %w", value, err) + } + val = &qvalue.QValue{Kind: qvalue.QValueKindNumeric, Value: rat} + } + default: + log.Errorf("unhandled QValueKind => %v\n", qvalueKind) + return nil, fmt.Errorf("unhandled QValueKind => %v", qvalueKind) + } + + // parsing into pgtype failed. + if val == nil { + return nil, fmt.Errorf("failed to parse value %v into QValueKind %v", value, qvalueKind) + } + return val, nil +} + +func parseFieldFromPostgresOID(oid uint32, value interface{}) (*qvalue.QValue, error) { + return parseFieldFromQValueKind(getQValueKindForPostgresOID(oid), value) +} + +func numericToRat(numVal *pgtype.Numeric) (*big.Rat, error) { + if numVal.Valid { + if numVal.NaN { + return nil, errors.New("numeric value is NaN") + } + + switch numVal.InfinityModifier { + case pgtype.NegativeInfinity, pgtype.Infinity: + return nil, errors.New("numeric value is infinity") + } + + rat := new(big.Rat) + + rat.SetInt(numVal.Int) + divisor := new(big.Rat).SetFloat64(math.Pow10(int(-numVal.Exp))) + rat.Quo(rat, divisor) + + return rat, nil + } + + // handle invalid numeric + return nil, errors.New("invalid numeric") +} diff --git a/flow/connectors/snowflake/avro_file_writer_test.go b/flow/connectors/snowflake/avro_file_writer_test.go index c5f187e2e7..41b76513c6 100644 --- a/flow/connectors/snowflake/avro_file_writer_test.go +++ b/flow/connectors/snowflake/avro_file_writer_test.go @@ -19,7 +19,7 @@ func createQValue(t *testing.T, kind qvalue.QValueKind, placeHolder int) qvalue. switch kind { case qvalue.QValueKindInt16, qvalue.QValueKindInt32, qvalue.QValueKindInt64: value = int64(placeHolder) - case qvalue.QValueKindFloat16, qvalue.QValueKindFloat32: + case qvalue.QValueKindFloat32: value = float32(placeHolder) case qvalue.QValueKindFloat64: value = float64(placeHolder) @@ -27,10 +27,9 @@ func createQValue(t *testing.T, kind qvalue.QValueKind, placeHolder int) qvalue. value = placeHolder%2 == 0 case qvalue.QValueKindString: value = fmt.Sprintf("string%d", placeHolder) - case qvalue.QValueKindETime: - et, err := qvalue.NewExtendedTime(time.Now(), qvalue.TimeKindType, "") - require.NoError(t, err) - value = et + case qvalue.QValueKindTimestamp, qvalue.QValueKindTimestampTZ, qvalue.QValueKindTime, + qvalue.QValueKindTimeTZ, qvalue.QValueKindDate: + value = time.Now() case qvalue.QValueKindNumeric: // create a new big.Rat for numeric data value = big.NewRat(int64(placeHolder), 1) @@ -58,7 +57,6 @@ func createQValue(t *testing.T, kind qvalue.QValueKind, placeHolder int) qvalue. func generateRecords(t *testing.T, nullable bool, numRows uint32) *model.QRecordBatch { allQValueKinds := []qvalue.QValueKind{ - qvalue.QValueKindFloat16, qvalue.QValueKindFloat32, qvalue.QValueKindFloat64, qvalue.QValueKindInt16, @@ -68,7 +66,11 @@ func generateRecords(t *testing.T, nullable bool, numRows uint32) *model.QRecord // qvalue.QValueKindArray, // qvalue.QValueKindStruct, qvalue.QValueKindString, - qvalue.QValueKindETime, + qvalue.QValueKindTimestamp, + qvalue.QValueKindTimestampTZ, + qvalue.QValueKindTime, + qvalue.QValueKindTimeTZ, + qvalue.QValueKindDate, qvalue.QValueKindNumeric, qvalue.QValueKindBytes, qvalue.QValueKindUUID, diff --git a/flow/connectors/snowflake/client.go b/flow/connectors/snowflake/client.go index 910ca285e6..3a825ad493 100644 --- a/flow/connectors/snowflake/client.go +++ b/flow/connectors/snowflake/client.go @@ -213,15 +213,12 @@ func toQValue(kind qvalue.QValueKind, val interface{}) (qvalue.QValue, error) { Value: ratVal, }, nil } - case qvalue.QValueKindETime: - if v, ok := val.(*time.Time); ok && v != nil { - etimeVal, err := qvalue.NewExtendedTime(*v, qvalue.DateTimeKindType, "") - if err != nil { - return qvalue.QValue{}, fmt.Errorf("failed to create ExtendedTime: %w", err) - } + case qvalue.QValueKindTimestamp, qvalue.QValueKindTimestampTZ, qvalue.QValueKindDate, + qvalue.QValueKindTime, qvalue.QValueKindTimeTZ: + if t, ok := val.(*time.Time); ok && t != nil { return qvalue.QValue{ - Kind: qvalue.QValueKindETime, - Value: etimeVal, + Kind: kind, + Value: t, }, nil } case qvalue.QValueKindBytes: @@ -234,35 +231,8 @@ func toQValue(kind qvalue.QValueKind, val interface{}) (qvalue.QValue, error) { return qvalue.QValue{}, fmt.Errorf("[snowflakeclient] unsupported type %T for kind %s", val, kind) } -// databaseTypeNameToQValueKind converts a database type name to a QValueKind. -func databaseTypeNameToQValueKind(name string) (qvalue.QValueKind, error) { - switch name { - case "INT": - return qvalue.QValueKindInt32, nil - case "BIGINT": - return qvalue.QValueKindInt64, nil - case "FLOAT": - return qvalue.QValueKindFloat32, nil - case "DOUBLE", "REAL": - return qvalue.QValueKindFloat64, nil - case "VARCHAR", "CHAR", "TEXT": - return qvalue.QValueKindString, nil - case "BOOLEAN": - return qvalue.QValueKindBoolean, nil - case "DATETIME", "TIMESTAMP", "TIMESTAMP_LTZ", "TIMESTAMP_NTZ", "TIMESTAMP_TZ": - return qvalue.QValueKindETime, nil - case "BLOB", "BYTEA", "BINARY": - return qvalue.QValueKindBytes, nil - case "FIXED", "NUMBER": - return qvalue.QValueKindNumeric, nil - default: - // If type is unsupported, return an error - return "", fmt.Errorf("unsupported database type name: %s", name) - } -} - func columnTypeToQField(ct *sql.ColumnType) (*model.QField, error) { - qvKind, err := databaseTypeNameToQValueKind(ct.DatabaseTypeName()) + qvKind, err := snowflakeTypeToQValueKind(ct.DatabaseTypeName()) if err != nil { return nil, err } @@ -311,7 +281,8 @@ func (s *SnowflakeClient) ExecuteAndProcessQuery(query string) (*model.QRecordBa values := make([]interface{}, len(columns)) for i := range values { switch qfields[i].Type { - case qvalue.QValueKindETime: + case qvalue.QValueKindTimestamp, qvalue.QValueKindTimestampTZ, qvalue.QValueKindTime, + qvalue.QValueKindTimeTZ, qvalue.QValueKindDate: values[i] = new(time.Time) case qvalue.QValueKindInt16: values[i] = new(int16) @@ -375,10 +346,7 @@ func (s *SnowflakeClient) ExecuteAndProcessQuery(query string) (*model.QRecordBa func (s *SnowflakeClient) CreateTable(schema *model.QRecordSchema, schemaName string, tableName string) error { var fields []string for _, field := range schema.Fields { - snowflakeType, err := qValueKindToSnowflakeColTypeString(field.Type) - if err != nil { - return err - } + snowflakeType := qValueKindToSnowflakeType(string(field.Type)) fields = append(fields, fmt.Sprintf(`"%s" %s`, field.Name, snowflakeType)) } @@ -392,24 +360,3 @@ func (s *SnowflakeClient) CreateTable(schema *model.QRecordSchema, schemaName st return nil } - -func qValueKindToSnowflakeColTypeString(val qvalue.QValueKind) (string, error) { - switch val { - case qvalue.QValueKindInt32, qvalue.QValueKindInt64: - return "INT", nil - case qvalue.QValueKindFloat32, qvalue.QValueKindFloat64: - return "FLOAT", nil - case qvalue.QValueKindString: - return "STRING", nil - case qvalue.QValueKindBoolean: - return "BOOLEAN", nil - case qvalue.QValueKindETime: - return "TIMESTAMP_LTZ", nil - case qvalue.QValueKindBytes: - return "BINARY", nil - case qvalue.QValueKindNumeric: - return "NUMERIC(38,32)", nil - default: - return "", fmt.Errorf("unsupported QValueKind: %v", val) - } -} diff --git a/flow/connectors/snowflake/qvalue_convert.go b/flow/connectors/snowflake/qvalue_convert.go new file mode 100644 index 0000000000..28ef745260 --- /dev/null +++ b/flow/connectors/snowflake/qvalue_convert.go @@ -0,0 +1,84 @@ +package connsnowflake + +import ( + "fmt" + + "github.com/PeerDB-io/peer-flow/model/qvalue" +) + +func qValueKindToSnowflakeType(colType string) string { + switch qvalue.QValueKind(colType) { + case qvalue.QValueKindBoolean: + return "BOOLEAN" + // integer types + case qvalue.QValueKindInt16, qvalue.QValueKindInt32, qvalue.QValueKindInt64: + return "INTEGER" + // decimal types + // The names FLOAT, FLOAT4, and FLOAT8 are for compatibility with other systems + // Snowflake treats all three as 64-bit floating-point numbers. + case qvalue.QValueKindFloat32, qvalue.QValueKindFloat64: + return "FLOAT" + case qvalue.QValueKindNumeric: + return "NUMBER" + // string related STRING , TEXT , NVARCHAR , + // NVARCHAR2 , CHAR VARYING , NCHAR VARYING + //Synonymous with VARCHAR. + case qvalue.QValueKindString: + return "STRING" + // json also is stored as string for now + case qvalue.QValueKindJSON: + return "STRING" + // time related + case qvalue.QValueKindTimestamp: + return "TIMESTAMP_NTZ" + case qvalue.QValueKindTimestampTZ: + return "TIMESTAMP_TZ" + case qvalue.QValueKindTime: + return "TIME" + case qvalue.QValueKindDate: + return "DATE" + // handle INTERVAL types again + // case model.ColumnTypeTimeWithTimeZone, model.ColumnTypeInterval: + // return "STRING" + // bytes + case qvalue.QValueKindBit, qvalue.QValueKindBytes: + return "BINARY" + // rest will be strings + default: + return "STRING" + } +} + +// snowflakeTypeToQValueKind converts a database type name to a QValueKind. +func snowflakeTypeToQValueKind(name string) (qvalue.QValueKind, error) { + switch name { + case "INT": + return qvalue.QValueKindInt32, nil + case "BIGINT": + return qvalue.QValueKindInt64, nil + case "FLOAT": + return qvalue.QValueKindFloat32, nil + case "DOUBLE", "REAL": + return qvalue.QValueKindFloat64, nil + case "VARCHAR", "CHAR", "TEXT": + return qvalue.QValueKindString, nil + case "BOOLEAN": + return qvalue.QValueKindBoolean, nil + // assuming TIMESTAMP is an alias to TIMESTAMP_NTZ, which is the default. + case "DATETIME", "TIMESTAMP", "TIMESTAMP_NTZ": + return qvalue.QValueKindTimestamp, nil + case "TIMESTAMP_TZ": + return qvalue.QValueKindTimestampTZ, nil + case "TIME": + return qvalue.QValueKindTime, nil + case "DATE": + return qvalue.QValueKindDate, nil + case "BLOB", "BYTEA", "BINARY": + return qvalue.QValueKindBytes, nil + case "FIXED", "NUMBER", "DECIMAL", "NUMERIC": + return qvalue.QValueKindNumeric, nil + default: + // If type is unsupported, return an error + return "", fmt.Errorf("unsupported database type name: %s", name) + } +} diff --git a/flow/connectors/snowflake/snowflake.go b/flow/connectors/snowflake/snowflake.go index 49300adf8c..0ce38070e1 100644 --- a/flow/connectors/snowflake/snowflake.go +++ b/flow/connectors/snowflake/snowflake.go @@ -13,6 +13,7 @@ import ( "github.com/PeerDB-io/peer-flow/connectors/utils" "github.com/PeerDB-io/peer-flow/generated/protos" "github.com/PeerDB-io/peer-flow/model" + "github.com/PeerDB-io/peer-flow/model/qvalue" util "github.com/PeerDB-io/peer-flow/utils" "github.com/google/uuid" log "github.com/sirupsen/logrus" @@ -92,7 +93,6 @@ type snowflakeRawRecord struct { recordType int matchData string batchID int64 - items map[string]interface{} unchangedToastColumns string } @@ -380,7 +380,7 @@ func (c *SnowflakeConnector) SyncRecords(req *model.SyncRecordsRequest) (*model. switch typedRecord := record.(type) { case *model.InsertRecord: // json.Marshal converts bytes in Hex automatically to BASE64 string. - itemsJSON, err := json.Marshal(typedRecord.Items) + itemsJSON, err := typedRecord.Items.ToJSON() if err != nil { return nil, fmt.Errorf("failed to serialize insert record items to JSON: %w", err) } @@ -390,19 +390,18 @@ func (c *SnowflakeConnector) SyncRecords(req *model.SyncRecordsRequest) (*model. uid: uuid.New().String(), timestamp: time.Now().UnixNano(), destinationTableName: typedRecord.DestinationTableName, - data: string(itemsJSON), + data: itemsJSON, recordType: 0, matchData: "", batchID: syncBatchID, - items: typedRecord.Items, unchangedToastColumns: utils.KeysToString(typedRecord.UnchangedToastColumns), }) case *model.UpdateRecord: - newItemsJSON, err := json.Marshal(typedRecord.NewItems) + newItemsJSON, err := typedRecord.NewItems.ToJSON() if err != nil { return nil, fmt.Errorf("failed to serialize update record new items to JSON: %w", err) } - oldItemsJSON, err := json.Marshal(typedRecord.OldItems) + oldItemsJSON, err := typedRecord.OldItems.ToJSON() if err != nil { return nil, fmt.Errorf("failed to serialize update record old items to JSON: %w", err) } @@ -412,15 +411,14 @@ func (c *SnowflakeConnector) SyncRecords(req *model.SyncRecordsRequest) (*model. uid: uuid.New().String(), timestamp: time.Now().UnixNano(), destinationTableName: typedRecord.DestinationTableName, - data: string(newItemsJSON), + data: newItemsJSON, recordType: 1, - matchData: string(oldItemsJSON), + matchData: oldItemsJSON, batchID: syncBatchID, - items: typedRecord.NewItems, unchangedToastColumns: utils.KeysToString(typedRecord.UnchangedToastColumns), }) case *model.DeleteRecord: - itemsJSON, err := json.Marshal(typedRecord.Items) + itemsJSON, err := typedRecord.Items.ToJSON() if err != nil { return nil, fmt.Errorf("failed to serialize delete record items to JSON: %w", err) } @@ -430,11 +428,10 @@ func (c *SnowflakeConnector) SyncRecords(req *model.SyncRecordsRequest) (*model. uid: uuid.New().String(), timestamp: time.Now().UnixNano(), destinationTableName: typedRecord.DestinationTableName, - data: string(itemsJSON), + data: itemsJSON, recordType: 2, - matchData: string(itemsJSON), + matchData: itemsJSON, batchID: syncBatchID, - items: typedRecord.Items, unchangedToastColumns: utils.KeysToString(typedRecord.UnchangedToastColumns), }) default: @@ -669,57 +666,15 @@ func (c *SnowflakeConnector) checkIfTableExists(schemaIdentifier string, tableId return result, nil } -func getSnowflakeTypeForGenericColumnType(colType string) string { - switch colType { - case model.ColumnTypeBoolean: - return "BOOLEAN" - // integer types - case model.ColumnTypeInt16, model.ColumnTypeInt32, model.ColumnTypeInt64: - return "INTEGER" - // decimal types - // The names FLOAT, FLOAT4, and FLOAT8 are for compatibility with other systems - // Snowflake treats all three as 64-bit floating-point numbers. - case model.ColumnTypeFloat16, model.ColumnTypeFloat32, model.ColumnTypeFloat64: - return "FLOAT" - case model.ColumnTypeNumeric: - return "NUMBER" - // string related STRING , TEXT , NVARCHAR , - // NVARCHAR2 , CHAR VARYING , NCHAR VARYING - //Synonymous with VARCHAR. - case model.ColumnTypeString: - return "STRING" - // json also is stored as string for now - case model.ColumnTypeJSON: - return "STRING" - // time related - case model.ColumnTypeTimestamp: - return "TIMESTAMP_NTZ" - case model.ColumnTypeTimeStampWithTimeZone: - return "TIMESTAMP_TZ" - case model.ColumnTypeTime: - return "TIME" - case model.ColumnTypeDate: - return "TIMESTAMP_NTZ" - case model.ColumnTypeTimeWithTimeZone, model.ColumnTypeInterval: - return "STRING" - // bytes - case model.ColumnHexBytes, model.ColumnHexBit: - return "BINARY" - // rest will be strings - default: - return "STRING" - } -} - func generateCreateTableSQLForNormalizedTable(sourceTableIdentifier string, sourceTableSchema *protos.TableSchema) string { createTableSQLArray := make([]string, 0, len(sourceTableSchema.Columns)) for columnName, genericColumnType := range sourceTableSchema.Columns { if sourceTableSchema.PrimaryKeyColumn == strings.ToLower(columnName) { createTableSQLArray = append(createTableSQLArray, fmt.Sprintf("%s %s PRIMARY KEY,", - columnName, getSnowflakeTypeForGenericColumnType(genericColumnType))) + columnName, qValueKindToSnowflakeType(genericColumnType))) } else { createTableSQLArray = append(createTableSQLArray, fmt.Sprintf("%s %s,", columnName, - getSnowflakeTypeForGenericColumnType(genericColumnType))) + qValueKindToSnowflakeType(genericColumnType))) } } return fmt.Sprintf(createNormalizedTableSQL, sourceTableIdentifier, @@ -768,19 +723,20 @@ func (c *SnowflakeConnector) generateAndExecuteMergeStatement(destinationTableId flattenedCastsSQLArray := make([]string, 0, len(normalizedTableSchema.Columns)) for columnName, genericColumnType := range normalizedTableSchema.Columns { - sfType := getSnowflakeTypeForGenericColumnType(genericColumnType) - switch genericColumnType { - case model.ColumnHexBytes: + sfType := qValueKindToSnowflakeType(genericColumnType) + switch qvalue.QValueKind(genericColumnType) { + case qvalue.QValueKindBytes: flattenedCastsSQLArray = append(flattenedCastsSQLArray, fmt.Sprintf("BASE64_DECODE_BINARY(%s:%s) "+ "AS %s,", toVariantColumnName, columnName, columnName)) - case model.ColumnHexBit: + case qvalue.QValueKindBit: // "c2": {"Bytes": "gA==", "Len": 1,"Valid": true} flattenedCastsSQLArray = append(flattenedCastsSQLArray, fmt.Sprintf("BASE64_DECODE_BINARY(%s:%s:Bytes) "+ "AS %s,", toVariantColumnName, columnName, columnName)) - case model.ColumnTypeTime: - flattenedCastsSQLArray = append(flattenedCastsSQLArray, fmt.Sprintf("TIME_FROM_PARTS(0,0,0,%s:%s:"+ - "Microseconds*1000) "+ - "AS %s,", toVariantColumnName, columnName, columnName)) + // TODO: https://github.com/PeerDB-io/peerdb/issues/189 - handle time types and interval types + // case model.ColumnTypeTime: + // flattenedCastsSQLArray = append(flattenedCastsSQLArray, fmt.Sprintf("TIME_FROM_PARTS(0,0,0,%s:%s:"+ + // "Microseconds*1000) "+ + // "AS %s,", toVariantColumnName, columnName, columnName)) default: flattenedCastsSQLArray = append(flattenedCastsSQLArray, fmt.Sprintf("CAST(%s:%s AS %s) AS %s,", toVariantColumnName, diff --git a/flow/e2e/bigquery_helper.go b/flow/e2e/bigquery_helper.go index d6cc6e2333..b15bf04e38 100644 --- a/flow/e2e/bigquery_helper.go +++ b/flow/e2e/bigquery_helper.go @@ -10,6 +10,7 @@ import ( "time" "cloud.google.com/go/bigquery" + "cloud.google.com/go/civil" peer_bq "github.com/PeerDB-io/peer-flow/connectors/bigquery" "github.com/PeerDB-io/peer-flow/generated/protos" "github.com/PeerDB-io/peer-flow/model" @@ -209,15 +210,10 @@ func toQValue(bqValue bigquery.Value) (qvalue.QValue, error) { return qvalue.QValue{Kind: qvalue.QValueKindString, Value: v}, nil case bool: return qvalue.QValue{Kind: qvalue.QValueKindBoolean, Value: v}, nil + case civil.Date: + return qvalue.QValue{Kind: qvalue.QValueKindDate, Value: bqValue.(civil.Date).In(time.UTC)}, nil case time.Time: - val, err := qvalue.NewExtendedTime(v, qvalue.DateTimeKindType, "") - if err != nil { - return qvalue.QValue{}, fmt.Errorf("failed to create ExtendedTime: %w", err) - } - return qvalue.QValue{ - Kind: qvalue.QValueKindETime, - Value: val, - }, nil + return qvalue.QValue{Kind: qvalue.QValueKindTimestamp, Value: v}, nil case *big.Rat: return qvalue.QValue{Kind: qvalue.QValueKindNumeric, Value: v}, nil case []uint8: @@ -228,38 +224,8 @@ func toQValue(bqValue bigquery.Value) (qvalue.QValue, error) { } } -// bqFieldTypeToQValueKind converts a bigquery FieldType to a QValueKind. -func bqFieldTypeToQValueKind(fieldType bigquery.FieldType) (qvalue.QValueKind, error) { - switch fieldType { - case bigquery.StringFieldType: - return qvalue.QValueKindString, nil - case bigquery.BytesFieldType: - return qvalue.QValueKindBytes, nil - case bigquery.IntegerFieldType: - return qvalue.QValueKindInt64, nil - case bigquery.FloatFieldType: - return qvalue.QValueKindFloat64, nil - case bigquery.BooleanFieldType: - return qvalue.QValueKindBoolean, nil - case bigquery.TimestampFieldType: - return qvalue.QValueKindETime, nil - case bigquery.RecordFieldType: - return qvalue.QValueKindStruct, nil - case bigquery.DateFieldType: - return qvalue.QValueKindETime, nil - case bigquery.TimeFieldType: - return qvalue.QValueKindETime, nil - case bigquery.NumericFieldType: - return qvalue.QValueKindNumeric, nil - case bigquery.GeographyFieldType: - return qvalue.QValueKindString, nil - default: - return "", fmt.Errorf("unsupported bigquery field type: %v", fieldType) - } -} - func bqFieldSchemaToQField(fieldSchema *bigquery.FieldSchema) (*model.QField, error) { - qValueKind, err := bqFieldTypeToQValueKind(fieldSchema.Type) + qValueKind, err := peer_bq.BigQueryTypeToQValueKind(fieldSchema.Type) if err != nil { return nil, err } @@ -395,7 +361,7 @@ func qValueKindToBqColTypeString(val qvalue.QValueKind) (string, error) { return "STRING", nil case qvalue.QValueKindBoolean: return "BOOL", nil - case qvalue.QValueKindETime: + case qvalue.QValueKindTimestamp: return "TIMESTAMP", nil case qvalue.QValueKindBytes: return "BYTES", nil diff --git a/flow/e2e/qrep_flow_test.go b/flow/e2e/qrep_flow_test.go index 6b0f7e6ddc..0a210d1656 100644 --- a/flow/e2e/qrep_flow_test.go +++ b/flow/e2e/qrep_flow_test.go @@ -94,10 +94,10 @@ func getOwnersSchema() *model.QRecordSchema { Fields: []*model.QField{ {Name: "id", Type: qvalue.QValueKindString, Nullable: true}, {Name: "card_id", Type: qvalue.QValueKindString, Nullable: true}, - {Name: "from", Type: qvalue.QValueKindETime, Nullable: true}, + {Name: "from", Type: qvalue.QValueKindTimestamp, Nullable: true}, {Name: "price", Type: qvalue.QValueKindNumeric, Nullable: true}, - {Name: "created_at", Type: qvalue.QValueKindETime, Nullable: true}, - {Name: "updated_at", Type: qvalue.QValueKindETime, Nullable: true}, + {Name: "created_at", Type: qvalue.QValueKindTimestamp, Nullable: true}, + {Name: "updated_at", Type: qvalue.QValueKindTimestamp, Nullable: true}, {Name: "transaction_hash", Type: qvalue.QValueKindBytes, Nullable: true}, {Name: "ownerable_type", Type: qvalue.QValueKindString, Nullable: true}, {Name: "ownerable_id", Type: qvalue.QValueKindString, Nullable: true}, @@ -116,9 +116,9 @@ func getOwnersSchema() *model.QRecordSchema { {Name: "asset_id", Type: qvalue.QValueKindNumeric, Nullable: true}, {Name: "status", Type: qvalue.QValueKindInt64, Nullable: true}, {Name: "transaction_id", Type: qvalue.QValueKindString, Nullable: true}, - {Name: "settled_at", Type: qvalue.QValueKindETime, Nullable: true}, + {Name: "settled_at", Type: qvalue.QValueKindTimestamp, Nullable: true}, {Name: "reference_id", Type: qvalue.QValueKindString, Nullable: true}, - {Name: "settle_at", Type: qvalue.QValueKindETime, Nullable: true}, + {Name: "settle_at", Type: qvalue.QValueKindTimestamp, Nullable: true}, {Name: "settlement_delay_reason", Type: qvalue.QValueKindInt64, Nullable: true}, }, } diff --git a/flow/model/column.go b/flow/model/column.go deleted file mode 100644 index deff60597f..0000000000 --- a/flow/model/column.go +++ /dev/null @@ -1,23 +0,0 @@ -package model - -// ColumnType is an enum for the column type, which are generic across all connectors. -const ( - ColumnTypeInt16 = "int16" - ColumnTypeInt32 = "int32" - ColumnTypeInt64 = "int64" - ColumnHexBytes = "bytes" - ColumnHexBit = "bit" - ColumnTypeBoolean = "bool" - ColumnTypeFloat16 = "float16" - ColumnTypeFloat32 = "float32" - ColumnTypeFloat64 = "float64" - ColumnTypeString = "string" - ColumnTypeNumeric = "numeric" - ColumnTypeJSON = "json" - ColumnTypeInterval = "interval" - ColumnTypeTimestamp = "timestamp" - ColumnTypeTimeStampWithTimeZone = "timestamptz" - ColumnTypeTime = "time" - ColumnTypeTimeWithTimeZone = "timetz" - ColumnTypeDate = "date" -) diff --git a/flow/model/model.go b/flow/model/model.go index 56d2d09aea..e8e985826a 100644 --- a/flow/model/model.go +++ b/flow/model/model.go @@ -1,9 +1,11 @@ package model import ( + "encoding/json" "time" "github.com/PeerDB-io/peer-flow/generated/protos" + "github.com/PeerDB-io/peer-flow/model/qvalue" ) type PullRecordsRequest struct { @@ -29,7 +31,31 @@ type Record interface { // get table name GetTableName() string // get columns and values for the record - GetItems() map[string]interface{} + GetItems() RecordItems +} + +type RecordItems map[string]qvalue.QValue + +func (r RecordItems) ToJSON() (string, error) { + jsonStruct := make(map[string]interface{}) + for k, v := range r { + var err error + switch v.Kind { + case qvalue.QValueKindTimestamp, qvalue.QValueKindTimestampTZ, qvalue.QValueKindDate, + qvalue.QValueKindTime, qvalue.QValueKindTimeTZ: + jsonStruct[k], err = v.GoTimeConvert() + if err != nil { + return "", err + } + default: + jsonStruct[k] = v.Value + } + } + jsonBytes, err := json.Marshal(jsonStruct) + if err != nil { + return "", err + } + return string(jsonBytes), nil } type InsertRecord struct { @@ -42,7 +68,7 @@ type InsertRecord struct { // CommitID is the ID of the commit corresponding to this record. CommitID int64 // Items is a map of column name to value. - Items map[string]interface{} + Items RecordItems // unchanged toast columns UnchangedToastColumns map[string]bool } @@ -56,7 +82,7 @@ func (r *InsertRecord) GetTableName() string { return r.DestinationTableName } -func (r *InsertRecord) GetItems() map[string]interface{} { +func (r *InsertRecord) GetItems() RecordItems { return r.Items } @@ -68,9 +94,9 @@ type UpdateRecord struct { // Name of the destination table DestinationTableName string // OldItems is a map of column name to value. - OldItems map[string]interface{} + OldItems RecordItems // NewItems is a map of column name to value. - NewItems map[string]interface{} + NewItems RecordItems // unchanged toast columns UnchangedToastColumns map[string]bool } @@ -85,7 +111,7 @@ func (r *UpdateRecord) GetTableName() string { return r.DestinationTableName } -func (r *UpdateRecord) GetItems() map[string]interface{} { +func (r *UpdateRecord) GetItems() RecordItems { return r.NewItems } @@ -97,7 +123,7 @@ type DeleteRecord struct { // CheckPointID is the ID of the record. CheckPointID int64 // Items is a map of column name to value. - Items map[string]interface{} + Items RecordItems // unchanged toast columns UnchangedToastColumns map[string]bool } @@ -111,7 +137,7 @@ func (r *DeleteRecord) GetTableName() string { return r.SourceTableName } -func (r *DeleteRecord) GetItems() map[string]interface{} { +func (r *DeleteRecord) GetItems() RecordItems { return r.Items } diff --git a/flow/model/qrecord_batch.go b/flow/model/qrecord_batch.go index 3675b92fcb..385a16f980 100644 --- a/flow/model/qrecord_batch.go +++ b/flow/model/qrecord_batch.go @@ -3,6 +3,7 @@ package model import ( "fmt" "math/big" + "time" "github.com/PeerDB-io/peer-flow/model/qvalue" "github.com/google/uuid" @@ -73,7 +74,7 @@ func (src *QRecordBatchCopyFromSource) Values() ([]interface{}, error) { values := make([]interface{}, numEntries) for i, qValue := range record.Entries { switch qValue.Kind { - case qvalue.QValueKindFloat16, qvalue.QValueKindFloat32: + case qvalue.QValueKindFloat32: v, ok := qValue.Value.(float32) if !ok { src.err = fmt.Errorf("invalid float32 value") @@ -111,15 +112,23 @@ func (src *QRecordBatchCopyFromSource) Values() ([]interface{}, error) { case qvalue.QValueKindString: values[i] = qValue.Value.(string) - case qvalue.QValueKindETime: - et, ok := qValue.Value.(*qvalue.ExtendedTime) + case qvalue.QValueKindTimestamp: + t, ok := qValue.Value.(time.Time) if !ok { src.err = fmt.Errorf("invalid ExtendedTime value") return nil, src.err } - timestamp := pgtype.Timestamp{Time: et.Time, Valid: true} + timestamp := pgtype.Timestamp{Time: t, Valid: true} values[i] = timestamp + case qvalue.QValueKindTimestampTZ: + t, ok := qValue.Value.(time.Time) + if !ok { + src.err = fmt.Errorf("invalid ExtendedTime value") + return nil, src.err + } + timestampTZ := pgtype.Timestamptz{Time: t, Valid: true} + values[i] = timestampTZ case qvalue.QValueKindUUID: if qValue.Value == nil { values[i] = nil diff --git a/flow/model/qvalue/avro_converter.go b/flow/model/qvalue/avro_converter.go index a391393819..568b169d92 100644 --- a/flow/model/qvalue/avro_converter.go +++ b/flow/model/qvalue/avro_converter.go @@ -4,6 +4,7 @@ import ( "errors" "fmt" "math/big" + "time" "github.com/google/uuid" "github.com/linkedin/goavro" @@ -41,7 +42,7 @@ func GetAvroSchemaFromQValueKind(kind QValueKind, nullable bool) (*QValueKindAvr return &QValueKindAvroSchema{ AvroLogicalSchema: "long", }, nil - case QValueKindFloat16, QValueKindFloat32: + case QValueKindFloat32: return &QValueKindAvroSchema{ AvroLogicalSchema: "float", }, nil @@ -66,7 +67,7 @@ func GetAvroSchemaFromQValueKind(kind QValueKind, nullable bool) (*QValueKindAvr "scale": 9, }, }, nil - case QValueKindETime: + case QValueKindTime, QValueKindTimeTZ, QValueKindDate, QValueKindTimestamp, QValueKindTimestampTZ: return &QValueKindAvroSchema{ AvroLogicalSchema: map[string]string{ "type": "long", @@ -98,8 +99,8 @@ func (c *QValueAvroConverter) ToAvroValue() (interface{}, error) { switch c.Value.Kind { case QValueKindInvalid: return nil, fmt.Errorf("invalid QValueKind: %v", c.Value) - case QValueKindETime: - t, err := c.processExtendedTime() + case QValueKindTime, QValueKindTimeTZ, QValueKindDate, QValueKindTimestamp, QValueKindTimestampTZ: + t, err := c.processGoTime() if err != nil || t == nil { return t, err } @@ -110,9 +111,14 @@ func (c *QValueAvroConverter) ToAvroValue() (interface{}, error) { } case QValueKindString: return c.processNullableUnion("string", c.Value.Value) - case QValueKindFloat16, QValueKindFloat32: + case QValueKindFloat32: return c.processNullableUnion("float", c.Value.Value) case QValueKindFloat64: + if c.TargetDWH == QDWHTypeSnowflake || c.TargetDWH == QDWHTypeBigQuery { + if f32Val, ok := c.Value.Value.(float32); ok { + return c.processNullableUnion("double", float64(f32Val)) + } + } return c.processNullableUnion("double", c.Value.Value) case QValueKindInt16, QValueKindInt32, QValueKindInt64: return c.processNullableUnion("long", c.Value.Value) @@ -139,32 +145,23 @@ func (c *QValueAvroConverter) ToAvroValue() (interface{}, error) { } } -func (c *QValueAvroConverter) processExtendedTime() (interface{}, error) { +func (c *QValueAvroConverter) processGoTime() (interface{}, error) { if c.Value.Value == nil && c.Nullable { return nil, nil } - et, ok := c.Value.Value.(*ExtendedTime) + t, ok := c.Value.Value.(time.Time) if !ok { - return nil, fmt.Errorf("invalid ExtendedTime value") - } - - if et == nil { - return nil, nil + return nil, fmt.Errorf("invalid Time value") } - switch et.NestedKind.Type { - case DateTimeKindType, DateKindType, TimeKindType: - ret := et.Time.UnixMicro() - // Snowflake has issues with avro timestamp types - // See: https://stackoverflow.com/questions/66104762/snowflake-date-column-have-incorrect-date-from-avro-file - if c.TargetDWH == QDWHTypeSnowflake { - ret = ret / 1000000 - } - return ret, nil - default: - return nil, fmt.Errorf("unsupported ExtendedTimeKindType: %s", et.NestedKind.Type) + ret := t.UnixMicro() + // Snowflake has issues with avro timestamp types + // See: https://stackoverflow.com/questions/66104762/snowflake-date-column-have-incorrect-date-from-avro-file + if c.TargetDWH == QDWHTypeSnowflake { + ret = ret / 1000000 } + return ret, nil } func (c *QValueAvroConverter) processNullableUnion( diff --git a/flow/model/qvalue/etime.go b/flow/model/qvalue/etime.go deleted file mode 100644 index df5f8bbeb2..0000000000 --- a/flow/model/qvalue/etime.go +++ /dev/null @@ -1,72 +0,0 @@ -package qvalue - -import ( - "errors" - "time" -) - -type ExtendedTimeKindType string - -const ( - DateTimeKindType ExtendedTimeKindType = "datetime" - DateKindType ExtendedTimeKindType = "date" - TimeKindType ExtendedTimeKindType = "time" -) - -type ExtendedTime struct { - time.Time - NestedKind NestedKind -} - -type NestedKind struct { - Type ExtendedTimeKindType - Format string -} - -var ( - DateTime = NestedKind{ - Type: DateTimeKindType, - Format: time.RFC3339Nano, - } - - Date = NestedKind{ - Type: DateKindType, - Format: "2006-01-02", - } - - Time = NestedKind{ - Type: TimeKindType, - Format: "15:04:05.999999", - } -) - -func NewExtendedTime( - t time.Time, - kindType ExtendedTimeKindType, - originalFormat string, -) (*ExtendedTime, error) { - var nk NestedKind - - switch kindType { - case DateTimeKindType: - nk = DateTime - case DateKindType: - nk = Date - case TimeKindType: - nk = Time - default: - return nil, errors.New("invalid ExtendedTimeKindType") - } - - if originalFormat != "" { - nk = NestedKind{ - Type: nk.Type, - Format: originalFormat, - } - } - - return &ExtendedTime{ - Time: t, - NestedKind: nk, - }, nil -} diff --git a/flow/model/qvalue/kind.go b/flow/model/qvalue/kind.go index 032bb63153..be1545d30b 100644 --- a/flow/model/qvalue/kind.go +++ b/flow/model/qvalue/kind.go @@ -3,21 +3,24 @@ package qvalue type QValueKind string const ( - QValueKindInvalid QValueKind = "invalid" - QValueKindFloat16 QValueKind = "float16" - QValueKindFloat32 QValueKind = "float32" - QValueKindFloat64 QValueKind = "float64" - QValueKindInt16 QValueKind = "int16" - QValueKindInt32 QValueKind = "int32" - QValueKindInt64 QValueKind = "int64" - QValueKindBoolean QValueKind = "bool" - QValueKindArray QValueKind = "array" - QValueKindStruct QValueKind = "struct" - QValueKindString QValueKind = "string" - QValueKindETime QValueKind = "extended_time" - QValueKindNumeric QValueKind = "numeric" - QValueKindBytes QValueKind = "bytes" - QValueKindUUID QValueKind = "uuid" - QValueKindJSON QValueKind = "json" - QValueKindBit QValueKind = "bit" + QValueKindInvalid QValueKind = "invalid" + QValueKindFloat32 QValueKind = "float32" + QValueKindFloat64 QValueKind = "float64" + QValueKindInt16 QValueKind = "int16" + QValueKindInt32 QValueKind = "int32" + QValueKindInt64 QValueKind = "int64" + QValueKindBoolean QValueKind = "bool" + QValueKindArray QValueKind = "array" + QValueKindStruct QValueKind = "struct" + QValueKindString QValueKind = "string" + QValueKindTimestamp QValueKind = "timestamp" + QValueKindTimestampTZ QValueKind = "timestamptz" + QValueKindDate QValueKind = "date" + QValueKindTime QValueKind = "time" + QValueKindTimeTZ QValueKind = "timetz" + QValueKindNumeric QValueKind = "numeric" + QValueKindBytes QValueKind = "bytes" + QValueKindUUID QValueKind = "uuid" + QValueKindJSON QValueKind = "json" + QValueKindBit QValueKind = "bit" ) diff --git a/flow/model/qvalue/qvalue.go b/flow/model/qvalue/qvalue.go index 5be793ae9e..1435ae27cc 100644 --- a/flow/model/qvalue/qvalue.go +++ b/flow/model/qvalue/qvalue.go @@ -3,9 +3,11 @@ package qvalue import ( "bytes" "encoding/json" + "fmt" "math/big" "reflect" "strconv" + "time" "github.com/google/uuid" ) @@ -19,8 +21,6 @@ func (q *QValue) Equals(other *QValue) bool { switch q.Kind { case QValueKindInvalid: return false // both are invalid we always return false - case QValueKindFloat16: - return compareFloat32(q.Value, other.Value) case QValueKindFloat32: return compareFloat32(q.Value, other.Value) case QValueKindFloat64: @@ -39,8 +39,10 @@ func (q *QValue) Equals(other *QValue) bool { return compareStruct(q.Value, other.Value) case QValueKindString: return compareString(q.Value, other.Value) - case QValueKindETime: - return compareETime(q.Value, other.Value) + // all internally represented as a Golang time.Time + case QValueKindTime, QValueKindTimeTZ, QValueKindDate, + QValueKindTimestamp, QValueKindTimestampTZ: + return compareGoTime(q.Value, other.Value) case QValueKindNumeric: return compareNumeric(q.Value, other.Value) case QValueKindBytes: @@ -56,6 +58,22 @@ func (q *QValue) Equals(other *QValue) bool { return false } +func (q *QValue) GoTimeConvert() (string, error) { + if q.Kind == QValueKindTime { + return q.Value.(time.Time).Format("15:04:05.999999"), nil + } else if q.Kind == QValueKindTimeTZ { + return q.Value.(time.Time).Format("15:04:05.999999-0700"), nil + } else if q.Kind == QValueKindDate { + return q.Value.(time.Time).Format("2006-01-02"), nil + } else if q.Kind == QValueKindTimestamp { + return q.Value.(time.Time).Format("2006-01-02 15:04:05.999999"), nil + } else if q.Kind == QValueKindTimestampTZ { + return q.Value.(time.Time).Format("2006-01-02 15:04:05.999999-0700"), nil + } else { + return "", fmt.Errorf("unsupported QValueKind: %s", q.Kind) + } +} + func compareInt16(value1, value2 interface{}) bool { int1, ok1 := getInt16(value1) int2, ok2 := getInt16(value2) @@ -86,9 +104,9 @@ func compareFloat64(value1, value2 interface{}) bool { return ok1 && ok2 && float1 == float2 } -func compareETime(value1, value2 interface{}) bool { - et1, ok1 := value1.(*ExtendedTime) - et2, ok2 := value2.(*ExtendedTime) +func compareGoTime(value1, value2 interface{}) bool { + et1, ok1 := value1.(*time.Time) + et2, ok2 := value2.(*time.Time) if !ok1 || !ok2 { return false @@ -96,8 +114,8 @@ func compareETime(value1, value2 interface{}) bool { // TODO: this is a hack, we should be comparing the actual time values // currently this is only used for testing so that is OK. - t1 := et1.Time.UnixMilli() / 1000 - t2 := et2.Time.UnixMilli() / 1000 + t1 := et1.UnixMilli() / 1000 + t2 := et2.UnixMilli() / 1000 return t1 == t2 } diff --git a/flow/workflows/setup_flow.go b/flow/workflows/setup_flow.go index b5db2b8968..0131bde450 100644 --- a/flow/workflows/setup_flow.go +++ b/flow/workflows/setup_flow.go @@ -203,11 +203,12 @@ func (s *SetupFlowExecution) fetchTableSchemaAndSetupNormalizedTables( if err := fSrcTableSchema.Get(ctx, &srcTableSchema); err != nil { return nil, fmt.Errorf("failed to fetch schema for source table %s: %w", srcTableName, err) } - s.logger.Info("fetched schema for table %s for peer flow %s ", srcTableSchema, s.PeerFlowName) + s.logger.Info(fmt.Sprintf("fetched schema for table %s for peer flow %s ", srcTableSchema, s.PeerFlowName)) tableNameSchemaMapping[flowConnectionConfigs.TableNameMapping[srcTableName]] = srcTableSchema - s.logger.Info("setting up normalized table for table %s for peer flow - ", srcTableSchema, s.PeerFlowName) + s.logger.Info(fmt.Sprintf("setting up normalized table for table %s for peer flow - %s", + srcTableName, s.PeerFlowName)) // now setup the normalized tables on the destination peer setupConfig := &protos.SetupNormalizedTableInput{ From 3c652be15f093874fe9a1d87c3dd6349c17d0041 Mon Sep 17 00:00:00 2001 From: Amogh Bharadwaj Date: Wed, 5 Jul 2023 02:56:27 +0530 Subject: [PATCH 4/7] Connectivity Checks in CREATE PEER Flow (#190) CREATE PEER flow now checks if the user-provided configuration can successfully connect to the peer. Note that these sql files are CREATE PEER commands of the three peers. ```shell amogh=> \i bq.sql psql:bq.sql:37: ERROR: [peer]: invalid configuration: unable to create GcpClient. amogh=> \i sf.sql psql:sf.sql:62: ERROR: [peer]: invalid configuration: Internal error: error decoding response body: expected value at line 1 column 1 amogh=> \i pg.sql psql:pg.sql:8: ERROR: [peer]: invalid configuration: error encountered while connecting to postgres Error { kind: Connect, cause: Some(Custom { kind: Uncategorized, error: "failed to lookup address information: Name or service not known" }) } ``` --- nexus/peer-bigquery/src/lib.rs | 14 +++++++-- nexus/peer-cursor/src/lib.rs | 2 ++ nexus/peer-postgres/src/lib.rs | 9 ++++-- nexus/peer-snowflake/src/lib.rs | 10 ++++++ nexus/server/src/main.rs | 56 +++++++++++++++++++++++---------- 5 files changed, 70 insertions(+), 21 deletions(-) diff --git a/nexus/peer-bigquery/src/lib.rs b/nexus/peer-bigquery/src/lib.rs index 95e636ddde..43615586d6 100644 --- a/nexus/peer-bigquery/src/lib.rs +++ b/nexus/peer-bigquery/src/lib.rs @@ -11,7 +11,11 @@ use peer_cursor::{CursorModification, QueryExecutor, QueryOutput, SchemaRef}; use pgerror::PgError; use pgwire::error::{ErrorInfo, PgWireError, PgWireResult}; use pt::peers::BigqueryConfig; -use sqlparser::ast::{CloseCursor, Expr, FetchDirection, Statement, Value}; +use sqlparser::{ + ast::{CloseCursor, Expr, FetchDirection, Statement, Value}, + dialect::GenericDialect, + parser, +}; use stream::{BqRecordStream, BqSchema}; mod ast; @@ -41,7 +45,7 @@ pub async fn bq_client_from_config(config: BigqueryConfig) -> anyhow::Result anyhow::Result { + let sql = "SELECT 1;"; + let test_stmt = parser::Parser::parse_sql(&GenericDialect {}, sql).unwrap(); + let _ = self.execute(&test_stmt[0]).await?; + Ok(true) + } } diff --git a/nexus/peer-cursor/src/lib.rs b/nexus/peer-cursor/src/lib.rs index 37065cf163..7d2525a7df 100644 --- a/nexus/peer-cursor/src/lib.rs +++ b/nexus/peer-cursor/src/lib.rs @@ -51,4 +51,6 @@ pub trait QueryExecutor: Send + Sync { async fn execute(&self, stmt: &Statement) -> PgWireResult; async fn describe(&self, stmt: &Statement) -> PgWireResult>; + + async fn is_connection_valid(&self) -> anyhow::Result; } diff --git a/nexus/peer-postgres/src/lib.rs b/nexus/peer-postgres/src/lib.rs index 32e4eb7817..35fb5ad38f 100644 --- a/nexus/peer-postgres/src/lib.rs +++ b/nexus/peer-postgres/src/lib.rs @@ -16,7 +16,7 @@ mod stream; // PostgresQueryExecutor is a QueryExecutor that uses a Postgres database as its // backing store. pub struct PostgresQueryExecutor { - _config: PostgresConfig, + config: PostgresConfig, peername: Option, client: Box, } @@ -59,7 +59,7 @@ impl PostgresQueryExecutor { })?; Ok(Self { - _config: config.clone(), + config: config.clone(), peername, client: Box::new(client), }) @@ -173,4 +173,9 @@ impl QueryExecutor for PostgresQueryExecutor { _ => Ok(None), } } + + async fn is_connection_valid(&self) -> anyhow::Result { + let _ = PostgresQueryExecutor::new(None, &self.config).await?; + Ok(true) + } } diff --git a/nexus/peer-snowflake/src/lib.rs b/nexus/peer-snowflake/src/lib.rs index 0b7d14ca8f..6d5349702b 100644 --- a/nexus/peer-snowflake/src/lib.rs +++ b/nexus/peer-snowflake/src/lib.rs @@ -4,6 +4,8 @@ use cursor::SnowflakeCursorManager; use peer_cursor::{CursorModification, QueryExecutor, QueryOutput, SchemaRef}; use pgerror::PgError; use pgwire::error::{ErrorInfo, PgWireError, PgWireResult}; +use sqlparser::dialect::GenericDialect; +use sqlparser::parser; use std::cmp::min; use std::{collections::HashMap, time::Duration}; use stream::SnowflakeDataType; @@ -84,6 +86,7 @@ pub struct ResultSet { struct PartitionResult { data: Vec>>, } + pub struct SnowflakeQueryExecutor { config: SnowflakeConfig, partition_number: usize, @@ -434,4 +437,11 @@ impl QueryExecutor for SnowflakeQueryExecutor { )))), } } + + async fn is_connection_valid(&self) -> anyhow::Result { + let sql = "SELECT 1;"; + let test_stmt = parser::Parser::parse_sql(&GenericDialect {}, sql).unwrap(); + let _ = self.execute(&test_stmt[0]).await?; + Ok(true) + } } diff --git a/nexus/server/src/main.rs b/nexus/server/src/main.rs index eea316c4cb..b1eb05c816 100644 --- a/nexus/server/src/main.rs +++ b/nexus/server/src/main.rs @@ -158,6 +158,19 @@ impl NexusBackend { peer, if_not_exists: _, } => { + let peer_executor = self.get_peer_executor(&peer).await.map_err(|err| { + PgWireError::ApiError(Box::new(PgError::Internal { + err_msg: format!("unable to get peer executor: {:?}", err), + })) + })?; + peer_executor.is_connection_valid().await.map_err(|e| { + self.executors.remove(&peer.name); // Otherwise it will keep returning the earlier configured executor + PgWireError::UserError(Box::new(ErrorInfo::new( + "ERROR".to_owned(), + "internal_error".to_owned(), + format!("[peer]: invalid configuration: {}", e.to_string()), + ))) + })?; let catalog = self.catalog.lock().await; catalog.create_peer(peer.as_ref()).await.map_err(|e| { PgWireError::UserError(Box::new(ErrorInfo::new( @@ -219,7 +232,7 @@ impl NexusBackend { })) })?; // make a request to the flow service to start the job. - let workflow_id = self + let _workflow_id = self .flow_handler .start_qrep_flow_job(&qrep_flow_job) .await @@ -297,7 +310,11 @@ impl NexusBackend { QueryAssocation::Peer(peer) => { tracing::info!("handling peer[{}] query: {}", peer.name, stmt); peer_holder = Some(peer.clone()); - self.get_peer_executor(&peer).await + self.get_peer_executor(&peer).await.map_err(|err| { + PgWireError::ApiError(Box::new(PgError::Internal { + err_msg: format!("unable to get peer executor: {:?}", err), + })) + })? } QueryAssocation::Catalog => { tracing::info!("handling catalog query: {}", stmt); @@ -327,7 +344,11 @@ impl NexusBackend { let catalog = self.catalog.lock().await; catalog.get_executor() } - Some(peer) => self.get_peer_executor(peer).await, + Some(peer) => self.get_peer_executor(peer).await.map_err(|err| { + PgWireError::ApiError(Box::new(PgError::Internal { + err_msg: format!("unable to get peer executor: {:?}", err), + })) + })?, } }; @@ -338,32 +359,25 @@ impl NexusBackend { } } - async fn get_peer_executor(&self, peer: &Peer) -> Arc> { + async fn get_peer_executor(&self, peer: &Peer) -> anyhow::Result>> { if let Some(executor) = self.executors.get(&peer.name) { - return Arc::clone(executor.value()); + return Ok(Arc::clone(executor.value())); } let executor = match &peer.config { Some(Config::BigqueryConfig(ref c)) => { let peer_name = peer.name.clone(); let executor = - BigQueryQueryExecutor::new(peer_name, c, self.peer_connections.clone()) - .await - .unwrap(); + BigQueryQueryExecutor::new(peer_name, c, self.peer_connections.clone()).await?; Arc::new(Box::new(executor) as Box) } Some(Config::PostgresConfig(ref c)) => { let peername = Some(peer.name.clone()); - let executor = peer_postgres::PostgresQueryExecutor::new(peername, c) - .await - .unwrap(); + let executor = peer_postgres::PostgresQueryExecutor::new(peername, c).await?; Arc::new(Box::new(executor) as Box) } Some(Config::SnowflakeConfig(ref c)) => { - let peername = Some(peer.name.clone()); - let executor = peer_snowflake::SnowflakeQueryExecutor::new(c) - .await - .unwrap(); + let executor = peer_snowflake::SnowflakeQueryExecutor::new(c).await?; Arc::new(Box::new(executor) as Box) } _ => { @@ -373,7 +387,7 @@ impl NexusBackend { self.executors .insert(peer.name.clone(), Arc::clone(&executor)); - executor + Ok(executor) } } @@ -500,7 +514,15 @@ impl ExtendedQueryHandler for NexusBackend { // if the peer is of type bigquery, let us route the query to bq. match &peer.config { Some(Config::BigqueryConfig(_)) => { - let executor = self.get_peer_executor(peer).await; + let executor = + self.get_peer_executor(peer).await.map_err(|err| { + PgWireError::ApiError(Box::new(PgError::Internal { + err_msg: format!( + "unable to get peer executor: {:?}", + err + ), + })) + })?; executor.describe(stmt).await? } _ => { From fc3719a96c46669b42e8b52e5d2de69d5d5791df Mon Sep 17 00:00:00 2001 From: Kaushik Iska Date: Wed, 5 Jul 2023 14:23:32 -0400 Subject: [PATCH 5/7] Support prepared statements on Postgres peer (#191) Co-authored-by: Amogh-Bharadwaj --- nexus/analyzer/src/lib.rs | 1 - nexus/peer-postgres/src/ast.rs | 5 +---- nexus/peer-snowflake/src/lib.rs | 2 +- nexus/server/src/main.rs | 29 ++++++++++++++++++++++++++++- 4 files changed, 30 insertions(+), 7 deletions(-) diff --git a/nexus/analyzer/src/lib.rs b/nexus/analyzer/src/lib.rs index c51bbcddde..8909987d84 100644 --- a/nexus/analyzer/src/lib.rs +++ b/nexus/analyzer/src/lib.rs @@ -12,7 +12,6 @@ use pt::peers::{ peer::Config, BigqueryConfig, DbType, MongoConfig, Peer, PostgresConfig, SnowflakeConfig, }; use qrep::process_options; -use serde_json::Number; use sqlparser::ast::{visit_relations, visit_statements, FetchDirection, SqlOption, Statement}; use sqlparser::ast::{ CreateMirror::{Select, CDC}, diff --git a/nexus/peer-postgres/src/ast.rs b/nexus/peer-postgres/src/ast.rs index e6ec466cbf..3e33a535d8 100644 --- a/nexus/peer-postgres/src/ast.rs +++ b/nexus/peer-postgres/src/ast.rs @@ -1,9 +1,6 @@ use std::ops::ControlFlow; -use sqlparser::ast::{ - visit_relations_mut, visit_statements_mut, ObjectName, ObjectType, Query, Statement, - TableFactor, TableWithJoins, -}; +use sqlparser::ast::{visit_relations_mut, visit_statements_mut, ObjectType, Query, Statement}; #[derive(Default)] pub struct PostgresAst { diff --git a/nexus/peer-snowflake/src/lib.rs b/nexus/peer-snowflake/src/lib.rs index 6d5349702b..385d1791a4 100644 --- a/nexus/peer-snowflake/src/lib.rs +++ b/nexus/peer-snowflake/src/lib.rs @@ -433,7 +433,7 @@ impl QueryExecutor for SnowflakeQueryExecutor { _ => PgWireResult::Err(PgWireError::UserError(Box::new(ErrorInfo::new( "ERROR".to_owned(), "fdw_error".to_owned(), - "only SELECT statements are supported in bigquery".to_owned(), + "only SELECT statements are supported in snowflake".to_owned(), )))), } } diff --git a/nexus/server/src/main.rs b/nexus/server/src/main.rs index b1eb05c816..1b87c12aa5 100644 --- a/nexus/server/src/main.rs +++ b/nexus/server/src/main.rs @@ -525,7 +525,34 @@ impl ExtendedQueryHandler for NexusBackend { })?; executor.describe(stmt).await? } - _ => { + Some(Config::PostgresConfig(_)) => { + let executor = + self.get_peer_executor(peer).await.map_err(|err| { + PgWireError::ApiError(Box::new(PgError::Internal { + err_msg: format!( + "unable to get peer executor: {:?}", + err + ), + })) + })?; + executor.describe(stmt).await? + } + Some(Config::SnowflakeConfig(_)) => { + let executor = + self.get_peer_executor(peer).await.map_err(|err| { + PgWireError::ApiError(Box::new(PgError::Internal { + err_msg: format!( + "unable to get peer executor: {:?}", + err + ), + })) + })?; + executor.describe(stmt).await? + } + Some(Config::MongoConfig(_)) => { + panic!("peer type not supported: {:?}", peer) + } + None => { panic!("peer type not supported: {:?}", peer) } } From 12311716045c048e24f929fd0b1cddf8f8642b40 Mon Sep 17 00:00:00 2001 From: Kaushik Iska Date: Wed, 5 Jul 2023 15:50:06 -0400 Subject: [PATCH 6/7] fix errors with invalid pkeys (#192) --- nexus/analyzer/src/lib.rs | 5 +-- nexus/peer-bigquery/src/ast.rs | 2 +- nexus/peer-cursor/src/util.rs | 4 +- nexus/peer-snowflake/src/auth.rs | 61 ++++++++++++++++-------------- nexus/peer-snowflake/src/lib.rs | 22 +++++++---- nexus/peer-snowflake/src/stream.rs | 2 +- nexus/value/src/array.rs | 8 ++-- 7 files changed, 55 insertions(+), 49 deletions(-) diff --git a/nexus/analyzer/src/lib.rs b/nexus/analyzer/src/lib.rs index 8909987d84..3cb13e3f04 100644 --- a/nexus/analyzer/src/lib.rs +++ b/nexus/analyzer/src/lib.rs @@ -12,11 +12,8 @@ use pt::peers::{ peer::Config, BigqueryConfig, DbType, MongoConfig, Peer, PostgresConfig, SnowflakeConfig, }; use qrep::process_options; +use sqlparser::ast::CreateMirror::{Select, CDC}; use sqlparser::ast::{visit_relations, visit_statements, FetchDirection, SqlOption, Statement}; -use sqlparser::ast::{ - CreateMirror::{Select, CDC}, - Value, -}; mod qrep; diff --git a/nexus/peer-bigquery/src/ast.rs b/nexus/peer-bigquery/src/ast.rs index 6a72f95d75..d6b47fe631 100644 --- a/nexus/peer-bigquery/src/ast.rs +++ b/nexus/peer-bigquery/src/ast.rs @@ -3,7 +3,7 @@ use std::ops::ControlFlow; use sqlparser::ast::Value::Number; use sqlparser::ast::{ - visit_expressions_mut, visit_function_arg, visit_function_arg_mut, visit_relations_mut, + visit_expressions_mut, visit_function_arg_mut, visit_relations_mut, visit_setexpr_mut, Array, BinaryOperator, DataType, DateTimeField, Expr, Function, FunctionArg, FunctionArgExpr, Ident, ObjectName, Query, SetExpr, SetOperator, SetQuantifier, TimezoneInfo, }; diff --git a/nexus/peer-cursor/src/util.rs b/nexus/peer-cursor/src/util.rs index 2bf7d0b7ea..6b134d3ef8 100644 --- a/nexus/peer-cursor/src/util.rs +++ b/nexus/peer-cursor/src/util.rs @@ -6,9 +6,9 @@ use pgwire::{ api::results::{DataRowEncoder, FieldInfo, QueryResponse, Response}, error::{PgWireError, PgWireResult}, }; -use value::{array::ArrayValue, Value}; +use value::{Value}; -use crate::{Record, Records, SchemaRef, SendableStream}; +use crate::{Records, SchemaRef, SendableStream}; fn encode_value(value: &Value, builder: &mut DataRowEncoder) -> PgWireResult<()> { match value { diff --git a/nexus/peer-snowflake/src/auth.rs b/nexus/peer-snowflake/src/auth.rs index 812bec8f0e..8001e56821 100644 --- a/nexus/peer-snowflake/src/auth.rs +++ b/nexus/peer-snowflake/src/auth.rs @@ -3,6 +3,7 @@ use std::{ time::{SystemTime, UNIX_EPOCH}, }; +use anyhow::Context; use base64::encode as base64_encode; use jsonwebtoken::{encode as jwt_encode, Algorithm, EncodingKey, Header}; use pkcs1::EncodeRsaPrivateKey; @@ -44,13 +45,14 @@ impl SnowflakeAuth { private_key: String, refresh_threshold: u64, expiry_threshold: u64, - ) -> Self { + ) -> anyhow::Result { + let pkey = DecodePrivateKey::from_pkcs8_pem(&private_key).context("Invalid private key")?; let mut snowflake_auth: SnowflakeAuth = SnowflakeAuth { // moved normalized_account_id above account_id to satisfy the borrow checker. normalized_account_id: SnowflakeAuth::normalize_account_identifier(&account_id), account_id, username, - private_key: DecodePrivateKey::from_pkcs8_pem(&private_key).unwrap(), + private_key: pkey, public_key_fp: None, refresh_threshold, expiry_threshold, @@ -59,9 +61,10 @@ impl SnowflakeAuth { }; snowflake_auth.public_key_fp = Some(SnowflakeAuth::gen_public_key_fp( &snowflake_auth.private_key, - )); + )?); snowflake_auth.refresh_jwt(); - snowflake_auth + + Ok(snowflake_auth) } // Normalize the account identifer to a form that is embedded into the JWT. @@ -85,26 +88,21 @@ impl SnowflakeAuth { } #[tracing::instrument(name = "peer_sflake::gen_public_key_fp", skip_all)] - fn gen_public_key_fp(private_key: &RsaPrivateKey) -> String { - let public_key = - EncodePublicKey::to_public_key_der(&RsaPublicKey::from(private_key)).unwrap(); - format!( + fn gen_public_key_fp(private_key: &RsaPrivateKey) -> anyhow::Result { + let public_key = EncodePublicKey::to_public_key_der(&RsaPublicKey::from(private_key))?; + let res = format!( "SHA256:{}", base64_encode(Sha256::new_with_prefix(public_key.as_der()).finalize()) - ) + ); + Ok(res) } #[tracing::instrument(name = "peer_sflake::auth_refresh_jwt", skip_all)] - fn refresh_jwt(&mut self) { + fn refresh_jwt(&mut self) -> anyhow::Result<()> { let private_key_jwt: EncodingKey = EncodingKey::from_rsa_der( - EncodeRsaPrivateKey::to_pkcs1_der(&self.private_key) - .unwrap() - .as_der(), + EncodeRsaPrivateKey::to_pkcs1_der(&self.private_key)?.as_der(), ); - self.last_refreshed = SystemTime::now() - .duration_since(UNIX_EPOCH) - .unwrap() - .as_secs(); + self.last_refreshed = SystemTime::now().duration_since(UNIX_EPOCH)?.as_secs(); info!( "Refreshing SnowFlake JWT for account: {} and user: {} at time {}", self.account_id, self.username, self.last_refreshed @@ -114,7 +112,9 @@ impl SnowflakeAuth { "{}.{}.{}", self.normalized_account_id, self.username.to_uppercase(), - self.public_key_fp.as_deref().unwrap() + self.public_key_fp + .as_deref() + .context("No public key fingerprint")? ), sub: format!( "{}.{}", @@ -125,21 +125,24 @@ impl SnowflakeAuth { exp: self.last_refreshed + self.expiry_threshold, }; let header: Header = Header::new(Algorithm::RS256); - self.current_jwt = Some( - SecretString::from_str(&jwt_encode(&header, &jwt_claims, &private_key_jwt).unwrap()) - .unwrap(), - ); + + let encoded_jwt = jwt_encode(&header, &jwt_claims, &private_key_jwt)?; + let secret = SecretString::from_str(&encoded_jwt)?; + + self.current_jwt = Some(secret); + + Ok(()) } - pub fn get_jwt(&mut self) -> &Secret { - if SystemTime::now() - .duration_since(UNIX_EPOCH) - .unwrap() - .as_secs() + pub fn get_jwt(&mut self) -> anyhow::Result<&Secret> { + if SystemTime::now().duration_since(UNIX_EPOCH)?.as_secs() >= (self.last_refreshed + self.refresh_threshold) { - self.refresh_jwt(); + self.refresh_jwt()?; } - self.current_jwt.as_ref().unwrap() + + self.current_jwt + .as_ref() + .ok_or_else(|| anyhow::anyhow!("JWT not initialized. Please call refresh_jwt() first.")) } } diff --git a/nexus/peer-snowflake/src/lib.rs b/nexus/peer-snowflake/src/lib.rs index 385d1791a4..ad4b262122 100644 --- a/nexus/peer-snowflake/src/lib.rs +++ b/nexus/peer-snowflake/src/lib.rs @@ -136,7 +136,7 @@ impl SnowflakeQueryExecutor { config.clone().private_key, DEFAULT_REFRESH_THRESHOLD, DEFAULT_EXPIRY_THRESHOLD, - ), + )?, query_timeout: config.query_timeout, reqwest_client, cursor_manager, @@ -169,12 +169,12 @@ impl SnowflakeQueryExecutor { ); let mut auth = self.auth.clone(); - let jwt = auth.get_jwt(); + let jwt = auth.get_jwt()?; let secret = jwt.expose_secret().clone(); // TODO: for things other than SELECTs, the robust way to handle retrys is by // generating a UUID from our end to mark the query as unique and then sending it with the request. // If we need to retry, send same UUID with retry=true parameter set and Snowflake should prevent duplicate execution. - let query_status = self + let query_status_res = self .reqwest_client .post(self.endpoint_url.to_owned()) .bearer_auth(secret) @@ -189,9 +189,14 @@ impl SnowflakeQueryExecutor { }) .send() .await - .map_err(|_| anyhow::anyhow!("failed in making request for QueryStatus"))? - .json::() - .await?; + .map_err(|e| { + anyhow::anyhow!("failed in making request for QueryStatus. error: {:?}", e) + })?; + + let query_json = query_status_res.json::().await?; + let query_status = serde_json::from_value(query_json.clone()).map_err(|e| { + anyhow::anyhow!("failed in parsing json {:?}, error: {:?}", query_json, e) + })?; // TODO: remove this blind retry logic for anything other than a SELECT. let res = self.query_poll(query_status).await?; @@ -200,6 +205,7 @@ impl SnowflakeQueryExecutor { None => self.process_query(query_str).await?, }) } + pub async fn query(&self, query: &Box) -> PgWireResult { let mut query = query.clone(); @@ -222,7 +228,7 @@ impl SnowflakeQueryExecutor { query_status: &QueryStatus, ) -> anyhow::Result { let mut auth = self.auth.clone(); - let jwt = auth.get_jwt(); + let jwt = auth.get_jwt()?; let secret = jwt.expose_secret().clone(); let response = self .reqwest_client @@ -440,7 +446,7 @@ impl QueryExecutor for SnowflakeQueryExecutor { async fn is_connection_valid(&self) -> anyhow::Result { let sql = "SELECT 1;"; - let test_stmt = parser::Parser::parse_sql(&GenericDialect {}, sql).unwrap(); + let test_stmt = parser::Parser::parse_sql(&GenericDialect {}, sql)?; let _ = self.execute(&test_stmt[0]).await?; Ok(true) } diff --git a/nexus/peer-snowflake/src/stream.rs b/nexus/peer-snowflake/src/stream.rs index 57c5e21e54..0417163957 100644 --- a/nexus/peer-snowflake/src/stream.rs +++ b/nexus/peer-snowflake/src/stream.rs @@ -207,7 +207,7 @@ impl SnowflakeRecordStream { self.partition_number = self.partition_number + 1; self.partition_index = 0; let partition_number = self.partition_number; - let secret = self.auth.get_jwt().expose_secret().clone(); + let secret = self.auth.get_jwt()?.expose_secret().clone(); let statement_handle = self.result_set.statementHandle.clone(); let url = self.endpoint_url.clone(); println!("Secret: {:#?}", secret); diff --git a/nexus/value/src/array.rs b/nexus/value/src/array.rs index f48694efb0..9b50c8c679 100644 --- a/nexus/value/src/array.rs +++ b/nexus/value/src/array.rs @@ -149,8 +149,8 @@ impl<'a> ToSql for ArrayValue { } ArrayValue::VarChar(arr) => arr.to_sql(ty, out)?, ArrayValue::Text(arr) => arr.to_sql(ty, out)?, - ArrayValue::Binary(arr) => todo!("support encoding array of binary"), - ArrayValue::VarBinary(arr) => todo!("support encoding array of varbinary"), + ArrayValue::Binary(_arr) => todo!("support encoding array of binary"), + ArrayValue::VarBinary(_arr) => todo!("support encoding array of varbinary"), ArrayValue::Date(arr) => arr.to_sql(ty, out)?, ArrayValue::Time(arr) => arr.to_sql(ty, out)?, ArrayValue::TimeWithTimeZone(arr) => arr.to_sql(ty, out)?, @@ -227,8 +227,8 @@ impl ToSqlText for ArrayValue { ArrayValue::Char(arr) => array_to_sql_text!(arr, ty, out), ArrayValue::VarChar(arr) => array_to_sql_text!(arr, ty, out), ArrayValue::Text(arr) => array_to_sql_text!(arr, ty, out), - ArrayValue::Binary(arr) => todo!("implement encoding array of binary"), - ArrayValue::VarBinary(arr) => todo!("implement encoding array of varbinary"), + ArrayValue::Binary(_arr) => todo!("implement encoding array of binary"), + ArrayValue::VarBinary(_arr) => todo!("implement encoding array of varbinary"), ArrayValue::Date(arr) => array_to_sql_text!(arr, ty, out), ArrayValue::Time(arr) => array_to_sql_text!(arr, ty, out), ArrayValue::TimeWithTimeZone(arr) => array_to_sql_text!(arr, ty, out), From eae6bbcba59563f1730b56ef7dfd629f5abdb797 Mon Sep 17 00:00:00 2001 From: Kevin K Biju <52661649+heavycrystal@users.noreply.github.com> Date: Thu, 6 Jul 2023 20:13:01 +0530 Subject: [PATCH 7/7] fixing e2e tests, and Snowflake TIMESTAMP and NUMBER (#193) --- .github/workflows/flow.yml | 2 +- flow/connectors/bigquery/qrep_sync_method.go | 6 +++++- flow/connectors/snowflake/client.go | 2 +- flow/connectors/snowflake/qvalue_convert.go | 2 +- flow/go.mod | 1 - flow/model/qvalue/avro_converter.go | 20 +++++++++++++------- flow/model/qvalue/qvalue.go | 8 ++++---- 7 files changed, 25 insertions(+), 16 deletions(-) diff --git a/.github/workflows/flow.yml b/.github/workflows/flow.yml index 4bc3ef44d8..2adf8c619d 100644 --- a/.github/workflows/flow.yml +++ b/.github/workflows/flow.yml @@ -57,7 +57,7 @@ jobs: - name: run tests run: | - gotestsum --format testname -- -p 1 ./... + gotestsum --format testname -- -p 1 ./... -timeout 1200s working-directory: ./flow env: AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} diff --git a/flow/connectors/bigquery/qrep_sync_method.go b/flow/connectors/bigquery/qrep_sync_method.go index 8dd998c973..92a9d4142d 100644 --- a/flow/connectors/bigquery/qrep_sync_method.go +++ b/flow/connectors/bigquery/qrep_sync_method.go @@ -94,7 +94,11 @@ func (s *QRepStagingTableSync) SyncQRepRecords( // col names for the destination table joined by comma colNames := []string{} for _, col := range dstTableMetadata.Schema { - colNames = append(colNames, col.Name) + if strings.ToLower(col.Name) == "from" { + colNames = append(colNames, "`from`") + } else { + colNames = append(colNames, col.Name) + } } colNamesStr := strings.Join(colNames, ", ") diff --git a/flow/connectors/snowflake/client.go b/flow/connectors/snowflake/client.go index 3a825ad493..ed64a9adb6 100644 --- a/flow/connectors/snowflake/client.go +++ b/flow/connectors/snowflake/client.go @@ -218,7 +218,7 @@ func toQValue(kind qvalue.QValueKind, val interface{}) (qvalue.QValue, error) { if t, ok := val.(*time.Time); ok && t != nil { return qvalue.QValue{ Kind: kind, - Value: t, + Value: *t, }, nil } case qvalue.QValueKindBytes: diff --git a/flow/connectors/snowflake/qvalue_convert.go b/flow/connectors/snowflake/qvalue_convert.go index 28ef745260..558a9b8373 100644 --- a/flow/connectors/snowflake/qvalue_convert.go +++ b/flow/connectors/snowflake/qvalue_convert.go @@ -19,7 +19,7 @@ func qValueKindToSnowflakeType(colType string) string { case qvalue.QValueKindFloat32, qvalue.QValueKindFloat64: return "FLOAT" case qvalue.QValueKindNumeric: - return "NUMBER" + return "NUMBER(38, 9)" // string related STRING , TEXT , NVARCHAR , // NVARCHAR2 , CHAR VARYING , NCHAR VARYING //Synonymous with VARCHAR. diff --git a/flow/go.mod b/flow/go.mod index 8bacdde3ce..e815d6132b 100644 --- a/flow/go.mod +++ b/flow/go.mod @@ -13,7 +13,6 @@ require ( github.com/jackc/pgx/v5 v5.3.1 github.com/jmoiron/sqlx v1.3.5 github.com/joho/godotenv v1.5.1 - github.com/linkedin/goavro v2.1.0+incompatible github.com/linkedin/goavro/v2 v2.12.0 github.com/sirupsen/logrus v1.9.3 github.com/snowflakedb/gosnowflake v1.6.21 diff --git a/flow/model/qvalue/avro_converter.go b/flow/model/qvalue/avro_converter.go index 568b169d92..8dcaca542f 100644 --- a/flow/model/qvalue/avro_converter.go +++ b/flow/model/qvalue/avro_converter.go @@ -7,7 +7,7 @@ import ( "time" "github.com/google/uuid" - "github.com/linkedin/goavro" + "github.com/linkedin/goavro/v2" ) // QValueKindAvroSchema defines a structure for representing Avro schemas. @@ -70,8 +70,7 @@ func GetAvroSchemaFromQValueKind(kind QValueKind, nullable bool) (*QValueKindAvr case QValueKindTime, QValueKindTimeTZ, QValueKindDate, QValueKindTimestamp, QValueKindTimestampTZ: return &QValueKindAvroSchema{ AvroLogicalSchema: map[string]string{ - "type": "long", - "logicalType": "timestamp-micros", + "type": "string", }, }, nil case QValueKindJSON, QValueKindArray, QValueKindStruct, QValueKindBit: @@ -104,10 +103,17 @@ func (c *QValueAvroConverter) ToAvroValue() (interface{}, error) { if err != nil || t == nil { return t, err } + if c.TargetDWH == QDWHTypeSnowflake { + if c.Nullable { + return c.processNullableUnion("string", t.(string)) + } else { + return t.(string), nil + } + } if c.Nullable { - return goavro.Union("long.timestamp-micros", t), nil + return goavro.Union("long.timestamp-micros", t.(int64)), nil } else { - return t, nil + return t.(int64), nil } case QValueKindString: return c.processNullableUnion("string", c.Value.Value) @@ -156,10 +162,10 @@ func (c *QValueAvroConverter) processGoTime() (interface{}, error) { } ret := t.UnixMicro() - // Snowflake has issues with avro timestamp types + // Snowflake has issues with avro timestamp types, returning as string form of the int64 // See: https://stackoverflow.com/questions/66104762/snowflake-date-column-have-incorrect-date-from-avro-file if c.TargetDWH == QDWHTypeSnowflake { - ret = ret / 1000000 + return fmt.Sprint(ret), nil } return ret, nil } diff --git a/flow/model/qvalue/qvalue.go b/flow/model/qvalue/qvalue.go index 1435ae27cc..cf0b6f11bc 100644 --- a/flow/model/qvalue/qvalue.go +++ b/flow/model/qvalue/qvalue.go @@ -105,8 +105,8 @@ func compareFloat64(value1, value2 interface{}) bool { } func compareGoTime(value1, value2 interface{}) bool { - et1, ok1 := value1.(*time.Time) - et2, ok2 := value2.(*time.Time) + et1, ok1 := value1.(time.Time) + et2, ok2 := value2.(time.Time) if !ok1 || !ok2 { return false @@ -114,8 +114,8 @@ func compareGoTime(value1, value2 interface{}) bool { // TODO: this is a hack, we should be comparing the actual time values // currently this is only used for testing so that is OK. - t1 := et1.UnixMilli() / 1000 - t2 := et2.UnixMilli() / 1000 + t1 := et1.UnixMicro() + t2 := et2.UnixMicro() return t1 == t2 }