From 2690dc0bf5eb17c8bdc6bbd109fcebdc0c2f3162 Mon Sep 17 00:00:00 2001 From: guowei17 Date: Fri, 27 Dec 2024 16:10:59 +0800 Subject: [PATCH] Release cce-network-v2/2.12.11 --- cce-network-v2/VERSION | 2 +- .../deploy/cce-network-v2-2.12.tar.gz | Bin 20804 -> 20834 bytes .../deploy/cce-network-v2/Chart.yaml | 4 +- cce-network-v2/docs/release.md | 9 ++ cce-network-v2/pkg/bce/bcesync/eni.go | 15 ++- cce-network-v2/pkg/bce/rdma/rdma_eni.go | 11 +- cce-network-v2/pkg/bce/rdma/rdma_super.go | 2 + cce-network-v2/pkg/bce/rdma/rdma_wrapper.go | 116 ++++++++++++++---- cce-network-v2/pkg/bce/utils/bce_utils.go | 98 +++++++++++++-- cce-network-v2/pkg/bce/vpceni/node_super.go | 34 +++-- cce-network-v2/pkg/ipam/net_resource.go | 2 +- .../pkg/ipam/net_resource_set_manager.go | 17 +-- cce-network-v2/pkg/k8s/watchers/cce_eni.go | 12 +- .../pkg/nodediscovery/nodediscovery.go | 4 +- 14 files changed, 262 insertions(+), 64 deletions(-) diff --git a/cce-network-v2/VERSION b/cce-network-v2/VERSION index 420fbec..d94526e 100644 --- a/cce-network-v2/VERSION +++ b/cce-network-v2/VERSION @@ -1 +1 @@ -2.12.10 +2.12.11 diff --git a/cce-network-v2/deploy/cce-network-v2-2.12.tar.gz b/cce-network-v2/deploy/cce-network-v2-2.12.tar.gz index 7849e5098e150e90a0013e75b050b0613fb4c80f..d839e401f7ce3aaed90e10aadd1509e779071eb6 100644 GIT binary patch delta 20760 zcmV)PK()Wbq5Hh2mojtVP^mX>^y6C6vvf(o?lUsKG;swBaM&*%zhFQ-ZN(n z)*?bqlJ$vux@%_2^vnH_G|uiG{E7kd^2QiHaBT0|4$g@I$BBalm_MT)iBJ9ux%XC8 zPxrheVZbM6>BA8-)m69ZKI*>e)>sxdJRYz5;R$19dUDUhcz>qg**7~&|HIS!KYgyu z?AtpvU74xOPVbqjOz)kp>|wJHWi>rd62)f7*q+e$<5B&#d;d@RSVjJ?m`=i@a@%yB z2R8>Ycy4B92>&ZHvwP=||I>5(=B7am_F&V@)E+kV;Fd7_`G3y;akj`CX5z$5vYs{k zkhQqumP_O8Sbs}I4E~wym-C08G2oSJ#xb{9LpU5eEON^+Lk?p@Vb&cO9{a4$SztyH zbQ2zYZTm^cV(tcx8G~q*l(bb#jE%Eb@K(5H zlP@QYhFGhzu`y`z10I^OAL3ZXLLNB6G8g>Bi>qv!+J9(_e2w5cUtjIKb^&|(k${11 z_A}<0E@z$7=Q{6RPp_TrT>9(g`>*L`oKqva1FG=3DWW)Rm+Pjmle)#rmhVpDz6qS9 zDLf+x{cOUGJ9jtVzPb`v2KU6qohutR*Y%?19I7WSqRR4{Pj@raVAU~59B0V~JYgmC z&Z^mtvVXarUEyIUY$SS%OUzkpHrZ;+k2tdxTZMdWwahSPW)K|bVI+L7tiZqkMO?%> z0$y1>sxm^ZA}OAXCU2Kn-p$CVwFST((-=P8|4DYkGdxd}xhZNetL` z2O5BdXtqFrDp{Kb$32TzbFF})8C$JGYw$WkB1Xlx20OulO_l>Xg@;DyJG}5dC%~-c z^}LeXugZQs=9)Ns>^W?OPQX@>?^0K4sG(olcPq1#c`kKf{^%mNGlqx!#YEf(za_k7 z?tgF#NOg>dZXXjzGdW{(+zxu#n`iJ5Kknmk#^t_tgt{WaI58dLL{f+4#ibbJ2Y~4% z7&+i$ht`0<(O9WeA|@;W%a*wn@)#wEJy$|W(aa)ZH=svfV=3a{im>>+WswpkmAo8q zOEd)Vc(uiWk0Gdk)sLC7Ba2#o;@GI^VSi3g>o$y(pR%1h&gMPOkLCK&ESR>9|Dj~` zfI}hg_SKSr=7&pd?KcJuLug6F%hb2l0LH??I7pmltHqEc!g3}61p)-$LC}$kV4ac( zM1%lq;$&v3r1u&UDe&$2-sNE<5FDf|ZZG&=%-3T1Vw^Rie+~U40E-094U%|3FMnLI zWd>$lI3gBOAjUxm?9dPNXM@c@^9+3kT|}#d=ERQ zX4Uv}pc$w(wGtv9!bjPyvIVKAYD>@(_#q7458e8<euib~cbHD(g9m|} zi~t&F6VHjeNQil^`tAZWRPUAP{fC7@@kP)lgGTiCFXc6# z65~wS%C8`x(m){dm6(EvCaz(5LeC+Er`t}=Q9U%c2#S4FBbYH*4w#$t+NsW&Pq*%V zuzB^(d#|6{ymPGttu}0phJR?vp<2hWgA0e)@udZ(`Z01Uv}qN#vLJP8^b?!hCfUpi z?Vv1xAqJ`Jo*}d~dF~bXZ|iS?JD6U$4qjY(b{$+&_^`2l{@%&coj1Q&z+r5DsfK;7 z+(S%61c3F&Qlg?6;YV1s(62b&p74TCeMw@;=QuiLzyG&@&6Nq=u&cyiN!UD--B##`-7H6dpd0lh#ASgD72977Kqh8&qZtUp7Lf(QjR@-n!Ctq4 zJtSj*vf<#~HS9#1u`eo{n~DbX0!NFl8aA}zo~p9FQ&KkoJphlbH~y5KxtgB(AicE? zyd?X*^|#kL>yw%WAq$xYq_-kSMVx)fbd$ygC4aL-U&0Zv6T!+%cNYBCJ2SM+b*EL0 zGli`#&s`*tA1tTLv$z54*}#FAsMn<3Ze72Go)E-Fjqa|ZuZDDwDl>cc z&p}&@n@-%)jR*GcI{=MMKZv?b2QZw3p?@yA&`D&V^TiGDfzf-fRt_APn;y&Q{mkq^ z|4>*_PUXmkOdn)iZe17*{`ub<-sT=8q&o^$K+_?Ftt-oJSzk3ZJtb*Ij(ZMj#P8kr5ps6tJ zcGqZ%z;u~3mt~+fZEzBpI@p~m+x+9pohzrorl;$d()B<7i%P9jGv@+C3E&_UFcVqc z1-#q92ZcUJua9FtChbpp@oeYI&wtXlu6C|`RO-gEXg%6qe(1CsDu>;B<6P&R8*(r@ z65oE4UU(}#cLAKD&ZQdz0HJ+?5E{%U8VN6qjG;ZGSwXw7T^~Soco%6RBHGfUdPy!& z@<7wC>#&Wp&Cg#30}e!H3r82xx`>Em8s`Svm^WR}NxChPo8RAle;M5q8-F}$_`LPq z>*<*38&7i0^;M&HZ0t|%` z$_Y4VVCE~v3~FJU(oy1!$GgKLfE-2xgjcQI|aV)u#MaArf1*D_$XTqS9sl3i$L-} zdz;5eCS&z5P{d{({6tX4Kp)`U33xvv_XD|h_ue?udHcJqZ(au<0)O@c@>Jt=^PE(V zvJ3E9eq{wJFc`FT=9SG?zLlf0vFJd{{EEm1PW#I z{0%v3>)MU<&Lz-;dicb~o!@ody$){t#?3D~fB6Jef?LweOg?@Q#mgbqD)wQhELPVZ)~K7rUU%zeDw;lt4NA25q~fo=p#em+j0qrBF!%X z5cJJU@AVTCg8y8+v47v()Ksgad{taZ=lo~sh5yUE)p6E2{YCoKrOt)_0jrdry1ezt zRT6@j437?mDg8E7(yc7J?f861i*DwUfeo}|BW`K5CG zrErq|>V{Tx694G0(TX?O1yl$8l7P-MqtZ{y(P{vH>*fjg8@u?~2RNml{x^&`Zibk- zWVOXZFfXWq7q}Id%EIR6r0fn&rSeK-f%7q0`mve2gQsWzXXmsJvJzy6^&k23|GhKQ zkLy1k#V#|H|KOiLC>hv2|7RC%v{m_PfyRn zlMM|Y5v5W(Av_4!7l2&8dsJlv7CDnW4JQ>$mFLPBI_jMHF^B|!J#J8CUg9{}lX(p( zf45xYGw_{WK!6Zp?59>lZBYxHqa)pU3>MX8B>)~cCyp8F{+MI2{M=D2G$?AQn8UPf zqOa*Jg%oKn_)g+_8RQot-&>+XoigHHHdjpH$YbU9ufUZmshS5O+x)N8S9LlUhw>~F z0ojzs4P@Aa2N28>cBuOI&XRj2AurSxe~-@W9QfQQ;P@Wl^-VCh!wC+p;V~h2phG6$ zJ@4fVG^#K|-S-`CdV{7=WOg`U1ET5H`5swnolDI{N(9wH$M&xi)fs#E| zWK~jRwRpD>3na#yU#cD7yIg3dd^5Pru_#uUg-8J#J9%o^RmxHIuSlMBWW-wb_Wvw_4qUz)%gb7-!;n{wIr{9yL zphP`^2qT)bG0Qy}iKbzO5WBB9lQW|Q@``WUcd=57^} z*AG8`S#pv6hsS;~fAq+p3cSOx7TjY9%Inf<5VBY~1iDJ%teP!ZH`b3n)f(Zx(j`*4 zW!9GF5BJcaf9i<=V1YEamn{>p$LTRiO8T<_f=K0dYDqA~*bo`X8@L=~dsi~0&GoJZ zk#XD207-~r*kL!rVzW7**fQ@_4cpM$03ND;H1JsotlDmfI86VL|FBC9q}+^Qv*$gG z?xTvXcO@n*Gdkjop0N@!f^;xLM)HYwIua!!JUGnh>X>4 zU3z29SOC^6=fum9KvgYkub;l+`Jzyzp}&5vBq!$faqY)rJpYvOf3Fxm+&sV$9&3b``|<(BmwkDGo|O*C4|rje zedZ+ z<y{Dj^rj*90$2 zNj_bW!-FG&rh742f2&Tb!TE#snCj+>K*=tFfwMH!Y=QDxgnX|qN#G3SZ2aSlz~hVn zdz=v%&VnD95h(BrZ%!4q3cSTTYDKj$S={w<+xEssg-wjl2*=;H@uu3K5T>wyx_RAC z-_JuMM*9+?N773fq;el5AbIaaxdBrg!K-=_h!opA*nXg+uu{_ogWG%Ng%Dp*z^+)?p{JVl=90iG%VQSR;LZix?4G>X&91n(e+YPHZCsSS@GMooy+6d<{*G z{0x{-CIvLEt!ld2YRl|jfYe)mh%JJ{R0!RmLJKBwry`Mt;Eq&Y0bSF^WI2Gw3Q>0u zcSegsKEGzXG%&%|hwOY$PEPYG4~rozb&+rguc#PCXOKYQ0@4Fzd0Vb#MT>`3=_MAj zZ;!^o0mC#Z^5_9sA#4XzJ@OLVFsXwPE;#IN;-a;nfSGs*+XinVT1XIoH*sPSIP!+v z6y@Mt2;zrIGDrvkK3FKpL>A*B(H5Xo5i~(5zsj~SO}capq7u<9UO5NsZUFp5MESg~ zju9wzR25Dn7y=PpB+j0sem~1&i-HP~#Z&pa%asw#9h^UrJi|*S2G%d~5U6Yp&d$oN zIrhrSDbISxx}$P7L=hK%Nt)fc9jcoUkS*nv}oc{X3$0nMJqaWy@xhNHADA`sdJ z5`1;NV&LjXXt}Vmtdp!Sl>L@`J3HO&TW1Dfu-q#pU1d}ErHa%Yhh+)}4&n`m zqw0dg_6)8K8>^u*6=l%P38rpN3_78ZprxkJ<>13`8NS5}8G)UyANuE=GraAy;JZPB zzX+f;A;Oc=He~Gw7Z%vTgA4a#>0NVeULC*e%*G!s48ZY!WTd<1osJY2iON9)YbXNN45b5hY=;NZa5;7nHWa=x7g2 zISC8scV?J>yz4ZP>5E|?e;{JTYgt2VYcZtHdj@@xvxb*B0RFIjNf&|XSk`MZDc&YI zo9g#d$PCF|Jo}jxt*ZuWsE6qub{JyO73r?2l7`fw#jU zpV$K+2H`*2`?lrCjVv)g^D7WFv(b{wD&D$(*%~_ZK(X0940Tfsvek<5ctZlKP$agi zfEobB%Sd|ghMyea2>W$Ez>j`+_6aPoDcp zZ{QV}T?TA3CbFUvPPz|P+Aug96?3}XgoX`W(~7ch5$OU1wO~UQd16(R(qhepjma>7 z6~Rv}(#3K{7ZyxCq;yR60=e8CMB1&OPNJV zJ)#A_j%EvhJXww-O&@513iCNUI|=ZA%j7i=v0TXAOltt^+dya;Wv7Wv#0zOt-B&qt z{B@D$d5pIa>QWS#QScUXb_mOr4d$HkyVYlSi-{3&8Zg462+p*hWDt8Nsm;EHa5bbv zVwvuxwmy>5FroTh>hi?3C|8E9HvEPKDC>ejUJo&Y3GY2XQkY5!Ca?xbouqYtl+_mP zn-hUfBF@VeOwwC2NpGVjHqDUO!C$B?#2C)`qR@Am1>zh{g5{Zk)siaE$>V%xEt7%I zXO5G`XBJv?!)Kl^8+K~hD&JYDX%nF;XO$T+VFeI;_-Le`Eu=2kL#2-z8MtP~AETl| zyGpz&T%t|yaY(7@h+Oe01>6*W)!D35Fyg)%6x9NHmR`lq@qPYgnc0bmqgQouw0Wbv;yXzn3pp%)Zs#VW-B^ED|;wVq6+uh-pDA zyhd)~FhFxVOu_;ZaSJf%hEnVoX%WsUyHwj2G4DTs7ijS|%{5DBxIJfo$Ue}~tkV$* zAg?g*L%T(dxF$I*sXHw_peNCSEg6)^6F5ti90~inRjQG=y(CVRB@|2!6T1jl%Q42O z&ukKw{z)&F$cZ+VP>Tba&!Y(%TVV2Nf&-4)0+tzZ+~$?J0istZEGIYV?BH#Ul+ z_3Sb#H3(LIvCS0+Tbv7Bh`mc)29F5f8TnMURyu%9C4%+cCen{s?sr_AOZXt#DJE5a z%W33Bf_YqVNG9F5=ISX|t915oLPXX?+Sn7|z2}$G=Y+C<@Em@Bfr9_0;QDbBk^;;L zQ(OZlujHn&oBgS~&${Qrrr^Eq(g-0BsIWbhNL3H1d!OdDL(036RxC?V=gz>5er%m4 z>U8vL%<)WcBKn<7$`;d&j62WGD|_9@Su_vhge?eo>|?uhyDS=)xZic%vI>>df*z}+ z38y-5YLS9}3bTtNwx&Q+Kz(9C!uq{B1Tt0+FGn!w zUkkXG83zJ5e5aeYgDVySo1M!6j^Y8KqSfidbC6(x3W|A2l0-jiD27NWS8;tH?>uiMUmrQ>^QY)P`zY#5#LPu4__vE9_{C;4e!McZCsk<+%bKxiFm7Ita&= zW;1sByqaTZh_-eoiO4DLv*stZw}Zos_@pcndS={{_&Zkqus#H zZ5X6BtuR04F&@e{&awE)mH#3M3N2oH=;)Za)-^O2&^rhwGTZON2DF>>xeaNs$gNj7_A1AJ z6(`zF<#<%Rg|rJ5r6fE+FxTn{vy97k%Qe=5^tc>auvC?eOsXa0M81{kG4Zj~-%V!4 z2(10Cid@oNo$JHC5*2z;iXgNNF7GQUJ)5VFH;-XgtqO+#o1Uoyd*O4LyS^4Tvs#d& z(Zwm#)MqWOE7;oQQelsGozJ22NpW_6ayq?WdjyMsu+;Optga6^oGLelHu7I_lv!mj zNuQF~7Fe?~vwCtSoKD1ssHUUAD&idAO==r!(51o@g+nVm7zoTF^175SWI4p6o1iNe1KtreK`J@q_@Bl zUa<#S9!?|ORR>+w$;0Q1+J|DyEKn&#Nu(x7=r-RX$5>=*RpIn&F3Uh?cE&tCHEB~R6Q$+MR{SDa`!*49_t)C5b*xK)b7 zJCbtH;XGGjbup}yOLeYEcnx@&`j|_xf;BTh?{hOsah|{7Ux~49+Xs!Zsv-&+t<#G zl3x{vuE%)xx$FDf^+BJz-u?+5t(qM$0I%T&1~lK`-ffZAkE^}KN9QCIQ+_}~WzMen zq!u*BCQag;9;*6(7~*ZzaeRzGXu6!m#9M*RTY1lG}4vx5H3LSE6<74q|1z3GUJ-4ZGy`5 zWn%GDK6zz`Q+KOT#z{QM;v1VCqOt&RYGR-V+@FwKN&6yyC*qfd?c`M%Hgb9IwG16l zuxlx*MHJAqp+~PM2rH?PntRU!2f6(M!6m@TY@(g(GM%kuL11$vKE7AfkBkafKgnV*Y16lLpJ-!_pS?mJg0q|n~Kede~Us=G-F_|gJauAb<^id*r2!Z z1f+!aMgCg${dU2ju)Kgw-080DNma3b$ygeg=KgbQulATXWW}FssJSzM z>!IF!ykuui&2;<1U!r%E*JAA1lIHs@#v~tG%loWz9Ji{zgtGd=>q?7NmpyTAjYgTJ z(wC(46QQvG3cu&{&H$!HXbUJKDgu%b@09(Gc=1c1FlawT*p|dZkRu%Z5pC2u; zERq3>JdN)nO3|Hmfw=3r$Lwt4evG zC90as&oo!b@f;wPH&pXLy(U1E#yb=#aL1It>8 zs4IFMM4s`-VLF{s0*BzW%+es1$(8oP?*{R^LxfQXXuSVa&LgrI2l;(6zRuErB>t<+ zOvR8s*<@OsJ?+XIlRnfS1eCkliGqWayTj2;p=gbh?uCe7YL^7731LFwArg;e77fRk z>)j`x$a;3)CmI&{3pM3=ZYffyLu_%Ut^h2Q3&~o*iddU9f|U@Tbk$o==W=P!gh1L^q*-#c+rGtEwGMNYkF$Bhrrdq` zJcp_m!-bJ6cFvE{{g5387Ga!KN+6Q9!9H@zevlf%WVy#PWM_R^&xM0Nwa%4d)Vx0I zIYqOPCa<=w0`gU%r(7LMTI%dnUE0tw=An{14$F5ED3YluZs$jR5B_L>E4U>}Zqds> z%2%i?i5D4vm0eO+^+CQ>$n8|=Xs-WupCN55rJw4t#5_eU}o$9VC zzsK{aoT4BIw_#5Ag>{y~!Nl_1H&58Zl3uyLhKOG7N}5~AX2Uf^EE(HJV@sKsDzR;G z5b+m=d*2M%df$xRH>3A|&FFnIRvGK>**9aB8W_IHl-{OSb%i8*{ldSe(Jk--8JStB z&^#%f0H#qbSzs8#ib}~rRX=;g>0M9wMinj@q0IWK(!d&*>c1pGej8qa^mu?KompT4 zZ3;GT7jWz*?iEwlVtQJqLc@`^wv}OTNhaGpk*t?S%5OY9LZS12Fs<5)s$Ej{W;VC_ z74Uk=?1^C>N6H80czk_)hc<8(1ml>%p*oo4A(2Nj7><;`#2?Qo*8q?|eEJL-_seKz zV4(aC;-P&I$t8*>EX(r6Yd(qW_`eA+fgH>+0%&blnf^IF!i;^Zf#yIj>LdaNRyd*n z!Ea9xg@bjXF$ocWV$tQ?IJG&s)vvlqbFP=Ek#bk22BFEN}GzbV<`UoXyG9=S-1GY2w)1dq>=MyM&7phbq* z0{l?uBLU*rhNra<_fyqT8HvhY zN^)1`QnDF;e_JwAv)7anp9W4RXV16x_3AE~#7JDP)j=1V^AlH(JCAS{##!SjJR z9;|GG{ef!c})W3UG)rm!zZHUd0ZcUBa zFKR13zn`m1yLDfytSMmAPLkZ*8dTFGn+2da}X>2RwS%!K@av9iFEZYJ1pBdnD#6Lbf$DS8ORvC$Y z^c)I-1mvaV9Op>u!GnBP`?cwyRU7F2cBNd6i{7UErh6Q?sWkyueEJAV*;E4Ung zeXt@S6T5`7#dWyDn{tKpE+m+Zc@HYB$%3{f8E&JyXdL{ABw$?3s8VTum6!e-ebSdc z=}XWjeX)OnM@#yGU$V)qJD7hCH$GpQvXr(c)36YLMP75NGkmmEFpq-nV|(esmWfqi zojhDhXZ4jMhQk|DgxSvB!DgP27Ql#qoTGZ{;gGVlNP*%%7!cMPH>E-72qxE=vNdc1 zrc&#L_TWv`u5D49?r{_@6F7?f5ncgOe}wxZOzjgNVaLlR7dpp;L;{tb6X?lEPe!^z z`>m6ao)cIZQh$W|BTVhr%L$m>x*bz0!YjjZw$gz%8UWzFvAZO;JGzEfA5a5-wITqw zGZf=ydqm?J9+bc#d)-qU_{w$D;&=P{wLRc;!#MM?)XWDho9?Lbg7R5t zH(%(DXNt<*aftCB@X8cJ{EC!xvz#^>9jb&67`hXEnX9GXU2X6#v2PL#xrmRrc(cnzj{^F_8f2RGtv(!qKT(8;cm9@N?t@as|m9iTJT{rnO62S*M#Ry2By5dpa z^w!P)2MA_|x=*sT`2W1z+uy1A|LktR-0A&)o}&FG{eNg1KCah~Od(kM`oJ#@4(CdF{ZX)4J!5tC~-3C`#dp*=$tB>G}w8NxiF= z3wq~2fR-@FI0QSf93353N^=O`mCA2zBjKu*qm@!jbF>dOCDqPxn-C#M9WGP220hd17nxaiBiqSH`JiY@~bcV-fc>Dd~>G=3h-!yYexW3`yW?z&xKRLT-t%8o-RjG4K zrXMwrv1f7+H7dm^YDEAVxjt0Sl@ff!lT5^<+BEJJys&?NtwLtKwW|02tCsRORWMEYdclI5qUZrts*t(H40s}+sfn)G z@x!}Km7Gjuk`o;JC0>!sM3t5J@Xj>zYq+DXu9>maQB%};8XoNKA8c>P|= zb|TVdi)SPGiOfWetAW`bNoB#}6XwB3se!exm?h$WCt;;HdG*WiC%JY%APD$7gTC`L z`K1$*Le6hN6qoql{`Y_V+yDDt|N4LaSv4P;E-REpqW35{eLJ1zX5u62>g?>kbW^7D8Y~7bYgnc9+olq6OyK5m z63;PzEf zd?vG~o~0B1>3s@2B)z)7SYR^b&o?b;U<5{gERZ66^8w1Kx>t6urE`GJBenb{$d}15 z33u*y@3Q-Ux}WZKVr6VTL*md}j+>r6RX&HEh++a%o(|J{4lIPlz<~%?Ju}@sRfX2Q zd_|Bn`I;DA~%uJLQCkJblu36M!`*-6-zCD zFRx&3Z{kp>xQlDbVjoaSrVnoqPrEFmB}Gg1i~z#~n-IGCoen$GuAdG7nwMwe>!TAs z2h5#Wx4Z$aPy>f5N2+w*QS=8eAAXr!?|jT(hA&=(yPdYX%aN?IOn|twLEuse+e5j* zx2Xbx{9TxjOe!&2CD;>_2xb8n1SNuhN2gxSNyqx}B#I`vbYrrleP7j4BMANqWZgp& z_|Kvn^{)soa(no%w9UD$W_*-C*RS!~adV$asX?8CT6hA5xoms10pje%SV39Nh-_)esKp;BX!h7i>xrX{@R$xv?=r9noW@XO@r_zHP#hcfJhOOHJmH-thU&!n#LS*~O#;`+N>k0m08qR39ih)HzUf;v0YNxVq`gl}!) zVSpd6q6E`dv-Jjh0I}=nGE;TuYxT*{J2#e7LyiQ@H~|Vazl;-G+6<0dJ4hHh%IMoObqZiY8WHDA!#79cl_}LVMlSR{mahiq+xw zfgyiCty}yD^pu~CD~ab((H;=uj{m^OkG)#_$BP#)p7-$|Ptksp@gL@G__zTfr%@rN zeMDnk2vueoA!6#%2pKWr&NQ)+8#c+t_3#qY>l>}{CTq&=X_}ya=z(F**Xb?YadFO0 z)}^SRXuWouF#4h_y^L$oQjqsi27Q!4&_@~AKf$9#84yF>!Wkc4iGpULdZ>1kg>jFd zbS<-UxdbT&ZAHU*bdA!)^)O0em&j``bRSX9fMGV7TdJA2`82`E!!qJvD4iyTbe~x3 zj5>23wJXb#TkwP2;9n7v8_<*W6mI6 zEMVUa=i%Kjyi9MR;VmXau%}FF+w2lvoD5}2qXz*3YP)_wtDoh0v0UxNEJE)m82Jv< zh77G`V%lf{>9dARno^59nh!p>YjO7qEv|e3Z8<$Gx zuo*+aC(-pDX0y1eIiYw~zJu`E{$9t$09j&34sr^m`dBF@;xS!s7F?&OfzvQ5!jB;A zxHLF>yN#!RE4vwT;mwv0ej)2~a$Pnm;0YWM+779dI~Ny%?}*ap1v|ieswC!=Rn6$0oDRi6{pWvsRC;S z>aGrp%_l3Pqe^UJIj}ygEVji{^|Qx#Nq*;(g2swHW=m7aA1Hq?e~FF)&EazklacC_%R z@9G5YQTr9!Q&xLyPfKYhHO`u_X`^VZE7rV=cxa~Q^KgMX z|0g7W&$NktAwOT({fDBOuw`8R4H5IE14pD)1lEJ1dgjzk6>YgHgX+X~gqs9emf!%WLdW4P`9@>mu^0x6( zm~?VBC1r=zrs%sDo%^t^}9KY~AdCAjPrTrx1ZLIR(bF zm;l}OzrDQ|FY5W<_V;%7di&o~wBMxt&$fH?>Mecbyug`WqA2lYrTTWC>OP`SqQ`|Z@u-v4f zl0aG99tE$2!>AInc8KsdmHZ@sZ6h(F%2=@Po8cpmVnFCz=%Y6=^(H3KY8+V8u0({4 zbpELBnTH98%F5Y(5=L{QTRV_PxOq|6O0*iusfsSSj7p(`?gm!Ma!2~AsWDVUh_`r1 z2}?6`g#t63q@u)7DFV^L-Dz{80gl2@94inxnz0s z5cd|c-a^(}$n@4*$gHimkYVdBWS#KqEo8lg%<)j1xi9*Iy}K!RdHTjz zv-A?{$}RY&8rz~3<;#-t$?RF-EKl(Ypqns{_%l?q*I6Q;ktNIG3G>+kjVeQZf;J`j zNF;rWbD<$LmLp>xQe=mJ(|B*0d@R=o6CE#Af~t<}9(yXEMC(hmR!tT3%Rp7SBLvlHR|6+djo-TKdl{8sCp+ESuAM z1B8QGc5f^D$>R6q9wt}G_}GBDDf*6r)9A)jqD-zBYyp9XuKkdI5xdj)4rBPw%9LB? zJP`IQ5JT$<4H-4ZH?pTtBK#cN2@+lQkBcE8ar?@`c zF&6r>a5KZ;HEs=m%d>Z3vJB0uITXe>%R`_8NS-lg-)U@vAQDR0GIOK(;cCK28k{0JsBtiw6|6LB-OHg?Iz_a?n!5;9bHa=_@{iZ-)W)9g;k-ui} zwsBgo2_Bvpn1Z16qu@F6=lfO&9{g?Qt`=RrXDH!~Z9NHp(g-Dz$Stl5qDlRggpgy^ zPND;0qMtwR@9A`e*q$YZPMI_u&A6%Y@*2zxQNVhVBuHI zs{jOzxu*ZKEom&|N&RDpSy2gsv5Ez+wt1XrH?F~;>)1SKf%z`U;eySlF@70`rTA3} zH0?r|&RT2+-`C-)>}#ciNd%;VjG=7liA^gV1B;(VdGhQTY$<}q>mm%L<>o`%Uj@P7 zGl95&$#z&ovzf>mEV^f8qXV?$4>fq%Rx1w|`w z+(kd`#Y9>)TVW}+PND+7FK`FRB+A?j0Jm?B|8zh~R{`t8;oI+y50oM=_!dbNc}^g( zA1w}@T-=vGtI#A6KAZ#WN9*YLTItk(g3j+sEo7J+TG^4GGAo9;$$GS$?lGF!(xFL!?_&SK`@6XHVUv8X^)~+Z7M1yQ zty}!3&?0mU19iuLzS!N~->$`fzI?f})5m{4Mf*+0f3|JI$IS#QlC1j0iMl&C!bMwV z#x$%EH99tgjk+OiJXOkA_xxLb<4CnzK)C<)ainrN`Z&@)j?c+%MIMP0j)By1} z5=W})Azk>z7dC>&%&9C@G?BWLY~CZ#0asKWWf57x*J&-Ju5`u6+S}wbY5-cpkDx@P zdy!G081{e+t);7p-lXc?#?wBOun#5dLkacPhZ0&_A4-U=4<+n`Umr?;*oPAK+2;FH zAhPwTK>Ac5J^vo`{Cm&8_x!uH_58cudj7rV-+!h2d!GuVPX$t$ZJ!FHPX*Ga0_jtM z^r=AlR3LpS5c_z2Dv&-Eh_&^pK>Ac5jn=0E=~IC;S^>@wG4G!Wq)!E+w>}lfCrAat zLD<52HyoQo_MAph!*K2)v9(7%e^ajx`dJ@N&o=_PoO26HUAljqTG$P-9zv6g4*ANb2$%w6Ohc z5!EH2mXFyIS&i*M(be9FxT37RfwVUHTh-+*FSwN)`PK678PtgKd}l@JwMU&6FooC^AoP={9W-A zo>^V|&~_v2wi{kpWa~9&1Z%l9yAnV@n@g9w)0?3B-=DrWX!{-Nja>UZ>U;Tl-=+Ql z-S?^Y9KG+c<+x_i6bzMl85-wo{j;XQ8QHQdAA z=(OF%-hh1W<8|N2{&%u3E%n$Dt&{0NAAg>o>G_$SpXvFTo}cN8PS4L6dOeJv>7Z-) zw5e`keOGl7=+oHt?N`>U`F|(TVwTdU}r|HK7OuktS;p(<( z3UxjIC|;5AI655D6@03;B#aAGOm!MBF~iDH3Rm%6VP2$ikuR#<&8sKkcj3g8;AF03 zLtt<`r+a(g+NjhAw`-cm@b^6ow$5qbbq*5$n?g) z{9=GXuU;&t)A$ZL4C+%lTh3-@6dUC}gp9#L^)45Z7jr%j>+vSTVCynYww(J7BcZSI z`g_A6NBVxX6%}IyuvK1c;t}0qjn_VYnGY4^`pQALs;^C=%jMNq+Vo*a;O1!@dRBcY zN^a^H^S(JcI{xA8+gCrF9)G?&zBoHPI<9e)B=Eh?Qd2zzbgMm&rbf?Fjk4b~UHiq$ zXJK)Dpoq{&ErS(PiS66NuaDnem{2`1l8~?TFABS&7H}bl_~y%tA5Oo0bxf=A^b0}a zd6G_|(|Y8yz6OM3_pjwf(*w~i-;Aj2)U-D*$Kzv}-@37_d6Hir9e@3BeERC_+mqAx z=1)8@@B1q3K?5Vcd4j+f)ysBNzg`v9%eA5!#fvsnk5A54L$wOqit2{;V3B5ZwXZ~D z0h<#eu9tOHh`QX);N^p0=MR6_uM5}kZjp{tkQm<|of)CSCT;_}yH}5RwSh&I8Rp*4 z&$9Hgrg6axU2md>{C~Zm!?AT8g|p)Nul0w7K^4V@AkEFdBp7&1@v6ow?Ce%Q|4lku z&ZBq8r!>CD9Dn80j9K2dW~nJV@O*%dld5PYIiz*C3Xx_oc zJP4leRx@)NQxZ^WF{1TNi#TKCAqctdi!Qwy`iml+MOj^Wq5b$E<%P_mdNd33ob>?t zeGZE>6j6@DF+7gPDv=6)w^K%4)2AEIZfu-zN{J;j5`QdzS#buRTJ=ZGuK&$hN_&Xn zSF|?$@6L;tr2pOC-rL!Jx&I%wclKZI?DhKJr)UoSFD2u>h^p$}6?L~J$|02iK1$9w# zZc^FyiGTHM>sBv)0%hQDLv8sh)hW*6DJ2}N=`}!%EiO#CilV!E0XA&L(@lZ7Tg|RX>${1TiLQ|=eVhB+Za%ujz5BsI)8~X4|o78sq6I^3_WRqk}e9oRFIUL z)kz>}!m}1AMAiWc=4q-+Bqf}rbxz$CTXtCaeSK9>w}>u%{P6?x(%0Uq!d&Lr5akKu ztHy_o`Ptt$KCHe!i!Zl8=3mYliMYiMj4y}H=MFjDtTb)tZ@+x`2X8sOv;G$648&+> z_kZPR8~(Rb&ut_`?3*2@ir4&%>Z-y#|M>p>S$$Z*=8u4Fh1aRC&)_nPqp2y|lGNYh zR+7&rNiNBisDk?%^9Hv{u6R|{#a33DUa*SmZJd0}D>Mri?d0sJ@taA~CC$ajtIBU# zM7ijU)b*3u+6yepnUIp;O$K-SSp-XluYab=Bo|CSt$j!BjE%^#Bi^exBPRL&+w6#P z7&f&Z1|rxpq@5ym8MIC@*P?}Tg7CPdd?#SI#XJXZUTMB#Hm|&vAvKD+Wf*ZVXfCFK z#4{qK{snj(W$zekc{TUEc|WRfPFK0O;%YoyJIpdTmsGK*AUCO6Ub(Ev*< zkoX@2KX23tTGOgpQ$c^~mHu@$>wozdY=6jX|AS~j%lY4X`Qiop|G(Vdeg1L}{r`92 zr~dpuK{KBJi;Bo+asOBHpU>$X4u_JquqX^%LA0R{o?V9HQMfFwK`Q<$@k^tRUlKt9 zw44}Z$b93IA*yBHWa)B&u}TBs2Gj5r`11fm<_4R=03~hyhs2fsV!AXB7k~6u^&rqs z(M=@&#SSETKLdnF`0l|Uk{2Tswm42pVWR(CBE++}iLy(Y)m2nrhg-C7;XkmfBmS3} zH5#0RA^aB=NFxBXr9n%deR&X#Xu6!`VrCN+2||C$_zQgc7|2cHe2jIR?!^XZAPRld zlFnk^H8L(k9NiUBLg8(W(SOEN+Cl!LRhU?DVB8>SHWA7VIdEY6AOHtXR4#kS;er*` zJ1{Im-L7cc8ACjEYUS0BaDAw*k^OhEyVc_;iK~CqJ5=L!)!+4Qc#`OeOk>U&!`nM} z0xD3Z@$6LxDV$JRQEcf~2dk|tQySa4mromuD>QIdvo2K2{(pt>Bu(ZqmCwtf ztC-5VX&c+Rol56HVaBew2bNe#_dk2D3U+wecwa4zTj^`8H;`Pl+&L*0X(Q91TnEQE z8#3xvzx)hP;p0>7V7|1mVyw)gA)KQCUs z=;J@1r0tA?HxZW?(SK?+2H%~(#X=(4GD*Ib~I+Q?d=YLN_ZAl)93#%EXK~%0ps`modk=+?`1x~LH3qf-u+nVb zt~#rmEZf3ZUKERsy4zvL;OTBYdNjjQMZcEsZ$1A%06KtbK7T3}k9_|3_TbDn&VR4} zeUkRmPg`GX9N#TuwQ35B5{1&nN^Ji>H2o81RkEij(OSPW~iAf!hd-$x9a z`WH9?P9VpkOMfAjl+Cg9(l<(1%~R#l(o|V{hR(#CV~`RleI_cYy*jP;)gzr}@0t3_qmLac{`$O~-EtBM_}E2|M_`A{vU=wf~Dv)Hp-!TCKQ ze(;t5*DP^VOTa)XWL@D|`LQE%i!4rxX)ySQeE1JY7k?|Tcu@bnW&USn$7!Du&&G{# zTnzJSl|AZ3*OBd$QkU|3*5{!wG5=foNd8QjPVhz?{Kr8s_+ilf@BqUOfcUnwpeL_V z#E);F>n%c=uIK9IB&IUr!CG%FZ3i11KsWHNteTE)BwvAyLF1L5;nZVk)ap!|P}nRr z8D8F3Ie!enqu@(^p7f3NR7L5WD_UQOp%F`FA{wj6hksp zwX^TU>)$iDBYjI6M9!`e<>EJ?b{)@R5Vm*>6$ll6y-X(P+H`dF)mP$2L1{|=DvQ8d zJ``dv67?ZF`>Myn(@q7&PBc z3V-eVAUI8vNVs%}PjMdI#L=ym^>J{sBXszjrShU8@`_kHkRdPuZPm|>JobVESLB-J zb0%HJ<>>tQ@O>Ff zWDH0jmi2Cncpjz8LOCz*ZU36~TCQcqb$@QQ*YKxts{w>{?={cQ*)U#ueBRxzpBI1n z?(CK47q^=Fe$~4`2FyQvH;jL4+aa7SuEWk}-hWg0n1P#n1AjI1t!uphYy2MzUF&lE z6s?8-+uhwJ|Ia<-`(9wbo#(qRwtN2XDOyFFzq?^1D3$c%@T`molm7OYNBSyGHh(sL z7aR(m2v!%rjDje}g5&}G@l3NXJ_|sI4ufa2_%aK#`)5+s_&WrknIY+C*3N}6>J0e~ z{2e7(DXD-=8>-=e+lnM zu^5EZU|C=>3oNz{SWlP9m|grar+<8@1DZrP(F`faD0qJz-9H1Ag}95lKNtg5n2QKC z`Qv+>K-ioopcW^{VFBC}gxzvrR6&&^)8ugI4B=6V`QHVLcoEIuqm1ganWV(43$Z>- z?zOcOYTqn^$wat_*t4=cY)R3O#gF_R^`lsVKuupP^@txyZJEeYpI@iT*?)w}%qQb4 zLQgVSl_(rv6WI&y(s#OY*0;X( bt#5tnTi^QDx4!jlE$#mUT5HK}09XP5pL5Cq delta 20766 zcmV(}K+wP9q5;IB0e>Hh2mnp{SZ4qO>^y0ARL7P6+`pnm`d~ZJYpJyXJ)eYx=kyta zHHeUtWPIY+?^Sn~d7IxNshycQc#8qEd18zg9NRN?f^%ZPapGVB=Fg~G;*SeiE{n zyMbfIAX+6QZPgNE%MorbM0*R(qFeecwH~!oEn)8sKV!_h~ltauA9P6>J~3szB`HgCUBCb@Qfhz(+M~4 z+}(QT>S|ya+!LF3u58}i(2JIHsGhipD$8#^-OW&gRmUK4oMj*Igq18fYi2u2=X!3H zhoP{M=zlFPF=w&aWNR%y;>=cT74o&!GQ*geL2#Uhk?_5;0s{jSaS`hXcxCaZ%6`NC zQ#RSIsmaMbWO0?nVZy0tjn`Yge?q@^lh}-f*DT_-I&J6|h8m#ygTM(}jZiBXFo_BL z_e9**vv|JE7kb)&Od)RoHMqT$gaB~GYH>Sp;D1xC>G^5%p>=L0F<_q!Gyn_HY=Hn( z(l!l_dls+eS^-5fwpxeQ;dO*WjEZj!c7g+&EC+N74~@`wc;S6cfLYD!c_p`BmHleW zHF5aZ^VkZVfUO|krA%t5p%iY= ztW+uy6PAHxE8Gfsj1t72FQKGpW)ZO)(4()h6!CCXSbV{66e`!F(iqwoC!dI0Ks<B7t**Bp%QUS8SPqSr?9og?|)? zaS#GK^aK6bU<=PaOP>K3`#H~P9|afW8L+%j8x(|_qjZ>f3(*nZ!w#xhHU11}2C7Z1 zgvf{RQFg0rQ7Wq1GPDGK2t)Tnx4tbo&VJ5;SRk5C8`c7v#N|L@BsXv%B5`S_n?D^+ ze1HxPH&=nO=t`LAN?1TdOZZ77sejP&k)`N$ASZyIVOjberW5(#K_Dk1fCk#cv*iC_ z93ghh_rP4huJJus5VWm{xW#~=v7^YMmomYk#A@O6he3_t{wkxufM>#NpRJi*0h8RY z#FN;@QcQfu@z+3T2s&sR>|)w40u}MSixYSp21aJso*?`~?)rp1?35$~`hV3!lW5Zb z!m}qcT0dDn^~b9lm#%vAeG%S zgtjKny#oJj<85#UI~T5l7uPwv0WK+g*xWdO@8s$3TVE{VFt)Hb;MC ze`0^Mb^dzie_q?ZdvW{Iv(gEr_wa`7iA5wf0yAz^*(7Y9$*dJRVmHeXAm~Ot32|8< zL&f%&0+5M0-Dn2Futj8na3jKcL9o|qZD0?{7@%x8xOWXZk!GyQ=BJ_oy};4pYlaQ2 zxTmUYW=iS?payBimwQ*=TEhHH9Y149Gf7>)_n zQ(0Ikg}rtatW6U8hBOhW2c)+mNJX4}Np+LX1|@%UL|?)YuoJ=RY&Hvi>zx_e=Cf&4 zBYEdJX#lRUd+%TD{_T}?*C@423@flU|03gUzBLvu_mfy-giUBT4boAPz~d;+xoN2p znn=JWp*@;htxO(7hibvLmqW1%0BXx1^*Km$la*+KzRLwxrY2l--B>e4oQY0lE}96; z5Nv-CcZ^i)%}qFdvx<}&8!`0~vYr13voFN439jUk+UP0g2FwUFzn>WwNvI#?j7vPS zLQ#i6y$)OHflWJPwiWQg`mn1^_407{6IG^^{@uRu+V=HJ=m|k=)W~)feKn+eRGFRG zKM!p!ZaQ&GHy+r(?*KG5{UFMk4q!M5LtTG#p_9l!=ZhQQ1Ecp|tsFQoKRuSy`>EN3 z{-Lma1S5gZ<7%7yt8cT6A{st9@R^7 zfszNBewo8I&bI#W3K(!8GFv>lgw{nwB-1!I*v7o+f=<$Hk=*?L_6IBIp4fljNyF#u z@80N~`J%S`{p}Ayjk~gO^X4sVIkolHpF8J10RZ3MKCKXgFam5fM!6{z^#)wq7*v3v zP(nEY2Mx>|rJ+)r3;RVPFYHJ;Zq%rPIiraoUe8O8{01S6_^^JeV*pMktO524`X7la zb5Fz&SO_;_B9h-|M5s_J)BAsN-ViA`e2*0Ce)S1#p?kO9+rIXhq~7M;kGf|;=udWT zyv-^MwiEVaTVHJ4yZjgQKJ{?WE3FWAZcA>6irP+r?>lVs_IsVP@1}f|t%fVSY}F!= z{7>KJagxbcJq#4FSqDE6)G^Qpcy|Kc&&vHkuHC&i&vf7UZu^@zz=wZ;{eV2xINdxa z)uZeJyp~^CfeH);ZJ&8{>(y`NsB|nk(6WDRnQ0KGX@g0}l&@24K**%7m2NY1FPl3i z22uM-+%XIptRVC*N4?kia07H*ZF%dRueY!Nee289o$uZN47-=Vfq(Zd{}nnm#5#x` z%##we$Klc}?_B?O`|f`g&<-h3P#PcpzWdHMoiiVAzjo&-iDc*E`Ocl+ZGC-f>(ppV++f+wObU!HwU%`DOPnpQ4Jqd4BWTw_u{q#ybFA;|FV< zKq8*{yk`0ROsQy^8EAk|KX20)_*9WC(m)E&)-b z`9%PNzIEySequuKpNlv4@0*{RYL%3)ic9I9|GabIe^YODoOMrs(fR69_riaJRqC9& zy#47_5`vfvj}C?@{Wg}e9ZYYXOY;rQStOh`p&Q_u!Pt*|y0i1sbN*MxqaQ&z)CV`m z=<~m+skxasI{$y0nwqIh?VHE*zuCEckI(-e#WT*9@klR}IaHrsLOT(j0nEpmK%Xw~B{=aVytudNu&?c$4-fdkW83aM#MT z-9J5DW|-q-zDFZ*Neui~O0$(q<)xKhuYfCyiOogdg*bnl9ba4lha!wh<)(-y>91UV zsa$_KoTR_Hq1BwkKl*F5>P=>V>VRJo(3xgb`bjxj3*c|vJOO`W7eD&|r}UHmf)U5f z5HpvowsZ*Q1vT&@x8hP+*xa0y-Jz*eUX3hpJ|;^)Hgk9J^z8rioc2Lhg6zHiV|HeG z_Hq5kqj)Zc@*n*32PFf0=l|T)zQ_E3B+p3x!^^@6@~>r3wAcEe9qj-7-0XbO{!dTO z!IKpY9}%@uIw3p=*cX9ZzI#+<1eQ3HMGYqvPL=1&7&_{l`7wwDfIV(dWnSVq>63vC zDSy{oaRLo)8 zHqqB~mP3j(7kwvjy%h3`k?$?jp-vfbFPp2TaOAOa`Z>}Pi$i&q ziGXZM;|4Np!UG6q2|HB%duPeLl8_f`OMgdacMp7i6mWcx@cJg0-{k~{*72ASJkTK% z@SgW_1{zhEq3-(*H@!hqC^9=7umRC@>wFM4um_T;oc66PA4%LIL+I!|n-MwORG?&! zRauo3SuLIwVu8eX3(K|RGb@E=$~S|{9E)OwNp5%NT3HdJClR3Rg@-_RwoRwjN`D-| zO_ytswHOctgd7WGxFv0yAcsY2gjFlb6Xa=*@Ro05*AgWGILYSbn8lLSZc=|m_ z3QE)yh%ll_8?)S#k!Tub2(e2H(0`MV;}L{GOZP~#m0f#0J-&aku+}~Sl@Do^5gb4T zBdaatP>DrGXkD#x^w7$&G)GL#=%qfVCvL{tW=;A-yWc# zv9L5DtvCQ)mDwVxdaMIV0vn5J%B9lQW|Q@``WUcd=57s> z;1554X>yVMyT^XMaP-Ka3cSOx7TjY9%FAdq2wAKg0$n9>R?U{I8|z1(YK?GT=@O~j zGHc5VhkNMIKlQ`_us|Bz%a#e)g|L8Nj!wIrBgY>15H4O|Ygy(gK{=6X+q z$hhrgfF#5*?68+%vDq9@Y?*hehHdB_01s7v8u+XRR&6gt9HxKBf7qi2Qf|hu+4CMo z_Nb!kJ&8%njE?xI-fPQy8;JFMd(vA2$ zqc_%!1z^o`PP_~WRMoQf`spj4FA7x}`m5(la$;^D*M2-U{@798@MmxFKlDr~xcfYR z!07mYZf@Uf&;6h2*_rvr_kSMA^A8#S_ln`e%>x|au|{~MFCS2R*_RjSS?Q4cfEPyT zr=CPX5x|2_7*IWg2TDNLh0wING(6hw3#dI&1~=reZx=)A!_O@p+8dN~HMz?b64tR_ zEDwdHLgnebgvtefx5&g?;Dq-E04pYcD@<)xZ?nb4L*UTGB-19WHy>!L5^|w@P4L2$ zrK7bveq$oJ}!1kO-S$3M;pJkAKP z#~Fd)Eck&Lfdaqq=2T&;z+1edR#Xd<#a%DAZEt*3*u?mZaQtl>Z>kLnVG65%ThRUV z{X8^cv@aogB)ybED)&JGlJ{Pe8!*KYys9UGNU_a>?FTvvD>ZE}xV>jy2=N6)&hP;d zqIri592wN59d(ii%-$1_=}{AU#l)cjRhXw0KyRUSc8p z&S)GQFifK&j~Yb6DJmdBX7t} zQ4Y?9AbzMMgM<*^gN2ezWHBxhZ2?LZK@*hntLzBVq)W#jDiPh{m2<%E2Eb27l+Vj_ zj6kWQs&FE~5Qyj^arPAT`)M9q6jX>Tp3dK0u8d&r;QWc?8D26muzrz;KxK1qc2;)H zu~%MBdCohQjmp^&MO-9*X?o{&sBS_)wv<3>GL%PcsKIF@e6osbeHWLm-eMHC3(6lF zKAPc=T4d0fuICVE1)@LC=Igtl*)uq*P@=O-k^M9z}7APV4d3se#OtAgnw|F5Vu-o-R|GaaCw|y3UH%Rao z0kkGWcv9Mito`8PB0G3+@qR45YpyS-L#AgzRrWb&M22elBqxt ztR~KWSm%M#0mABCFnsrvST4Y0M~ZTgziaa_u%TN>$IzF1PToP<0Ux@cv;#&*dtk~* zSU|rs!{l9mr-@Ww3 zO;l5mK)>F!))Js(5q3Ke78T}ThiCq6=}0g9ft5Ph`Z56YLvZOaL~g+p5Y*c$J?ERi&?%dRC5k(5ThDis@D> zq@M0ruP_$wh2qqFBkjU>u-3KXc2%L?IIo)nT!-lSCMaj1abpf1OkRb~_u{5PLUvq9_GE9Yk z^HYm-v7F(B#ikx&x+a)hTryn3?+RB>b~mZR?nz_IkQ9ju>MP*Z3Yubz{g`cd>63Uz zEK;4_lp1ln7%Tm(hS7ZrlK)guqz6e9jp;&X_dIX_?5V=s*)BkV*&uk6qD$IPWRa4O zaKWe1Y{4Lpm!nA02U?)A`5c{}2JmHn@`{I8&gE{RH4N+9fNL2=r?E}M3vpB3>zq0K zx=8aphT8~qDH6;ecndi@xaGTl_9GQy*apbyR4y ztHev=5^Z{qBTP+)+`LAlBAjWvRM{3W?LP)DP~vUsYnIMnd(MD=eZZqxr6U4B zt}*X@yG0GTCNV9cJ1sq+$I*f;8JNhEah5VU67+SeR3myGmL!gmBX@3P?(VC=&`;%0{8#YHGK~KSoum0wL(x)L~wqF{4}_`0g({)x1a;; z^B@;wfp|$lbs#nje#kA{&tqzSU^|#W`-T-j(;XG$3aO4PQku1Y>2+)zc@SVXGKz!s zY#5aq1S`MTJ+9Qfl43m;7wZxN@qiyFE2kawC{k58>hpGA#v zu8|?`$}*GQd+08(#difV5aOfxyf`NTt5D9q{(X?^6Q@BenUzdY1JXg6vO#}HS_VZ| zQF9nXy_*xFoEk)bOns0umR>!xXu_D4S(seYu7p&~MZ@P~^cGy?=@yi{Ynj!L<^EpBMyUlRmD2-bI-2tQ)6-(hiv?1Lz$7*+iZ zr;!>8=CS6GjJk2f)l;li>FnWz2&{>)u_ui8o)3l331;DcIr?-7g8wGx`e75I0?Y}M zUjwJF#HO*E{js~xy64=c;Jxm`2q6wAw>^|d>4((4PxHzlrQL`tmZhL`C*VduwpJ5$ zI{MY-c+NQy^-e}*i|Iy&o#*D2y>8?znul>h76drXgr^`;iVPO0Pb^4SzgLSuM*Z+o1cUmu z822)xKmdpDbp3X4#X>-{b2*HoctEJAKAm{ZBv?iT`Md;4!k-lsL!gwaxJAp`;~2;h zCw=CyPXl^Kwzn(lEQ;E8TqxX~H<DlhE0v6J<_+D@=s2KQm&_v0NRmhPYZ2UfL_c zG`%GiIqEq9H{ChKy1qbdu*OBKvX|((CUm#lj=+fx$vDQVP(&zj@&n~mt{AIkwTp}P=kywL3Pj_7eCFtAH?ULd z2dPafOpj@dhw_bcD86##zet=y^Vc3aI%KYO4NV2~4ui4C_WQ5_?IwL{ecUjsvYwm; zw?a3w-wN*?kA+jmLg)Y%s}hHs#ppql$+c8}{gUn(pT?$L*)LHtJasN^H6kZR`Fb1` z>;&YB3`KSusURT7^3$Sg<6mvmqP=Vva*p$wPc*gd!ZiVA4~pSXI6~B zn(wN}CEZoIKIkh@!54)Hg4lVTGY&HL5fD_ zr%X|wHMp)oYnDs7J>GRb`_d=H*~#gD^aAY>ECSq8Pv=r!A9OgCZVXN2UvrdMWiJVz zlGqklvof=Kdd{6r_=eEa(Lfb(3h+j?jWy^}V#tV4GR~*rwe3^6_Dr8UD)redE8J+T zU6CL5>N!sPjEC|7=-jJ~`%704@l{3F#Vj0Mmm7%q{s@YDVsZfKDe91wIBo-f?q9`q zoRD5Y!~~1-xgwj*Veqy8YC?PoQbuX$P<@QqXk#cQFGGsIt}i}=VS#-#x=zwt#u8qU z2U;FZBGpv~T-DLT=j+OcBFrpMF+_={CP?r$-$KV&U~6UQ0Qsfysfj_vXI7_gA0%I5 ziCf5Ha4%mSRF})FylXhq%TDrt>?F@l^6Vr}**eLylRQ_PXgk){SKL$tOUt-A#i1RE zIp|=XE3vu|)=8y0*Caf(W86NTA7eOu1M3fAq==L_1 z!~ID$acG;D8Rw8$yegXF*%=RHyQiAw%!uX+W4glD*frF#v73r$m%`bGe~Uexi;kK`DNPyvtelC!R2+HP z@93=eOD(m>n&K;`Bb0Re9BIB0GxW;RnfHHlVSp1Yv zUm4=m-D;Fk5|6X^#%71m7J!|an9&33k4UbheGwD!%ffbl`l<{Y8Qyy>Lkkq_S_-|0 z0+=?`=oJZJB{fog?|I-LwqL-x1hz7pU}v04tA(on4${$Rxv)pFh%9LjnFph>xPZt2 zd(7bbZ}LF)N5ceO5s}KvuYdxPRlnT?DDcZG#t#V@vW1l@WLJF5?jDyu+yftQ;PKp7 zfQ~estz|)f*yc!le6O${8Wo^^lEq%qq;G3K(b^oJzXBeDvY8<|tmk=(S{BRB1+I*B zl9Cr@JB-5dSM8_Z(2w&&)^sKCS)5UCEH=ac%?nB4j2U|!oY+39n?4WO2EC2POiE~8 zR27j*%Rm1sFhhNeMw3` z5e)m+_&uk024iZtwg5At0w59bj@jRk7auYT1NKvZZAna2E|AmwyI^A0nU#XqCV77O z;SyPYAyMiEJw9ln@XXr@1>s(nRu#n2IWu@h8+_xre&#?@jY8VUJTm;d6W0Gf8>DOH^9Bqt;+EMKR;YzQ6z&c z@+7{CFhzGhV)7n4er`1JJAzwGoU30;W=Ir&I|rUaZmoNJAgbHlx9|K0eVX1wHbqg) zD4TSSSrn zex|uhj^_X=y`ky{>NNtwG~OXkfqP+8-I^MMh*K*IFufeN*c5KaxLuy$VlS-KNuWu8 z5^w?$;YKF~(NWnC)-@;$2g9)p?lovSF-<7<>FYb5zYCLPXbTI04rEkvbT~c7`xT51 zXR!+s_rG-cWc7oKSMQD$q9GX3#I6FcGd$Rr>K4*{~8JI<;(T7*nd$!VfQV}Rr# zI1k^fE2U+yNh5Y#_N?YWg46++PgCiCB7{E^n{31pnqQU65)=Dc)KOX`&x@Ar#>>0< zSj&&Y5i60D=75A1rWSf)x;-bk!SA=W=P!Xbs?bYY7QbxCblM2{6ArN*JVU}F&wr?<2t;1a9<7{5DDYsuf&!MWt zaM{S3o%1@n?~~)eB8)Sw1R`o1>?5b-2dN;8mU}oucGjoyTsY`c<6JRD&FjNnU^E+Q z@+#XZKwlMnO4Xr+rOr;(g$*5I9<enu1%x52sFdvK`q^Vj?|Mo%DtE~UMb=lD2G+P#|0N0X+wh7>kB4B=nFS`$reO1C z0mp9QUNLnoq^ET%G#qJb8yWV7WU}oO$$Dv|{Kiuw6j~3{sJ+mC?UJH5vAOzJfa@i* zCx&?(DW5UNqv^>V*uZ5FjAH^v@?fHeL>|r9aH#x6{&cH#j8D;!Y zg=6c4VNF4nW076x|56at>ol0Ubv4z-UcwTj_M za6U~ogR``N|D4=G!2M?MDvfgX6aF~}-V~e8{2gzteie-iLHK$t!%;%#g{Wm2c9F}9 zfO7M&y?E+K7n6zU@*G=B)F1$8;Uko!NgqwiHP|lMo>mWk;H&HB7>0(hf;3F)O#$HF zT7hCvKxl;st`Rdf?hMcDCeJS+a@3$*F<4}fEr1UN zJ`zlvok!UfXu246v(4CLsi2IrBD2+trFD1Dv zb1B)3zYQ6GsmW`~h))ft)AJWw2WoW}MPfv**Xp1P&H0J-<4z--hcRw^yj~U|FM)CM z0tnI;rbD}t*x2@!myODr%ff)0tRNW5+ZXb<(9dig=va{hqXTAu9!z7lC7GxeqNxO9 zwM2<{%A^%T6~#Q7EN48CjT53V6P3s=6Jp*Jp7aKP(CMQgIiI;oNG3yG(w<)&{ zUysCp;xPeo)#}A=Zi)cS%MjH>8K_OdIa_S+UGN&ai<1eVZA#svL2v@OnR+jJ=6JBO z^$vQjCGeV`Gp|VtICuk0teTECkk(;6bX7?426M9*xzkU1Im9+qqoBRxyp|d@F1Foi z%{p3tAK*k#-0CVcYAMdyORIq#qQ2dm;dD{+83GsrLqkl zHv`NiAM6bP!{5ssFxs5KgnJX+Eh_>y&Q-NF1*xbel(l%=#mnFfUbEb^LDo#CUUV)H2QKDL)GY?zo1>-6DL zI;pQ5F&y5IBFJ{`4mR_Iv|xip<0nzb!*(>;#DWz3FZcZ64f)E(jO2xI%qN7(VQ$%W1_A)Y{`;{-Y~(vgw2(7tps z(s2SSL+XxjcZ9M1b~ypFTeo9MMR;X6&Q?0mL<1PO@9i#$&5o|&)kn}ktq8z>?F>b@ z*&gAzhKDQqLz&5hZcB~IlL9Jq@F)jgwK$Kq5{M|9ZH1#*K>wt=Pjt2nSdx7Jy!3At z_1rIxWdl!wcHAR-!&)@P9rIk*VT)$|aQ&bX>DCVv+k*ql5>ythHPx*(8bw5V z7se1d8Qm>rp>!7FzKFTVw=^PuTM&AtQYt4@g<8I6eHdpxmWuhHVbdMgUXVTu&E^ZW z(M&Er{$m znz2F6QKNcrh}xH8LPGd6Fovu($9+1^(L5YY<0M*PLd)D5E$<4Th4|}#?;zLGDzCj` zQ#3deHotgg^q*;d?>x1VCD&`VdSwl7W~+S$X{BsOLDx+_jYROlPB9$P=dO6vHMO<# z|6v5PMcpUS8vK7=?j7t@{C{@0_xC#gpQmVFr2h|Y!^idd5h(;qUmx~M&BF%}AwnnJ zcc@tNI-PbH#2_pRcD<8-W2&XB4|?seM`u;f9alA<%21fX6SCRJiqq8**d_I@S}y3F z{{TwD9ODq=#BzLmqLt?0zAM^qO(Vgom7|qXOntPE=f{T8KDHW7I)$^1Iv+x{?8IGu zNxpEJjH5ekE)S->{BrZCz&pRKT&PPXI{mw_{seN>RhG~|ix2NY8=|Zp2 zp!RJgh^S!PV<>;L5m-va)TOT8ZHnYBU#z;a#^cAY<76_6R&&(w%ptLoP2?twXQ;iP zq!*DqRBe_Fl3Rqc*(JUR2~5Wq6lB(flI=YKk?XLriuPDUu9*}xB0={goaHvu&GImg zl(4JL5MoqGd@p-{!b|Wkp3MfsUc|a>Qo1L4^4;;VDQrcl4p0)PGJ*yZtzg(XA z<59q=UzP;k*Ey6cIo(2hj5dp^?3f2Gt;na=YIJ;O4TDo;qdh87*S_=4B5pUvFjKh{ zP={4S_T9#ol0=9VqZ)(Y)3gi>21*H&8D%LuE(AWO%mREYs9H+nSiwT%U5FJj#_@!4 zMUV>wc_Mp%qWXsQS<#9#4@cIIsZ-*2vM0PG=>-XHlA;G#sY2u$GvEn)rXspR$B*we zWpXl}Nse>uL%1TBi7YGe@ttYrQ+7vPO^LBoQIphp5+3dz9BywPZjTT5cMeA{TM=oq z#j}?DL}a3d)qre|sInmO3G(2B)Ii!-$P)20w^E#czWTNQvsk+y0R;OyV}0jo@@p$3 zxt!mCC@k^6{qO(!xBvIQ{`LR-%isRxzx~^P|L4E`^Z)(V|M@@KjJ5E;vAWBuyR5p) zs%g~jvZ_BcT~;8AL?1wMdOw-uX5z!^>g?>kbW^5_3M>LHYFMTC+olw8OyK5m63-#y z+;%H}Svqeg;6AGg#4=AoMczGql|!~Cpzw5i8rGc>9TfpoVq!3ovK&|in7lgw+`H=E zE$;si#l_up5)XTZ*GMqyXQ4e1LMS?v>pu=^R++fn0tQ z*F*_3%rAC!N;+G}GHSdyMY0L?ZT{JSu>;=|v5BM>T0p0y>qf3L3U1=8SSopWIdgk| z6Ng;IT}&~HeSj&MKE6FVYqN}&6fOE0VGI-7gsiLI>98~H>gixW^YUzbcYI3cfVea3 zmN&rVYT!`jNEOaIlKwEv``;wfo$LH%xW6Cnw%YD4M>1`h7~;YP0ZYYg59J2m#tI1H zcWyp1sl;%V*q#_iY!>W-z(nxi)Jr*k=}14GMA0}GZcLW6@2fg$1i@b!S@)0x{xk1} z{Wal5Y!CmHwmH{TjE~~y>NQ?@tQ!>7p}m~&>|DMyB04C~d6bOpx0P|1NwAa$_pK>E zu%DpAPK}nVml2vG&F=YuqDN(-Gi$XA5UaCrbj>rent0eIQOL1W2D~4XdR7{LTUYk! z{f%W2cGXYp2UfU!4tLd6c>3JLg`-Bs6BVp5R_t|37xVD$6c1ytYvbEZJgH-sO4^Yy zO4^2SNSGbrFd@?s4yAR3qun6C8NwkP94;c{f=x*xja5|zHx}l3I*L(~%-D5a&K6}7 z5J5IUGDVLa6N$hOAxeZw6=e8-+fu&S4Dytbn35x%r#F#9c`xd`2*6x{FXWrDga-r9 z5JRVoG#dl^n+D-cYOE`?U?N3~)^MuOmy1NmC=hVRKMrv-xQ!5700@3JnF*riSc2K; z6TntkN>HPrkix)BmKk#B6L5nmZ4h+bw7voNs8tEHD zL1S&Jz7aiz1T9_Io(Wy!vs}th$o03m9!o?FMUkD35##8t0d;by6L_Nn2;SO+!@z#L ziV{d)P1YOe!H8W;ml@riuhAz%?Oa<<1vwNjX_{ytcLmyggQb+xW?6?6i}2Q#7{vLb>Mp?@&`<6WXh;w&M4yQ>+TV3k>=D zY3<@aSWofUsFHXd70m%5?)VRg{Mf6+f9&t?9dz*@Ptm@}_z&|oeB6MLv#1c$KE^RG zxhk`a5HWSBg^U<+Cz@Ex4Vz@6YIuq1^^L}OlQre`G)<8AfH3EOX?lxyT%41Wbtx)H zTCdzDgg!4zFXCF16y#l$K^J8ZbWsNOPw;3_2FQ>%aK=Yhyr3Dc9;zH=ZrnpCZOiNo zmrRNQTTy==O+lKN_M;?riM;kg_Yvj{=x5`(rJ8w@PZJxtUq&4Ch0{bI?-OgCQD@Gh zW@VW()e>M-Lt~YH+)L}g14aJNH$D@o`g5yFw_yOr;`B!h2)mW9Vq2lq$DBdDSg?K9 zpNDt-aG2gi{aZ+gU{9IUwAlr`IO&U$Mh^nSsO|azt$vp0#d5V5vk1JO*vNN~Hl%MY z6VpTsP@mN&(v(`<(R}cMT?@OHD`)$m%U#ipltZh>pr=58BN~YKIq#)h{trjS#X_z22R1K2(Ov2>l z?be>I>}E)Rg*O{M_@$`NNp;z0)QBhtT$tZ!X_j zL%jNmURaJMVyx?GD;xF!?p*0nMm93#KmpN;!FzdvS9;b7*iaj`x%{kq$ql7x$2Ebjgh2+5m#`3QuWQHCZg*}L5CD9j};(N3GK0H&;XcE zeu^{EPu6YbnR5uA9K$NdIR8ccw0 z``_N){(d$8+rjS3{m%aP6zz+&|5>-;!%ToQMN9A(_9$(AmR>b;pAA)_5jBh7LNNV* zrNz$zVM7}I&V)x9{ARZM5y%xRewJxEjedYIXJ^#=REeo#^9e?{lgMC3fvU1|XZh_B6!5$GGXhK80*59 zLHJr7M6t6j6tNqq3m6S?f^pm?$H`UKHdiOc;xK%T6w*bip{C zlU&27Kybxa;zT~P@a}`)b(#ecMKf+9&|t5RA$L5~aWfDLg!$T0qT4|tU;v4jp>wCy zQKJe?YH{f%fT+x!t(~py!{8T;@7v-3t@FJBf3nB^`wRQ)9DgbQOFjBQJW=s~+<~dE zO>UNsL9;&sG6~|}FhU|YUe3thkp@GTMg|~4Ja_(+=!0pp~x$iK#O&@mAr$>UJe0WT^88 zbH@dYm^2lyp(6u71268H+ONLP?RN&n}N?GcNUo|y`ig5834k==3 zX0AX$$rwu-;q^XFM>*W=Ag1SAFrk|`y4?binH`Hh5a>St+*<^6>v#Bn-@aI~yncu~ z3t49&>nvny>nvo})>+7)br!N#_;nVt&O+vR$j@98|9;84VO}}z#wXw1{?diFX?vK)Tdxm5|4P& zw>ak-LSs2H;vsoC=*n*WcSEZ`66OMPD27iJ*l1c0*E9OQJ8Xo zGI3@Wq4;D^CRhq6Oop=92|auy$=CAIy0d5oT$A+S!`tR5K2y?vX3^+=G$Yv@*PB5& ztYr7LvY#Y=kMG&!w2Y56sGGd+AUKO|OeM<1im@%g@ldrNB4T%c65l}#|9P2m%bW+o zo&}?ZeyPkLhhNOW}H!DSlyd;HjBKP{a>^V>7gzN@$P} zO!Us8t1uhScotzb{FT9=9V*hS(e|}uE%C@T**rr|dN8#I3`0AH0UGVR`m{E=-o8c{Tgo_-1*?=m3ys$k}%m+aU0S61Gg9{ovx&yQB8Y?z(2k zwzIlVgoB;a3sK_Y9Pvb`jOPE4v+czxyn5hC{or5^=u{gYHj91}Mqg$Q*-w(cdhoV! zTCWivoESC*PU#213*gTWtPni-+ss`pTEAy7;f-xQ5z+vEB$CK2u5+S^{iTGEL)A{C z17V_`KkDTNG$ORfF9Nwcp(8{nIS^=W-x)87HL=TEjpBo0*PWjGERjl4Q1MxbO7M~dTlg-Osko>pdA8TT)cFOyqK!4IQoNW?6frvMia(5AV0 zLTeKCp!H#YU7XJmqeh|AA@!A=?2>5%5ZCL5RMokxT*LPrGC5p*OM`3%QOE=VdTts~ z7}b<&z+8=un+J@P@#J35R$NxjA-Lu`FA^)U3l&5uJ*nhwA!BLCo#iM`kO3mM)RBC~ zbNI6p`4gn#{rV?d3Tfek-$tX!a=f#*xE=k45owcu=00SLO1Z?wiu?eJNGt~wepS2* zn4mFN^nbP`jk!FjehfYx#gACjD1u=ym0FXONjze*WR zyX2;`2AjdBDqNX;t#mMn7^y&GC|P=9(@MvT#ZRL=dG?HLDTGGr!VRV6<^$WGhG6iV zBW|*P9Tw4S#NtgZjr;;>bG5S5G#Q^)NNbbFqaP%Q<_*UK2ki3i+f0a z#=NqkRib=)er8V~*%XFK#kZM(=b0v!SHIYgwhPe7J%=8f4nFKMJsRI zMnCSwL|QdlVJWqaqk?^3#vLT%D04Fa)V@FY(;+HdIjoOIZ+|#Bl#0CIJ&-8$oB&`y zSsXg4xG#NHu1P?ACGs zb9QG#5}N_A95gnYn%%k z*RZ_xADKXsLZEp z?czVV7NKPrs5}01e|P&}yAuET^5ErO7ytPb?Td{6Y}$s8n+cXDS@nw(b$70Xi#E)R zXjm<3bYut{bwk>Cs+6(r`8URYkt(+UcmM0+NX2q=aim=wX%|P@#gTS#q+J}T0pb@D zM=I+fUHZir)`G{(sVrqQk-U^_-Xp97tf)B3JhFhU(@IEP>531vx6x_T0JMT1LWxlK zBBDYe>;V{BOIH)UN#(n>r(Gyv7fRTL5~{5WCA79Kln`1MO4tg&E|jo;3nlEb&3CCl zMC($4bg4i({ypgU_l|$>_;+jT_;-uE)__Z3PhW2mkOjy1=6Jg=~97o zsX)3^AYB3P5D@Pw5c_ytDv&M}h_!X8K)O^QwbrEq=~98zT9*o>O9i5~E)~dUNCiSc z*xY*8ADKh;oJLW@aPA?YHAg*Tf3Ft$SszZ%Hv+nxa|27{7YSX0b45c#b45gZBjt{a zF2T40o1u9`Nc#h`M@pA48s_B{F%7L<)U@~bj}$!(?ej)ZLtB3oH8kHy>hc>ju>DOD z)g_>YkJ%7e4edeE)!vA>qO84vG(}t2rt_ePYiK@E*CqOHy60);vv%-`f5G-cx(QwT z*J}7q%~?{QJup+sdgj=eHKm5>>X}p4gRxH*6=)qlu?kWhKhg0M9Y4|W6Rzp}UGWp1 zSzY|lb|dVz8(vyu>osQtYq&Mr5u+5qw~Zr;bbd)wWGPTcd{=n_1F=OUA%AR^(2OScosYlsedhU|GMVuCH=2e{$Cg7zFL5e zpXvCSj-ToHnU0@ni%!SS7PB+#X??b>gwe_8YYj-$maz3+=? zH%n??ujbDGyR)6vCAonZpg|{mWt7ok+#Kf}s*HQ;tr)oxCjPcemi331mqs%

!K>-?PECn){u#mmWg`~xt z&%9>>9F`b$xAQ^lC}&GGTckLT}S{djit?&R|P==h|2I!jIU z6u_FQ13S^iUF^fm{YFrUKiye@EY)yuCD`dT1me->P4f?TSjk zg&5-dZ!Uj4d;jVLSL5jy0>$$r9Y<%?$Y*s85Xtsm%Z;W7qHVqzQrU@VZw^PJ6OrG# zwykNB-yI+ScyjjY{Qc?K2lFTHnfJX8yH~@AZk{0UMRnMW>UXQ6I$SHNQM_nE_2l$? zHB@!je^yjCln0A6tEznk8Vj~LA>w*jRfWjQ?VP=Q80`Gv4+mA@n!Q`3qm)UEACAwB z(Ag$#0=v6cjd-e`d(?zBNlt*^v)}>hJoS{I)Z`sf%~~ z7A*HkWjog=IG%@GTgjQ%@;%N(dc)!jp$VMWJG@KBK(%C`f`&N z6_~S*-sx*uQji_`)`gh(ng}SwVhjcrX}&DNb&Mk5n8pyGxlBMn^?mPnXOu;S*DEa< ze?&M96uJF-Ohos4QrjPXYoPC@ciBmb{(|oYy@*LQ%RK5sPK=&m1=Q!?!pnRx16WP+ zbN&o~<3%{PFu3qYf5ma*)Xz1v>b>yIA9kxukNsYFM4DYEZbZX1k#CyjM1M5|7B*U5 zr{jH|m*`!%&|z>w^NxMY!{EiPo|)5_f06)Giy^JATf_+?4?)OvU$p5}(_a+nEXu0N z3+2ZLDKA78)#F*1=cEV7?{l_DeIDi5AF;>rNG4K&?{-G0EBbT;+O>@nPB5{ALW0FF zE6(6^tNyUr^}jhuX%A8S^46sP-PwPM`rqyCy`Als2mfJv=iuc|SO4uvnnVAKf5~_+ zBVGNwqVCp2IfN3xM`>K@es5wx_Kz{xrtaUy^BAsYk-1eok4*-O;w4cgp)MRsl8&$4lBQ}f36DX7Qv;D zK7L?c`pR26%w?YSL7p(aYJ6CmpZ$H~L;d|(JltY3|8iDK#LahLJnYw>JLGh;(zK?( z{pRH#yyf)P`dgSYBSt&BF9+M~e>>IOMqI?c*>S3P&Cjr|Dx2pYKYTc^4$H9lBg0nX zb*k&r8)k7dF=bm4`g_<)f8zN#$pyLMRd8QJ-rz>b6|Rb^*h)&%OHy&Yjg#wKqgk+M zr{~AD-%OG&YA#M+X}@I==AtuF*Ux5aFQ6=Ef=Yrn8N1uhBerDlYMP95&h+Ejx75zq zh#WcMy^1qrlJDPV$C$&guKh3&!Gjjc z{qJbL2F))uxBho~e|v9#x1#^uKiKZ{zfaOCiVRZy9t(0Tx&Q@jncH*{BkdEJg8tkq{o8ET^UvA-kl6kQ(SnBazxQ%~ zzv}k1w5)_7(6N3zxZ+OxNwd|WLT`nM2smI-5YQ6%0?m@_0Z!_qDq)q=2xZ+<#m*(LD z|I!Zv{uJFr{9oiiqV_XDh`8?_`$O zZ-5%2tdCsMS?s$;#$|}3yCO<3yv;G%2unMNpOgv{D-H}BB+bTLxgiD)Y#)TdffJRB z9%8s)h4l^$%TTo|+IB`CPMutN`6IhNWY@_4JKx>%e{qz=`XAK}dc3auyV?y-5y{aI&6G|hB4gIQMm6at*V_Wy~X>D=22JQ-Xf#qlQCAr7HHlC#M zJjU{QS#%X+SvO^4Tenl`Jjl)1ntNc0m2m&F_tLO~%f|a^sohH7LA`QCzpfbZVgONll-@R zQ1$=Wf4Seqe?Cdu83b=4DlekcY7Bn3cngI@vSpHh+bxx0oTKlLCm&Di*Def&Ue?B-yoK(;Jx;+JBuVVJ@b0hJK7outm z+X9T0Ci}MTtZuSwb7Of?EH3`-ULTE4&a{Qt<%VW{SVV)4l5e{YYS z`P%vK^uJHie*Ssu>y4AUg{W4IVNtwL+6cY2Hoo5C+m6Z^@{I}|dqKZ^qM3 zfBWOTsLT8dxE&;;;9w8`8PB=DSFhjGclX8yqDGcoteZw7OqR3ReE`)0<4Fv~uoeqM zda&^+BG}l!fDvEgMrI)^!x@wv#m6pcJ+B0w_rW}KqNa;DRr1t9U68Zuk z1mxwb!N%!C6gVazvZ^<(82uI(GOZSte`O1y9ugqWu{EzMbkJ8;A%@X-hb=`y&rq+4-aG5e;FXU zEiLGYs}%5~8)&+PDC704Uru}~0~)OH=2CXB!2xsw?n>QsbR+l*U<_)n{K8H>lt$HO zT8Bbrsd0aJuX7ljM}e38Jn3ueDT~rMSG2kieIu5{MATN14*$SBGxXt2Oq`QzCs)Zj z=j;Qn(FS_QwzPFJRqrn&Y0|q5e@N>X{Hk9L&-JO6v~KW~4N#)aO`!$?WRL%&=LMj4 z;y?H8`}v!d|1(DHt|kpO@qdil)A_&Mo&DXNj{kd#wnEfCe+8{A{S$#ZR=1=<Cy!WNAo1EIq2mdO}gn~tx(fBlyK$SF`9)f~xardBB#dXvEE2xDr&mduMT%ZnfB-++nkV9tMu@@Y= zBG(k3JMobKsp3?1&PDg?f11;JPu$exh$!vA^t7~3Y-lf)UW?^Mtn^VLdI+a_m*b0* zqYq^;kue~>U)H-V;(3%V3+cSPxBXk%Yq^pY*SXnVvpJ~xws3uKQ;!a zFc%SM@<;bDfv`SLhFY8ehsEHgOxP_41{G8}GENSL&Hx^!f0+MWu!tAYjD3_rbvEM^ zd37$77xTGzVPwXSuoYhCMF*SglV hu63b**b%>)Kk{{|Eae{W$>vOD diff --git a/cce-network-v2/deploy/cce-network-v2/Chart.yaml b/cce-network-v2/deploy/cce-network-v2/Chart.yaml index e0fd470..a77004d 100644 --- a/cce-network-v2/deploy/cce-network-v2/Chart.yaml +++ b/cce-network-v2/deploy/cce-network-v2/Chart.yaml @@ -15,10 +15,10 @@ type: application # This is the chart version. This version number should be incremented each time you make changes # to the chart and its templates, including the app version. # Versions are expected to follow Semantic Versioning (https://semver.org/) -version: 2.12.10 +version: 2.12.11 # This is the version number of the application being deployed. This version number should be # incremented each time you make changes to the application. Versions are not expected to # follow Semantic Versioning. They should reflect the version the application is using. # It is recommended to use it with quotes. -appVersion: "2.12.10" +appVersion: "2.12.11" diff --git a/cce-network-v2/docs/release.md b/cce-network-v2/docs/release.md index ca72a09..1b69e3a 100644 --- a/cce-network-v2/docs/release.md +++ b/cce-network-v2/docs/release.md @@ -7,6 +7,15 @@ v2 版本新架构，支持VPC-ENI 辅助IP和vpc路由。版本发布历史如 2. 增加 eni 安全组同步功能， 保持CCE ENI 和节点安全组同步。 3. 增加节点网络配置集功能 NetResourceConfigSet，支持指定节点独立配置网络资源。 +#### 2.12.11 [20241227] +1. [Bug] 修复 VPC-ENI 模式下，弹性网卡预挂载 eni-pre-allocate-num 配置不生效的问题 +2. [Bug] 修改开启 RDMA 场景下 RDMA ENI 对象本地缓存过期状态相关逻辑，解决因 resync nrs timeout 而导致的新增 RDMA 节点初始化慢，大规模集群扩容速度慢的问题 +3. [Optimize] 修改开启 RDMA 场景下 RDMA NetResourceSet 对象拼装规则，以及 ENI 对象的 LabelSelectorValue 的拼装规则，防止 RDMA NetResourceSet 名字超过限定值 253，防止 ENI 对象的 LabelSelectorValue 超过限定值 63，解决因 Node Name 超长而导致的 cce-network-agent panic 问题 +4. [Optimize] 修改 RDMA ENI 对象更新逻辑，解决因 NodeName 变更时 ENI 对象未正常销毁而导致的 RDMA ENI 对象无法被更新而导致的节点无法就绪的问题 +5. [Optimize] 优化开启 RDMA 模式时的 RDMA ENI 状态机处理逻辑，支持非终态 RDMA ENI 的处理流程，避免非终态状态 RDMA ENI 卡住节点NotReady 无法恢复的问题 +6. [Optimize] 优化开启 RDMA 模式时，对 HPC OpenAPI 的请求逻辑，大幅降低请求频率，降低大规模集群下的 OpenAPI 请求压力 +7. [Bug] 修复创建 Node Interface 对象时因初始值未判断而导致的出现 Instance is out of interfaces 导致节点就绪慢的问题 + #### 2.12.10 [20241213] 1. [Optimize] 优化 VPC-ENI 模式下的 veth 单机路由规则目的地址存在冲突的判断条件，解决残留本地路由规则时创建的 Pod 容器网络不通的问题 2. [Optimize] 禁用 VPC-ENI 模式下的 IPv6 DHCP 时使用的网卡名修改为udev生成的原始名字，避免生成的网卡配置文件导致虚拟机重启时 network.service 服务启动失败 diff --git a/cce-network-v2/pkg/bce/bcesync/eni.go b/cce-network-v2/pkg/bce/bcesync/eni.go index 3b565a0..6a437d9 100644 --- a/cce-network-v2/pkg/bce/bcesync/eni.go +++ b/cce-network-v2/pkg/bce/bcesync/eni.go @@ -90,7 +90,9 @@ func (es *VPCENISyncerRouter) StartENISyncer(ctx context.Context, updater syncer // Create implements syncer.ENIEventHandler func (es *VPCENISyncerRouter) Create(resource *ccev2.ENI) error { types := resource.Spec.Type - if types == ccev2.ENIForBBC || types == ccev2.ENIForHPC || types == ccev2.ENIForERI { + // Remove "if types == ccev2.ENIForHPC || types == ccev2.ENIForERI return nil", + // because we need to support the case of RDMA ENI already have RDMA IPs + if types == ccev2.ENIForBBC { return nil } @@ -110,7 +112,9 @@ func (es *VPCENISyncerRouter) ResyncENI(ctx context.Context) time.Duration { // Update implements syncer.ENIEventHandler func (es *VPCENISyncerRouter) Update(resource *ccev2.ENI) error { types := resource.Spec.Type - if types == ccev2.ENIForBBC || types == ccev2.ENIForHPC || types == ccev2.ENIForERI { + // Remove "if types == ccev2.ENIForHPC || types == ccev2.ENIForERI return nil", + // because we need to support the case of RDMA ENI already have RDMA IPs + if types == ccev2.ENIForBBC { return nil } @@ -326,6 +330,13 @@ type eniStateMachine struct { // Start state machine flow func (esm *eniStateMachine) start() error { + // ENI for RDMA (ccev2.ENIForHPC or ccev2.ENIForERI) need do nothing, so return nil directly. + // esm.es.remoteSyncer.statENI(esm.ctx, esm.resource.Name) can not stat ENI for RDMA, it will return + // error like [Code: EniNotFoundException; Message: eni:eni-tzjatpp7gbh6 resource not exist; + // RequestId: 148ca1d1-174f-494a-8192-5bae2a3bf0c7]". So we need to check ENI type first. + if esm.resource.Spec.Type == ccev2.ENIForHPC || esm.resource.Spec.Type == ccev2.ENIForERI { + return nil + } var err error if esm.resource.Status.VPCStatus == ccev2.VPCENIStatusInuse { if len(esm.resource.Spec.PrivateIPSet) == 0 { diff --git a/cce-network-v2/pkg/bce/rdma/rdma_eni.go b/cce-network-v2/pkg/bce/rdma/rdma_eni.go index 5027534..61b5417 100644 --- a/cce-network-v2/pkg/bce/rdma/rdma_eni.go +++ b/cce-network-v2/pkg/bce/rdma/rdma_eni.go @@ -74,7 +74,12 @@ func (m *rdmaInstancesManager) ForeachInstance(instanceID, nodeName string, fn i return fmt.Errorf("list ENIs failed: %w", err) } for i := 0; i < len(enis); i++ { - if enis[i].DeletionTimestamp != nil || enis[i].Status.VPCStatus == ccev2.VPCENIStatusDeleted { + eni := enis[i] + vpcStatus := eni.Status.VPCStatus + // Do not process the RDMA ENI which is being deleted or in attaching/detaching status. + // It is not useful to process it, because the IPs are not assigned to the RDMA ENI. + if enis[i].DeletionTimestamp != nil || vpcStatus == ccev2.VPCENIStatusAttaching || + vpcStatus == ccev2.VPCENIStatusDetaching || vpcStatus == ccev2.VPCENIStatusDeleted { continue } fn(instanceID, enis[i].Spec.ENI.ID, ipamTypes.InterfaceRevision{ @@ -87,8 +92,9 @@ func (m *rdmaInstancesManager) ForeachInstance(instanceID, nodeName string, fn i // waitForENISynced wait for eni synced // this method should not lock the mutex of bceNode before calling func (n *bceRDMANetResourceSet) waitForENISynced(ctx context.Context) { - ctx, cancel := context.WithTimeout(ctx, 30*time.Second) + ctx, cancel := context.WithTimeout(ctx, 10*time.Second) defer cancel() + wait.PollImmediateUntilWithContext(ctx, 200*time.Millisecond, func(ctx context.Context) (done bool, err error) { haveSynced := true n.manager.ForeachInstance(n.instanceID, n.k8sObj.Name, @@ -111,5 +117,4 @@ func (n *bceRDMANetResourceSet) waitForENISynced(ctx context.Context) { }) return haveSynced, nil }) - } diff --git a/cce-network-v2/pkg/bce/rdma/rdma_super.go b/cce-network-v2/pkg/bce/rdma/rdma_super.go index ab1d863..02c1ee4 100644 --- a/cce-network-v2/pkg/bce/rdma/rdma_super.go +++ b/cce-network-v2/pkg/bce/rdma/rdma_super.go @@ -849,6 +849,8 @@ func (n *bceRDMANetResourceSet) updateENIWithPoll(ctx context.Context, eni *ccev return false, fmt.Errorf("get eni %s failed: %v", eni.Name, ierr) } eni = eni.DeepCopy() + oldversion = eni.Spec.VPCVersion + eni.Spec.VPCVersion = eni.Spec.VPCVersion + 1 eni = refresh(eni) // update eni diff --git a/cce-network-v2/pkg/bce/rdma/rdma_wrapper.go b/cce-network-v2/pkg/bce/rdma/rdma_wrapper.go index f80e0fe..4222ac6 100644 --- a/cce-network-v2/pkg/bce/rdma/rdma_wrapper.go +++ b/cce-network-v2/pkg/bce/rdma/rdma_wrapper.go @@ -47,7 +47,7 @@ type rdmaNetResourceSetWrapper struct { *bceRDMANetResourceSet // rdmaeni is the eni of the node - rdmaeniName string + rdmaENIName string } func newRdmaNetResourceSetWrapper(super *bceRDMANetResourceSet) *rdmaNetResourceSetWrapper { @@ -86,42 +86,96 @@ func (n *rdmaNetResourceSetWrapper) findMatchedEniByMac(ctx context.Context, iaa // ensureRdmaENI means create a eni object for rdma interface // rdma interface has only one eni, so we use rdma interface id as eni name func (n *rdmaNetResourceSetWrapper) ensureRdmaENI() (*ccev2.ENI, error) { - if n.rdmaeniName != "" { - eni, err := n.manager.eniLister.Get(n.rdmaeniName) + if n.rdmaENIName != "" { + eni, err := n.manager.eniLister.Get(n.rdmaENIName) + if errors.IsNotFound(err) { + goto forceGetFromIaaS + } if err != nil { - return nil, fmt.Errorf("failed to get rdma eni %s from lister", n.rdmaeniName) + return nil, fmt.Errorf("failed to get rdma eni %s from lister", n.rdmaENIName) } + + isNeedUpdate := false if eni.Status.VPCStatus != ccev2.VPCENIStatusInuse { - _, err = k8s.CCEClient().CceV2().ENIs().UpdateStatus(context.TODO(), eni, metav1.UpdateOptions{}) + isNeedUpdate = true + } + if bceutils.IsCCERdmaNetRourceSetName(eni.Labels[k8s.LabelNodeName]) && + eni.Labels[k8s.LabelNodeName] != n.k8sObj.Name { + var ownerReferenceNodeName string + or := n.k8sObj.GetOwnerReferences() + for _, ref := range or { + if ref.Kind == "Node" { + ownerReferenceNodeName = ref.Name + break + } + } + labelSelectorValue := bceutils.GetLabelSelectorValueFromNetResourceSetName(n.k8sObj.Name, + ownerReferenceNodeName, n.k8sObj.Spec.InstanceID, eni.Spec.MacAddress, string(eni.Spec.Type)) + eni.Labels[k8s.LabelNodeName] = labelSelectorValue + isNeedUpdate = true + } + + if isNeedUpdate { + err = n.updateENIWithPoll(context.TODO(), eni, func(eni *ccev2.ENI) *ccev2.ENI { + // do nothing + return eni + }) if err != nil { - return nil, fmt.Errorf("failed to update %s ENI status: %w", n.rdmaeniName, err) + return nil, fmt.Errorf("failed to update %s ENI %s status: %w", eni.Spec.Type, eni.Name, err) } - n.log.Infof("update %s ENI status successed", n.rdmaeniName) + n.log.Infof("update %s ENI %s status successed", eni.Spec.Type, eni.Name) } return eni, nil } +forceGetFromIaaS: // the hpc or eri api do not use vpcID, subnetID and zoneName vpcID := n.k8sObj.Spec.ENI.VpcID // the macAddress and vifFeatures is decided by the NetResourceSet's annotation macAddress := n.bceRDMANetResourceSet.k8sObj.Annotations[k8s.AnnotationRDMAInfoMacAddress] vifFeatures := n.bceRDMANetResourceSet.k8sObj.Annotations[k8s.AnnotationRDMAInfoVifFeatures] - iaasClient := n.manager.getIaaSClient(vifFeatures) - rdmaEni, err := n.findMatchedEniByMac(context.Background(), iaasClient, vpcID, n.instanceID, vifFeatures, macAddress) + var rdmaEniId string + requirement := metav1.LabelSelectorRequirement{ + Key: k8s.LabelNodeName, + Operator: metav1.LabelSelectorOpIn, + Values: []string{n.k8sObj.Name}, + } + labelSelector := &metav1.LabelSelector{ + MatchExpressions: []metav1.LabelSelectorRequirement{requirement}, + } + // Select only the ENI of the local node + selector, err := metav1.LabelSelectorAsSelector(labelSelector) if err != nil { - n.log.WithError(err).Errorf("failed to get instance %s eni", vifFeatures) - return nil, err + panic(fmt.Errorf("failed to create label selector: %v", err)) + } + k8sRdmaEnis, err := n.manager.eniLister.List(selector) + if err != nil { + return nil, fmt.Errorf("failed to list enis: %w", err) + } + for _, rdmaEni := range k8sRdmaEnis { + // only one RDMA ENI per RDMA NetworkResourceSet + rdmaEniId = rdmaEni.Spec.ID + } + var rdmaEni *client.EniResult + iaasClient := n.manager.getIaaSClient(vifFeatures) + if rdmaEniId == "" { + rdmaEni, err = n.findMatchedEniByMac(context.Background(), iaasClient, vpcID, n.instanceID, vifFeatures, macAddress) + if err != nil { + n.log.WithError(err).Errorf("failed to get instance %s eni", vifFeatures) + return nil, err + } + n.log.WithField("rdmaeni", logfields.Repr(rdmaEni)).Debugf("get instance %s eni success", vifFeatures) + rdmaEniId = rdmaEni.Id } - n.log.WithField("rdmaeni", logfields.Repr(rdmaEni)).Debugf("get instance %s eni success", vifFeatures) - eni, err := n.manager.eniLister.Get(rdmaEni.Id) + var ctx = context.Background() + eni, err := n.manager.eniLister.Get(rdmaEniId) if errors.IsNotFound(err) { // the hpc or eri do not use ensure subnet object var ( ipv4IPSet, ipv6IPSet []*models.PrivateIP - ctx = context.Background() ) for _, v := range rdmaEni.PrivateIpSet { @@ -164,7 +218,7 @@ func (n *rdmaNetResourceSetWrapper) ensureRdmaENI() (*ccev2.ENI, error) { InstanceID: n.instanceID, PrivateIPSet: ipv4IPSet, IPV6PrivateIPSet: ipv6IPSet, - MacAddress: rdmaEni.MacAddress, + MacAddress: macAddress, }, }, Status: ccev2.ENIStatus{}, @@ -173,19 +227,28 @@ func (n *rdmaNetResourceSetWrapper) ensureRdmaENI() (*ccev2.ENI, error) { if err != nil { return nil, fmt.Errorf("failed to create %s ENI: %w", vifFeatures, err) } - n.log.Infof("create %s ENI resource successed", vifFeatures) + n.log.Infof("create %s ENI %s resource successed", vifFeatures, eni.Name) (&eni.Status).AppendVPCStatus(ccev2.VPCENIStatusInuse) _, err = k8s.CCEClient().CceV2().ENIs().UpdateStatus(ctx, eni, metav1.UpdateOptions{}) if err != nil { - return nil, fmt.Errorf("failed to update %s ENI status: %w", vifFeatures, err) + return nil, fmt.Errorf("failed to update %s ENI %s status: %w", vifFeatures, eni.Name, err) } - n.log.Infof("update %s ENI status successed", vifFeatures) + n.log.Infof("update %s ENI %s status successed", vifFeatures, eni.Name) } else if err != nil { - n.log.Errorf("failed to get %s ENI resource: %v", vifFeatures, err) + n.log.Errorf("failed to get %s ENI %s resource: %v", vifFeatures, eni.Name, err) return nil, err + } else { + err = n.updateENIWithPoll(ctx, eni, func(eni *ccev2.ENI) *ccev2.ENI { + // do nothing + return eni + }) + if err != nil { + return nil, fmt.Errorf("failed to update %s ENI %s status: %w", vifFeatures, eni.Name, err) + } + n.log.Infof("update %s ENI %s status successed", vifFeatures, eni.Name) } - n.log.Debugf("got %s ENI resource successed", vifFeatures) - n.rdmaeniName = eni.Name + n.log.Debugf("got %s ENI %s resource successed", vifFeatures, eni.Name) + n.rdmaENIName = eni.Name return eni, err } @@ -217,7 +280,16 @@ func (n *rdmaNetResourceSetWrapper) refreshENIQuota(scopeLog *logrus.Entry) (Rdm if client == nil { scopeLog.Fatal("K8s client is nil") } - nodeName := bceutils.GetNodeNameFromNetResourceSetName(n.k8sObj.Name) + + var ownerReferenceNodeName string + or := n.k8sObj.GetOwnerReferences() + for _, ref := range or { + if ref.Kind == "Node" { + ownerReferenceNodeName = ref.Name + break + } + } + nodeName := bceutils.GetNodeNameFromNetResourceSetName(n.k8sObj.Name, ownerReferenceNodeName, n.k8sObj.InstanceID()) k8sNode, err := client.Informers.Core().V1().Nodes().Lister().Get(nodeName) if err != nil { return nil, fmt.Errorf("failed to get k8s node %s: %v", n.k8sObj.Name, err) diff --git a/cce-network-v2/pkg/bce/utils/bce_utils.go b/cce-network-v2/pkg/bce/utils/bce_utils.go index 5b156cf..20cbfb4 100644 --- a/cce-network-v2/pkg/bce/utils/bce_utils.go +++ b/cce-network-v2/pkg/bce/utils/bce_utils.go @@ -31,6 +31,11 @@ const ( OverlayRDMA string = string(ccev2.ENIForERI) // Overlay RDMA (ERI) PodResourceName string = "rdma" rdmaEndpointMiddleSeparator string = "-rdma-" + // a name's max length is 253 and a labelSelectorValue's max length is 63 in kubernetes + // nameValueMaxLength is a name's max length + nameValueMaxLength int = 253 + // labelValueMaxLength is a label's max length + labelValueMaxLength int = 63 ) var ( @@ -40,23 +45,65 @@ var ( OverlayRDMA: {}, UnderlayRDMA: {}, } + + defaultMetaClient = metadata.NewClient() ) type RdmaIfInfo struct { MacAddress string NetResourceSetName string + LabelSelectorValue string VifFeatures string } -func generateNetResourceSetName(nodeName, macAddress, vifFeatures string) (netResourceSetName string) { +func generateNetResourceSetName(nodeName, nodeInstanceID, macAddress, vifFeatures string) (netResourceSetName string) { + // a name's max length is 253 in kubernetes, so if nodeName's length is more than the max length like this: 253 - len(string("-fa2700078302-elasticrdma")), + // we need to use node's InstanceID as node's identification to generate NetResourceSetName + var nodeIdentification string + lengthLimit := nameValueMaxLength - len(string("-fa2700078302-elasticrdma")) + // NetResourceSetName use nodeName for nodeIdentification if lengthLimit < nameValueMaxLength + // if lengthLimit >= nameValueMaxLength, use node's InstanceID for nodeIdentification + if len(nodeName) > lengthLimit { + // cce-ujgjc8tg-lca6hnyh-fa2700078302-rdmaroce for HPC or cce-ujgjc8tg-lca6hnyh-fa2700078302-elasticrdma for ERI + nodeIdentification = nodeInstanceID + } else { + // 10.0.2.2-fa2700078302-rdmaroce for HPC or 10.0.2.2-fa2700078302-elasticrdma for ERI + nodeIdentification = nodeName + } + macStr1 := strings.Replace(macAddress, ":", "", -1) macStr2 := strings.Replace(macStr1, "-", "", -1) vifFeaturesStr := strings.Replace(vifFeatures, "_", "", -1) - netResourceSetName = fmt.Sprintf("%s-%s-%s", nodeName, macStr2, vifFeaturesStr) - // 10.0.2.2-fa2700078302-rdmaroce for HPC or 10.0.2.2-fa2700078302-elasticrdma for ERI + netResourceSetName = fmt.Sprintf("%s-%s-%s", nodeIdentification, macStr2, vifFeaturesStr) + // cce-ujgjc8tg-lca6hnyh-fa2700078302-rdmaroce for HPC or cce-ujgjc8tg-lca6hnyh-fa2700078302-elasticrdma for ERI + // or 10.0.2.2-fa2700078302-rdmaroce for HPC or 10.0.2.2-fa2700078302-elasticrdma for ERI return netResourceSetName } +func generateLabelSelectorValue(nodeName, nodeInstanceID, macAddress, vifFeatures string) (labelSelectorValue string) { + // a labelSelectorValue's max length is 63 in kubernetes, so if nodeName's length is more than the max length like this: + // 63 - len(string("-fa2700078302-elasticrdma")), we need to use node's InstanceID as node's identification to generate labelSelectorValue + var labelSelectorIdentification string + lengthLimit := labelValueMaxLength - len(string("-fa2700078302-elasticrdma")) + // LabelSelectorValue use nodeName for nodeIdentification if lengthLimit < labelValueMaxLength + // if lengthLimit >= labelValueMaxLength, use node's InstanceID for nodeIdentification + if len(nodeName) > lengthLimit { + // cce-ujgjc8tg-lca6hnyh-fa2700078302-rdmaroce for HPC or cce-ujgjc8tg-lca6hnyh-fa2700078302-elasticrdma for ERI + labelSelectorIdentification = nodeInstanceID + } else { + // 10.0.2.2-fa2700078302-rdmaroce for HPC or 10.0.2.2-fa2700078302-elasticrdma for ERI + labelSelectorIdentification = nodeName + } + + macStr1 := strings.Replace(macAddress, ":", "", -1) + macStr2 := strings.Replace(macStr1, "-", "", -1) + vifFeaturesStr := strings.Replace(vifFeatures, "_", "", -1) + labelSelectorValue = fmt.Sprintf("%s-%s-%s", labelSelectorIdentification, macStr2, vifFeaturesStr) + // cce-ujgjc8tg-lca6hnyh-fa2700078302-rdmaroce for HPC or cce-ujgjc8tg-lca6hnyh-fa2700078302-elasticrdma for ERI + // or 10.0.2.2-fa2700078302-rdmaroce for HPC or 10.0.2.2-fa2700078302-elasticrdma for ERI + return labelSelectorValue +} + func getNodeNameFromRdmaNetResourceSetName(netResourceSetName, rdmaTag string) (nodeName string) { // 10.0.2.2-fa2700078302-rdmaroce for HPC or 10.0.2.2-fa2700078302-elasticrdma for ERI index1 := strings.Index(netResourceSetName, rdmaTag) @@ -66,21 +113,33 @@ func getNodeNameFromRdmaNetResourceSetName(netResourceSetName, rdmaTag string) ( return nodeName } -func GetNodeNameFromNetResourceSetName(netResourceSetName string) (nodeName string) { +func GetNodeNameFromNetResourceSetName(netResourceSetName, ownerReferenceNodeName, nodeInstanceID string) (nodeName string) { // 10.0.2.2-fa2700078302-rdmaroce for HPC or 10.0.2.2-fa2700078302-elasticrdma for ERI + var nodeIdentification string underlayRDMA := strings.Replace(string(ccev2.ENIForHPC), "_", "", -1) overlayRDMA := strings.Replace(string(ccev2.ENIForERI), "_", "", -1) if strings.Contains(netResourceSetName, underlayRDMA) { - nodeName = getNodeNameFromRdmaNetResourceSetName(netResourceSetName, underlayRDMA) + nodeIdentification = getNodeNameFromRdmaNetResourceSetName(netResourceSetName, underlayRDMA) } else if strings.Contains(netResourceSetName, overlayRDMA) { - nodeName = getNodeNameFromRdmaNetResourceSetName(netResourceSetName, overlayRDMA) + nodeIdentification = getNodeNameFromRdmaNetResourceSetName(netResourceSetName, overlayRDMA) + } else { + nodeIdentification = netResourceSetName + } + if nodeIdentification == nodeInstanceID { + nodeName = ownerReferenceNodeName } else { - nodeName = netResourceSetName + nodeName = nodeIdentification } return nodeName } +func GetLabelSelectorValueFromNetResourceSetName(netResourceSetName, ownerReferenceNodeName, nodeInstanceID, macAddress, vifFeatures string) (labelSelectorValue string) { + nodeName := GetNodeNameFromNetResourceSetName(netResourceSetName, ownerReferenceNodeName, nodeInstanceID) + labelSelectorValue = generateLabelSelectorValue(nodeName, nodeInstanceID, macAddress, vifFeatures) + return labelSelectorValue +} + func generateCCERdmaEndpointName(podName, macAddress string) (endpointName string) { macStr1 := strings.Replace(macAddress, ":", "", -1) macStr2 := strings.Replace(macStr1, "-", "", -1) @@ -99,13 +158,14 @@ func getPrimaryMacFromCCERdmaEndpointName(cepName string) (macAddress string, er func GetRdmaIFsInfo(nodeName string, scopedLog *logrus.Entry) (map[string]RdmaIfInfo, error) { var ris = map[string]RdmaIfInfo{} - var defaultMetaClient = metadata.NewClient() // list network interface macs macList, macErr := defaultMetaClient.ListMacs() if macErr != nil { if scopedLog != nil { scopedLog.WithError(macErr).Errorf("list mac failed") + } else { + log.WithError(macErr).Errorf("list mac failed") } return ris, macErr } @@ -116,13 +176,27 @@ func GetRdmaIFsInfo(nodeName string, scopedLog *logrus.Entry) (map[string]RdmaIf if vifErr != nil { if scopedLog != nil { scopedLog.WithError(vifErr).Errorf("get mac %s vif features failed", macAddress) + } else { + log.WithError(vifErr).Errorf("get mac %s vif features failed", macAddress) } continue } if _, ok := roceVFs[vifFeatures]; ok { var rdmaIfInfo RdmaIfInfo + nodeInstanceID, metaAPIErr := defaultMetaClient.GetInstanceID() + if metaAPIErr != nil { + if scopedLog != nil { + scopedLog.WithError(metaAPIErr).Errorf("get node InstanceID from meta-data api failed") + } else { + log.WithError(macErr).Errorf("get node InstanceID from meta-data api failed") + } + return ris, metaAPIErr + } + netResourceSetName := generateNetResourceSetName(nodeName, nodeInstanceID, macAddress, vifFeatures) + labelSelectorValue := generateLabelSelectorValue(nodeName, nodeInstanceID, macAddress, vifFeatures) rdmaIfInfo.MacAddress = macAddress - rdmaIfInfo.NetResourceSetName = generateNetResourceSetName(nodeName, macAddress, vifFeatures) + rdmaIfInfo.NetResourceSetName = netResourceSetName + rdmaIfInfo.LabelSelectorValue = labelSelectorValue rdmaIfInfo.VifFeatures = vifFeatures ris[macAddress] = rdmaIfInfo } @@ -135,6 +209,12 @@ func IsCCERdmaEndpointName(cepName string) bool { return strings.Contains(cepName, rdmaEndpointMiddleSeparator) } +func IsCCERdmaNetRourceSetName(netResourceSetName string) bool { + underlayRDMA := strings.Replace(string(ccev2.ENIForHPC), "_", "", -1) + overlayRDMA := strings.Replace(string(ccev2.ENIForERI), "_", "", -1) + return strings.Contains(netResourceSetName, underlayRDMA) || strings.Contains(netResourceSetName, overlayRDMA) +} + func IsThisMasterMacCCERdmaEndpointName(cepName, masterMac string) bool { macAddress, err := getPrimaryMacFromCCERdmaEndpointName(cepName) if err != nil { diff --git a/cce-network-v2/pkg/bce/vpceni/node_super.go b/cce-network-v2/pkg/bce/vpceni/node_super.go index 0d0c166..3b54598 100644 --- a/cce-network-v2/pkg/bce/vpceni/node_super.go +++ b/cce-network-v2/pkg/bce/vpceni/node_super.go @@ -76,6 +76,7 @@ const ( retryDuration = 500 * time.Millisecond retryTimeout = 15 * time.Second + retryDelay = 15 * time.Second vpcENISubsys = "vpc-eni-node-allocator" ) @@ -191,10 +192,14 @@ func (n *bceNetworkResourceSet) getENIQuota() ENIQuotaManager { n.limiterLock.Unlock() ctx := logfields.NewContext() + var instanceType string + if n.k8sObj.Spec.ENI != nil { + instanceType = n.k8sObj.Spec.ENI.InstanceType + } scopedLog := n.log.WithFields(logrus.Fields{ "nodeName": n.k8sObj.Name, "method": "getENIQuota", - "instanceType": n.k8sObj.Spec.ENI.InstanceType, + "instanceType": instanceType, }).WithContext(ctx) // fast path to claculate eni quota when operaor has been restarted @@ -540,19 +545,26 @@ func (n *bceNetworkResourceSet) CreateInterface(ctx context.Context, allocation inums, msg, err := n.real.createInterface(ctx, allocation, scopedLog) n.creatingEni.add(-1) - preAllocateNum := math.IntMin(eniQuota.GetMaxENI()-1, n.k8sObj.Spec.ENI.PreAllocateENI) - preAllocateNum = preAllocateNum - availableENICount - 1 - for i := 0; i < preAllocateNum; i++ { + preAllocateENINum := math.IntMin(eniQuota.GetMaxENI()-1, n.k8sObj.Spec.ENI.PreAllocateENI) + preAllocateENINum = preAllocateENINum - availableENICount - 1 + for i := 0; i < preAllocateENINum; i++ { inums++ go func() { - n.creatingEni.add(1) - _, _, e := n.real.createInterface(ctx, allocation, scopedLog) - n.creatingEni.add(-1) - if e == nil { - scopedLog.Infof("create addition interface success") - return + retry := 0 + for retry < preAllocateENINum { + n.creatingEni.add(1) + _, _, e := n.real.createInterface(ctx, allocation, scopedLog) + n.creatingEni.add(-1) + if e == nil { + scopedLog.Infof("create addition interface success") + return + } + scopedLog.WithError(e).Errorf("create addition interface failed, retry later for %ds(%d/%d)", retryDelay, retry+1, preAllocateENINum) + retry++ + // attaching ENI is need to wait serveral seconds (<15s) for the ENI to be attached, + // so we can retry preAllocateENINum times for every retryDelay(15s) seconds later. + time.Sleep(retryDelay) } - scopedLog.Errorf("create addition interface failed, err: %s", e) }() } diff --git a/cce-network-v2/pkg/ipam/net_resource.go b/cce-network-v2/pkg/ipam/net_resource.go index 5ea891a..3f547e7 100644 --- a/cce-network-v2/pkg/ipam/net_resource.go +++ b/cce-network-v2/pkg/ipam/net_resource.go @@ -521,7 +521,7 @@ func (n *NetResource) createInterface(ctx context.Context, a *AllocationAction) // not track as interface allocation failure. There is a // separate metric to track nodes running at capacity. n.mutex.Lock() - if time.Since(n.lastMaxAdapterWarning) > warningInterval { + if !n.lastMaxAdapterWarning.IsZero() && time.Since(n.lastMaxAdapterWarning) > warningInterval { n.loggerLocked().Warning("Instance is out of interfaces") n.lastMaxAdapterWarning = time.Now() } diff --git a/cce-network-v2/pkg/ipam/net_resource_set_manager.go b/cce-network-v2/pkg/ipam/net_resource_set_manager.go index 686a4b7..febf262 100644 --- a/cce-network-v2/pkg/ipam/net_resource_set_manager.go +++ b/cce-network-v2/pkg/ipam/net_resource_set_manager.go @@ -262,14 +262,14 @@ func (n *NetResourceSetManager) InstancesAPIIsReady() bool { } // GetNames returns the list of all node names -func (n *NetResourceSetManager) GetNames() (allNodeNames []string) { +func (n *NetResourceSetManager) GetNames() (allNetResourceSetNames []string) { n.mutex.RLock() defer n.mutex.RUnlock() - allNodeNames = make([]string, 0, len(n.netResources)) + allNetResourceSetNames = make([]string, 0, len(n.netResources)) for name := range n.netResources { - allNodeNames = append(allNodeNames, name) + allNetResourceSetNames = append(allNetResourceSetNames, name) } return @@ -291,11 +291,12 @@ func (n *NetResourceSetManager) Update(resource *v2.NetResourceSet) error { }() if !ok { netResource = &NetResource{ - name: resource.Name, - manager: n, - ipsMarkedForRelease: make(map[string]time.Time), - ipReleaseStatus: make(map[string]string), - logLimiter: logging.NewLimiter(10*time.Second, 3), // 1 log / 10 secs, burst of 3 + name: resource.Name, + manager: n, + ipsMarkedForRelease: make(map[string]time.Time), + ipReleaseStatus: make(map[string]string), + logLimiter: logging.NewLimiter(10*time.Second, 3), // 1 log / 10 secs, burst of 3 + lastMaxAdapterWarning: time.Now(), } netResource.ops = n.instancesAPI.CreateNetResource(resource, netResource) diff --git a/cce-network-v2/pkg/k8s/watchers/cce_eni.go b/cce-network-v2/pkg/k8s/watchers/cce_eni.go index 6e6fb47..86c5566 100644 --- a/cce-network-v2/pkg/k8s/watchers/cce_eni.go +++ b/cce-network-v2/pkg/k8s/watchers/cce_eni.go @@ -60,9 +60,12 @@ func (k *K8sWatcher) eniInit(cceClient *k8s.K8sCCEClient, asyncControllers *sync // Select only the ENI of the local node, // contains Ethernet NetResourceSet and RDMA NetResourceSet objects values := []string{nodeTypes.GetName()} - rii, _ := bceutils.GetRdmaIFsInfo(nodeTypes.GetName(), nil) + rii, err := bceutils.GetRdmaIFsInfo(nodeTypes.GetName(), nil) + if err != nil { + panic(fmt.Errorf("failed to get rdma ifs info: %v", err)) + } for _, v := range rii { - values = append(values, v.NetResourceSetName) + values = append(values, v.LabelSelectorValue) } requirement := metav1.LabelSelectorRequirement{ Key: k8s.LabelNodeName, @@ -73,7 +76,10 @@ func (k *K8sWatcher) eniInit(cceClient *k8s.K8sCCEClient, asyncControllers *sync MatchExpressions: []metav1.LabelSelectorRequirement{requirement}, } // Select only the ENI of the local node - selector, _ := metav1.LabelSelectorAsSelector(labelSelector) + selector, err := metav1.LabelSelectorAsSelector(labelSelector) + if err != nil { + panic(fmt.Errorf("failed to create label selector: %v", err)) + } optionsModifier := func(options *metav1.ListOptions) { options.LabelSelector = selector.String() } diff --git a/cce-network-v2/pkg/nodediscovery/nodediscovery.go b/cce-network-v2/pkg/nodediscovery/nodediscovery.go index 70cae0e..e309e88 100644 --- a/cce-network-v2/pkg/nodediscovery/nodediscovery.go +++ b/cce-network-v2/pkg/nodediscovery/nodediscovery.go @@ -554,12 +554,12 @@ func (n *NodeDiscovery) refreshVpcENIConfiguration(nodeResource *ccev2.NetResour // update old nrs if nodeResource.Spec.ENI.UseMode != string(ccev2.ENIUseModePrimaryIP) { nodeResource.Spec.ENI.BurstableMehrfachENI = burstableMehrfachENI - nodeResource.Spec.IPAM.PreAllocate = preAllocate + nodeResource.Spec.ENI.PreAllocateENI = preAllocateENI if burstableMehrfachENI == 0 { nodeResource.Spec.IPAM.MinAllocate = minAllocate + nodeResource.Spec.IPAM.PreAllocate = preAllocate nodeResource.Spec.IPAM.MaxAboveWatermark = maxAboveWatermark - nodeResource.Spec.ENI.PreAllocateENI = preAllocateENI } podsNum := k8sNode.Status.Capacity[k8sTypes.ResourcePods]