From 5a15875e45728804c0e69915d1b7bb8766a6ce04 Mon Sep 17 00:00:00 2001 From: Jelle Teijema Date: Thu, 31 Oct 2024 14:42:28 +0100 Subject: [PATCH] Add metric for loss value in active learning problems (#66) --- README.md | 42 ++++++++++++++++++++ asreviewcontrib/insights/algorithms.py | 30 +++++++++++++++ asreviewcontrib/insights/metrics.py | 20 ++++++++++ figures/loss_metric_example.png | Bin 0 -> 21228 bytes tests/test_metrics.py | 51 +++++++++++++++++++++++++ 5 files changed, 143 insertions(+) create mode 100644 figures/loss_metric_example.png diff --git a/README.md b/README.md index f32e578..e8032eb 100644 --- a/README.md +++ b/README.md @@ -133,7 +133,44 @@ pinpoint hard-to-find papers. The ATD, on the other hand, measures performance throughout the entire screening process, eliminating reliance on arbitrary cut-off values, and can be used to compare different models. +### Loss +The Loss metric evaluates the performance of an active learning model by +quantifying how closely it approximates the ideal screening process. This +quantification is then normalized between the ideal curve and the worst possible +curve. +While metrics like WSS, Recall, and ERF evaluate the performance at specific +points on the recall curve, the Loss metric provides an overall measure of +performance. + +To compute the loss, we start with three key concepts: + +1. **Optimal AUC**: This is the area under a "perfect recall curve," where + relevant records are identified as early as possible. Mathematically, it is + computed as $Nx \times Ny - \frac{Ny \times (Ny - 1)}{2}$, where $Nx$ is the + total number of records, and $Ny$ is the number of relevant records. + +2. **Worst AUC**: This represents the area under a worst-case recall curve, + where all relevant records appear at the end of the screening process. This + is calculated as $\frac{Ny \times (Ny + 1)}{2}$. + +3. **Actual AUC**: This is the area under the recall curve produced by the model + during the screening process. It can be obtained by summing up the cumulative + recall values for the labeled records. + +The normalized loss is calculated by taking the difference between the optimal +AUC and the actual AUC, divided by the difference between the optimal AUC and +the worst AUC. + +$$\text{Normalized Loss} = \frac{Ny \times \left(Nx - \frac{Ny - 1}{2}\right) - +\sum \text{Cumulative Recall}}{Ny \times (Nx - Ny)}$$ + +The lower the loss, the closer the model is to the perfect recall curve, +indicating higher performance. + +![Recall plot illustrating loss metric](https://github.com/jteijema/asreview-insights/blob/loss-metric/figures/loss_metric_example.png?raw=true) + +In this figure, the green area between the recall curve and the perfect recall line is the lossed performance, which is then normalized for the total area (green and red combined). ## Basic usage @@ -467,6 +504,11 @@ which results in ] ] }, + { + "id": "loss", + "title": "Loss", + "value": 0.01707543880041846 + }, { "id": "erf", "title": "Extra Relevant record Found", diff --git a/asreviewcontrib/insights/algorithms.py b/asreviewcontrib/insights/algorithms.py index 11fd3ce..20b02db 100644 --- a/asreviewcontrib/insights/algorithms.py +++ b/asreviewcontrib/insights/algorithms.py @@ -19,6 +19,36 @@ def _recall_values(labels, x_absolute=False, y_absolute=False): return x.tolist(), y.tolist() +def _loss_value(labels): + Ny = sum(labels) + Nx = len(labels) + + if Ny == 0 or Nx == Ny: + raise ValueError("Need both 0 and 1 labels") + + # The normalized loss is computed based on: + # + # 1. The "optimal" possible AUC, representing the area under an optimal recall + # curve, is the total area, Nx * Ny, minus the area above the stepwise + # curve, (Ny * (Ny - 1)) / 2. Combined to Ny * (Nx - (Ny - 1)) / 2. + # + # 2. The "actual" AUC is the cumulative recall sum, calculated with + # np.cumsum(labels).sum(). + # + # 3. The "worst" AUC, where all positive labels are clustered at the end, is + # calculated as (Ny * (Ny + 1)) / 2. To normalize, we need the difference + # between the optimal and worst AUCs. We simplify this difference: + # + # (Nx * Ny - ((Ny * (Ny - 1)) / 2)) - ((Ny * (Ny + 1)) / 2) + # + # This simplifies to the hyperbolic paraboloid Ny * (Nx - Ny), which is + # the denominator in our normalized loss. + # + # Finally, we compute the normalized loss as: + # (optimal - actual) / (optimal - worst). + return float((Ny * (Nx - (Ny - 1) / 2) - np.cumsum(labels).sum()) / (Ny * (Nx - Ny))) # noqa: E501 + + def _wss_values(labels, x_absolute=False, y_absolute=False): n_docs = len(labels) n_pos_docs = sum(labels) diff --git a/asreviewcontrib/insights/metrics.py b/asreviewcontrib/insights/metrics.py index 2898a9b..ecee421 100644 --- a/asreviewcontrib/insights/metrics.py +++ b/asreviewcontrib/insights/metrics.py @@ -6,6 +6,7 @@ from asreviewcontrib.insights.algorithms import _erf_values from asreviewcontrib.insights.algorithms import _fn_values from asreviewcontrib.insights.algorithms import _fp_values +from asreviewcontrib.insights.algorithms import _loss_value from asreviewcontrib.insights.algorithms import _recall_values from asreviewcontrib.insights.algorithms import _tn_values from asreviewcontrib.insights.algorithms import _tp_values @@ -169,6 +170,20 @@ def _tnr(labels, intercept, x_absolute=False): return _slice_metric(x, y, intercept) +def loss(state_obj, priors=False): + """Compute the loss for active learning problem. + + Computes the loss for active learning problem where all relevant records + have to be seen by a human. + + See the inline documentation for detailed description of loss calculation. + + Returns: + float: The loss value. + """ + labels = _pad_simulation_labels(state_obj, priors=priors) + + return _loss_value(labels) def get_metrics( state_obj, @@ -225,6 +240,11 @@ def get_metrics( "title": "Work Saved over Sampling", "value": [(i, v) for i, v in zip(wss, wss_values)], }, + { + "id": "loss", + "title": "Loss", + "value": _loss_value(labels), + }, { "id": "erf", "title": "Extra Relevant record Found", diff --git a/figures/loss_metric_example.png b/figures/loss_metric_example.png new file mode 100644 index 0000000000000000000000000000000000000000..47e1f5f39df86b0b753a3c8f30d8ddc2f87b7a51 GIT binary patch literal 21228 zcmeHvWmr(#*6sp95F}JWkP=Z!x*J7MK)R8T?(RlWP%x2DS{fYCm_uo1DY=sqb%{k^A^}b{5hsuiA3GgWK5CkERy>U$yK`;Xm1Y->M zI9%cTIy4S{eehP-c2zaHKks+kY3LzKf=Oz1?n&4881c0cnp-@!#@{Z> zz}~?$$k)jh4U$@c<=wNAl2VqHlKOK+u#Pyt=VCV+B&iw=)k;)Z3EhcLeNm2|OQxrm zCk?y(x|megjk976KZciqnf=LKn?L?|+W00YtSFB#?=IaI#`n3g)G7Ll2OP}B57x)Z zX%`p77Dh0xro2-xV#LdgF-!f<#T9#t)GvR@gzDIo)HUPugc}^@?QXuq4Gq0joNY7y z3vFDP;|j+;uYG6gBna~i^0e>l5L!Hgqa&;OpzUGC$x6EA3%zF1qk`nrWS-{jA5(um z40>-|5Hvw(eo^XT!bL*nH!|B-d(7^yl3A3fn&kSdkRM+ekc*k+Z6aR6c|uli>R48Pz9wH71h4 zO`e@}Do$Vgt=^##-CaRHguAPIg}~7f_a+&$Pfc^xu87a>=FRLn+JQ{xcCz!0xp4B_ zt>A=dE8G+|aj;`IGIcOEWB0UkgrkQbq7t5tMkY39u5`v`7FPCR^ovz>^mJCHV)R;k z3Y-d#Qf8J`H@uzA)VvkdO}uSPgiPrr#PLKug<$|YGgl)zPdi(C7hz8^`lE4$;b-(R z2R+@(;$k@TnRg9hIg-snytW1S}{b+8=YsO`lzjpFc zh>J^r+g#{xLnt|0!KpN|{rjxYp-f>YZeC74J{~S^HckOkPBva+Ga;CbDL zv5|=Yzo5y{Q0P7gUs0A7qvvMl{Kpk#TO(I<2WLAt>Q?rq4jwN5xTS7oXQt+Ago=$z zfQOSufQOq)K#-qLkcaml4{4e?yTFk{N9E#V=Mg-*V`?HS17jM&(Xp~KvM}Rtw6{3A zfo_a2EC)nugq~Cw`RF>VMp(+(%*fTjS>3_GR*W8nG^##yqQ5r%hLsC^-|OhoFP%~| zbNcn$uU~Dgj+BXx?no_!jZA(m!Nthk%=BoUFxIbICYDC_7G_{SejnIBuUq|#!@$pP z#4BjR%f-gSC&bIf%VWgFCTPrK#Ad=P$Rl84%*SVH@=M--jqc)L?&@LWY<9&0L6@50MGbbNpT`2Wm0?T&*a_f02ji(F1=oG4S56 zYhb#-5_0@y3V+KB=KBBhf4}$P|K}&@=>GGOe@njq5!Zjj^>0bw-+K5zr|UoB`nM$T zZ$12<)Aj#MTzLOTrp)Xi0`h=ViMxW#`4#%A&2NcNi`WS;K~Wd8#*osLS%&g zi;*djVFed)TxAucai;KS@d+*lZ!mvB5IRKm+7)%rfw^HHJ$1K(rsYk<`2@u&Y=+mW zdDe0h->nTwy9-NWy|TJYOTsG6tY=P}-ja!C5S2rx{O+&qGs# zP7;E~vn0yw@8U*jr@F8xSw89+v^52V@L04OnTt<8TK%b4OZ&p;Y%t7aj9-)q1OCP! zhCi?*;SXX2ewReB{_Tr_2rj&XNe6%62Ere72quF3UtRp4CsXhC>gFW^TJa(GfW56{ z?pHSlYJ3D5F<{z(%}(OV`+JPs+^RL+>m7HF;V=;+>VJftMjWv($UJIy>($Tr{@vKj z)YNjM+H0V~C8N|~lwCqX0uKvaf8Ni7-9@c3`(fM7kdqY20ee4aV}xANhtF#xNJt4mCAO-swnFXlx9rFPk`cq}w^`2_^Df{qiYnwqA<<@skT z1tv`lvKKF2q~y1K2~XMV>Qf85wpV+tGKnG?^)!+Q`?T+pO?%QE2CL3=h04Q&-R^QH z69OthWq*Hv-4g4_<3uz$g@xSv(@Fv13=@-+**3j-0|h1_G~QF^ORT%)yu7@0tK4%p z7KXDgDa0^wa43b5bM(xYjposac~7lPckV0(9A-Ds2I#$h{d%S&4WIRB!>tD&o?&#I z>*I($e}d z$6VUPGN?t|E#Y>l)g(pmd0FeB@`8=UQ4U&K+Ch^x@!cHz%IQlAuek4{rvVqylB)CH zpIc~p|9Xk*p*H14P=YPuIxY0i7P?viQhcMG?_2sH5=c*w`@xg%Q=Z|T{-24$LBK2%LONX z{-M)KdbhLapO~9Vhu>bD7 z20N_QaDLKzwM8H@E-pu8v0Cln!-v)#sSm5Gt9jjK)Tzb1BO)=51h6l`&%tq#kuh5- zQCv6hy(M)>gOp21kewVwpcKC?*EL$j1 zJu~OYlSp(|m50Mc28O8yA_;Y67%FDalpEm&0sC(IaELlzD2ZqB+4M|(IOos>o3;)cXt^@}X{Iad zg?-gLKbv8#p2x6TbfxQba!$@=zrE!q>;A&D(6BIOQPebaMKUJ(s!mT&doR{(^19Dy z=Ni=La#Szfjbhh-qn4q_EGVe`{P}ZAZqo;L6h-|x@F~kLFS&MBnmN{|GZI}OkIeyQEyXIp+&etyj#O;0Vt zU|1|ki|2`n`I(taVuyRH-7x{ZdHUXKU$1wkE1W}R?YFz3Atx95vo*e16YRWR%qPy1 z4HF{~pHingGi;{XUz5jKe0+9Zx6Z%DePPHd)*bd~1}unfk-0)i0wb7~D_5=*YF}pW z#xEaY>V4JG8J=oWy+2{KGX4S+$#j$Ps*qFC@;&t2-EAJ%5gC=p+u8SHt{!r`(ER0# z`qis^Cg_+sd=I=I>kdtR;W`1-ZH7+POwQG$?=y~>?Fg%qz(P5kR1mfuCDaWlf zwAa!%sWCpA9XEuj;1{M=j2p-YkZq<)ii~?_46jvkTpSf@z77r!BBNyGl$|3X$l5^_}qhePxN1ZJ>eM)k&-j3K)wm}9Wjx(&Zfl|8SRo8g< zX?xvzJ*PLygGt_hqBh`HJL7Kmf`K56D%oUxX1dkBOyw-*?UoaGc&hgHdBepfO;4B^ z7*api1cZ;z&axUB8kSNbj-0(cn;omm{v_`{MV5H!X3+Ynhp)8`)ipQSrZbhiw`;<% z9j+{?J=n3SvNH7FPSY*5{q$=en&&XSw6*na%n#aa5>X4Mme};NT)uqs>({T8!mgQ$ zae}Y0@yTw4%RHt`8$H-4^Zx#jac%sClJ%$Xi@JrT4-eM!11i>!tJp`_nEDfaiY3Bv zEP06^YdM!n{h5^|3zGHax?~*m!MvF4yd>H18}m5RsCRr3;LBN*z0X(s7pH`^{-9QZKEp zPx-vpxX*I+qHXSlBJzQ1B z&Y${iTP&bXFWti-kG=BQH-e_-hHG2LtYf>aBN_PCSKFTr+9%OPe{5<}Bic$#CuVN5D4%ESw3jmIBeMA}@g}0&;6KzZ`QxWwcJj)M z+rVWJ58Sh;2O3Ki@AuU1Z;bYLn{m6N_wgt)QOuR^wBemQ(Cm&WONb5`AlZ}riW>j= zSA5m!3I1xn)jRl(-D0Ta(yJ|8agNk^EbMI`M0)hHRwt1m)>PYJf0l7xE)vMXg&OFh z}V52;yx`sg_C1)Xl z3Z0^!hKlT~J;G~(UuFy94x%D4L7UaDSE%GY-xg!|klc8YN&Hs2QJeYfxXy>S#~i&* zW@~scGyConBP!&eJ(|xwo-q90jI+@v$;o{$e;_-U@eyvFU0LSL+GXpNV==mjL6|pW z8t$RkxT$1yRaC85Id?qUditbCTKD}>n1Wbyj<-E?iZ|UIc4ykTgt{bPb(EFp7O`l;m)$E zY7$D@-d+3KG4aNjI?MA45-!MQ@!=M5RySn-6jKBoN7#F?uwh7HFW6_ND@isZ#b&$3v+6n}KWq2}C57EWlQA4cPGTw`_kwN{ zPsOO1=Vlshf{pp9yNW#-hUbuUPKo;UZDfchNb(GV5&gBm?7?x~ioS&UP7*Mi-Tx5j zP=~UkU)Rt0D(QrW4ZSy_@_yKZ1%p3RGNzk2sa-q zAn4luaH(X4>iIWo%BJ5<3!vr zxK2*z|9E1Pk*01ItZ~wv0CC$2;MSL%A%XjPY3y_f!OgY3>5L}D^G&BoZxKXNT=IH# za+vDOXBh->8;jPLte(bn)+oRs=3nT-HeJpnw4RwvtDlp~@kBCBTrtigm|D{K&8NIP z9YLYt9O=pERA`)HyfvV(yUbwMi+cEX7fB-{tlk$jMfhysHX=w8o*ouv?{2oagW5zw zjbm6@?73~nEplXRWHd4Y$zyq&AbX4UL_Y7ZDg7P>y2WAKr#8TMV1@*Kb9TOL!Gwt( z2ld7t?DzLcA``lo4(CiKFawYAW*(t6h9%QX zB3Hv_-7uEdpBX4-wuNFNIM>OaRsI2?{;xApe$P4kb%Rab*MkdHOrThG@6tAqC+7qd zA@<>pZL#jni`UH2cKW&vfn2s`uI?vr@{rJ=S%r$qsayBY3<$QhzuflWwd-6CJ#m~5 z1F4@lcH7&#vZJGe*K_H1u5O83OAMcyv2pSf8Zq68Ju54#7;dxB1kdj$>_)0}IO_Ha zVc4{c1n>6*?E`1$Si)|RK4xw&S=7q+sYWHFzOKEgALLltVV0uGuG%l5HA z;wbO(P%N;&zdt` zS$KFj2?>em?!(Yf=Yh>dIep0wd!ziMQCfQI-{z$k4H3sFiGgX%t4I#Bk&#h&Dg?Gw z(=dfQcQibLUh-OnFI2B8qTyEMde5!(42gYNC9(Ag^Ol&4+17{kFh;^4(HR+XW->Ze)yM3T9hv z41T&jSvrgt5*)kXN{#)Ubu}ZS*Yhl+1~uMR5bKq0J3vCmrdxai{*H2=A4uCA4KRcp zR4tlae|(~Gs}F3xTgA-|3NGV&&Mq!|CLd1L?960CINup3_x^{D^ z@Kx9LB_ZP1jl1*tdo_8oNv`P+BJmNzx#%FtSPHBd@PsN{xWz6dN~~*3>C zl=NPdgoiP8BYro{f@KY8f@gf>%1W%>g<>I$YWCcpL^8hbX7Dmx8-cJPjj#Zx{V{=q zpaIRL$^L9I+VKpcMf~UL?>-+x7|s6-976hz!F<46Q^|krcMd%g!os-WAWJJn0i|Xr z1}c<=q}Lve=9Cn^1Yon#B|2MF%`#*DO(Ze8x|aI^IV>y!sI}yK;Dii6D00J~*l*R% zZDTJS-dx`ZFQ+?;#1ZOC;s$0)MOFsBB#?a~LcI1VnOnAlpw}}?R0V8t7a5@=>&UwrnrXF%S>Rg5utzT|(2X{Qv zcW~|CCLF!2Xl=VgMs?@XyEn5Lvm(A9Z6>R*zVE`jr#lJO6wZ(D&iG_|j8X7w9KGNs zAVAQbdaDJ}#z5Uak4riIAYD6Q&T;{hpEF%zBsRkS{;h6gbllDJumt=_IM=S#PV46E zpNoeH*Cyw%U~+X-SOc+FlBh993++Fu;M`$nW-YRd2zZiWL=5o9@jLERa~Ss;rro9d zws)4wPW+Z&7p{9zQfuzvQ;o1XVRm_0ir->hT5OS+vV7j{F3~q9rE;WTk?~Lcj}7p4 zw0wQ+mUw6t3Y&oLkIAe}_wa6nARk1&1C0V~N86b9;dxacuh6NKxy1NG9K?|twPDGG zLeAvR+W2YS4%>gd{1RQ&J$AY$VJ^rpM0OY_R#%nZf8EcaLmX8=(wAYnQkjd50Z=*x z3PYB2eDV+jaV&AdJ?nPqvVdYuRBjTSb3L;;jBD}zD+*Eb8!g|~nPGRZB+J8yX!eiXzcTzJlSlQSVAidK76p+`dgW=+(OW^5hRlzQ@wiv3a zCZm}+C#N#N6&kLt#kKnzx9;D&SKftJ3Lf7*uD5tDv2ka&r^I&$slQL5nZ2T_N;N%; zlXn)GY>m$bM2_j=Md|RhKX7Q7goSlqFe>%LI@Ib|9W4K7%6|J+`5eFPvoepypA_We zlg_N=s^r8)=d zCj4~yI!ZB4d4Kz9v&DcJh@S$Q@mAB+!a_cro}PX#9S0kGdi>K14R#j*3sDpy*l23l zW3;>dwbVKQM^##J6wfIr0Xo2VvVs4$S=A^lDQTmoI)T!jThPYD zAdbYDAYu$enNjaMYhP2-^%Eyf=$6^X+r@NSYskn1!_WE?+z>H!fLP2*4!XkJ7e{LF zh=}sJ_hiHK^Yc+`XZHl&_wV;-3Q07&DzXwo==W>7DqolFjL=*+LQkJTEyl6qmS2;l zV>pfOmf8=$@!#Lcg1t2!5gs`L~F(M-M-MgzOusc}h(3_)E#B0$?wLaY$ zlABDXaHCp>m8MdYR-*lyQ`BWVu<6u}uUwGpkIudx8q#Zd$t9IvyYuGz_wVRS_d`PD z!<}38L;VfZ)zMHRE!x0aR=?6!MO*tNjqlPGg_oRnazT5Ek$7oq>o?$Zvwi0#LqgiB zb{DGL0Oczp?J9&Ibf`!ZVx+*JFLl73epCk|;?I4hv?6+6}bR(*;6lzx-K6oP1- z)0d<=eN7Doqo0C#Iwu%q==bBT_9w|IwD9>F!zfS%^zW@h!!D9%xG5KZ*eHPg_ zi&h=OTz4ovJ>(z$k%$2zez%V88N24ai@c*n5W>4A&R~Pk6psBfKW?!H^`1sSM%E?& za(#&e^Dh#Jqrj1mi>Tb%xs!rZzdqZmX~_DSo)hL(NNonAH2R%!i%}kV;4*5ymDaS)$<1M6l~oR={V3F~Re0$@sNa z^I+@k%@F1zkM9$NLv#tL%=EI^=(iWJd;hQw6-(Q** zieh18y#DRmHwc~|U}9pDkdetC)hFG$u~RL5-jXeR!z4x=wZIh9xcTF5fvM=tEgN|= zH)jhb&9cAHY@o~`5rXBE=NGOQ27~M%kJFlrYo3Vp#|tP>Xvx7v9C>~p0nD-cbAx3{ zpFVwBS<3WemX?+d4GrzU=crEd81V|>bG^?}8K^hhO+`DNYoroqfQKwRVyDBZ4_n#T zh?=Y*VBq9z@41-j^y9f$kBvx3$OGVy zO{eghlao`L{@kF?jD@)V`Bp_y3F9D6Hc5^gGKCj=z=g2h-h)oRQdpy^)c|lW+9HbMB~B6t#VTH$$LYYS)rUQJ`m`5NZ4FIL z^H@zy&2(U3vLIoBB|7chY}~OKw^P+)pC5w{nY#ItF4^SQq(d;DrID=(nL-HzGY^lN zMQhwwAeXeI6wRH#D{$+$pl6LRr*#v#1k6DgM{T=Z*>HMwwP?JMiwtZ{2B6645Uv*m zgGyPyjJ{C);GBK=6+oG9+X*r^Jad+t!UXSFLjGkBL;my5qZ=sy~E#W3pls{B;Md!o_>`z*zcUztVfvE%e)a5ds1yB8k$RZ zM2LNBwdM1h2?=1&JdJROXuAA1hqbJ{e|&fhnJIG*c%8s(?E1HFIqgPkYdY4_W z>F@H5V-ZqwBtg_6_*}RSZwp*rH*0=%axlR;x$xTE1re0>KsgP-*>$88Z*Px?%Bnh1 zl%T<^Ho=Qa3gO+>-9bd+nRoAEu1|fH!oa{loiVr886oTd4yB2xs4NpxIta?@Pjqdc zzy0D>{apl>UTtunV4tF z&-0Mj&kU4gGh_iFfrpFR4rJJg0KQaX;;Dgrv*`-5da$)`Quw0Z9Se*#A-0@UwwOf7 zNJu{R_V&7uwzakKlbnoe8!qEL^_3byCY3W7`yotw1F8_7nw(58w%(<-vbgWiALax?rWRa1N)bQ-<`QS3ci zeaXDLT%EmYj!0mti@s#|3uY|HG+sj}r<9_Zkde8TQ<}i^oB`XbXd4Nv+BG95c0lpW z4_tB#Xf%P=0;mXpwe0G7v(K#Esx&QAfPwI%;q^NL-ho{L>7$cXhSHrMKi+_N7RUlc zR?VEu9%nR>zj^6QkjgPIP)R(pN=jO~x-lq$3??E=r^p;mUKa#99F;?`@sVIhQO)O5 zZ}`?7HwZ=zqy5U4co8iz?=@whiB6wBjmD{&5Y8N7EYZxYteq)O#owS@GGHOL1l@1# z9P;K%%ap_*VnHz8+DAl0SOS*}NI<8O!*Kl9K4Cg`87An}L}*!fy6= zW}aF_LoCItmj1T;#>yHUc-*m^p%s=f2&U0v%~89R42DT{e&Uj198*}M@os-zK;2+> zt2N(N7i4zhdd5U!+pC*}g@tqJw=R=mA3v@c698D-K$S=SaRSQJD?!Km#mu>1MZ9{I zbpu@SgwsvKG-K~7nVHWed{%#vzK%4AcIz+Ogf}M_;S?k9v4&RrH8_3Sq7Ss9j zS7v*1ixi42RO&etK#!H3eMYs2P@HamhbMJK$Qg+lL_(TV*tRyyRx-7uyJr~K0KlT z=a$VoClBb(#ZS{AL0MCns=Jeu>vNQ>)ctLYxCq@t)XfTWgOa1nIK1J#-mOIn(X)`t zPbN`Oz3>Q^1r9)6;K1O^Dkx}JTU$py-QQbs)BrN5zGsQ`b6AbKx#Kzqg0wy{sLw2o zASW8MkL`mH42{dEIappWNaJ`;4>ZI<9*CkGCVNSx-z7Ow75SwTa+pu~wTy(^(1ZIx z*kTq_wK+Kbu#2a^X)p{`@xLLe@49C&RXt0MHx0v%uanq(N?t<7exi8Ry`W53T2;2I z>(eMb1QU5HMNSOF%u#~vUG*+P-!_f#NCDXs5{p{g2j>12OPx~~)9i@#; ze}o zN|HVzR-<>Vk*=LDAb8}}ch$l9RvGsD$B{k=1t;IrNUApji|7QN@2|D1qJR$9>ZUSW z1Mq-+aeq-{a9+kgZWPrwV)pzmJ8C%30>sIbeoa^3^^q0B>eHsbfkIB*UGC)1AY9L7 zlw;yW>VLx=CBxY7C#xI>=c8KgG6^C{#yA-Il6si&l6sPMZWuy~M77u?6o~Q(eaSd- zAH+#zeOnKWu^}8lzdKbk;ffq#N@dQXb=u+T+KQ0a!;Z4ORCiL5HTJ#m8=fJ>>c95u z21%sT966j5Iw~1E4-94?W_}bH;}QJciWz0NXl4DdXt=hjfq;bS=;Y0VOE3FD- zh@i^&hy+3~@qOqwD0Huim7`1*rUPPOR1+UigcuIKA0Nlg{^I?Ke1*$N=JcsMzxrGalwJeCoU=cJw9z=9rRLAngp<@12joX4t?k`HxMWVG;Pu z$6a?lcmKd9;86-XoxdeGj_GT~q17%7vi@o3!!uM;>JAU1=H3o(?0&-qg5*Z{XN5~% zlj^>Q5yAI94vr=!3v$JX2^Ji2m+D26)1=XwdRV8w|9I^^VoQSre*O|h_Gx@t_I%t~ zNPW^`8}eTNZ5i_YmG!BIb6-;62%=D-xvzhpyT}ry!;u*!&dbvyC65p#9+c!58J9mg z7s9m=8Z!e6U|ys%KTy&cO3KPExCp?+8z@O);^kHURol}nJ?X)?U-{u_3iZ05YI(5puGwAMJ*YO+c1Fz6f2b|Z;Xx$<9bsbs z6*=(_JscrJqUn-^g3@ULnP8T+-5OtR1kp$aM5Nv95e1J7t5#muU2OdE$w`ktwo!{5 zZRp>RmN1|d9~Gn3=f!RxSmmcH`nDa%05xl0zJLF|Z1~?ije%?b__%2cIN8Oz!&<1| z@%-xL=jS&u+q}etdcTVI8ufr@$U!kwrQ57+XO3nzq#s@IGd}y~Tu~beT*Zj_l){Cd znymf@hweq|s8+k$|7*A(Dh4jf&U9x-2NP0T#~wnUT$HYqm`85ts{&kp3d{@;mbAS+ z-!koGoG0;745a6L;h@67$vSmG#uCV#7%r1wD3y8x$*Og0oX}BnH8DOu1$j#8HaV20 zKwK-fzt&l~P&sQW(I#8xNHU!6OyD0hDW&X z*i`z>^yYbb$nL=Cuv}Ln`vn?0lH+LZSSj?pa@p+r<;2(>MxVbZ9o#H?nnd?YM+ZW- zW@rvO(b}Is{SFkqnfbOo1J4+UOp&00U;Wp%{SvMW?eOvGPuO%wprM6!?q_+$bYd*Y zuKO`lnBi{8_3l3H)G8#T$Wk`Usd5VIyJ=oA{*Bq-k(rrKuWuvuX|M4g*Z?JdWT*3{ zc1=$1!Z&dP?I()%NkjbPn+bkf=K8~-{sih2%%d1qviwDC@;S95a|&-dGM+s4W6U;4 z(#LS8aAZ{4a^@X13mcE?n=Uyn45Zr}otA7xTs|!^;*`chfN96ipy5DOUrbbHqY<^= zUH6z$??U`UC`_ku-aXqn}Iafh0dPsCM@w@yqVs=&3?v@-t*C1Gu9?Odo7e98s=!U=uHn5?J zCL4kjJ(@gB_tUjFQWNtok5e0fMRM%kdf7s92YHY^0aQ4Rr%TXh(HRvWx^q#}L+my+ z3B2;-g=B78rR@ZfnZd5Q1lr48%Oec{f&0j?p1z97DYfW3q_o@06xZw5Q#ck0MyAA9p=Yt!#z_34Z z3(Z^U5KXiH9iI;kYn3qjOoEoOf^&n60qh(9EjY+vgu?z0-b-S8{0srDxE>U5=4<8~ zssR$x_Q;VP{uFA&FbqM^mW$@1XS}n_l+)jk3Cu*eHna9qRTs?zXAv>E!^n zGWfL7I4g2fP1kcHLu8m7ok$)Y$BT#b<;|7u_XVEE$Fs?XK6vmpmWG5R6-|vQz5pOZ za{4s%T?z_rCBJ5eJVq`3X4bysEd%3QJRuHo%&*J~)6iiQMV*QCuzgSbEx8Gp_ubk% zrm@t=#zN7dIpSpprNv053VQ(#=fA|zBVF=_Tf7(*XsZtuyxHJ^?W4{WzeO|xOIp#qWfc0Gc!%4aU!MDKU! z8%jXwnDtSez+cpJqVI~V-B7s-#1&8!9UDwUn+G_m&G^UXwxpn6X-65*f6qHd5^jz_ z5%SKg;o%!7n?|W;kf$=#Kmg*__E&9VD`(YBD31oLG`!JRr6HIAjsJZEM6mNP!T!)e z8u8D^9v1K~FJ@jeh!|UmcStra&_+2(#qWEYr6K}S7Z8Ixu-&q%>gptoFzpj{O%P`c zt<-elZ{ca@YNGX`5N||$GEJsa+|jxXXgw53FP0vjP_~#kIWzYF#g9Mwl*cGA1>b8+haQJmfo7wO31*(bOQ>u$3>fU-$~QKGEwjx{=$-kKYj<- zpmCtOjyC`gQ~OoDK2TzlLCUJB=QyHe=%)c?>&j5DeL?n>RV8xEZ$=z z3(Cf=raRK`PMlDI+8015CAnE;S{UFJXdSE2-}q8u&by#;jZF9KT&f zt0tiux#!C(p6)C)MtDOBIG^N~(G7^6Hh^(lpb3NfD=W?^FtZN;a%Gq7^%q$@e)f#m z1hRUPqTpHIjX9kZK*2r$mTSH?*WX$Gh z&ZmRyK3c+38gnkmet33X%HOR93k$1H!Dgho;uH;yA_m7XALLshy!Thljs6G?P3Y67 zTIS~FCH~i;2n3+{vk!{NW<;T;hR?EH68O%v2PNv05WVSv>MDcufcootqrv(Z3W&_i z%t(rl&-ymAm7(?ar48>zJQlisw#1^?AD(UyWFVUDEgp;H2#@-Q@McteD^ z-XbtMRnlvq1}tJnXD72YLAMte5hy>N>CAX~gdBt5x`CQxwj_8n^K=R}&9YC)5apVGE()XYG3!#~T3GNm9uoBO}qWb2J)9qg)7j zrwW>AI^W&LXI3PLkqD?YZ%fF9kR1%%P2hk#VcC2jwa{ZBjWm+wIe-?+szNituxnUj za(#V$7<()f8CAGU&g^fFY5}OkY);VQ_1}X_&)@abZ%|QDT_z$T^77hWGxXEbi+e&P zlnh>HduvO%k1}&LRrZDJWW2`=@H5d2!``#G!hSo&Xdz8h)FmRKjgl@bz`k`ZlT}$s z>@5W`6l^YXztwnW2Cx4MG>W5q2r7TG;TYOLDlyZOd+vVDH(^GwL0}%;B|L_lLWDgQ za=>Zl92~p_ zI@E5F=!fzv(^0qVzXoC)W17M?H8Yz=iEZG6if#M3v9Yn4q2e8yAwb8`S^~5j01Y_8 zyP!d5A6!odv>S~7_>oe#H^G4xl+pOiag|Os|8c+F(G5i*frxRrKu1@9UfWPW-~9OCQeiSw}d5Wx&%3a${#> z;{*3-|Ni~^v%nq5K(hY$V1M@T(?bR*P_?K+b+gb-=@`ijz0 zLTFi@0j?iRJo^7%Jqsuw0jDs2p?P=dTLYz_&~!EEKe;_218h8$klYUreq&hY?~YMeUapPyh>@N;bsmHRcF+=- zz|yA4wnPr#ao|L2L6?{YihD!dBy>Ybi@49dov)w*QZn^(NMQCZ<%<_D?DChvWo~!3 zKZ}WZ)7PhsBKi;u19Ma;Mn*xA1+H8jpPU_n2vyLPHzkBWzr0HM8|u@O8(Ry8S8mmB zf4vu_=H@HNFKgf4+yaJReRuTG0HSJGnD^Kn9BB6{z!!G__#N$Gg6+4lw9pHcN2B65B3&Y^U`OLqZmF)lcc393b@}VP6KB}9W1&rE7Tgl# z5BtUL^SZ+Uhx_1YJAiOykVOv?+66TXfvK?X)?4$|IGx-WauO0IE-n=mID)6fr>C>7 z(|BD2TWVhm!bPh=P#O*vkcsw2H9e(`W&oUi26z|sn>Qcz|l_ zw!{gk!CC1B;*%HHgqr=0!JQ_FgV*T16a9CxCCHs({cr;l)T9D`I`Fdstmz}4g^J0$ z>kuv0EJN@KQ4M+#a>Rj!QJ`;7^PR1~`3CBv&V@e$(VY!8It$K$n}NdQ!=r*0HC
<4K7 zn~$0Xa)`_HBXI-g$sdp#6!|zzjyl}9@e$i_ce@uNR_&^3E-o$^AoYfeEfTywfMsrB!d0T28s*q zvO`-M2hWShTKD9<0?#~0*a?kvtl%~H=b>gRngKGqMt(M^Qv>xY8(st!3feY{p~7Ll zuRsd?VV1;TLs60V_QsPlW)YafdF!u!eJsjybjPijx8S|D#0YE zHyBU|gF*)=D#^2?HZ?V!LYr=FZF3;AfYwJ7k+>Zf$W@Am^R)64pyu3dtR6$QBSFju zT5kElI1aXf2HNiK*A?3j3(Km;^4rZt_1F4UL9IKBtg!3UGcZ#c)BBp&!A{I?I0DhB zg*xM5m;KSh{lp_g81_pQiubNwy_%P~F{z{`xCl%-T3RxfUSiwd;V#0!a199Ko}?>5 zT|gP5)!i^K@gWz&_XT>1pQO1=wt7QI2EAX|Xte}*l_H<&JdPa8_M|Jh`}PnNRqeXD z_2q-JLbDM#r^(65>IMcV-)h()Eda?6S}qR#m+$4C4uhZ~)1jLEM)<*I-Jyo={eu7W8@nR(FKg)O>6KV_ zrOA`ak~U1n1&kP0LM~O}GO1W)1uOw18vFo%d*l_g<@+D`R z0C07vA8_x40?a|V)%^vJy3R%aO%YjD*jtpfhK?4>_vdfde-c~$!UIJ{J$B0DfAQXp!i%hM21GH-b^|ol-0i6cWlBce! z+991N0j-#50SP3?Zcr2nY^vc`n`#np^c@+A{%^Xv)S+*73am*d7{_GjvzRFiuyAyI zi)N4TC8)ocL3?;ns&%zBQNj>4#-JCxkT0T8$y#4QsVstE`k}P|Q(wbVA^$)dzpp=~ z(XDV!18;v0We}hVLPbYMCvDUX`Y^KKrktW;Pgj;2)B~{RY8PBZ?UDY}GTJ8y2NZ5< z!gir$0}8SH`aQe8a|KOD{e4hDGXVNiySE}=v;5(lH`qT0SrnB5B%nLxIuTlEg=T!v z-<$%Ozk$j;&dIY$7v-bSUhZ}{(R{XjoK;fQAo&i^qY~ICaO~HicQgUCjg*4oQwVjY zY~1k^f7t>BT`_T!YuU<)LLguZ7(#o~j}dk*dAEZ?sJ_uELdi z><&#{i)Oh5exD0Z`q`JoRu1RolyHB@G|0E98V=sDIQ4Pc*Po*5vOHg`LZed-Qz$RZ z)6r1LgK^3LO_KX7qKETw1R+uyETj}B5$Rc%UVM_@9VNK~b**VuEGCZ%FLzp(cvmD^ z7&Y4RtL19qn{Y%oN3YTl;s{NWG&xPFk=&x~!9N8@ql}Jd}?k z7#=?2vgW0vJb9d!1+#zN-NCMNT3sjVzD4+C#m`kQO9`5%-})&{)N?gU4DuyJH|z55 zWcb(7_84V%tLFFOOVed^%C=U#C49l~eA8faH=YP<>0#}@r!0oA(Y(7zr9u8z^CBZ4 ztZXr${^JPMSxPIw1{ioL@!M}c0mA