From 4eafcf08af8cbc27a0a0ca55740e40fb54958b98 Mon Sep 17 00:00:00 2001 From: Michael Farrell Date: Thu, 21 Nov 2024 23:53:46 -0800 Subject: [PATCH] New cli command `tr-pull-datapoints` (#365) * Export datpaoints * ud * ud * ud * ud * ud --- .pnp.cjs | 10 +- ...ypes-npm-4.92.0-7a3e95a69c-44f44a1286.zip} | Bin 356919 -> 357026 bytes README.md | 85 +++- package.json | 5 +- src/cli-pull-datapoints.ts | 129 +++++ src/cli-request-export.ts | 4 +- src/constants.ts | 1 + src/data-inventory/index.ts | 1 + src/data-inventory/pullAllDatapoints.ts | 480 ++++++++++++++++++ src/graphql/gqls/dataPoint.ts | 64 ++- src/graphql/gqls/dataSilo.ts | 16 + src/graphql/syncDataSilos.ts | 14 +- src/index.ts | 1 + .../pullManualEnrichmentIdentifiersToCsv.ts | 10 +- src/requests/bulkRestartRequests.ts | 8 +- src/requests/pullPrivacyRequests.ts | 22 +- transcend-yml-schema-v6.json | 3 +- yarn.lock | 11 +- 18 files changed, 813 insertions(+), 51 deletions(-) rename .yarn/cache/{@transcend-io-privacy-types-npm-4.91.0-9ec526e569-e338b15df4.zip => @transcend-io-privacy-types-npm-4.92.0-7a3e95a69c-44f44a1286.zip} (93%) create mode 100644 src/cli-pull-datapoints.ts create mode 100644 src/data-inventory/index.ts create mode 100644 src/data-inventory/pullAllDatapoints.ts diff --git a/.pnp.cjs b/.pnp.cjs index 64a165ca..239c4e42 100755 --- a/.pnp.cjs +++ b/.pnp.cjs @@ -32,7 +32,7 @@ function $$SETUP_STATE(hydrateRuntimeState, basePath) { ["@transcend-io/handlebars-utils", "npm:1.1.0"],\ ["@transcend-io/internationalization", "npm:1.6.0"],\ ["@transcend-io/persisted-state", "npm:1.0.4"],\ - ["@transcend-io/privacy-types", "npm:4.91.0"],\ + ["@transcend-io/privacy-types", "npm:4.92.0"],\ ["@transcend-io/secret-value", "npm:1.2.0"],\ ["@transcend-io/type-utils", "npm:1.5.0"],\ ["@types/bluebird", "npm:3.5.38"],\ @@ -682,7 +682,7 @@ function $$SETUP_STATE(hydrateRuntimeState, basePath) { ["@transcend-io/handlebars-utils", "npm:1.1.0"],\ ["@transcend-io/internationalization", "npm:1.6.0"],\ ["@transcend-io/persisted-state", "npm:1.0.4"],\ - ["@transcend-io/privacy-types", "npm:4.91.0"],\ + ["@transcend-io/privacy-types", "npm:4.92.0"],\ ["@transcend-io/secret-value", "npm:1.2.0"],\ ["@transcend-io/type-utils", "npm:1.5.0"],\ ["@types/bluebird", "npm:3.5.38"],\ @@ -781,10 +781,10 @@ function $$SETUP_STATE(hydrateRuntimeState, basePath) { }]\ ]],\ ["@transcend-io/privacy-types", [\ - ["npm:4.91.0", {\ - "packageLocation": "./.yarn/cache/@transcend-io-privacy-types-npm-4.91.0-9ec526e569-e338b15df4.zip/node_modules/@transcend-io/privacy-types/",\ + ["npm:4.92.0", {\ + "packageLocation": "./.yarn/cache/@transcend-io-privacy-types-npm-4.92.0-7a3e95a69c-44f44a1286.zip/node_modules/@transcend-io/privacy-types/",\ "packageDependencies": [\ - ["@transcend-io/privacy-types", "npm:4.91.0"],\ + ["@transcend-io/privacy-types", "npm:4.92.0"],\ ["@transcend-io/type-utils", "npm:1.0.5"],\ ["fp-ts", "npm:2.16.1"],\ ["io-ts", "virtual:a57afaf9d13087a7202de8c93ac4854c9e2828bad7709250829ec4c7bc9dc95ecc2858c25612aa1774c986aedc232c76957076a1da3156fd2ab63ae5551b086f#npm:2.2.21"]\ diff --git a/.yarn/cache/@transcend-io-privacy-types-npm-4.91.0-9ec526e569-e338b15df4.zip b/.yarn/cache/@transcend-io-privacy-types-npm-4.92.0-7a3e95a69c-44f44a1286.zip similarity index 93% rename from .yarn/cache/@transcend-io-privacy-types-npm-4.91.0-9ec526e569-e338b15df4.zip rename to .yarn/cache/@transcend-io-privacy-types-npm-4.92.0-7a3e95a69c-44f44a1286.zip index c849b119b29889d48d9aaa899216994612992e56..c2f6606ee875cb6b7bbfa0aaad6ec7e2a4c0d667 100644 GIT binary patch delta 15778 zcmZ9z1yCH#_Ww9f}owtka7xWEZUFw8%k?j~BZRd$ zlyF=I1%GR2O1QuSokR@yn9oPNQ{-#p!VZ9?Ce9u6K5c&~rOzuEqDP%yTj~=wB65KC zXwM^XOw(z)6V%sLs$J_Z0ujJ7ufc%YyoRLAJya_1f*p2<611fG8d6ni&A>pqN0j+{ z8J%EopuXxWIT;Suc`PvDtI5uJma~!4m1NRV$UM7Weg>;RjFQHWypRIJ=UV$izsQl0 z8OSZzlc`{2W`iJIrdR#Q&ood37BLSK4)_<^%g+;kCae`aWJQE zY4zbDiXX?k2K8}IKeB8cd+K{Si0nd)v=r@uuO;p>H?U<0Kasz=Hd`93&Br|r(8H*} zlw2Qqi!(%|o`?yzq7>0-jn(VS8!G4K0^=g}a4-$x1e6Dpz9pk@(L;5nB0~9bXIK%l zP~Nu;)IQhONM|pUGur}(ZXk|(mLrG!$nLZ>=u4*LFP0|Mho>6V%ujUaJ5lLbW6QR0 ze}1~AdE)I&KRY^dm8x=Yr3_r8U9HvM$1BQ1P#8O@o-SuB?9e;;)Uoq#rmc`H4#r#NU$RYFw*l#b{a| zIB(pnnwe&6so(eBqt^+Ar4W1NuP6s~?N~5gr(=NTYKVhJu$KY-jr!E!51B zL7yFk1MN$ek0e=JJ)WE&w_~&zQh&2#5@qO!ekF+&`){7>^c}FUi+znd_hjo#<2b!h zRj_ac!|(Q+FdW?8%Pg7(_w+>re|{-$h3Ei-0>+h!E4ck@v6LE1g=)<))9J2f>bg4 zS{E_^=JlUZgump)=EuK}_D|>`r^Y}vNI`aMo-)n);lNc|8v6a&Q;L1fmS^H{Mq1q&CmfZYkP-93dQo)*5g^zPR^+={zfA??8yFP)L;uL zivIaFWjjipN<8TIwj8tc8SMqseipW@*V;Y35JU5ozFxOd zt>g$Q085GVLvNwYEb!dKCc&A%^2)@UFKZ(jscWHc0|JPi`hnv4si)?$=JIpvq!L900tMOWhpq(Fh)X~~n8?W; zEFQ{Z*yEQ11V>2bFyb!wUKl2dj0pGM@=!VwVc_p!R zELuL_N{Wc5%Ls?2pco4xF<8_o{LD8UTF~nY;UEB*YmG&%f?o@o31r>aGc38SI|+JY zICzm6wX~wxMda22TQWOv8~~tu&TfyHxz6KHQhP*#o^E7AP6VZ1rNIP+3Ucbn!cmVB zEs02uN~?a?crh7`-wPf7yuD{7L_w8!$7%HYw=_d@;uYW&f=0>i)`7nmX ze!|fc=89Edc;OF@8mGOC$PhRdC%IJp7EU#UWGl?=g53y1@m;-n#Rg#Z0Krx|CX0Es zL4tfNqdz}=&)|6t)!Hzq1%3djz$4y0S>?$5HsT&ER^tXR?q^AF8NU0Iv?cwL@JP2y z>(Xnk89{oZrBu^z)Y0lq|2BK!b-X-tNihIot#+$ByMB;)3M-&|>hF$Ep)fwrT|ni{ zNX9LtrJt?_&s(ipmH;28uGcId%Yyg1NSbve?nLgK_{jtuwYf17ENF5Zjqs<2rq(Fu z{{Y>Yb|+jZR8kt>f5LZkQ*M=O%P2Zyp-zJo1bkNm>Zg>I40ip9>Thn;8A(bnzdIZ2 zBb@8$lWZutOB;W2JDv6L;-`z$5aZTWed)KrBa89l+1C>P<_%;GMR_@~xI6fXxNfDb zx~sK3So;wOTV*C)7Ca$+AUamdFnT{f+o+4p?$`rNU+AsT);ey-FZ;8g#usFqt}{?2 zNPdW1_D3ibb1{~!@;^b`&RL|Y*Q1Q!GZgq^*KslKhfiHi-0 zNfG)JC`Jwd2Rzk1wO{6%ddunxkSE?qyH;{Lw~izpJ8stA#^C*#i6d46hZQbYpG=jl zxAM5S{PvQ%87V4Nux3QPD2CfyF5)P@V;}ORZ!uoVRt?~O-n>Ra`f~nJn*G6e2a~nVu+KIttG0RA!o>Z z$F0^SAgYthGDGod=x~`r(eY;nYH{SGyQgm*-e}bz;#JH}tP!=04!&pi>d1+Q z&&X;>%1F@|{zzl|d3^R7af{-Q0?ObBPR%3O;Zea2Lz%b42Nr(rS9QAe3vjHSaix02 z+$HVpD{vRxLo($q$tI8TvgQrGlZ@=JBN1aBewdL8g|8K{^xISz7xYD>BvxV)&oKqG z<#9iy{rqx%qAE^nDM|1SB{M%&7YX`3GxNO@UvUg4V&Lj!^~BP)#_LU^&bbc`w21v_ zX!{o3Tb?LVxXW&^$xTmeBQO=vxc;<|ziHz}{pMb2= zaL;{3uQ66I8K1f83n|9A+ejgH>-Sdpyj9f`dm=Z^JVPmCxucK78lEWUaygK16lz|2 zo<^dpq8*hfO`dlx**1{oO*xiFp>E+Fw7U_f&C`s?4tPx)`lVJLP!W4RXgS zKY0!!ApgF|Omsk0irvbLPuI)$&s_cFo&uypMsl&Gsj0R8P-q={&;EkkUTN!K3 zy=bGa9j2l;yEzplm+RjOWck>friIhF-LMTS-3lX}t`x0)7#{44v(BCS29%*pbighp zq#I}8L&~zXbvF>56Kbo((q8itKF`7JHTNggeF7*h zc;j+Wj(Xg}P?A32+xq3=7l|epL*9+;$?tp`#`MW4OoA7qBJ`eYmsVHep0`j{*~DG@ za!1PJL5I3Ref8qao{til_7&oKW{u9EYOTJg!i)v)!sd6p}- z=v&){6}fUrdu=B6sl~yOx#x-)U??6+8y#9Bg3C(TPRiqnd|~7LvW|L8NacNB0F}6- z3a&o`tI}_dLk4Nw(nK?acr~k8@<0+BY-)|ZXtkLx=*dbth}R^41R-l-9D1S#LTvVGr5sae7<*0Q?+13%N4mHYmsTGN=dKzKJ}DL;;H4QnzUG-E84&WT4F`EJZUa1jV+|U z_4f1cTYCP@7t;da3{XZq1*&Y@TF%f1&n;HV@J8p6=Gd8*SyUO=K<(2MQ(6?+X9m?e z%(`XgGOY}JwnW+b&PyUo&tL0Kx6>Qs5|JBXW(F1+8M)X=L1~^EnQEqA<`^v>w z+p^R3Mnzo|RsZ;OCn$G=^ra|EvVIHRjNdUz^2}-rPug?~?>>%WX07pPv_@zu3DI%@k*jhpt?W9Pvejo;N=X6t>+-plzu}^ z>IZU+X-cpW2>(7`2*`C8n|MhL5KWn>_M=gHIXXuOV;MkOp&+QyLaP+wTbQ-K%juiY zi0`KM4G8yS9%^YH%jwberBP%C0l5X_w+vIh^65h#?MFV0UU+i15i(KNl?qJ=X-4CW zhmLHHnmju4>5lDXS0-t}0E{u@_|s%@g*?7i5di17SsQHmn35xCnhZKYmzs zw<|Q^7?+>YIJ~%cc4EzBNcr=KqHa{aX~d$(@G}?DOIlj*(yICS!k3l5+S16d3k;__ zI&?nt=hB(Y4i-b4S(UhMgdYB?T}~XP8i5=#Nqi)Iz=A7NuBc1nP;gS4aqfZb^<&bJ zH)Moi;pPLZ9uUaV_Us{0uYMDFuo4gTSlNh-;`e4G6=0iA7vrZnpxsK$mp6bip zMa)>ezx@YCr>d`iaNlz&91|rUQLuoN(iBZsrpP!Bm6IizN^EutZ;k+247+XHLAu8l zQ5)qZwZrr_m|5tma5P;$aW{1F<_bo3ZPdo|^5bI77|?rowU1PP=IJKdJ?zcH8LARO z7H0lFB&@YdifjP`3tdpCZ+ zBFudX!nOdZh&!d)SE8TgZV>1{R(m1?)Q^fkBin=j*d3NP zUz$4IhyfOnXoAWEu=pRgE0Bt5mDK}JK>D_ENcJz?@@@w!9Bg`GmuXx(^sDh)QmFt@ zP=p+e#sIH)aBNHUhwLaj&`CE5d7L6n(Wm~sUy%u@iUC*;F>jXIgY&*+5Y3LlgbYrGb18*I^20{3*y!S=gI3BCOh@1RqBas;%4ynL>Zn^!%O$&Hb9U`fuhs`sK$RB zY!)8mAp#jvj%5S-=P(v^oR}oU(!*_>qb=fvvlX4@NW13l>7MpbVr{pCXn&Vc4sx$q zuR+2Kd6g`7<-PbAtZm;P5@!+oVPXUhNgz^NCog`Zja~5(e{9$IJuA9XW`aM{G1qNi zi>-oqx2S$|Y)G3@@hCKHulC#XB#3uwV;SDGI@F4oCy@($W*|P|!(yi>e8){X;zeQ( zX&2{VW7jUS@2?u8iL0Rbyn(krdodGD5oM}i1P8G2aRL%>*O(7wvuq0SH6j7=jke4h z*kF4q`RKP5N$!Y?#=XfGe8d-{20uwT>c?S90C%ny^`u^+{nUs5F|0=;}0$;7%lb@DP5jTKt3SjYLgZ zK!Z!Yw#lO|(S10?du`U@vHA#%sc*VPj||sZbBF0OM(An5J2MMMM$+X=c%)&JEVO$mZb?gGQj?Y{*2qhz5Mdis}G(zqX8AL3AZHFX9)f3-= z$6l$JVR#p7{-yVffN|()B-{X@iFJr|v9nE6*HzO~vW1)uyiHe+uqm>X+xYLoQ&?0v zv&oVZK7Lg9I1Z;H-`#{OoSBM12&0FPMr3>qr+qXj4~#CegvEHD*Ja1tYl#vW!bwCu zJdpYF{6!ltNGD~_uR*6kEETVpM9GdTr5yiaUEwaJtu@|FB3aQ-fVj~^G+6bJN{?Gb zGo(hvp8!_zm>%kp@|rJL8iKg0HNR<;O-QBTCTIr7;0f@^Hyf{@YIw$d{Vk9eQH{6y z$UOi7)Oe)3U&4t}a=}SYNhImEVe8=>^0Q;| z;^pc6S}}GFKSp}n<~A}2f2jIGF>pmG3CrB7qRHg)B-U9}1iiwCq!HEL23Cc918q`& z`zVhHN20%K+T$i=-4|Hc{GhQlDiIfRz9DDPUdY}#MhQ#4{00p2pM+ADX_UC8y9WIs~OSyya6lRZ)IrR zk@`ae%t6#E?gme5U3=HSv#5q|do}SSlEHEs={D{HOcu%4ZG|i`_av9+<(OBzvTs~! zWV48D2k_df6ZT_(q(84WFohWQq9_Ur*=J#?nE5@(39)=&%leJc({1pvY?42ftyd=$0t4fT9avB^MiClri4-{ zGo0)bpr`-{+~KC>IU7NGb&>)QBy~-;d<$e_o#)G_uuh|PVar~(Pv&k|{grOEAzxwZ zbH}&n#(GLWT0woFrC}*HV0=`eIADl_U+G9D#5vG>qe9#+7Raehj*XnM0RPc;d9B4eizcx8kIoJKHZIjZ{|vi+*m9!PMGlbb{vL-^DR zx!H3Fd!Ud#P$+VsbX+g|ho;}FM>^4`X0`3KqH;R}xUS6AN7vKGF*=!?2q@L~rl5v$ zrEjw}0jv>P8 z7bBq}YowR16ut$ded2OiyyDAUW;EA}JDnA#RrjorUT&mu&0J4XbyGnSLGcX}=kB2X zK4&7(hCf1U9=1;tMv&7rLLui0FQ&{SaQ-aJdIgMUXuEX1I%~7nqFt`>R8z;~4!@*h zn2FBn>1WhlHTsu_ z9OuW5+ilNs0z`I}s=fy@-!d(Ue)1XhmW&d)KQtgbEDp2^EtUlrjt)g@rp}Sc>x(@DN4P3N z=~N?IZjBFD1I^vEJ%qG(Ivz)NLtgfrg4pAiW_)`MYY2% zPO?(Q>C-8BXm!N+PWze@_8MD&`_i2Rkl^qhFaA{HbpLB=B7z!Ym!fsqjQVC_iPSGCJv%$p zi4V2f;`OWfU6duK@kqeV-N|=yAQWj{b#rb7hwWt^H_;;>MDnIulMA!>q!DXXgBDKTzW}M!)*zZIa)m>#8k{gSJQeGJ??^Ws%GAs7-UD z|K!WrTq+@3pb#&d{jcew-p8}9v#MxDSSObG-Bz~%^`xH;&WQsA?bak_t9WQVD}Izr z&inPQgDVBLr*3J^(+#0ZU5(tU2nI(6r&bTwGMZj(ZZv4t-P7l)z|+O4K1zvW!HYT5 zJ7d$B3Z``=Jwgfl-8mxOHK{^ebJXVAGJ`&mtmkthidh3e&T?9U1bij^QAb%u0kMmT zs5qjBMmMMiVF!KAg_93oM?$r%v7?vsUMvasiy=dwc(e1f8UjiQD{htB!R4(wa;A_j z7g6<9kKB3`33*Goz>q4F^|=fFIz#OhHFJ_mvCPJ9mq@3DBwk=B_uDLx!f~=E@J$iKu8^xmj9`PO-{^YI`YUPS3H(Fh4fo!{{QJG|C8v5<1?>k4Wj!17g*rU3Yrs^)9np(Lt8T%I?gDK8@rW+DmO*vYb% zQXtkfN`Z(uu%Juu+JZnIF64^kIp-VVamJREgdmYul2-wzcXo zDP)fNCRFs3yKRw(<$-hzygW7Csjr8ei3^aZb7miGfr6K68$KKY`2bO)k{LtFZ;2&! z@}fvkp+0Isf!UF}e#RvLQF47H+){87I7>v8UHMN&ugU|m!GlMdBG$%!3B5Uhj%0DP zYxXNv0&z%Df)A5`5!CBw|3kb0oO0Ke5odbU9M;dzXBi&nHJIEh;H z&Mc>Vfnx__X`2=UQ9kt3pNnyX-pI(2$b*&-rF{LJ~ZR#xc4q|G5>iLm^4TYq}Ct% zbvv(ob(U?yE0ndf+U7=Oub!vFOF;Sr4)4s;G6f3?9#Ze13G_#n{4_PsygR3cU+WdK;HxGW#MI&KbNzVp zhT#jA^R~VB^u3I^bHC6IKzkL$`+uj*b$e)`#tD6&JqT{N zo%3D$It2i~yB@-R*^`A_2O1JT-^JG&tP#e{CtQKssrbAJD3f$Ih#+<~l;j}o{g8s- zO$j|QbJ~Adk3Ec)ZCdr3#E6!b1K|NIUO)NteGShy`odjeM=h;j2x336aPI<6XD9&sLbiQ@GKeI^d>?V=62qOtENY_Lv{J9vE$czk6?eM1f)saQKr*9^DkI zag#}+Cqm*xPNB^W(Gm835FtKCi~ zvBvW;{@Sw0q`ZdIt-jA&d6U02$8V@3vf-x;!5EbkZb-GfI%}|&oucG@x?qsp2!%(a zCu9f7OXt-Kz3-QoM7GD`j*o*MI+Gc;sk;E|4;bV(v`WOGI%McJcqc~*n7e=CQ29E( z#^29BUsDyiNrwP&#oXiLKCU^q2A7HE25IViZ?+nnGWzf$Swks`Q%B@|x3?j^x8L8} z((4M!#l}eM&b%b;t4tESda>Cf@MrqsEm-Qk?6_5E@Eit%+U2Dj6)!42wrnLBX@UsLHqC8(A6`Ls#Z}zhP`(r=z`djdh+@o+1 z_OEZ2SE%c6{yTbU4Bm5$hAv&NFpHwenFQ>E9&*+deJA)LsV2cuNR&K$@Ys?474FjG zC#6u~ynxHHE1Ym6S%#}$Ht}xcz7o#7jc$Y=&{tj4yM^$XTW}ydigX$@tG;I#?WNgw zzA?tXaVf+B4Hk|PDV9}TOk^blzq4NwZEHYB5E(`

;Kod2+B|_&?9FCFqQs)fdo) zI3>IV(!G6?str5OG3=y9+IC7ipXTRL*v9nKRz=kQttyGguZpO{#&)X-ptf;L81Vkm zGDEP_e|ELRz(iOe;n~tNy9b7pK^jg@RSjq3d&#HIa8#Lu~11mXEU(lAB?}6L&ZHV2)T&t;*)keBSglOvEyG zkp#UJ2d8c?Y$Uv6$S%2y`it{F34BeOPkQ7;BnxMPZ~CG(b*@@U&(y%lDdG#ad`-~nZz0nD_O{)9DkTg#XcO7Mwxng`lodAF7 z{15~2F8(rp$gi}JUrgAN&5g}{<@^aV*;q*>z-C3>1tWN|Mdh4MN2oWTPD;v0;S?k4 zCJ6@u;ZmInb-E?qP@)Vyt#9}VD=fQvJvwn0Q4GjpF(j~N-xv0xy#!wtXYOhvnnozG}nQ-m% z0dslX&Z8O9CE;T1OUK4iWDWw!wR#=khaonHSP{@2seY!Tr-f6Z!TGqt-!a{bwbE8Hn%F~Ur1KU;xQ=u z)aKVz-tr@{`aFL&PuFr{w-5&9zrx%$h(HSapd@U2uQ20$B2z zWlJUWgqUrkB|a_RM7e2W7rFz>?}B?i?(EaJVc)c4@6|^RS!y3e+XA}AeHe9HtL(K7 zRmFbrl7eo_PT#UBl;FC8ByJO6*!?8L;JrFw` zwVbdgA@q zixcaslQu*2(Ay6Ky|}lgMGkjiVuu$>O!*O}&_*BQHL^lfBf+tnDf<;sG!y5Uffaj{ z<+DGvKHRfVsXBi$$gY4v8;8LfCk|2Fz8FeM*|ORB+CLS(6x}$3|H6Z}l7wW(QlTZ- zUN;UN*~_>J{z}dkG-Xtu3JO){v=(-~Hu8P7Iy%SqCPl-YDrFY$mY{cu+6=(C;j z`dAJ_NOw`;O>D?jMVGc`Xsz>4T`t#`JEYUMi}?yvsGllJl8taVfgxIfZAGP4oO~{_ zAPrBbmJyxfex*Emdws|`n{Jn5@cwUzmdCG8ATv7-OTJW;PP)CIhE%vkr5{MDNLEO zteEj(V#+CE-McTk9ziKo^eLkS9@1%vrE`l<2O~;dT31BnomjbHHky}|lAcdEosol1 zs}gh=$v=nTeHwzU^6!K#ezPzLm#DDC*>z17WUs(@1-Gcj6ZQcUE;S|#)F#>ijo56v z4Z8sY30>N($4ZQiGevDzX~QSqtnk;+g4hgcj{AVQ$oa9Bo7!6&rzIq53F?;~M5Q@)73G5Fsv zM+9GlY%abmSC5~e=UH$v166v{wb@7wb-dG0M_8l2J0@D=Uf2gQgu`ps3<|#jl^}FVX!a)W5ol2cV1}AUW8>k&%a4s$ zY$hIdI0KIbat9lgAEZ6lT;G=1=u}Fl3uB8`=Ul<(O5MR=CxuY%dZ_sy8Uu0%RYLto z-I+(!JIu?EMSO~~2}gtZFIDt<^(9b(M4Ss<K zcnj9J`l+qR)K-_@{VL%L4djHU-iQsk0F@e zzFqcLEtFXtV`VesxJIjpy)wJnN#K?7NyG6y;qeHupJ{=aT*+o=ZD-*J(7^{arEw;V z?a;<4nhl}HKnI29ycl?lJ6mh%^*Xdsg$6NLu(R5;v-2Yd`s%qw0=y3*`O(j5vMiYi zmf!p;BUNb5b_xU>j&E6Il6aXgDV|bK=@Jj%BdCu)$;D$VH7x~g zK?z_S>#pu?Gb28LW*L(^uF;d3%y00Lgq-^cAdW2!_mcJ94 zbV=VcQ)&5nUQ`O4Sbx;xaBut5Do|1F^7!FytkurHH>l=7%A@P#EV`3aVn1T?CHW(H zrYNpIAyC&1aO9GovA=Ec$8s@jGCcV>AZl!(8(K~EgZxpm6$O1C<*x_=C5Q~jT7-v` z=Fs&eYVzvCNNmz+rKp+%N<2j5FA#zT*0vTU8gMt_uNvqf)r_Q0<{?Nvm)6*?4mVq8 zx3`-E@~`Xz<-o#tdhI_X_vKe(r?1(Mt-#AfAP_iNRc-%vHU8Y(aQx(QW7Mv>Eua1B z`*gc{`m(tx()L7^dhGgKHDd2^$a%WDU4Hz^`Q(QFdbDS_y6Wrwa1RJ(_BE>3TUmKc zI3rnGS?vVA?3thXuD%`Z9d{8D52g==+S>td)ZS)i`})pttaHc&J1^Jm*Vd~aj+cA5 zfF$}IK5rL)`iN3|Ze}C5p9P;=fgkd5{>X>AbqQQ&TPDPkw08LbIp?UM=7AUHe0;CB zIp~huI*%( zaKTi;V8I&%^PSk;*zNG7DvWcFagvk~;Ggz=kLr2%wLjh)58Z!+$M+qQ|EN}}2fHCv z;|o~2FI}|lx{aL2$aB429&S}d03ZFYHbwRlQJ2OAO6-2C)~v5B7H9v_C)DEpxJE7V zG{tcJqoIxc^cCIByANDndQk0|T)D;qw4U?^=96SanG2?l`CL8Wm;sfP?>Q*ifIbq} zVzFYUD&JL4RbA9)^=BPaM^t@mR(uXfy$xALLjf-9wr5vF$9P@Ghq0M_nRmmmy&zf7 z3|)coS3BTZ)@hLEQs$epH|za50u9FfnV0&v*kzI@n;-j6Gj1{mb=1?vA#bj+WIy|; zegAaGeUItOI0rlyc-4=tw;yir1IHmV&trnWeX-=YEc?X$dpqbiF2o>TaBVH;nE*?| zQwof{2qG89p;t(!;7boLv{VbBpaaJC?o-^@&UlvF`r*L(N7A9g8ImjXKM#M_G3#%U zx?c|ftp2`LUgXbs!kc~T>Kv^v#YDeY>JOwQy72feHoHTiM!yN*R}$~X3IGSc$M2Jm zi>=AG$ChhqYV;Y)spV2FqeC@Dz+C#SevvHL)eu10+_snNc(b**xuhP*+0rQo*J?aZ z8^K1tF!Y#*3oH-=-K%zn$Xoapc$Vv+dTL;RM)5hvX)9@|cQRSvFsvT} zOIDWVR$D(>pYZ{VdbFPuMu4+++!6z?%qG1)qy$62>CpL?SphPMq02jm=XSWk*YnWZ z_Z3kW2ND7M>L91zGC0P0X(?MuZ)gl{#nz{SLP`9P6o}q3!z76fTC+%Zb0+>B?+-g9 z^^|S9yUx7L1E{j0^yatJFUiZxgTuF!pX8S_B?jl}7;fUt(M#kyeggO8I?xd8&VxSP z{;2R;ZgY&S*aFWQGu`gt)#>SOa`H0vnLo4SjV7a#1gs2EV7FX-liR9#w)@!D6rFg# z?QM}y4E0D?;z&wOXsc4~icyTW=!5KrY()fz@O1IuV^8oIq3w&mz)<7LEHO>9jyqC` zg#ecw9}|TjAqAb_s{)W1SVH1Unj-LQKmxo_cg&_W#J)?#Fdu6AskTxarMku9ZedP> zbIToq+~XF7O@4J>dbNOCb-5Wjbh-1-*Brdq`OMGpgP1@j7P)9Bwm8-}M%|Lr{*mh*7@PgbH9gdMWvu^DQ~kVv zE1e!ANSPXQuLtyT^STpqCh6*Ud4qj;L%ljjy^2+QWaCO5%Mav!n1AtHwj)N_-VeQf z+`H9#n~YwsYReB>UA4h2c%Ts{2|i*H9fi$3>eNk=iaYC_Iqy~RV5FJfs*wr5i2EL% z+?qL>oQZe>6cA;|UASPU_oQZy5B!ly2d3IVj!g>KRsUo@DfpS4S*u{cRCl2JMeZoZ zhLeyx^pXTC4^w?O_J}<4#?mlmXQ!l6z|hCi+-LU}p)jnejR(hcJH}ymX*Cw2Hy-r> z(WS!k7DC`3h!0@Y`%kVz(Aef}5$^bMu7wAby$<+P0&J1Jalu$p%~VNOB%# zV9io9w44J~znPvoq%mob>b$2u{F=KIR2RlEq_STzy&@@$BTl5JiGo&*w~5p@9A^_{ z*1|N*zNP#oUszZ={Vm~dw^SiT?9UhkQ{YJSVcX7V>{FyF;>G32mB#8fn9L4QYX7VCH9!R?!{^x>L%xjoT z2(<$nSxDJByhap*aReZVQqy)S@3TQqTiusB)Q1~PvM*~7TFZ-RzdN<7G9XkF zyr>pV_)M_v$#%}XvkR1P2f_Tfut%scI*=(FLCJ&Cm5{HIjY2{$l@x^V6mj=yUgXW> z1f-dPw)1iO_*n7aK;Uk#v}(Nhp*1TPw@bfsx;Vns!*a(+)~$hVN{dO8G>81AR+E7{ z_UYUjK@(F1!eT2jW-H_CtREmh^LmLmZUdVF5Ajealnp&7cZ}nWvHL53N|zYf2~1LC zJL=tfqWg&HiTU(!5kmppCKXj1P~i`^eg_N+z_(STeF$M*WSsn;%W&EzSDM_tpZ|Pn=!SHjX8zvUg^eW zqeua!8@!?MEe*%F8>mSK1Au2r%|(Ll&jFzybv4B%#xVj_MThQmY{)+)P_5Fcq;Ma? zUYQoIn5javVQ=|u@Jbd{gB8W?P#4#|Puw7`%^$+{{s4~&nGC09&;}19@%x#^ZxVFx zQcY+E?H2|{ejB=>PD#IeLf%$WS#J8TupcCpt1@?ey928;a0ZY-o3j%Zj1t4sigaV{X{?ArE7g>Nl;s0&EfPfHq zfBpQwM}dC^9SB_k!1~V%=rI4;j{#zk17H#U|KC^tKZilae;x1${%?*R0)p?qHt**G z|1(gJ9KaG1NAVwPL6EgPfDY2}KSB47$D=+ z{t6s%01n7p@!f1%{U7obgrW%z4;oW^4@IE)FF^)HkiN&$RC+fQ)cT8?lmIM{joN<& zgED{}a!cnQAqXN;dAI#f0=>%b4%YhrVgblmgMS4Qs6^#G6_D}&kY0rMd^lC#y#!4E zVkT2)L{O&cdp`YU|3gAZ0azevwRgi!^M5fcC|vD5Vy5N)kQX(;XUIOQzr?@>8U^I7 z{vOQ1=3jyVI^lay7+&K&Shek6>}L;+1ai=LH;l9Ym!N@;{~@Oie+in?duIRXrlaP& zE2-1J7!|av`EJPb{V$1gejieKt#^{+{4YTU{X%%Z>b%ywYrV@~%&Pr9q?c}gMS=GF zkg|IG75F*;X2>G1zal~h@CB0C=daM$doQXDKWJ#s6VZD-Jl*$rM}Gg)cEJO{1Eu;y zqappbcnbV){r@$916}LBXGS0NFX4c!3kH2K07!t=^#Dwe;E=yUT_3;-xe@x0kOIx> z|Kk(>k8p?NiTLaLIR+XH1U7j0iH`gq5*-N*_qTow4d0V%i~6?>12ks%9!DtVUxEXY zH+mG95g0q^dE8>_b^45;!SqLKbTL_Yn!2mh>m zLRx41i{U`o|B$uJe+kmxTS{&5-ph-uzqxE=K_i1=E#3{`a{he}@&P2pa8u5%2F&;Qt*&YW?0p zzLJ0JWI_AZ@0W@!1?5TsSU^=a0QkQ(YiRQxtF-)I%n5l_@mJv4zE`13^%1*xa4R)wQ zz2-+^O6g_}QC(Hcb!@x{0>f>eY7oN17rwZr3e{cs)h>W#>o_V7#H(f)AtIF=YHQ7y z@6)OfKIdFv!58Og0-qE1SWSUEE>30l7UwzlB zv_zi2>He(R0z^)N^ZL|9U{OYI@6sY5J@&(roTtWTvQbi0X0_CV0~waJp_e+pNCw;j zWtYWzir{|P(CA^E;;?KVd+NK}i|&DqG!^U*+vh#Wg~L%IeN+T$FS9eBOAo$mV?gpk zI(xbZl417A-13|2!YC(|Q!7>&FRFaKCpw87F@ivtHrGpD&V_>y-wkGfRG9V| z;lt~ylB)05lWwZpBOu+1gFvTe=64jag=f1KLrb>yxPdd`JxLELI}pLY;&<$M zNVAOkuiE#nPKw4LL}@`YXp7ldD*JB2QaKaT0BH`-Ekv(aV0rbxw6R~Y8hL)xQnQh&6 zYnUE)v-MMIiG}`+za$A}Eq})jEVe`Q=hambWOlQ2JbnO1DO&1WVnaucL1^P4^#0Jq zA5ri90d3S@%fxMh6}|=a{o`kPGva>P90MGkS4;0A@$W7NTti6#Ms~&CJeSR%R(=6k z7P^VmFwNFIO{AR;#}|4}Bf@i4JSe#53c0s1#A9V6p>r`Y2YK5j?xv!srIWP>j4QFD z*l|+i9ZLp)_!a~G@a!j^*PC{=qieXWhJEd(RP!~w18ak*n)PfuWD^2Up{aG8DXeHyALZfMAxhv3J7Xzd314UYpox2>IvnRm1nRM z`hkz`;wb_SN&elrU?D~QXFTeSq%f`G#m9*ofGFHFuxm`Qy{I=B@nv$lc|=pUC{6+y z!)GQQa# z7aD$usfx0fkKbNk-0!rHTN$3(v}%x!m&TP0vJM|wI1jc;ym{IJH6PJeiQ;YSLlf4# zcN!b_fXv)b@|R~M%LMViSFTEGb;4#DbH1mUfWVI}@J{@hrZ9ai8=MFh3ok{h9&YQv z zrW0gzpUre`GLG;wu=S%=VNwXA2P8OHDO(EL2^-mdj_MXHesU0n7G-MJ)QjzS(}0#% z_mtvVnS_{ZYzbn$9yz1qnQbAD?QcP_~W56yi!aV`rjfc|p!r z`&0E&eJ7TydC|K>l|e0Bdd)Ks$5MlfuESRiZnJQWr~WzVXn2wkoXA=1kIBH$?~AQ5 zY4*e5^GI}cTQH&xC){{lI!;T{iX{|?jN`mFS0I6nD=uywJ9cS^%m%=S(h*7; ze7UZUfe*NI(Bt3AU=0Py6}Uz%2`h~qA<7aVBlbDXqv-u+!x`n+Yc@_wN^f+s-brBS zrh?S17U?po>g1JZXhmjbIui_|MTpIpVx^OYe{3a|M9)IS_5@G2psz%eBb%4b<21$m zIi0VttWpPW1uoy(ct>l5q9=T~zG8Y#FhadoeF4}zs@KJxMwkZSrRE={Z%o%}r9V%J zjw5+ZdPwV1JaGcJ3`#IMhm!~;<8(?mzfRnizKK7gy-cLtDy0{XwPbLq&NHfS^`?89 zyRbT5QMn}R1ASL-(R6wA&i4^JB7f@bj7ugnw)>%h%$J^zRm|vCkd?Gi`j!P9^_YE# zhY3st`i{GjXoj3+c#3)Nk~-=*+6}fjl(S{a=e9{zm;*0$&^r#H3nWd70lgefaf9+h zC0|N?@b%Siw~LQ=(J(T%@#*>xv&$=m93QBoCZ-TQrp)dhA6GGL8rYMY=nBTe52aED zhG!0ou3*G_P#j=Yc+^>Ww-a#c4$P%nsLX?P@6}cukt^yM=+iDsfgIXOjDdP03)8EC zK{#*C%bX(&GFlf|#?UaK?95t+{^T`NOt|(A` zpAu}+jKgQIa>UFz-(FO1f=m+DS~cSZ0uZY3nwC`8o=cibnjgD&$aBcUMHixDHVE20 z#J}z~`93c@kR8wGnwa_k5drL^mFxJjcECc{Rks6zM4h0n)Zok8L|=U%j3NHb;H9Hf z?ntK;zvn6$Fg9wB;x|KBPhdQe`FXTh->VVl_tB8tp$A97biJtkT>&Ygpk$(@S+{R3 z35yPAQ>($Zm@6C%$+Be_$u-U*i6DVx@3k{ z*8nb!V~$c6?_j&PqtI3$X#~udd13UX`&ZNkl(_q@@GzT)-DEU)!nLPa5{FOp5cfgk zuu?MRbEy7x)gtk?$7@rEy4x$&c=x;P3<6|8eAIg;hI>c;++?uUSd%2#^~gzsgFE0XaS`8d^Dgg;q?9dO>IXHI1T21MItb(ffW|V?Rz| zX)lDjpVP21zbY;$+70hTEepEk)|=b>Vvp8+6J&ME*QmyEYyLrqv;+Mtt%}Hy{Z-Jt ztC7F7Zj_jTz$p3*80O~-j%%|2!=nVGjn>x%r-$Jg@zzo|Z{96) z4<=oTM83@owf|jMPp2|;{#B<~o9mlwM!F-N+55Ga#9>0Ist8BjgKvYoqtw z+jq(SSKZiO3Z7=0KQ685#vmDLaF=C^7bnjEZH=ule!q5CM&S@uAQla)v|0$g8u^>{ zEjY5OA$^>Uf$D&c=c?3^^ku1oer?2<-*J9D{2FV)^YpOWIXDdbEK?JOaH!9k`BuNc zpF8yD*)?A)@ITiqV3#czf?YNuP|bQQ+EHAQyH|{Nn&#M)q87Japiz>~76poW)xTHdzv z(dEJA+(w2>#_m&8g{qO1X3-s(A5+Nn zq`6O?X0zG2!RKtIqO649+JS2>=3T9u#2+tmE>&{K2XWPFdun`?OL&WU{=f*gW?VfSPE!^<8-_1 zloh*XJ35_CMYuDQ552M#tgk47ISmMpb^XDEv%c_sX=*5E-7C;|JUbZBOirPAdtu_4 zGX;XKJKpXc3d|x{qGbH}GU?S4I=@#jQu$S@Q2O~HE7=Rwi*Z->f}T=@kAeF7rT1kWI8MvMaOVLU@CA5X!5qw>Cm?eRfit{pO?ahYmtB2%g%g(06`UKpQ zaQN=E)%CFF&YH<&HpiqBo&bIQ@KtaidVEZP!To1x*Y-}xbVDO^d89` zrw^^K-kuM~)&>$X&)E%;V zjhw*i%j}LMj>zxy4IJJWg;Cn^WaxNHxSp7{=bin?Vx08xp(ES|BI$}UeM)$Lh-jlQ zB1kHak!UjQdd)cQVipuI&EO$Bf1_!>5juAwq_)~F16dK*h`L1y0Hx)spsO7ry+wl@ z&J)1-(MnS>7w5db!H(L#k?&x%RDY(xnpb!w`F`G*LUHaSpCEv>kGFpH8`2^OJM&fyp=3 zpQbAJ-=AE04Ctb5}N|{$DV?Ouz zMP;hAI4y?E@+g@!B=r&3SYPxQPEs`DeMyUz#_m4*6x3 zoJ1-XB{iRCVfa}r*)>F@fgOgiaumpHTe!wr{cIbB{zR<0xd@c_;a9`z~Uz1 zPU|w#VAy(V!dyj9Vy`>ge%)w;Yv@Rvu{THW8&&OB_3*}N7#&}807O&A&ey3uEtyfcNBkwnY^>3w*gI|imElw9G(~Qb!4Mmz=T~%H z=lz(z=@z-zMQzd+>2W~u0b5)~1quV}0qYk}jw`E3J6q({*Xk&G>LAUeUhY1xJ}+YM zuSgtE>c2bSNBQuDPJ5he*7LWpEjCS}SX<(*B-hNd$0Pc5WPxH8O){ds8|=&)IDolM zA6L~EOO2YnLdHqwG>lrVhG&NADi;yWYlSgkb2R87fzhu`O#P>n2ZB!~mz&!%owsNq zV^NI<{8hs&UH}!$1-^a{4*vQcFk_rVY!PrRB7s$CwS(`M<}%#xHvwT;8J)`HN8+Dx z=BE55JL@m4Y``qohAhST+-fAF)g5DxX`&_X26eXzVL9Bu44YtlV~-2rQhf1)3O-CT z84ub;ak*yob3eY0khmR^c;;OwBMm8t@L|mI;;3WSLkO)|DyrS{Z~>t$V{TGG@g|Ke z%!MXz2sxqE94_gxzRWkNAw+Y>r+|X`4N#lPCnavzk|wlS#Qq`&xdntxsRB2YJYfBhV&~hRF{Y zJJGz^e!A{8tq`km?@UVtnK>g>qhxlv8lsOptNG617h*GSM0xTrMN1_Q-uTzGLJ&>~ zlY0FG9TYe{jyUHOH@yz!={e9S^vslH$0RNyCj)`gTop@SuVt~F(9WSDn+s!Kfy;k-1A^+xhH z$0qK(4q{0D5uVtRNR(z1x)H}JkEKZbhE+*?|1nfcWR7yK4cYIwg5-ePfg_8)G_@Bu z+7QU`eMCf?&R!04l(qvL6JMg3nD>bmmQo>ONYhzO>?pIvJem$ z!*pJ=$(NtB)upuRO@C=CCOP0{|CxPt5>CG`0W$6d2<=x?h^x)`zMg`JDY@i~8MRJ( z*>~nzrR(qa;e~t~Vq}nd1pA^S9Kljub_TdEco$vyBI|%ySfCw3E54&bwGpzJCgE=V zsnY}*We?S5nZ#Lwm#jbF^KSodVH9hgLhaRjoaS3dZqx=7SW{Mt_-pHI20avhN~>{W z*`CmRsZ8t~Ny$OP_E?3@03?ku46ZS#(k9_f>PdLI%TpLi#={~W@#ak|?`v_02x*{I z#;T|7@6KHsnKD~yy)aAK1p@FQuSQxdC~$c`U;tG8idiC{$U2C>jVp`w^6QJyjjtQVeF?G`cxp``}~hHUl?zf zVbP=IggE&pwf{qfDklP&YbYFq=SyBTJEV*oMEHm7Ta#qib&B}N8D=4Xz*i{!gf{oo za&<-N8QjByQ+fub8f4f#AC%_(CAY+~2NLO-pU0_Vi#OA;Bx&osqWZ{A$ueW6W|u>1 zyH3TcUWIw;QLv;Fu9_DDM(V)STOlbj<*}Zj<{S^D&od7%lw5_JcphsA&nflg$C*^u zas|5iDRWS|kaQW!m$6!ODh}U+py_mA6=Yd#1(IMYNi&Tw)6{ST_)OtR2ZDRpO_jG= z#L4*9p4N5ZZPb9TDri34K)5Pe;otR7* zStm5CF2rRwe~?pc;Fkp+G>oK}oA9|PG|>sD1=(n$>t}2vs=BBkh$8uhQo4D{@0Sf( zdve4p%Mnc|qO6DlK@#aFXyMH|-j7#aLxXSPH7Cny{VV6~7J1eK54FwVgHoaDKMrrR zv&4!iP!|d?km?Lia!-1A&gRa555KExcV{}V{ z-Y*+ZWV9COYAuFc!^P>6s#NqJ$uGn)6fC}^TLsGB={MBcMxVN);aYWG+*4gF3h6^? z>5iO#{!o1=W@B(EaOiaG3{*_uyO_4Zn68ab^{Y{Yhfj;Ks%4E5&8!H&2RUfStCm1J zor)f=C78S6dAzTG?6x(J?Ln{YJEpX}6gSrUH9vY3MwgVb7R%FYshgKrsru4ZVpv?# zxlXd%QyQhXY0LE+_CqULRh*SrH4xF*zM zPFuDIEa0L9>g(`oUsLm9PadtiK2qdIceM>m=|F48D;c7opU=gu#Cnp4H(s46g9VZ_ z5f-j4hl)7wEV$7}dy7AKng3y>Rc~v~Ud^5*`c;=hm z_&|9WJDxzr(eI+!1|Ezp#PNt5*I{yH_<}W)zqeDwJU+Q;pXVW!BSJ_vJv^9F4~$%d zKUUf_Onh%(YoM2W?G&WC;sPUe??p6CvWQ4{bUqjXQwi2Soo~Yy4;XT{z4@{#n-&mn zr?P7hPUpx@A~AOwLUDRa^z^266 zd2%UygmQBU7(*gv_jayVn)=DGoYC*#>#=DM{3ShGE5VErFlQUCZdo$6r^2i7#3Xq< zoHaj3N>g%bb@1g@7I6a}uVKl?Wq&Gl&H@go(?T60+cd-Z`=Rm-FRT`;jE74nDv-Z? z1vf$ohE_Qo7ymv?%V9?Btor!Vq|A$nI!ws1-?R_*PgYN3jpaTs@q-~ryuoSE2>mmy9p72 z7d5aj<;kCS?0&(JhTL!CTOGe8E1kp;mx|Rxy%FLcmFC2d8`qJW0p&CaV zwjd?b-8&$_9EMd$2@Q65+X*ADW~u16>)2t-;LDIk7bO*?Ke z5O|ccQD77#u49hC?Iu0hz}`)!FZ#M1p9~gMrD$_p`xC?A$+cb|HObQYt2q(r;(33F z30L?61%p_5iJ16VTT1V~`*+UEuh&DaNb6WTY`CAugTG)%II@TFwup(b*)JA~0lDvs zHEy|s-~z2l)lq;JkbeBu_5q3L%AG!e>&`GKD^VEjLr-AL1UU; zi@fj+IO8BV`PraEU2`;8eKOxlB3{Wm&Dx&h?3K$chOaWV$R|fyyK&>_{#f;ugiqz6^FL*@xx1l#EB& z8R{lUHsB-X1d}8uF{-d|=1)KNNd5toXp3bSg*xB3B+Q>e{WEr&{9qix8D}L`N5qL= z+-U1z(uDBUgXH=0EY$ZGmM4Q*z>qh0YC6-ep2NAXa_jJ;iP{JB=0{$tz|V?uz{BcS z#)QS2oIRn(J9fecmUS9e*I^zep(re(!i9*vG56mV|2#qqDhF%4jo%pIW@h>x#*Hno zUA<iOORaH5jUeY!L%B+bC9`JhiPx1-cr{cz*y%(R4O~$YB-j_z>}f&TZ9x*XgHLSF!w=HKqK1^>L2Tn}^&zE^ z{{h~IPi-aTf!Ic&&=6k(FFOsE$AorB>Ns7@7{#M7AAg1GPL?f0VQLot3`?ynl@g>! zbY?ly!b^$RtWofKi-_;Grz@~e(0N&%m(JY7(F_fPio`+_HF)8=2HZzvoh_65W&UA6 zTWX2L4r<^%K81UBqr&R*LiYF%Mi1Q*=r$`}4vhjz7A$mV-LE^k3)#L=BTl~?kNI0x z_gGV%i*d`;Ym+G=SIg^qbWLz&KUqkB88C5y&MIEn(u2Q~I=9DmM}-1qp{utth!vba zU*JMau=Nz7WVe#idyKO;rc`>?Z`yK8}viO#@;4($R~c#dZK8lLjd(>szB zm4q6gPRwRZ`4c=`$Y{3+C*@HRZi|FjVbaHjd~eKoO!_;9yEfahvCU@gHU40W93Ab}NNPR*GGZ5`so2MONAyV)LaADhO^FrrFyNrPCjB+b6V0x~_>o^< zN5xi~0QyT{zRBC%U56*$PqQBWY{K}~iH99Y(-blT3f`6i{5{O1z>WvegF62B#v-c8 zg{gD8jgD=u86l}$;Bs2m(Pg;_j?e=`(GUowSU&EGdlxRupvOQRlun%$;=Or@5#HOe zU&zFgqRrY$4eX8x9K8sxrBk7RqZr*z?NN0~#)LnR52qlh`F^_D!UZ$IB1DBbj?vJ9 zP#!x8bruPOW1IgRcW%zM0qdNy^(3{o68(`0u=APS{eCUYNQ|7dc?h047yjA7{l_QK zJNr+Q{b!TowlA9#=yiPdtdatzjJuy5HQl#q7=F<7U<0G6GH0lKA6}Wr-KLRA%RUKy zNdVrzB9q@un{3tEekgsq;TohYIaw`Y8F$h$CNNvT9ww(VF(IDXk}A?5-|y7+G?OQg zT%`$BPimAzw=#FTJ|Mx>+qMiDlJftlFeJPD-IPg=FEbxCRx$SrS9i zWoNP^BVz9@?c&;CB7#*nt<|x(On51wrU2|agNS#E{m|nzJ0Sd}N98)_g9`G;62i>6 zBk)2Z5X+$1)Na43S3li1X?DoLD`-*AZt*!=qq(og@yZwYe9C;MT@{H}v*5}cOMq70 zvA%R~AEI*1Q(U?Q_}9c$0^7yOk4Qkg1Jsu6;&SN^w@Xw{`~+&JN^14;R@3H9>*WAt zm2-Sg%u0HR3JWBS4|&`KI2Ir6rZ=%mhk5YPUs$k*NPcFI_@)S;-46su61lID$(__2ZK{QK5WrDD3j>5<3IO#B9TXH5sb ztbcDZ!bO!2|oiH}_9pt!{ySa4hm0Jcy7SMV@VC4G!#hT8Hv2JPCfZ87xt%1Q<| zlt~v~R?!zI#k=FME*5VLX}G|Rw(!)aT6YrW6!CYD&pEP;ZboapeUMD`vm%w}ae zA1O_ICVEN)O3f_&DXZ`b)Ruwz-W@MNym;!C102O*`e~eg116Vg=)F?5vHdQ(PH@vZ zMUo*~T)`IC2$n;KJCv#UjCiewxEP$RQpVem(?eoe9S`t+x zl#uyPor8kRheZO-O3d{G%=Ht@ zHxMJZ>yW82+pCz^f?7V)e+dCO;;7P-p^Sv~Ua^;HsrZi8@SB^8`T3sXn{ZHRmIu$l zb;i&X=dgi4nQ(k_=gfA$WiG=F5h|!dHz&g)O_rOrt;?T3Sm@d;FtLv?oO|5L@KMv_ z{FTUrj`o9uy@H7tT44YPEW8~AKxB43%kFBWZVdKI)eh-xbKFFLg7Yfr3NS-O-S&R~!-Pn0CX!&>I*ovW)GmK6|}7O^_ACF1oq|K@H#<4!MI=NMI; zL(`;D7eGL5^C1!#XQxfINF6ScMXj}L$c5)QmoFro$#XK4ukJbAyMd< zZc=@IU+ye$@km^Dd!4IYsdRFq*ppr@3tc})owL3O$Bbb;|2yJBqGD6jbo%2+yxn*8 zkog^5oo)4KH52`FX6l5Q`CW0H*xC5DbKJOHWrsvPl(36Bl?n zX$so-cm(pL0vHE3e7>*I*jT6j-)asuxPBQe*1bLn#o{(HD?+(@0*)R;(}Wt!N1%M^ zfxT2-0@7SOgHuv4*i%}(_Ta#vBMemH>q0c7vnp{BY=OqvTf0XmLz{YsRJZA96%8Wqt2*#0l;Xe|EjU`?fVVt`-}WuRFSN~)bnQJw0aDo7ixYe#KU+M$oRVS;$hLzuK7?>U(p}z=IL2C%bug3c8 z*7s!8bt{pIZp3d2Q+$nXh#9Kct7C5OBi(9CwV8}ThgpV8&BSq-5@O)F>x~_t9XM)Z9+uIhqZ_1j0C^#4yrI7JJMw z!YHRO)MmKc!F`)y!YfwTxpIdwJh^0F0$%6uYPwc37H$lE%4d*&y3M=5qfh*b9FjM4D*%ZP-D<9Iw9*&l~MF*4Bv#1?yV$IR;4ZB{MzXN zC`Mo5e*1lEbl=!1)8XsZ*xKpSf}QXD*}_YuZTsuUmSL-h_x=8@y4{Vwt^ohDZ8bFR z)ehd<%l^wtYDYUkB=GujZsiVHNNCk?NnO8W_AfBN{+=B>1p!g@_&6zKzdk_kq_$S&U~T_k~2%M1m%Q57{p#0Z08h7*D4e zK94+aeLs>CwtQIYn8rk6G+VT1&WIcaxXNVG!e2|&myQUo8!^Z3xE{BL+;0;OSE)Me z3~#A8g6!C8fGZ(l`oA@1viS^`qx5`y0I{4yfqZ=*7g}Us-rJk zGG7;mW<0XIiM^Nd2A1}&PtI?Doi5206|OzV4^RcXReU4$J8yS$`Tg^ydLr1PUEXIP zP|%Cj$MW9p4jY2cdz)+HnEe;`=!0IieDmj(=dm~-Ia6WYxWo@R(zMY9t}E+MH{^xBaRWnND_?lSbAGr%3w z zP8~VC9srH`u{;>Bbar2QpMIt$1#$52n_y_w5U?1OenmN75J(Wyir(8n5q#aBecA|C zBLXTR(2m!a&`0zMF}PMOPM~=w9|W9UhWz^C`>e7Z*85RP=sIx!b4layO?C){!%~^U zb^3IwqsTf=0C9D)Q0Q>E&02X{_5QdrF=lH$ay?MzG&w-!?-7}jk^Z$+$z40 z2Ts-_x~G2rBS8b-!uJDq`gd5bdRTI6V_M1)itg10w|;vOfYg6&E~+Hx_71@D`Ni~|mGXsyjm*lvYuZ`iEN8U+x`U6*&CZ$9M8*y^~u{>#M{qDXQA^X;GW*|z#HJd3y%C6FwN=j zJI7_-&mwUou;9~!zBvLAfJrJNwkymit#@>=L}k5)|jmTKK&y{$bnQN~nCx`;=^TgvXjcu1L5e%;OE zK77GuY*=LXa^yxucnLSPz_LBbW23mMa@^yDGWW8@aM+Z}q*&5eLo4K?vzPz;nH(HN zb{@YiQ%>UxGnuqG?TB=C51Jb@3Vul@p))hO7Zq$iO)_{6c!eOuCfgos6;QYj9ES0k zd>op))xGU8i)+pcLbtZ@mO>i*Q5JYH%&hdA_3*nag8yW&T1@%8x`t!f z)3WOjS|5$~HY190z(+1v0;akFu=muC38Xf0M}wo*d6+ZLldqU1);#rt zUT?i)tu*bb=!ZKmnx^>8!3C3I5h0D~%9C#Jn%ndXuEXZ7eKk0U^yriebjbw3Z{&vp zq)Gkj(A=ghKp+4APCbBu34;CKC(kd679^|+UtyKv(%o;6Y-F0D5o=kb@$?3*xK#p9Ugj04#_@>D{$T<6p#A^F4K39RLcb zR|$X%F{}BvjU7T-`>z29%2a+2b<_EWNP}=x-VHGQKSrVofEB{h;2%R4grWMLB|Yeq zD!?0J#^fI+40=-qut8{={WU&-eAV9LUCsZHb2R`9I4g)m9pDC`Vfhae0!?TD;6OX- z?_L1we-Ryx_e+V{{3V5=2nr$9IWDwBIF?a`}UiU7=t=W`ys! z?jIcN_WxlW00+c?`yV4n2fzfu=k>?9(Rr7l*87jKuKPYfAL94u`)~ir1NiN)12;s3 z-(Le7q^k#@hv*FW7n#%pFoR2i!1dpi(+~dBfD{ac_)kfogO>H*({c&My|rHUC2% zqyLc37^sgRZsT|2ANw!TVEo>5W!zta3}QEVFV?RhLlXcHqA2lCgKg4#693cIBl$n- zfF=LSq0IEXFG|o~f&ij2dmmuw_kWRWGXOKha>^f)oeBjH!Z&}f&{NQq*?S$oXM+Lq zOou}LSHXaRy37Gi5Kb9?V_-m^GT&wTUkSv2eHS$-^H1A(&U>xgetpj(BkNzp*5X~O z>g>M+9&}>yt^-ZZUxNInv&P*2l&T;o&Jy4Wv6%PAFEbws9b{nj9tm6Uhm03MA%T|u zAyb9_BL6GV8vqA*2uRTeUd$29WC gm&g%-OYon0#DCoS!2of7wEs*i@S*D=0q<@94}{i<;{X5v diff --git a/README.md b/README.md index 0cce518a..84b89652 100644 --- a/README.md +++ b/README.md @@ -114,30 +114,34 @@ - [Authentication](#authentication-25) - [Arguments](#arguments-25) - [Usage](#usage-26) - - [tr-upload-consent-preferences](#tr-upload-consent-preferences) + - [tr-pull-datapoints](#tr-pull-datapoints) - [Authentication](#authentication-26) - [Arguments](#arguments-26) - [Usage](#usage-27) - - [tr-pull-consent-preferences](#tr-pull-consent-preferences) + - [tr-upload-consent-preferences](#tr-upload-consent-preferences) - [Authentication](#authentication-27) - [Arguments](#arguments-27) - [Usage](#usage-28) - - [tr-upload-data-flows-from-csv](#tr-upload-data-flows-from-csv) + - [tr-pull-consent-preferences](#tr-pull-consent-preferences) - [Authentication](#authentication-28) - [Arguments](#arguments-28) - [Usage](#usage-29) - - [tr-upload-cookies-from-csv](#tr-upload-cookies-from-csv) + - [tr-upload-data-flows-from-csv](#tr-upload-data-flows-from-csv) - [Authentication](#authentication-29) - [Arguments](#arguments-29) - [Usage](#usage-30) - - [tr-generate-api-keys](#tr-generate-api-keys) + - [tr-upload-cookies-from-csv](#tr-upload-cookies-from-csv) - [Authentication](#authentication-30) - [Arguments](#arguments-30) - [Usage](#usage-31) - - [tr-build-xdi-sync-endpoint](#tr-build-xdi-sync-endpoint) + - [tr-generate-api-keys](#tr-generate-api-keys) - [Authentication](#authentication-31) - [Arguments](#arguments-31) - [Usage](#usage-32) + - [tr-build-xdi-sync-endpoint](#tr-build-xdi-sync-endpoint) + - [Authentication](#authentication-32) + - [Arguments](#arguments-32) + - [Usage](#usage-33) - [Prompt Manager](#prompt-manager) - [Proxy usage](#proxy-usage) @@ -2209,6 +2213,75 @@ Bin data hourly vs daily yarn tr-pull-consent-metrics --auth=$TRANSCEND_API_KEY --start=01/01/2023 --bin=1h ``` +### tr-pull-datapoints + +This command allows for pulling your Data Inventory -> Datapoints into a CSV. + +#### Authentication + +In order to use this cli, you will first need to generate an API key on the Transcend Admin Dashboard (https://app.transcend.io/infrastructure/api-keys). + +The API key must have the following scopes: + +- "View Data Inventory" + +#### Arguments + +| Argument | Description | Type | Default | Required | +| ------------------------ | ----------------------------------------------------------------------------- | ------------- | ------------------------ | -------- | +| auth | The Transcend API key with the scopes necessary for the command. | string | N/A | true | +| file | The file to save datapoints to | string - path | ./datapoints.csv | false | +| transcendUrl | URL of the Transcend backend. Use https://api.us.transcend.io for US hosting. | string - URL | https://api.transcend.io | false | +| dataSiloIds | Comma-separated list of data silo IDs to filter by | string | N/A | false | +| includeAttributes | Whether to include attributes in the output | boolean | false | false | +| includeGuessedCategories | Whether to include guessed categories in the output | boolean | false | false | +| parentCategories | Comma-separated list of parent categories to filter by | string | N/A | false | +| subCategories | Comma-separated list of subcategories to filter by | string | N/A | false | + +#### Usage + +All arguments + +```sh +yarn tr-pull-datapoints --auth=$TRANSCEND_API_KEY --file=./datapoints.csv --includeGuessedCategories=true --parentCategories=CONTACT,ID,LOCATION --subCategories=79d998b7-45dd-481c-ae3a-856fd93458b2,9ecc213a-cd46-46d6-afd9-46cea713f5d1 --dataSiloIds=f956ccce-5534-4328-a78d-3a924b1fe429 +``` + +Pull datapoints for specific data silos: + +```sh +yarn tr-pull-datapoints --auth=$TRANSCEND_API_KEY --file=./datapoints.csv --dataSiloIds=f956ccce-5534-4328-a78d-3a924b1fe429 +``` + +Include attributes in the output: + +```sh +yarn tr-pull-datapoints --auth=$TRANSCEND_API_KEY --file=./datapoints.csv --includeAttributes=true +``` + +Include guessed categories in the output: + +```sh +yarn tr-pull-datapoints --auth=$TRANSCEND_API_KEY --file=./datapoints.csv --includeGuessedCategories=true +``` + +Filter by parent categories: + +```sh +yarn tr-pull-datapoints --auth=$TRANSCEND_API_KEY --file=./datapoints.csv --parentCategories=ID,LOCATION +``` + +Filter by subcategories: + +```sh +yarn tr-pull-datapoints --auth=$TRANSCEND_API_KEY --file=./datapoints.csv --subCategories=79d998b7-45dd-481c-ae3a-856fd93458b2,9ecc213a-cd46-46d6-afd9-46cea713f5d1 +``` + +Specify the backend URL, needed for US hosted backend infrastructure: + +```sh +yarn tr-pull-datapoints --auth=$TRANSCEND_API_KEY --file=./datapoints.csv --transcendUrl=https://api.us.transcend.io +``` + ### tr-upload-consent-preferences This command allows for updating of consent preferences to the [Managed Consent Database](https://docs.transcend.io/docs/consent/reference/managed-consent-database). diff --git a/package.json b/package.json index 751834da..88eeae26 100644 --- a/package.json +++ b/package.json @@ -2,7 +2,7 @@ "author": "Transcend Inc.", "name": "@transcend-io/cli", "description": "Small package containing useful typescript utilities.", - "version": "6.7.0", + "version": "6.8.0", "homepage": "https://github.com/transcend-io/cli", "repository": { "type": "git", @@ -27,6 +27,7 @@ "tr-pull": "./build/cli-pull.js", "tr-pull-consent-metrics": "./build/cli-pull-consent-metrics.js", "tr-pull-consent-preferences": "./build/cli-pull-consent-preferences.js", + "tr-pull-datapoints": "./build/cli-pull-datapoints.js", "tr-push": "./build/cli-push.js", "tr-request-approve": "./build/cli-request-approve.js", "tr-request-cancel": "./build/cli-request-cancel.js", @@ -65,7 +66,7 @@ "@transcend-io/handlebars-utils": "^1.1.0", "@transcend-io/internationalization": "^1.6.0", "@transcend-io/persisted-state": "^1.0.4", - "@transcend-io/privacy-types": "^4.91.0", + "@transcend-io/privacy-types": "^4.92.0", "@transcend-io/secret-value": "^1.2.0", "@transcend-io/type-utils": "^1.5.0", "bluebird": "^3.7.2", diff --git a/src/cli-pull-datapoints.ts b/src/cli-pull-datapoints.ts new file mode 100644 index 00000000..0356cc29 --- /dev/null +++ b/src/cli-pull-datapoints.ts @@ -0,0 +1,129 @@ +#!/usr/bin/env node +import uniq from 'lodash/uniq'; +import groupBy from 'lodash/groupBy'; + +import yargs from 'yargs-parser'; +import { logger } from './logger'; +import colors from 'colors'; +import { buildTranscendGraphQLClient } from './graphql'; +import { ADMIN_DASH_DATAPOINTS, DEFAULT_TRANSCEND_API } from './constants'; +import { pullAllDatapoints } from './data-inventory'; +import { writeCsv } from './cron'; +import { splitCsvToList } from './requests'; +import { DataCategoryType } from '@transcend-io/privacy-types'; + +/** + * Sync datapoints from Transcend inventory to a CSV + * + * Dev Usage: + * yarn ts-node ./src/cli-pull-datapoints.ts --auth=$TRANSCEND_API_KEY + * + * Standard usage + * yarn cli-pull-datapoints --auth=$TRANSCEND_API_KEY + */ +async function main(): Promise { + // Parse command line arguments + const { + file = './datapoints.csv', + transcendUrl = DEFAULT_TRANSCEND_API, + auth, + dataSiloIds = '', + includeAttributes = 'false', + includeGuessedCategories = 'false', + parentCategories = '', + subCategories = '', + } = yargs(process.argv.slice(2)); + + // Ensure auth is passed + if (!auth) { + logger.error( + colors.red( + 'A Transcend API key must be provided. You can specify using --auth=$TRANSCEND_API_KEY', + ), + ); + process.exit(1); + } + + // Validate trackerStatuses + const parsedParentCategories = splitCsvToList( + parentCategories, + ) as DataCategoryType[]; + const invalidParentCategories = parsedParentCategories.filter( + // eslint-disable-next-line @typescript-eslint/no-explicit-any + (type) => !Object.values(DataCategoryType).includes(type as any), + ); + if (invalidParentCategories.length > 0) { + logger.error( + colors.red( + `Failed to parse parentCategories:"${invalidParentCategories.join( + ',', + )}".\n` + + `Expected one of: \n${Object.values(DataCategoryType).join('\n')}`, + ), + ); + process.exit(1); + } + + try { + // Create a GraphQL client + const client = buildTranscendGraphQLClient(transcendUrl, auth); + + const dataPoints = await pullAllDatapoints(client, { + dataSiloIds: splitCsvToList(dataSiloIds), + includeGuessedCategories: includeGuessedCategories === 'true', + parentCategories: parsedParentCategories, + includeAttributes: includeAttributes === 'true', + subCategories: splitCsvToList(subCategories), // TODO: https://transcend.height.app/T-40482 - do by name not ID + }); + + logger.info(colors.magenta(`Writing datapoints to file "${file}"...`)); + let headers: string[] = []; + const inputs = dataPoints.map((point) => { + const result = { + 'Property ID': point.id, + 'Data Silo': point.dataSilo.title, + Object: point.dataPoint.name, + 'Object Path': point.dataPoint.path.join('.'), + Property: point.name, + 'Property Description': point.description, + 'Data Categories': point.categories + .map((category) => `${category.category}:${category.name}`) + .join(', '), + 'Guessed Category': point.pendingCategoryGuesses?.[0] + ? `${point.pendingCategoryGuesses![0]!.category.category}:${ + point.pendingCategoryGuesses![0]!.category.name + }` + : '', + 'Processing Purposes': point.purposes + .map((purpose) => `${purpose.purpose}:${purpose.name}`) + .join(', '), + ...Object.entries( + groupBy( + point.attributeValues || [], + ({ attributeKey }) => attributeKey.name, + ), + ).reduce((acc, [key, values]) => { + acc[key] = values.map((value) => value.name).join(','); + return acc; + }, {} as Record), + }; + headers = uniq([...headers, ...Object.keys(result)]); + return result; + }); + writeCsv(file, inputs, headers); + } catch (err) { + logger.error( + colors.red(`An error occurred syncing the datapoints: ${err.message}`), + ); + process.exit(1); + } + + // Indicate success + logger.info( + colors.green( + `Successfully synced datapoints to disk at ${file}! View at ${ADMIN_DASH_DATAPOINTS}`, + ), + ); +} + +main(); diff --git a/src/cli-request-export.ts b/src/cli-request-export.ts index 56b7a47c..fb5c83ac 100644 --- a/src/cli-request-export.ts +++ b/src/cli-request-export.ts @@ -71,7 +71,7 @@ async function main(): Promise { logger.error( colors.red( `Failed to parse actions:"${invalidActions.join(',')}".\n` + - `Expected one of: \n${Object.values(RequestAction).join('\n')}`, + `Expected one of: \n${Object.values(RequestAction).join('\n')}`, ), ); process.exit(1); @@ -87,7 +87,7 @@ async function main(): Promise { logger.error( colors.red( `Failed to parse statuses:"${invalidStatuses.join(',')}".\n` + - `Expected one of: \n${Object.values(RequestStatus).join('\n')}`, + `Expected one of: \n${Object.values(RequestStatus).join('\n')}`, ), ); process.exit(1); diff --git a/src/constants.ts b/src/constants.ts index 417ba9bb..c91b4ac4 100644 --- a/src/constants.ts +++ b/src/constants.ts @@ -5,6 +5,7 @@ import { TranscendInput } from './codecs'; export const ADMIN_DASH = 'https://app.transcend.io'; export const ADMIN_DASH_INTEGRATIONS = `${ADMIN_DASH}/infrastructure/integrations`; +export const ADMIN_DASH_DATAPOINTS = `${ADMIN_DASH}/data-map/data-inventory/data-points`; /** * Override default transcend API url using diff --git a/src/data-inventory/index.ts b/src/data-inventory/index.ts new file mode 100644 index 00000000..26b6976a --- /dev/null +++ b/src/data-inventory/index.ts @@ -0,0 +1 @@ +export * from './pullAllDatapoints'; diff --git a/src/data-inventory/pullAllDatapoints.ts b/src/data-inventory/pullAllDatapoints.ts new file mode 100644 index 00000000..7f1f1b41 --- /dev/null +++ b/src/data-inventory/pullAllDatapoints.ts @@ -0,0 +1,480 @@ +/* eslint-disable max-lines */ +import keyBy from 'lodash/keyBy'; +import { + DataCategoryType, + SubDataPointDataSubCategoryGuessStatus, +} from '@transcend-io/privacy-types'; +import uniq from 'lodash/uniq'; +import chunk from 'lodash/chunk'; +import cliProgress from 'cli-progress'; +import { gql } from 'graphql-request'; +import colors from 'colors'; +import sortBy from 'lodash/sortBy'; +import type { GraphQLClient } from 'graphql-request'; +import { + DATAPOINT_EXPORT, + DATA_SILO_EXPORT, + DataSiloAttributeValue, + SUB_DATA_POINTS_COUNT, + makeGraphQLRequest, +} from '../graphql'; +import { logger } from '../logger'; +import { DataCategoryInput, ProcessingPurposeInput } from '../codecs'; +import { mapSeries } from 'bluebird'; + +export interface DataSiloCsvPreview { + /** ID of dataSilo */ + id: string; + /** Name of dataSilo */ + title: string; +} + +export interface DataPointCsvPreview { + /** ID of dataPoint */ + id: string; + /** The path to this data point */ + path: string[]; + /** Description */ + description: { + /** Default message */ + defaultMessage: string; + }; + /** Name */ + name: string; +} + +export interface SubDataPointCsvPreview { + /** ID of subDatapoint */ + id: string; + /** Name (or key) of the subdatapoint */ + name: string; + /** The description */ + description?: string; + /** Personal data category */ + categories: DataCategoryInput[]; + /** Data point ID */ + dataPointId: string; + /** The data silo ID */ + dataSiloId: string; + /** The processing purpose for this sub datapoint */ + purposes: ProcessingPurposeInput[]; + /** Attribute attached to subdatapoint */ + attributeValues?: DataSiloAttributeValue[]; + /** Data category guesses that are output by the classifier */ + pendingCategoryGuesses?: { + /** Data category being guessed */ + category: DataCategoryInput; + /** Status of guess */ + status: SubDataPointDataSubCategoryGuessStatus; + /** classifier version that produced the guess */ + classifierVersion: number; + }[]; +} + +export interface DatapointFilterOptions { + /** IDs of data silos to filter down */ + dataSiloIds?: string[]; + /** Whether to include guessed categories, defaults to only approved categories */ + includeGuessedCategories?: boolean; + /** Whether or not to include attributes */ + includeAttributes?: boolean; + /** Parent categories to filter down for */ + parentCategories?: DataCategoryType[]; + /** Sub categories to filter down for */ + subCategories?: string[]; // TODO: https://transcend.height.app/T-40482 - do by name not ID +} + +/** + * Pull subdatapoint information + * + * @param client - Client to use for the request + * @param options - Options + */ +async function pullSubDatapoints( + client: GraphQLClient, + { + dataSiloIds = [], + includeGuessedCategories, + includeAttributes, + parentCategories = [], + subCategories = [], + pageSize = 1000, + }: DatapointFilterOptions & { + /** Page size to pull in */ + pageSize?: number; + } = {}, +): Promise { + const subDataPoints: SubDataPointCsvPreview[] = []; + + // Time duration + const t0 = new Date().getTime(); + + // create a new progress bar instance and use shades_classic theme + const progressBar = new cliProgress.SingleBar( + {}, + cliProgress.Presets.shades_classic, + ); + + // Filters + const filterBy = { + ...(parentCategories.length > 0 ? { category: parentCategories } : {}), + ...(subCategories.length > 0 ? { subCategoryIds: subCategories } : {}), + ...(dataSiloIds.length > 0 ? { dataSilos: dataSiloIds } : {}), + }; + + // Build a GraphQL client + const { + subDataPoints: { totalCount }, + } = await makeGraphQLRequest<{ + /** Query response */ + subDataPoints: { + /** Count */ + totalCount: number; + }; + }>(client, SUB_DATA_POINTS_COUNT, { + filterBy, + }); + + logger.info(colors.magenta('[Step 1/3] Pulling in all subdatapoints')); + + progressBar.start(totalCount, 0); + let total = 0; + let shouldContinue = false; + let cursor; + let offset = 0; + do { + try { + const { + subDataPoints: { nodes }, + // eslint-disable-next-line no-await-in-loop + } = await makeGraphQLRequest<{ + /** Query response */ + subDataPoints: { + /** List of matches */ + nodes: SubDataPointCsvPreview[]; + }; + }>( + client, + gql` + query TranscendCliSubDataPointCsvExport( + $filterBy: SubDataPointFiltersInput + $first: Int! + $offset: Int! + ) { + subDataPoints( + filterBy: $filterBy + first: $first + offset: $offset + useMaster: false + ) { + nodes { + id + name + description + dataPointId + dataSiloId + purposes { + name + purpose + } + categories { + name + category + } + ${ + includeGuessedCategories + ? `pendingCategoryGuesses { + category { + name + category + } + status + classifierVersion + }` + : '' + } + ${ + includeAttributes + ? `attributeValues { + attributeKey { + name + } + name + }` + : '' + } + } + } + } + `, + { + first: pageSize, + offset, + filterBy: { + ...filterBy, + // TODO: https://transcend.height.app/T-40484 - add cursor support + // ...(cursor ? { cursor: { id: cursor } } : {}), + }, + }, + ); + + cursor = nodes[nodes.length - 1]?.id as string; + subDataPoints.push(...nodes); + shouldContinue = nodes.length === pageSize; + total += nodes.length; + offset += nodes.length; + progressBar.update(total); + } catch (err) { + logger.error( + colors.red( + `An error fetching subdatapoints for cursor ${cursor} and offset ${offset}`, + ), + ); + throw err; + } + } while (shouldContinue); + + progressBar.stop(); + const t1 = new Date().getTime(); + const totalTime = t1 - t0; + + const sorted = sortBy(subDataPoints, 'name'); + + logger.info( + colors.green( + `Successfully pulled in ${sorted.length} subdatapoints in ${ + totalTime / 1000 + } seconds!`, + ), + ); + return sorted; +} + +/** + * Pull datapoint information + * + * @param client - Client to use for the request + * @param options - Options + */ +async function pullDatapoints( + client: GraphQLClient, + { + dataPointIds = [], + pageSize = 100, + }: { + /** IDs of data points to filter down */ + dataPointIds: string[]; + /** Page size to pull in */ + pageSize?: number; + }, +): Promise { + const dataPoints: DataPointCsvPreview[] = []; + + // Time duration + const t0 = new Date().getTime(); + + // create a new progress bar instance and use shades_classic theme + const progressBar = new cliProgress.SingleBar( + {}, + cliProgress.Presets.shades_classic, + ); + + logger.info( + colors.magenta( + `[Step 2/3] Fetching metadata for ${dataPointIds.length} datapoints`, + ), + ); + + // Group by 100 + const dataPointsGrouped = chunk(dataPointIds, pageSize); + + progressBar.start(dataPointIds.length, 0); + let total = 0; + await mapSeries(dataPointsGrouped, async (dataPointIdsGroup) => { + try { + const { + dataPoints: { nodes }, + } = await makeGraphQLRequest<{ + /** Query response */ + dataPoints: { + /** List of matches */ + nodes: DataPointCsvPreview[]; + }; + }>(client, DATAPOINT_EXPORT, { + first: pageSize, + filterBy: { + ids: dataPointIdsGroup, + }, + }); + + dataPoints.push(...nodes); + total += dataPointIdsGroup.length; + progressBar.update(total); + } catch (err) { + logger.error( + colors.red( + `An error fetching subdatapoints for IDs ${dataPointIdsGroup.join( + ', ', + )}`, + ), + ); + throw err; + } + }); + + progressBar.stop(); + const t1 = new Date().getTime(); + const totalTime = t1 - t0; + + logger.info( + colors.green( + `Successfully pulled in ${dataPoints.length} dataPoints in ${ + totalTime / 1000 + } seconds!`, + ), + ); + return dataPoints; +} + +/** + * Pull data silo information + * + * @param client - Client to use for the request + * @param options - Options + */ +async function pullDataSilos( + client: GraphQLClient, + { + dataSiloIds = [], + pageSize = 100, + }: { + /** IDs of data silos to filter down */ + dataSiloIds: string[]; + /** Page size to pull in */ + pageSize?: number; + }, +): Promise { + const dataSilos: DataSiloCsvPreview[] = []; + + // Time duration + const t0 = new Date().getTime(); + + // create a new progress bar instance and use shades_classic theme + const progressBar = new cliProgress.SingleBar( + {}, + cliProgress.Presets.shades_classic, + ); + + logger.info( + colors.magenta( + `[Step 3/3] Fetching metadata for ${dataSiloIds.length} data silos`, + ), + ); + + // Group by 100 + const dataSilosGrouped = chunk(dataSiloIds, pageSize); + + progressBar.start(dataSiloIds.length, 0); + let total = 0; + await mapSeries(dataSilosGrouped, async (dataSiloIdsGroup) => { + try { + const { + dataSilos: { nodes }, + } = await makeGraphQLRequest<{ + /** Query response */ + dataSilos: { + /** List of matches */ + nodes: DataSiloCsvPreview[]; + }; + }>(client, DATA_SILO_EXPORT, { + first: pageSize, + filterBy: { + ids: dataSiloIdsGroup, + }, + }); + + dataSilos.push(...nodes); + total += dataSiloIdsGroup.length; + progressBar.update(total); + } catch (err) { + logger.error( + colors.red( + `An error fetching data silos for IDs ${dataSiloIdsGroup.join(', ')}`, + ), + ); + throw err; + } + }); + + progressBar.stop(); + const t1 = new Date().getTime(); + const totalTime = t1 - t0; + + logger.info( + colors.green( + `Successfully pulled in ${dataSilos.length} data silos in ${ + totalTime / 1000 + } seconds!`, + ), + ); + return dataSilos; +} + +/** + * Pull all datapoints from the data inventory. + * + * @param client - Client to use for the request + * @param options - Options + */ +export async function pullAllDatapoints( + client: GraphQLClient, + { + dataSiloIds = [], + includeGuessedCategories, + includeAttributes, + parentCategories = [], + subCategories = [], + pageSize = 1000, + }: DatapointFilterOptions & { + /** Page size to pull in */ + pageSize?: number; + } = {}, +): Promise< + (SubDataPointCsvPreview & { + /** Data point information */ + dataPoint: DataPointCsvPreview; + /** Data silo information */ + dataSilo: DataSiloCsvPreview; + })[] +> { + // Subdatapoint information + const subDatapoints = await pullSubDatapoints(client, { + dataSiloIds, + includeGuessedCategories, + includeAttributes, + parentCategories, + subCategories, + pageSize, + }); + + // The datapoint ids to grab + const dataPointIds = uniq(subDatapoints.map((point) => point.dataPointId)); + const dataPoints = await pullDatapoints(client, { + dataPointIds, + }); + const dataPointById = keyBy(dataPoints, 'id'); + + // The data silo IDs to grab + const allDataSiloIds = uniq(subDatapoints.map((point) => point.dataSiloId)); + const dataSilos = await pullDataSilos(client, { + dataSiloIds: allDataSiloIds, + }); + const dataSiloById = keyBy(dataSilos, 'id'); + + return subDatapoints.map((subDataPoint) => ({ + ...subDataPoint, + dataPoint: dataPointById[subDataPoint.dataPointId], + dataSilo: dataSiloById[subDataPoint.dataSiloId], + })); +} +/* eslint-enable max-lines */ diff --git a/src/graphql/gqls/dataPoint.ts b/src/graphql/gqls/dataPoint.ts index 694bcff6..9566233f 100644 --- a/src/graphql/gqls/dataPoint.ts +++ b/src/graphql/gqls/dataPoint.ts @@ -4,12 +4,12 @@ import { gql } from 'graphql-request'; // isExportCsv: true export const DATA_POINTS = gql` query TranscendCliDataPoints( - $dataSiloIds: [ID!] + $filterBy: DataPointFiltersInput $first: Int! $offset: Int! ) { dataPoints( - filterBy: { dataSilos: $dataSiloIds } + filterBy: $filterBy first: $first offset: $offset useMaster: false @@ -18,7 +18,6 @@ export const DATA_POINTS = gql` { field: name, direction: ASC } ] ) { - totalCount nodes { id title { @@ -54,21 +53,30 @@ export const DATA_POINTS = gql` } `; +// TODO: https://transcend.height.app/T-27909 - enable optimizations +// isExportCsv: true +export const DATA_POINT_COUNT = gql` + query TranscendCliDataPointCount($filterBy: DataPointFiltersInput) { + dataPoints(filterBy: $filterBy, useMaster: false) { + totalCount + } + } +`; + // TODO: https://transcend.height.app/T-27909 - add orderBy // isExportCsv: true export const SUB_DATA_POINTS = gql` - query TranscendCliDataPoints( - $dataPointIds: [ID!] + query TranscendCliSubDataPoints( + $filterBy: SubDataPointFiltersInput $first: Int! $offset: Int! ) { subDataPoints( - filterBy: { dataPoints: $dataPointIds } + filterBy: $filterBy first: $first offset: $offset useMaster: false ) { - totalCount nodes { id name @@ -94,19 +102,26 @@ export const SUB_DATA_POINTS = gql` } `; +export const SUB_DATA_POINTS_COUNT = gql` + query TranscendCliSubDataPointsCount($filterBy: SubDataPointFiltersInput) { + subDataPoints(filterBy: $filterBy, useMaster: false) { + totalCount + } + } +`; + export const SUB_DATA_POINTS_WITH_GUESSES = gql` - query TranscendCliDataPoints( - $dataPointIds: [ID!] + query TranscendCliSubDataPointGuesses( + $filterBy: SubDataPointFiltersInput $first: Int! $offset: Int! ) { subDataPoints( - filterBy: { dataPoints: $dataPointIds } + filterBy: $filterBy first: $first offset: $offset useMaster: false ) { - totalCount nodes { id name @@ -182,3 +197,30 @@ export const UPDATE_OR_CREATE_DATA_POINT = gql` } } `; + +export const DATAPOINT_EXPORT = gql` + query TranscendCliDataPointCsvExport( + $filterBy: DataPointFiltersInput + $first: Int! + ) { + dataPoints(filterBy: $filterBy, first: $first, useMaster: false) { + nodes { + id + title { + defaultMessage + } + description { + defaultMessage + } + owners { + email + } + teams { + name + } + name + path + } + } + } +`; diff --git a/src/graphql/gqls/dataSilo.ts b/src/graphql/gqls/dataSilo.ts index 0890c803..4990eed4 100644 --- a/src/graphql/gqls/dataSilo.ts +++ b/src/graphql/gqls/dataSilo.ts @@ -31,6 +31,22 @@ export const DATA_SILOS = gql` } `; +// TODO: https://transcend.height.app/T-27909 - enable optimizations +// isExportCsv: true +export const DATA_SILO_EXPORT = gql` + query TranscendCliDataSiloExport( + $filterBy: DataSiloFiltersInput! + $first: Int! + ) { + dataSilos(filterBy: $filterBy, first: $first, useMaster: false) { + nodes { + id + title + } + } + } +`; + // TODO: https://transcend.height.app/T-27909 - enable optimizations // isExportCsv: true export const DATA_SILOS_ENRICHED = gql` diff --git a/src/graphql/syncDataSilos.ts b/src/graphql/syncDataSilos.ts index 8cbd4849..fac0c0e9 100644 --- a/src/graphql/syncDataSilos.ts +++ b/src/graphql/syncDataSilos.ts @@ -147,7 +147,7 @@ export async function fetchAllDataSilos( return dataSilos.sort((a, b) => a.title.localeCompare(b.title)); } -interface SubDataPoint { +export interface SubDataPoint { /** Name (or key) of the subdatapoint */ name: string; /** The description */ @@ -300,7 +300,9 @@ export async function fetchAllSubDataPoints( : SUB_DATA_POINTS, { first: pageSize, - dataPointIds: [dataPointId], + filterBy: { + dataPoints: [dataPointId], + }, offset, }, ); @@ -356,6 +358,8 @@ export async function fetchAllDataPoints( }, ): Promise { const dataPoints: DataPointWithSubDataPoint[] = []; + + // TODO: https://transcend.height.app/T-40481 - add cursor pagination let offset = 0; // Whether to continue looping @@ -376,7 +380,9 @@ export async function fetchAllDataPoints( }; }>(client, DATA_POINTS, { first: pageSize, - dataSiloIds: [dataSiloId], + filterBy: { + dataSilos: [dataSiloId], + }, offset, }); @@ -404,7 +410,7 @@ export async function fetchAllDataPoints( } const subDataPoints = await fetchAllSubDataPoints(client, node.id, { - pageSize, + pageSize: 1000, // max page size debug, includeGuessedCategories, }); diff --git a/src/index.ts b/src/index.ts index 863b18c4..d6e2a923 100644 --- a/src/index.ts +++ b/src/index.ts @@ -11,3 +11,4 @@ export * from './helpers'; export * from './cron'; export * from './api-keys'; export * from './ai'; +export * from './data-inventory'; diff --git a/src/manual-enrichment/pullManualEnrichmentIdentifiersToCsv.ts b/src/manual-enrichment/pullManualEnrichmentIdentifiersToCsv.ts index de293a82..b7302a52 100644 --- a/src/manual-enrichment/pullManualEnrichmentIdentifiersToCsv.ts +++ b/src/manual-enrichment/pullManualEnrichmentIdentifiersToCsv.ts @@ -87,9 +87,13 @@ export async function pullManualEnrichmentIdentifiersToCsv({ // Save request to queue if (hasManualEnrichment) { - const requestIdentifiers = await fetchAllRequestIdentifiers(client, sombra, { - requestId: request.id, - }); + const requestIdentifiers = await fetchAllRequestIdentifiers( + client, + sombra, + { + requestId: request.id, + }, + ); savedRequests.push({ ...request, requestIdentifiers, diff --git a/src/requests/bulkRestartRequests.ts b/src/requests/bulkRestartRequests.ts index af5ac768..6d614f9d 100644 --- a/src/requests/bulkRestartRequests.ts +++ b/src/requests/bulkRestartRequests.ts @@ -168,8 +168,8 @@ export async function bulkRestartRequests({ // Pull the request identifiers const requestIdentifiers = copyIdentifiers ? await fetchAllRequestIdentifiers(client, sombra, { - requestId: request.id, - }) + requestId: request.id, + }) : []; // Make the GraphQL request to restart the request @@ -180,7 +180,7 @@ export async function bulkRestartRequests({ // override silent mode isSilent: !!silentModeBefore && - new Date(request.createdAt) < silentModeBefore + new Date(request.createdAt) < silentModeBefore ? true : request.isSilent, }, @@ -243,7 +243,7 @@ export async function bulkRestartRequests({ logger.error( colors.red( `Encountered "${state.getValue('failingRequests').length}" errors. ` + - `See "${cacheFile}" to review the error messages and inputs.`, + `See "${cacheFile}" to review the error messages and inputs.`, ), ); process.exit(1); diff --git a/src/requests/pullPrivacyRequests.ts b/src/requests/pullPrivacyRequests.ts index bd841b6e..65013148 100644 --- a/src/requests/pullPrivacyRequests.ts +++ b/src/requests/pullPrivacyRequests.ts @@ -71,16 +71,18 @@ export async function pullPrivacyRequests({ dateRange += ` before ${createdAtBefore.toISOString()}`; } if (createdAtAfter) { - dateRange += `${dateRange ? ', and' : '' - } after ${createdAtAfter.toISOString()}`; + dateRange += `${ + dateRange ? ', and' : '' + } after ${createdAtAfter.toISOString()}`; } // Log out logger.info( colors.magenta( - `${actions.length > 0 - ? `Pulling requests of type "${actions.join('" , "')}"` - : 'Pulling all requests' + `${ + actions.length > 0 + ? `Pulling requests of type "${actions.join('" , "')}"` + : 'Pulling all requests' }${dateRange}`, ), ); @@ -98,9 +100,13 @@ export async function pullPrivacyRequests({ const requestsWithRequestIdentifiers = await map( requests, async (request) => { - const requestIdentifiers = await fetchAllRequestIdentifiers(client, sombra, { - requestId: request.id, - }); + const requestIdentifiers = await fetchAllRequestIdentifiers( + client, + sombra, + { + requestId: request.id, + }, + ); return { ...request, requestIdentifiers, diff --git a/transcend-yml-schema-v6.json b/transcend-yml-schema-v6.json index a02f7862..86a34b2a 100644 --- a/transcend-yml-schema-v6.json +++ b/transcend-yml-schema-v6.json @@ -313,7 +313,8 @@ "managePolicies", "viewPolicies", "manageIntlMessages", - "viewIntlMessages" + "viewIntlMessages", + "llmLogTransfer" ] } } diff --git a/yarn.lock b/yarn.lock index 2dbe06ad..068ac3aa 100644 --- a/yarn.lock +++ b/yarn.lock @@ -515,7 +515,7 @@ __metadata: "@transcend-io/handlebars-utils": ^1.1.0 "@transcend-io/internationalization": ^1.6.0 "@transcend-io/persisted-state": ^1.0.4 - "@transcend-io/privacy-types": ^4.91.0 + "@transcend-io/privacy-types": ^4.92.0 "@transcend-io/secret-value": ^1.2.0 "@transcend-io/type-utils": ^1.5.0 "@types/bluebird": ^3.5.38 @@ -590,6 +590,7 @@ __metadata: tr-pull: ./build/cli-pull.js tr-pull-consent-metrics: ./build/cli-pull-consent-metrics.js tr-pull-consent-preferences: ./build/cli-pull-consent-preferences.js + tr-pull-datapoints: ./build/cli-pull-datapoints.js tr-push: ./build/cli-push.js tr-request-approve: ./build/cli-request-approve.js tr-request-cancel: ./build/cli-request-cancel.js @@ -642,14 +643,14 @@ __metadata: languageName: node linkType: hard -"@transcend-io/privacy-types@npm:^4.91.0": - version: 4.91.0 - resolution: "@transcend-io/privacy-types@npm:4.91.0" +"@transcend-io/privacy-types@npm:^4.92.0": + version: 4.92.0 + resolution: "@transcend-io/privacy-types@npm:4.92.0" dependencies: "@transcend-io/type-utils": ^1.0.5 fp-ts: ^2.16.1 io-ts: ^2.2.21 - checksum: e338b15df40848a87a81d010aae09ecf52c3f89c95e7ce4141ece69432653f3fc36206ce647354a4f71fab9c52837ee9c817f71323d573c2d0fc97ee58243001 + checksum: 44f44a1286543a99f72781e9e884ff96b4822956c592a9e54871ccffc5c3c0f2b342db2abe5edf7d882e144a8a4f400b8152c66df0eab1d7bc2fbd4b4d2d314f languageName: node linkType: hard