From a31c49099281b75243dd3cd535fa99820e871cb5 Mon Sep 17 00:00:00 2001 From: Ricki Jay Date: Mon, 18 Nov 2024 11:03:48 +0100 Subject: [PATCH 01/29] Optional Month Fields --- alembic/versions/749172173f94_month_fields.py | 62 ++++++++++++++++++ data/wikibase-data.db | Bin 21401600 -> 21401600 bytes data/wikibase-test-data.db | Bin 172032 -> 172032 bytes .../wikibase_log_month_observation_model.py | 29 +++++++- 4 files changed, 90 insertions(+), 1 deletion(-) create mode 100644 alembic/versions/749172173f94_month_fields.py diff --git a/alembic/versions/749172173f94_month_fields.py b/alembic/versions/749172173f94_month_fields.py new file mode 100644 index 0000000..bfdd56d --- /dev/null +++ b/alembic/versions/749172173f94_month_fields.py @@ -0,0 +1,62 @@ +"""Month Fields + +Revision ID: 749172173f94 +Revises: ced691eaf66e +Create Date: 2024-11-18 10:59:19.124446 + +""" + +from typing import Sequence, Union + +from alembic import op +import sqlalchemy as sa + + +# revision identifiers, used by Alembic. +revision: str = "749172173f94" +down_revision: Union[str, None] = "ced691eaf66e" +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + # ### commands auto generated by Alembic - please adjust! ### + + with op.batch_alter_table( + "wikibase_log_observation_month", recreate="always" + ) as batch_op: + batch_op.add_column( + sa.Column("wikibase_id", sa.Integer(), nullable=True), + insert_before="first_log_date", + ) + batch_op.add_column( + sa.Column("anything", sa.Boolean(), nullable=True), + insert_before="first_log_date", + ) + batch_op.add_column( + sa.Column("date", sa.DateTime(timezone=True), nullable=True), + insert_before="first_log_date", + ) + batch_op.add_column( + sa.Column( + "last_log_user_type", + sa.Enum("BOT", "MISSING", "USER", "NONE", name="wikibaseusertype"), + nullable=True, + ), + insert_before="first_log_date", + ) + batch_op.create_foreign_key( + "observation_wikibase", "wikibase", ["wikibase_id"], ["id"] + ) + # ### end Alembic commands ### + + +def downgrade() -> None: + # ### commands auto generated by Alembic - please adjust! ### + with op.batch_alter_table("wikibase_log_observation_month") as batch_op: + batch_op.drop_constraint("observation_wikibase", type_="foreignkey") + batch_op.drop_column("last_log_user_type") + batch_op.drop_column("date") + batch_op.drop_column("anything") + batch_op.drop_column("wikibase_id") + # ### end Alembic commands ### diff --git a/data/wikibase-data.db b/data/wikibase-data.db index 202e9624e8fcf087067f764a8408b7a3072caf4c..6db3cfe977a3ed3d1c18db8f9e6cb1caa90cad4d 100644 GIT binary patch delta 7214 zcmd6rd3+V+xyR3(ZML(7?1TV;kZ?j+0z?9YBq!%2VNt+nLxh0g17(Qd_Lb7e)GKV z%)8C|{N}V94@vD{&wzgYU9QwdmrHDNxq{nVF8}xTjvUYRKfLhY1_w%tCygx`H@0Ma z{iNb2a{e0bb3rY7x`^YFGJGdog1gJ*?g||UbOcJJll(N^>nZQfjB?NTyg~+7+ zgDfxrWP^cV5Eu+{zz{GLJ2- z2iyv71Gj^@U>=wc?f`d!1z;gq1QvrOU@2Gz>cMi*02)CPXa+4{1!x5;!78vCtO095 z8@LOs1M9&Ca5uOI+zU2>O<*(F0@{HBRB#{o7jQq&z*g`8_#W5>wu2pDC)m|pICb~Q zX97N#%jF4wiZVI)^MH9S98Tty^-zAL6x-O-W+v@P+7d*(jq|y}pNEf!4}|Xy&k7F< zeHnT?^rO(e(Du;M(5<0SA%F18;K#vZ!KZ@Tf}4V?f=hyvg8hQtzU%8URkC&KJ zvgB4roA?S6m6dLe>m2N>=CZ6(=*$^V%_V2Lbgx%#aWsfE)2-<7P0mOoMd7}+U2o>5%8M+48NqBs7G8~vmKQi8@|{j`heJf>Uu}~r z&vQib=ThWsy*VUUaJwC0Q=^|<(3K?zs&jG*Z=*URmz(*cktJuQnSO$&0#aJW3O%kd6rYmbc!3C;(CY3n_&xm!M*07RC&51 z7MW%VeQX1p;Hi#CUJXSSHkdg>q&r{xczDzu8YMt^vP^ZQeT7A}(nTdUD>hNGIC?i6Vb5lN*s z|2uIhj$lDD1!>~?IU;hBL*#{RVNYC$TG7M>ofQKVp^1|n5m|DGh)C6F;sg>jaXbl{ zI6n!RIG-i1oj5O5VH3yadq~h+xk<3O3d)?_n4OB=dtI)MQWfFFP#g0AlNjvuu*CNLRE9Z7qTgOkj!XD2X2IUTE(L$;e=pJvgV ze~o&#dfOA=Kfq()$KWU6aqv@c5c~`r0*Ap9;7RZl_&InQbc1KWv*0;!1UwI3055`< zz)^4vybS&m{1^BIcm=!)UIWL$>)@B*4e%y-3%m_ZfOo*JKo9tDa1y)=ehq$uQT_Jo z%ewlj6=T^hczdvB^lHZMG~+$ZcwaM4X~t>IIHMV7HRJc1@quRiK{L*2#ve7~Pnz*( z&G=9=&TGa;n(?t_T+ob9G~-jvxTqPQX~thP<8#gUt7d$m8GqA^ziY-nG~-Ln_)0S_ zX~x%@@r`DDs~MNK8Qd;|18(2}Uf=_Mzykq@KmsxdfFKBgFh~OZKr%=HsUQucgA9-f z`hzSm0Az!KU=VlE7|b2QYHpe8dfgR15}I2aW{R z1sVdmfi(Gq{Ia|e#wa5FUV22@B~6nii+>kC6n`lGyEt163a`NgMulA+w7a@{la_L+v3globv4V)Oo`0ey_XlcdvJk<1WD6?dR&a zu)gQw>Ip(jp)95!T9zn-veVe7YfYx?JXX|ZmZi%IB{E7qamCJL_qLhksA;FN7f=(* z&Sl?1Wy=aBGin!8D4V&M%M^f=zUC~L}>M%>pb0D&6Nmo z#ZHJ97beOv<&+VxEET0h>bze2uvwfHp5b=O%9ZTM&gqz9XGQi9bk3O;V~U*@S?7eP zVkbs!cW)`$QD)SmyL-!o3WZXmp0wDUj9yW0)N2=+<+vv$$1A;})TqC-tG7T;+GwVs z+ms-wLR7Icq_q(G-nm`bdP{mwf858JE#nHMOMT-;>lu_SuRMd&rG9EBo-y!gGg&?_ ziE+iwo6;t1-MB)DQ@;TU_jYE^xMHWy@XlT~tCTzG-Z6!eCoO~i*|DDu)W5&k%)%`w zkzT3hqctD1*?Jc!gIC9kuD-MhO;{FAS?E$pj}eK${W?Q?~n3%7;GgwBV$LU)Gx z2YZ4$gVTbZzzcy5fpPMAxl8VlYh_vbg>;`(A$}=7DK?A4g!ctqxJd}`ukZ?A>i+_t zxGVhyzH`1#-|fB>?_1vO-Wo6GdERr-)9A@_pK(9pp5vb8_HZw7>$yhGm+N}pwQp}_ zCN`=)hC!>biQ@_lLVt8=f^7mCg??bQ8R^`dS;Hk~1sgIlWTnkHRAnvA!pLP-n>D0b znTjc^NwISjdtgM69JxtodpT+poz|SN2XLscX;C z25PcxxpkZ`CuND-!v7 z^8<-=%-+I26&74up+dQnE&jH8)hhrzw$+$b4sH-btLj^1;;9 zm@<#rx3*$bab+&4jApB^x09+^nJDN?>#gSc1zcV1JiCpqKiq7!xs}v=tWA|Nht!~! zLq!uw%t<;g~YDq-#teZ(;CJH+LaNN54%X!1%%1v~A#0qOpYDr;Err-^|k(52! z*_QgwXmRgZRByp*Gfnq>W=6!!m@<>Dv5{gM-9T-v-xuP_^%Q%y70;|zW{|qnYV~tE zDa-=fjMGRdE3L6lC55pU=pU_2uzXlUm)>86T^CcTN!f2k6)AL!y%}**cr)0IW28P@ zW4%L_r0_lz>a*9FDX1Q$OD$`y>J_AHb5Y)>aoKkpPr2Io>URT|#QamKXFKyxcZElV z{*3cuVQ4_`-Qa6MHCPe&D)4b25!fHNIZ!QMmOqn!BG=19q*Ky9X`3`vnk0TMo)bI8 zz2fzvUpOi}A*>J<2w6B$Uge+V+xU7uhfnsu?ceH;`HOuQd8-u>P>Z?fl2 zJeK`+cSN@f9-%_DeMv^#=#)LxK88gl!-D>B_>IGnfK{h=iL17EVL2QtW_y>~op390 z)%GsP{iXdA9*?Y&eW5cpuG;nm_5s#F?PFh5+rBW{ftl*#T-1N&T+}|!MYWv^98fu? zo}7zLu1d9?i{+|fs_j@5=CR_c?O2fX3-t@TdsFcIKDI?A+d?blFnp@hz{FLa-Zo|# zF}gmkMYUZEnL)LWYf;IySPnN7Rc)(s)h_3BE|^urrG?@u`4;NNHKti57ef;rQ*0Z< z{3Q%7`51d)ZyeAwA?&VH$j?|8ul4Me3i%rfcD4_%;fkf4l$c^W9p<-jVA@uvV@ZO2 zDv;Nq&aqF05^+^bu{}@r(nOi#04mxQ9m4I34h6Yj7{~+pU^s|yFGdSE?}-$_|As64 zd~jsoU8!6=FFYl7`n!)UbRXx~hwlV$XL|YwZf{uJ}KRC9yeBG;*8jo>o$!jZ@i$0l>x`q80giheQQO!bb;kwkq(4VR_Qsp1Cd zjWyhU{m>XQtRJo7Y64P$QZtp$C@JC8LjCj%E@St}%n=-lLRO4q#VA&cX2lp*6tQ9~ zE5@;6JS&P>aUClru%d(&6In5d6_Z&}%8DthC}TxAD=JtKWkn?`VyuX>qKXyO+)F3n IuL{@y4~AsVt^fc4 delta 5884 zcmbW5d3+Pswa0ZelJ3lCB-zFr28?XWi)@2U00Ukm8?aeYz^q~4_dN?+8p6!v^(G;y zBLW{!lh<^H_o!)FN|H9gSz1W)pdlN9P}9<;kiBV|gfxZHq%TQ-SDKMa`uV&+oF)ey@}yVRhg8tK z2@~9R%rl;==qfQk&#fedZh48ZXDG=v{8e3f#^`8QU*q8@w3@MGpm{m3lC1H_(a~k* z(Senu^vy3t_R-&lfYy!1+;^yU>cYXW`LPs7PtY-26Mn% zFb~WJ3&29K2rLFmz*4XbEC(yVO0Wv725Z1tunw#T8^A`e32X*8f}6n2;1;k2Yz5oE zcCZ873cd|)13SU(U>8t<26S)-_z&)EOOp!w~TrTnhmn+L>D2dKCw?w*U zBsymaZb|4qw72tlcUG2+rvX5L1jIYl&Q4%}?|}!vgWx~G_rXKp2fzdt*bn{-{5Lqz zt#Dd!dlLO~*p=YOM`&)oooFUG2NDs_QnA=?Yy9oeBLk zba!ZJs4AH*lc`^CFORa_v>5C@B;qMLqBKc#=97ibSXOn*qfPxsOd zbUB?zr_m^N`!4u;e2@6Ld=L5d`8N7i_~Jg7_gSyzZSlH2&wF&w4$ml0t#Cf-=^kfXfdJT*0gO$+O|Dy z+h*Ic=vI7$E>RTYTd1q0#Z8Dg(A!E1l2SC=UkH<2h2CO2#crnN{NrH~HGWx~$X4i0 zj{Qbzz6RGOA-0(skCr5wgr-f@*ie#a%%U5e6*l}IE39{BT6gUVYaRQV^a`t;6;`EI z5MnDG%L>P`+_5aPEs>?v*fS^*4he!lm)Mg=7N?9w+_=a#(uIz)z;T=ZFJtG~#^Slu z*jbt=l1k^m!V_n_wdgq&GNo&2p5Pwi20x!zWaCQ{?AvP3wGPOvSJ@laX_A$u2IUy#heILEh* z8mFof;T+m(Tj*G4&=_ijsuLwb$!KbHmnVva@)l~mS)M2pBF)qYRNzr*oOaeEB0?la zjSlWx+C+_eD_|*Zq=s6PC{k#HGhaP5KEfcWcoZxxM3VLnnNr~KMn+O27RA`e2x>GB zf>K^bjr*fmr*t?qjz#%w!>I9LlqY*AAIp<{YSMv%jEZyNFul%RpnM25CRXy-tEI-a zN=#H%Lyi5FiPAzk*!H1=9HrV;B2}r+L7|nlr8Jt#dJe5{oXc&cs4SIQDN+i{H5rXK zE+tMzi=EL$wh{^38I25d@>gi*Z$K)4QfYraFXG#)wG~j;pC*hi?w3kr7RCR`N-gfo zeR}hkpGsRH&9fb7uA}7GN+jC}vX5;k&2oawbeuD6r6}YC8HDAUAOntz;soh;M$5Jm zk(?k!CrD}s>2reg@_DZb(vu2ODi-*o-q_r!*mhB3PtTA4iLUdm(6NvnY6%I!=Wyt6 z4XzJX1&ac22F?Zc1$N_@_bJDfrMx{n6^}pjk?>GE&{Q2^E`FnDQJV_oS6X~k- zgmjBEREmlph!@0%aEkXAv*>BMhfbv#zEi&4z6n0j`+}EwTT8tSo-aI4dA4~rc&ddz z3qKZC3uW&2+y~rC-C=T(JVfS^0+*pFhhv_Y)<#YL1{{V^Td8R+=4xCUOHHyiOtOtP zE_X$}S#fO)HGdB`K4LUAi#Dd*TBx~eNtjfb2RDUDAB=0J<{>=eZgEV!G&OOocSO@S zM>O`1Xy_eLPtClksllUg#&R`CwUKsSbl$y;m zF*ciOPQ}JFlq?@gOlwjb8AgY@-A!5k^V3sU1SxPTH>Q=KE6z$8o#KsY zMaau5{B>el*qQUhm*D-WdRq#Lj@W+_Z3asYz0jl7z9bB$^J>{;OH)NxGfYp>^N z)bf$d**u(>mPgIgP@6mrTCUx&u~d%Tu`xk*dZ|9xUN7->eC+YAtjrVCOErEa<4ZPb znbiDx4yMEir?lf*h?-~Sr*g%sJK6a)ly$Q88yDvs=x;1O(&aHJ!6$-039b!R1l|w)D6l$Ero5|I$^xap z|67~{GyQ$!*X4vfSQGOp16+GB}*Yl zbyqscJav3qV+Y@EPhzz9J6q>RN$gA+{=LGQG~6l=p+tDK;98li7S!*p$-RR5t=`VB ziBqcDh-5B&Emg(1YVsrOvRZnX$W-WQtQe1cbPe znuBn9@i;cv2brfyTija5@iJ;IvDKy29I}#Io3tg={210o_%80XE^@35+CpkR1K)Z$EWl^X zhskkmelqts)8_TGlHq~F+1W7-_tE6GFq(HCtn+%~+FWX$T9sOS4mIo7^N*uxH9ii! z<7{fqh05E}4b(KD#^h!Vchi;@0n_fe&`QFle@&S5^)zcUsaXv*?rG6xQ1fP}%@{X5 z6;RZC7Y1xOZCa|Y@vx?*0w^;#pwZ=`eA|1&m|};45tHpsXqK-FlS=Fo++}-X2;mTO z?zD03dTc^?#qbgnsreaH{t^>Xf$?rJo|-CRR=QhX8nyJq)^%qS2 zB~ycDCYei?iw_*n3!;Obymus^=Fo(^;b zCI$QfqCBe{QMM@Slu9M+zv%Dr@Acp9pXm3=C*)`3?Q)%TMe3HeN+ZRq;#1;wvA&zM zOK($8gN~-WA79n#Juw|knSX0TN72!g9V?o$V;$UUMI&~sXvB_hRmvK7qwPL&BDrn! zsIzT?$HtHW=A4c;l4sWZDongMx6+MRPc~vfk~%Zbj_|bT=)w2Fr&(^)afimoE=k6o z%(KHW#%aWHy@Wr(1@Yg%i{fBQw`Lu!Ir*+IqBW;Ft&X0Y{3;5g)01O5da~Vo^mI?w z(Ua{qr8_-2uA?U>8yeDvo*aBJxwUlkWZwH4@($vgqVC+U<2o+b-|U>PVn{v~Pj_h@ zT{_uwVN7>ty|+8->F%tfJM(@X^EB$g^o2kVI2VGLj&{89<^}&>krJjz^seRdXrgiG3#w+y~C_`ne`sC-e=Yyne_p){=}>gne}I8eZ;Jf zne_>?E-~vgAI$n+W_`)5ubB1qZi{qS z1h|0!JirTlfC3RnKn8xGfB*=B5XbO8o z)NoPD#Skv8<6!m%Pj2BESpWb4 diff --git a/data/wikibase-test-data.db b/data/wikibase-test-data.db index 0f9d6540f74f4a54d1de59f3a37d94e7b5579bce..c2c2892c5d8e0339207d94ceb4d17024178c7316 100644 GIT binary patch delta 278 zcmZoTz}0YoYl5_(2?GOz77$AUF(VLZPSi1GG})LiMV`mp#M02*$k5z4&C+C(!WVv2 z0RzSh;;h#gIG9#5FssATIeXJ}TQjEqM8}%8P9F?c4i%_%PI;+~$2ZO+plurfL_s!pGHjZu8M z?j1(a>GoHE+zZT%T$3ALim>P?aB)s|VqvtHzWolP;PiSn#*XP5?lbaFKfue#Jz4It L?e<+WnB3R_)BQ~F delta 245 zcmZoTz}0YoYl5_(5d#B*77(ifF(VMEP1G@FG}@RjMV==)HO0)*Ff}pF%q(@2!WVv2 z0RzSh;v7;8oJ^}3m{#*GjZ90=2qvWJFV$*lAGKx=ky22 Date: Mon, 18 Nov 2024 13:21:14 +0100 Subject: [PATCH 02/29] Optional Month Fields II --- .../d6c7d96ba079_month_optional_fields.py | 42 +++++++++++++++++++ .../wikibase_log_month_observation_model.py | 12 ++++-- 2 files changed, 51 insertions(+), 3 deletions(-) create mode 100644 alembic/versions/d6c7d96ba079_month_optional_fields.py diff --git a/alembic/versions/d6c7d96ba079_month_optional_fields.py b/alembic/versions/d6c7d96ba079_month_optional_fields.py new file mode 100644 index 0000000..f7974ab --- /dev/null +++ b/alembic/versions/d6c7d96ba079_month_optional_fields.py @@ -0,0 +1,42 @@ +"""Month Optional Fields + +Revision ID: d6c7d96ba079 +Revises: 749172173f94 +Create Date: 2024-11-18 13:18:58.190822 + +""" + +from typing import Sequence, Union + +from alembic import op +import sqlalchemy as sa + + +# revision identifiers, used by Alembic. +revision: str = "d6c7d96ba079" +down_revision: Union[str, None] = "749172173f94" +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + # ### commands auto generated by Alembic - please adjust! ### + with op.batch_alter_table( + "wikibase_log_observation_month", recreate="always" + ) as batch_op: + batch_op.add_column( + sa.Column("first", sa.Boolean(), nullable=True), + insert_before="first_log_date", + ) + batch_op.alter_column("log_count", existing_type=sa.INTEGER(), nullable=True) + batch_op.alter_column("user_count", existing_type=sa.INTEGER(), nullable=True) + # ### end Alembic commands ### + + +def downgrade() -> None: + # ### commands auto generated by Alembic - please adjust! ### + with op.batch_alter_table("wikibase_log_observation_month") as batch_op: + batch_op.alter_column("user_count", existing_type=sa.INTEGER(), nullable=False) + batch_op.alter_column("log_count", existing_type=sa.INTEGER(), nullable=False) + batch_op.drop_column("first") + # ### end Alembic commands ### diff --git a/model/database/wikibase_observation/log/wikibase_log_month_observation_model.py b/model/database/wikibase_observation/log/wikibase_log_month_observation_model.py index 5582ab6..145cfa7 100644 --- a/model/database/wikibase_observation/log/wikibase_log_month_observation_model.py +++ b/model/database/wikibase_observation/log/wikibase_log_month_observation_model.py @@ -44,6 +44,8 @@ class WikibaseLogMonthObservationModel(ModelBase): ) """Date""" + first_month: Mapped[Optional[bool]] = mapped_column("first", Boolean, nullable=True) + first_log_date: Mapped[Optional[datetime]] = mapped_column( "first_log_date", DateTime(timezone=True), nullable=True ) @@ -59,13 +61,17 @@ class WikibaseLogMonthObservationModel(ModelBase): ) """Most Recent Log User Type - User or Bot?""" - log_count: Mapped[int] = mapped_column("log_count", Integer, nullable=False) + log_count: Mapped[Optional[int]] = mapped_column( + "log_count", Integer, nullable=True + ) """Number of Logs""" - user_count: Mapped[int] = mapped_column("user_count", Integer, nullable=False) + user_count: Mapped[Optional[int]] = mapped_column( + "user_count", Integer, nullable=True + ) """Number of Unique Users""" - human_user_count: Mapped[int] = mapped_column( + human_user_count: Mapped[Optional[int]] = mapped_column( "user_count_no_bot", Integer, nullable=True ) """Number of Unique Users, Without Bots""" From 7036b285b10647d02a9d46e5166d49c54550db30 Mon Sep 17 00:00:00 2001 From: Ricki Jay Date: Mon, 18 Nov 2024 13:39:48 +0100 Subject: [PATCH 03/29] Rectify --- alembic/versions/749172173f94_month_fields.py | 2 +- data/wikibase-data.db | Bin 21401600 -> 21401600 bytes 2 files changed, 1 insertion(+), 1 deletion(-) diff --git a/alembic/versions/749172173f94_month_fields.py b/alembic/versions/749172173f94_month_fields.py index bfdd56d..fe3a2fb 100644 --- a/alembic/versions/749172173f94_month_fields.py +++ b/alembic/versions/749172173f94_month_fields.py @@ -43,7 +43,7 @@ def upgrade() -> None: sa.Enum("BOT", "MISSING", "USER", "NONE", name="wikibaseusertype"), nullable=True, ), - insert_before="first_log_date", + insert_after="last_log_date", ) batch_op.create_foreign_key( "observation_wikibase", "wikibase", ["wikibase_id"], ["id"] diff --git a/data/wikibase-data.db b/data/wikibase-data.db index 6db3cfe977a3ed3d1c18db8f9e6cb1caa90cad4d..c58f76b3d4595d41a47f0b9bdf9b9c350c3c5221 100644 GIT binary patch delta 8019 zcmb7Jdwf*You6~(dFS4ln-`FG1_DVY1UNjBKoCeWlL*LD1p#@Byx;QDf}A_rpOo$5 zP`4Sjtz7D+VO>Q;Eh&glv;}!7ZiHpq3ev7tZK?87v8(iVW=`(C)4G3bKA+!7&iDN8 z?>z7Ke1A7=*N^-REnbjW>Tu*NayaD04o6_C!{OPny=ovS-8cJRO4NBZ57f>ZQZsiB ztsU}G=!TXC#3)b#R08S*>H_Kp>H+En>H{hR^#fIasz3ukgFrQ)89+0EW&zCxngg^8 z(5^tc0nG)P2Q(jO0nkFAML>&zmH-U_?GCgD&{Cj1f%XDg2DBV#1<)|iN}#=gRsrn; zv@g(pKyf_%fervV5a=MF)j(^2)&d<2bO_KophJP)0dyG9;Xp?Ktq0lwv=L|oXcTA@ z&}N`_0v!o-6wuK??*cjo=vbiRfQ|<`0q8`acLTi#=)FKE0i6u=KA=;8P6av*=>0&a z1DyeMCeT?xX9JxBbS}^bfX)N@Akg_h7XV!dbP>?SK$ier3UnFJ>eT?up*(A7X6 z0=fq1TA=HIt_QjS=)*uC0s0-F8-Z>D`Y6!NKp9XSXbk9MKpzMCU7!H;37}hmeh=tY zp#KE)`#}F0=(aZcCI67Do_#*Ie;J1Ojfu9D}1r`RT1;zxrt6!?0s7KWws*Ba(YPRx`@*Cx}(yFv5 zOO%LGt(5!!>c8TD-T#We&A-FH#9!~v^1I{<@?QBdd6Hb|yX`ygd(QW?Z;fxHugJUk zs`r?8hj+O*;;r_Udv1HKdX9O1;fZ@6BQ;#}HX)0jWVPumCKp+iXBwWJ-dwHDvNx+U(`qy9 zwea-xNAuMC?ak%WtY+3LUL35PY67^ZiOaKE^EI_86snqH;)#x4@zQGbK6@N>a$0SY zy%xULdK4#DG}!HStM}MF)VtGa6YaIi3D(nnd&j#^RL9$!)p2RHv1zq2X|=oTwX)Ho zhAW=YH(pezj=~VBu`;;}CK8bgF$BW9i z4=ELm7I3gUUJ|ToFab%D5@$!Y=C@SDJxaLV#1k16@lddGgbCmlJ@s}|ej?N_?oz_T zO+2x^3cIKrW&uxD#JdN}?=S%(%f|bU;~SHor4AKn>a5k7K8a6xcjZG&03Uj)|M+~> z9Ug3AiH0zC6|S{_B^;=#v49_j}R zEUGvk3aJAGo*ZUXf72^5wlZF-gsBBQQiAB;&98GFy6$tUeeHqNK54Zo zdoA4C48dQ^qC$RhEA1YYVatP`&T_TF-mI3}Yh`7YGe2{|a4!?EpR%5550#oe;*|BU zH>=(4wQ$ICPM)%Gi3Ow%Pq7K4AD$u;Pad9NWuXahN4+^e%klB3k>v#@Y#yPaEHz)C z$-ODhv-^k>lWSrLen6_#ZuUpiu4%O{_F6c{deq#7Y`aHgmgSM!g-m<1nqjY%X_j+p z7lI~W--SThL#pW`c0sW>tA2YeEL+a0UGP~zY8SjFkiH8Z6BoPSHi5KVaG9{V3tCpW z(~gRDmrN|V?m?9ZAh}iTS4Z5`R<_mO{C^oLm+Q0IrFhHZL<&5c*s{Iln&h9;zVQbw z*PL0h)8X(hM6Kh+-wROfkSAD63t9qg2im`){Rr*HXg@)F8f_<EIp^jFEc2HZR^$C6$+!dS_%nqCmKwt!_p_lNs@1Z|d7>V4Kb*IVE@=h^1D)8ld#KBMRw*eQ;r;Tf)bfgC0ZGMqNlB-U)pm zM22V)-L8M+MCPfzMaGP~LnI%)h4Qg`Tk=ODc5m!GT>|Nr1~TFpuMUT7sUSn0mx3%E zWK86c)}UKTh{(Rg>F@5Y#!*^o2;Z(m|1qsaiGHx#ZNx7(kt|JhIitFzkBs66`E;YY zrI3uni%_f?^us2Sne8~zt7lXfY6)kDFNN{`Uz$j@b|G77)P-u|)BX$t4bh^ykWT;T zD^g`Av-E`${}@QK!M-CYxK zM|B~vc+7Wr1dJ&p7Ker{<#QXhFhsiGDhrLp8)hI>hrJ1vg^&|>?Z~*+BICe&O(Zw# z%KNS+U8t^PD>1fTY$72;!R5+Rn{=VUc(8s5-Jv$?LWpr_9YVu!EJBBIh_5rBpb%pS z;p!$vJc+wlpMz&g$TIHA7q`?{wis15E}Kwj$rxhUghoRsxop^x&}rPM>k{5kL>FQW zq0R*>bgO!rXp8DXwejcn^5<|GLb`G2CPI7~LcDQ^FDZ{M6dXc$-i@yg;XY+vmZG}Q zaEO~LsvK@2O5(liZNXqqS40=Ojt2`aNl5XsZom zMRlR{_~RGxaXuEI^~B>xK5wfD;7Ekv*dbuA0Xq!r2(VuQI|}S|U~d5X6|i3eI|l4H zus4C70Cp1CDPX68odNb1u(yGo1@;cG{{+?n>|J2zfSm{SU%)N^`)^>s0rp#9zXSFj zu#3Ph0ec_VWnjMt_6K191MCA}9|HR$u#bR!4D1tNe**SrU{`?s1=y#+J_B|Y*fn6+ zf!zRh6WHg#z5wHTBw;jS-TAG@Bx zJ2B7M;ru?P^VwfZ?b0f#H~D~^Cvh^GI2|_<{jVO$#ls`A1AcgB@xz1HNoR5(E;ErD za1VZ&l^Jhu;Z#2H8>bY}MVf$(67eP2t+rY`-y&5=f`4~ab!H36*b!ai3Osh_RVC5{ z4xJOXm&g@3l)S2N*dklt(5!n>@3_bp5c-yjDhn^FW?f_s=z=Mlv0{A_vY>Wvh;+kO zw~uwv_eY4ox+0n2ovcTwPHWUfMv?49eu|7OO5Fas&67A-(2*H*OAWfnF0g&)q!1a5 zbSyFq-Y4qtiP1vi`7Sgh)4sYXN>xkfTHF?Nk@CWEwUNov$Z+JOzm?=a{P9GOTw zI24+k+O@Q-BchA6gL_=Y{_3#5w8SH#i_9a{Ut-@&Z8aIrh%Pda6lz>r(v-YLMH<4R zO}Gyi2cvZ+BI1;Y`~yAstuyh*ZG(Dfqa{hzJM$2+<0AjC^;+mu^|bsWqKgECcb6Qz z+MtX4gG2Etd}SlL$Ur(nb?c}JIO}zhiEzhVGg2Zzq$A022o1$E-kFn#(TjA1d%Q3; zHNVcBB9 zR6COwu>oGQcSE0^Iy#+Mi|`OR3y;xPcy#6~!b2n~+@mu_XPzSFD6$mpkz56o84j!C zw{%b6&Xb;Ccmp^^VDd#w9%m*&{k_zm>LoQ~drn{G0r>@^$%nOp&I_qh+V>hHsB=r|&*rruU@x zRqtBwd~e8`={e;&|#w^lx%5qC)$KYVh zqFa7RY@5@JpPH9vc4#?!rKu8ah*{j5WP;3&Y7CpkzR{S)zTbLM3e;FS8OJO#PR$ZM z1QVyuNhD@baPrGGI;1ghBxW&i=W82F=iZpby`8UZERA@hF&ptF=>xakM!cEGXPDH) zEaElUsE4#0*(fH+SG+GVi+It|#&l;eP%h3OorPmI78cYQOK0KO|H;C!bQX@;SeO%f znQ`JI&(RuUf{QsZiF+hw@i7+;`;zxaBxYmbRi|*?YBXlCu)+DE!nlYH*|_`ZBpkCy zm`|JEg(bu{X7u-LKoN;`XB*6_494*eAZYO9iBJk3+7@Ci0LIGw{C{ZkyCgdMwQKN3-?7eTL zB#e!;lM+U2)-BfNZ(h{~{tuz?y_+%poB4K1q}f=Ui?lMM4I8@Puq-m?!%KkJ<7X;r~(mL~_ZS zoHyFZ1o=~fe@bL~Ls$P}^Tv6*jT$9EL*i&fTd8rYYbC+7!O-(7k+^8LO0q04f6?lP z)`S|zj~~<2FfQ?2Ra-%#UzCh6u48LvBBM`R*C(rTD+w08c~QlSKD_A5i+;SIyy(x1 z0lXN;w7?*Qm!9Vr(l$TbN zln*T_AJ$k=dNS~pFNM$$MjS>wMs|z@j2sv_F>+z##z@3S!brx*gOP%f7b71=X&9wr zlz~wuM%^&V!l*k&*%UKsVpD1cEPjQV1fk5NC23NR|fs6R$Q zjEXQCfYCsViZL35(O`^*U&VbEJoun8jsNgj2^&fB1V%inv79BMhzHE!DuQ*(=eKj z(F}}cVl)e**%-~iXf8(cFnSQ9hcKFt(E^MXVzdaO#TYHYXemaG7%jtSIYvzwt-xp{ zMyoJdjnNv6)?%~{qxBeV!02I&He$31qs*u}az3hF~d(gYjyTm)wJIL!)?kS%sXO)+f zy~++{ow8V|P|_8_^P%U2XSZjbXQ;<5|6RT#za_sS$K)n?o;+FZCud8yrJGWxbY40v zt&<{BnIwxJi7$)$#D!wH=yQMUe&4<8xchPUTsLw%UB7c3cVX97*D6=kHNxd}an28% z?>f&oUv#!R4>{L37dodo$2jwxX->Q2ua1k3XB?XyO^#~EP)C8Ihj2%@C3FZI1!TW% zf6m@$@5^83ukugwEqoO}nERAVaErMaTn0R8%3tebmtWo}o1EM24nJ%-p%BM$@wUH2U3%tgvP^(JXT!%b9OXWEu5SCL)C!DX$xOOR4{zyd{)3^A=M- zH7~fx;B6sY-4V<4+wFFFp-I`Gtic5-@lq0>Z;8tfS)B(hPHZIR>@s+uK&d>{5|pP{od%0jR8KGC zIZi(^`>21i?S>%0`_Q$;P)aW1>;R8q)&PzmX|G$vjl@Xpxswk^Iltnv+lxj~TA#WF7OHS0^ILYsdimul(@*Z7N*d%j zQNPpH?pK6WuEGk+>bJJQ+6PB4U)l;Y36?Xz7#2|)SjPMXTVwfB(QxMP-%6%X%KTTh z#(F75!>CVI^U}nl{OK6Usw++S&BIZxn77Hi?hEU%QW1z`L{blx7 zzjQm~!InYfK~|^O;sgiM%gKus@XG@%!J;A>q`NJc+-*Tij@;kk6co}6lG{iM7BJt~ zIsL53`7CJcoW7Qz+{fYs11uAEP9L#1bBztri}}We$fJJhhRCI~eq=jYjh@UW8zNiw zn?5OfVGau#OW%X}WT{K!Y)i7--RfjnoM1P0nNi+MORy+|22;zMZpo3;EKY%sUSO8j z%Y37}iZ$88f<}2|OHh_9PEcf-$@03HYn0c;e51Th>Zg|1L20wR0`p0Er9wONjf(Qj zClys>&h$z7JGkR*_vd${9dG;m|I@ZS#@o3LKGyaa$BS>1KB zn@w5{08yhn2mf8TyQQrSpaTv94gn4Weh7FP@Xvr}0CB((z>few2K)qY6!2_EOIv%( z#RNQ6;QKYgZlNUs_!q!)fS&?>26!IubHEFLV}Rp;6M&O|7XdE;egXI;pabwS;8%cG z0H*-20$u~W4mb@s12_vf2lzGM4Zxd#^MJPi7XTLlZv%b;9miq zfPVx0JK#OQZvpQEeh2sf@O!|AfNOy3fE$3DfLnk+06qfz5%4kKKLGy;_!HpIfKLFQ z0zL!W2HXMs1@JlGF5nBmmw>MTUjzOM_#5EA0QUgj0R9g6Z@{;Je*o_96I!?f0AhGz z?8FGfIEZl)<08gQj7W?`j7*G&7=;)wF+O6_h)E|VgP2TWx)GB_Om||kiRnR14l#aW zdJ>aMOdc`4xMM;~Z|=&jF}AmDzEi#zd`-Tf_apC*y$^bGmES6DN`vC_T=bmsZ1ybo zg9sucew{#pEq_>?$VRNNOp=+?N0x^BB}xDL7YfylKv zPdR_?T<)CX%yL|He9y7dF+%uSI3cVO3hmeJhwRhrK0f_z{&BvUAIjYZ1$&5_#`*Nv z=j+SluxfV4P)KiCo2UfB?2tg52f`ax&E}X?vOhRemlyw_KEU~NWR-SF@*+o6vqy$P zabE+dP`J?SlA#dKx%HNN?d34K*(qW4!HW`=FbHjxWRS3GHcA-eE(}7ZWA;d3%BKJW z!k=}6y?f+H2bEJUh=@vJbegGIO4HeQ7*&l5$sXb z?4WRq$x{Q|C~rhH+b9``)34@r=Eq-$NES@T?5U7@1DHy0NHrTP8DIPXH!)AY|82W7 zK4Dsb%k-}h-$W5807dnP^w&6wRYODYPQt`AzH@h$} zyJnDQjHqTKHtI1yr?WiIC9AX>8_RF@Vk4#DhwY_#zVSRStF###>ua`RdNY`>wHb$1 zvm1BK*CW)ZW=n>P?q3XWBjB;krVLE|EHdY6)$Gi`yf81ol|zxu_6*GDz*NCjqdnS~ zu-Ty@K4iAB2((R;c#Udy=j6)it#jbhYgPBo92myhsUxb{s$m!(Ope@!Hf!THW~+wy z;5iV7iJAQxm`#L%@|qnxS-be%zEQ3;z&{6HHmbFIM^v+Y8zr7}?pUtg+|izs`%|wZ ztF)IJWi~rG+(LcJ*cP;x8*#IR8)Yte8MXzi6z$_iirK~K5b#{Ij~mZrc5#R&pBt9I zX7qJnl1m^(b}=Hk0#fx(Mwl5p7-42?XM~yY2qVmlZHzE8wlc!Z*un@iqnQ!OjBI(c zLD-T{vWZ2LU(d;3GDg}C+I+A0Hu{EmKk***KIF|&I+Z7tv5MXEnrDk=sQih1RBn?S zWLbJsdQ_?w?};ypE5&~9Yi`{=#qDvOcd4!_=hyI~!&+yN<6}qMG24+TToU#PbpmI9 z)&7FL$zH(UBH%#7bM75+dv7pChpILqs3=?WBmBAx}FA-eE^nwUCKaf zX*hYh7sk`}M!EEihW&Opme$Z1oPhe|M-!!rB!tz~lue%Wa8RzIL`vF9%F>3g!XT21 z6IPpO*k}lFo-d~i=_>agOOP{u8RbB!T3Ui!qc+lresghR1f(vd1WEl)J4lCi5K))V z3{uu-aALUkH&YuT>S9XkCzd2jxrh>^lsE6d(e8aeQx2&MDX&*AO;kY57f|9miSsF+ zOoTV+Ax5M%Cde7~AR|gvB*@t@j}eC(fe5Q}8F8yIxlQIUqW`kwC9@f^1_-FYEJ~11 z;CcHJ!<8~VqRynOUa~4t7(aat+GG0dn?kKRgGTJ|oUPZR@JrT(SbMij$HALEPpvxL z8a{gt4zEnxhbLS@RGmg6@SPtodlMq*{19#fwaDT&;tfxoCfLy z8YZ{*T|Sl@FM2u3_3-hd1U{;cr%`?G%ETyGka3KF>K4X7Jp~Wqvpq|+%vcusbY*gB z#xMd(Or}!Dh%MwD535l|ke42|PAw(KY%YD(Uhd0?s1Yj(?|ol=-D;>tSPj#F{`{Kc z9Yc(O*FV^{CXoqKu3=>FHOXmIGs2XBqZwh2Gl~(GaYizdjFW&bxqA58*;QrVN0ja+ z-yrXw;Ot!B?Ww${Tu?Nn+VhR)Gf%>E$TQUwmG8@U<)6uoav$lsbWqwWjgczEFU60= zxY#OA5S{Kb?vw7-?)h#%oThKNe&yQeYIFr$8P3bj-OjMH)N$L9a4d2Jgx?E?glR&C z{av^%`Rj1ji*CDIt#t{<^k0J_@P;+3a5R3BedY}ij*!cQW1Qw%Rby&yms+H`OfABZ zlA7D47HO_UEvDo~G^P~e&%>;$ytNvWiR8ueo{n&N@!9+9xa>ZDvBJ*FT8e1Y`qSmG zi(*@PXSp2Jy7Xe~O(JY-G^Q6pj_R%Hox^iv&8iq98dHqMkIA{RmLeB5CKpp57RE7U zqj7L+Ofn{aP=;ZwaxttixtKhL4F#g5Xhp3{E1DukvueeN#>r#=1;0Jb7hYk$K!~8HrSpY|EBu@t~acP>L0*m>jR71v*LBx1GriG z#9FR@{1@;~6nFL*z@_OqBRNI)4rrgNpQ+{Q4x9uM%s^s_i5W!9U}A<4gNP|1W+*Yk zh$$szI5B0!loK<8m Date: Mon, 18 Nov 2024 13:46:33 +0100 Subject: [PATCH 04/29] Populate --- .../e0cad634525c_populate_month_fields.py | 240 ++++++++++++++++++ data/wikibase-data.db | Bin 21401600 -> 21401600 bytes data/wikibase-test-data.db | Bin 172032 -> 172032 bytes 3 files changed, 240 insertions(+) create mode 100644 alembic/versions/e0cad634525c_populate_month_fields.py diff --git a/alembic/versions/e0cad634525c_populate_month_fields.py b/alembic/versions/e0cad634525c_populate_month_fields.py new file mode 100644 index 0000000..2877b51 --- /dev/null +++ b/alembic/versions/e0cad634525c_populate_month_fields.py @@ -0,0 +1,240 @@ +"""Populate Month Fields + +Revision ID: e0cad634525c +Revises: d6c7d96ba079 +Create Date: 2024-11-18 13:21:35.815575 + +""" + +from typing import Sequence, Union + +from alembic import op +import sqlalchemy as sa + +from model.database.wikibase_observation.log.wikibase_log_month_observation_model import ( + WikibaseLogMonthObservationModel, +) +from model.database.wikibase_observation.log.wikibase_log_observation_model import ( + WikibaseLogObservationModel, +) + + +# revision identifiers, used by Alembic. +revision: str = "e0cad634525c" +down_revision: Union[str, None] = "d6c7d96ba079" +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + # Update existing records + wikibase_id_scalar_subq = ( + sa.select(WikibaseLogObservationModel.wikibase_id) + .where( + sa.or_( + WikibaseLogObservationModel.first_month_id + == WikibaseLogMonthObservationModel.id, + WikibaseLogObservationModel.last_month_id + == WikibaseLogMonthObservationModel.id, + ) + ) + .limit(1) + .scalar_subquery() + ) + returned_data_scalar_subq = ( + sa.select(WikibaseLogObservationModel.returned_data) + .where( + sa.or_( + WikibaseLogObservationModel.first_month_id + == WikibaseLogMonthObservationModel.id, + WikibaseLogObservationModel.last_month_id + == WikibaseLogMonthObservationModel.id, + ) + ) + .limit(1) + .scalar_subquery() + ) + observation_date_scalar_subq = ( + sa.select(WikibaseLogObservationModel.observation_date) + .where( + sa.or_( + WikibaseLogObservationModel.first_month_id + == WikibaseLogMonthObservationModel.id, + WikibaseLogObservationModel.last_month_id + == WikibaseLogMonthObservationModel.id, + ) + ) + .limit(1) + .scalar_subquery() + ) + first_month_scalar_subq = ( + sa.select( + ( + sa.and_( + WikibaseLogObservationModel.first_month_id != None, + WikibaseLogObservationModel.first_month_id + == WikibaseLogMonthObservationModel.id, + ) + ).label("first_month") + ) + .where( + sa.or_( + WikibaseLogObservationModel.first_month_id + == WikibaseLogMonthObservationModel.id, + WikibaseLogObservationModel.last_month_id + == WikibaseLogMonthObservationModel.id, + ) + ) + .limit(1) + .scalar_subquery() + ) + last_log_user_type_scalar_subq = ( + sa.select( + sa.case( + ( + sa.and_( + WikibaseLogObservationModel.last_month_id != None, + WikibaseLogObservationModel.last_month_id + == WikibaseLogMonthObservationModel.id, + ), + WikibaseLogObservationModel.last_log_user_type, + ), + else_=None, + ).label("last_log_user_type") + ) + .where( + sa.or_( + WikibaseLogObservationModel.first_month_id + == WikibaseLogMonthObservationModel.id, + WikibaseLogObservationModel.last_month_id + == WikibaseLogMonthObservationModel.id, + ) + ) + .limit(1) + .scalar_subquery() + ) + update_query = sa.update(WikibaseLogMonthObservationModel).values( + wikibase_id=wikibase_id_scalar_subq, + returned_data=returned_data_scalar_subq, + observation_date=observation_date_scalar_subq, + first_month=first_month_scalar_subq, + last_log_user_type=last_log_user_type_scalar_subq, + ) + + op.execute(update_query) + + # Populate empty records + op.execute( + sa.insert(WikibaseLogMonthObservationModel).from_select( + [ + WikibaseLogMonthObservationModel.wikibase_id, + WikibaseLogMonthObservationModel.returned_data, + WikibaseLogMonthObservationModel.observation_date, + WikibaseLogMonthObservationModel.first_month, + ], + sa.union_all( + sa.select( + WikibaseLogObservationModel.wikibase_id, + WikibaseLogObservationModel.returned_data, + WikibaseLogObservationModel.observation_date, + False, + ).where(sa.not_(WikibaseLogObservationModel.returned_data)), + sa.select( + WikibaseLogObservationModel.wikibase_id, + WikibaseLogObservationModel.returned_data, + WikibaseLogObservationModel.observation_date, + True, + ).where(sa.not_(WikibaseLogObservationModel.returned_data)), + ), + ) + ) + + +def downgrade() -> None: + partition_subq = ( + sa.select( + WikibaseLogMonthObservationModel, + sa.func.rank() + .over( + partition_by=[ + WikibaseLogMonthObservationModel.wikibase_id, + WikibaseLogMonthObservationModel.returned_data, + WikibaseLogMonthObservationModel.observation_date, + ], + order_by=sa.not_(WikibaseLogMonthObservationModel.first_month), + ) + .label("month_rank"), + ) + .subquery() + .alias("partition_subq") + ) + max_rank_subq = ( + sa.select( + partition_subq.c.wikibase_id, + partition_subq.c.anything, + partition_subq.c.date, + sa.func.max(partition_subq.c.month_rank).label("max_rank"), + ) + .group_by( + partition_subq.c.wikibase_id, + partition_subq.c.anything, + partition_subq.c.date, + ) + .subquery() + .alias("max_rank_subq") + ) + + # Will always be the last month of an observation + last_month_subq = ( + sa.select(partition_subq) + .join( + max_rank_subq, + onclause=sa.and_( + partition_subq.c.wikibase_id == max_rank_subq.c.wikibase_id, + partition_subq.c.anything == max_rank_subq.c.anything, + partition_subq.c.date == max_rank_subq.c.date, + partition_subq.c.month_rank == max_rank_subq.c.max_rank, + ), + ) + .subquery() + .alias("max_partition_subq") + ) + # Will always be the first month of an observation + first_month_subq = ( + (sa.select(partition_subq).where(partition_subq.c.month_rank == 1)) + .subquery() + .alias("min_partition_subq") + ) + + op.execute( + sa.insert(WikibaseLogObservationModel).from_select( + [ + WikibaseLogObservationModel.wikibase_id, + WikibaseLogObservationModel.returned_data, + WikibaseLogObservationModel.observation_date, + WikibaseLogObservationModel.first_log_date, + WikibaseLogObservationModel.last_log_date, + WikibaseLogObservationModel.last_log_user_type, + WikibaseLogObservationModel.first_month_id, + WikibaseLogObservationModel.last_month_id, + ], + sa.select( + first_month_subq.c.wikibase_id, + first_month_subq.c.anything, + first_month_subq.c.date, + first_month_subq.c.first_log_date, + last_month_subq.c.last_log_date, + last_month_subq.c.last_log_user_type, + first_month_subq.c.id.label("first_month_id"), + last_month_subq.c.id.label("last_month_id"), + ).join_from( + first_month_subq, + last_month_subq, + onclause=sa.and_( + first_month_subq.c.wikibase_id == last_month_subq.c.wikibase_id, + first_month_subq.c.anything == last_month_subq.c.anything, + first_month_subq.c.date == last_month_subq.c.date, + ), + ), + ) + ) diff --git a/data/wikibase-data.db b/data/wikibase-data.db index c58f76b3d4595d41a47f0b9bdf9b9c350c3c5221..a5991ce7933fc44b76f80fca5a96b9570e0f0b19 100644 GIT binary patch delta 14524 zcmcIqd7Kp0wXUk(_t|%bhUsC`f!jUZvor!Tzzq8`vUCrO11=E-6x@S~P>s265e6rj zQe0Zi*rPFVz!=fEfQZ2@pdlKcFBpkQUfeNJvpwVcZr!Tt>Y361UjKey{n7W_@7%ig z+;h)8r?%g_$Gsissr1zfLeYhS;94&Tz8iK=XcTMrt^L7h?{~<#XP?(lAB{y~=RO+z zJdpE8iclg_B2i+Y#7c>c5<4XhN}QCqC~;Hbp~Opxj}kv60ZQ^H$)}`%l0r&~C@H3- zgpyK9$|xzPq=J%4N~$OsMM*U!qbUheGKP||l+;iXqNJ9Rag>awWCA4lBJY%P;w?E%P3h+$qGtVQj(w~Nl7Oq zt0-AbNf#w&QPNGx8cNnuayBLBP;xFM=TY($_DcL~DMoKQCqmODMUNlFKN$oRTXjxssBrDETfWTPV4jl4~frmXfWMTt~?^O14uXQ<9=&2PM~2 z@;yo@B{xuVBPBOcax*2jQ1X3BZlz>rhU_}E&1WM*i`Xj!-VWRzNCZm!zxMC)FYr5k zFZj~FExu-7q4za!k9WOywpZ}H?NL0JcoukS-S4^YcVFwC>iWCuG1q2So$~|dVdp*0 zZfBFlVtLcD*K)I^-C5u`=6J}lLpmYtk&@Q!)>YOL$4vWw>`&OQluE=m#9o`j8v3O@ zV*Awgu>ka;dp&i`J^v1s@A&EF{4=*Ef!ra?{Lv@Z+J;{jjg3gmjk8KGfXbhr1-Y< zR7{w;WV!fjsfC!NUD6O~5|1=1i?8mf7!PS_jMO^CE4cV@EuuJFxJe;P`bRZcp8lKD z7MiRv3&l>S^E9(CiN~7a;f58<+RkjP)%6#Pq7rEDFYR5spehg3K69ecB8kewv->NR zOlN=DHsSYHYiNR*J2;#>%(z42%~GR>Ns(qIHGQ0+$K`TW50`4-e>K#F8-wAfa!*Hp zzHer|(Fub~FX}HWj|~XBi~PGF&R6Y+u}8U36tg z*eTpP;|xO@i?6Ji-(Q-3@A@{ebTWH|g7H&zm6e!l{}3{mQEbt?Wy|I*o}=sY)Pn%fL+p^fY!U6hehCJx=MvyoE6^=*iVLp?I`eQnU_vg)RY1BE`K2k<4#DYY& z_LO!qEB{!U-JmxYOOoU=^Qqe`qOCD@9@e-aPDni5Xi_0vCTm+kaqR!r z)3T~Wal-K4QRTU={_1Tun=LfSOfhZ*QB_gMELAy7il{qkdX1qLd!>8?DgGDsNU zytjH`UT0{mnYwI9r^a}!Q60b-vsCGDDRn+tgT??vvGHo8B;U2CltW4S3Q?Zusl2EBx*HDv$E}||}iCJpmFwT0_zC*=ksov-2YIa+s4+!pl znB6wU>l;+NZD4j=Xl58{w-8kgVrGFsh&6m=p81f-Hxmp!BtTHFzD}N5YUm*WDfOkN z2MjeM91@Lj-GSsB60Lrt6O47r0`+KHfTJzsGwU3>*&5^FCN{F^UZeX6n~h2J<}Ix5 z40+6C!?#zMzoy$PHGF%8)&1S-GE`#3A1xfRy_}}j5w;g;&fQ)PLmW$U&!RHLvaP2y zdra7k<`K6TiDtK09@}CzLmqTX7nk<#Xs_Zs%4)QXxT8pf@2K%?RtsiW%+fh=6t-#u z{Dc-qNxIxfdk1ncb??m9_Kv7YtNS$P(C4h`KD8kbv7WN{5$Yv;Jv`7f%gL^ z_KQnhM*_hDUn-0Xid&up0lU0a5ZqVE%hAW>yc!RS^4x2;%-El|x9`xDZ9e-{Auil4 z*}kytwOwSJV;d9LBV8jkN%x8W5PvQHL^^^H_;T?A>0@!GSY>_3`h@j1>qhH1%O5QF zTduVks*k_%`_pyua|C<+Xa=5uO+BcHHV%=LpyzwRd`+@T~DP zyMOO~(!IuQb3N?paQ@x7^ls-Y$NP?MN2$YUKV(0I_X3PxXrp;L3|A6vV5b79$3ACk zk=US0xmsHVQTg-o{`z$A#B9-?erHWkEQ*GfnK{Or9SsicNIb#~`%JTxaxmFnl+UGjo;%c4dapv! z<&2wk##s0jBg&*JdMd_Ycje9O)Mc@hVsw~nv{^a1roVcew=Ec)oy^Kg(&@uwalmW? z?>07aN1tV@<=RlfJj9%PiaNs))l@?(%~FOzYZo;bBcI6?W~rRBk#nhr)wlG z{Y061c~3LQZcdgKte(dV= zzmW&l+Z-nixjf&=;rUYM@vc5E%cCq_i9NFHvYv=xdFSSITe?zL-+B)3CJeFQ7*3OT z6eEbKfvg(sovn$g3u&BVVJ?ak3wH`@zlkv19IubWqdYR+#+>RJGoj(JKo}0k!}akf z&wg~9r3{aimEvA~iCL;f--(7_3#&V5db>Wjvf`n!DBd*^Ct((g)`k|F*~SiMW4?5M z>lZIBk@n{;bv^3a<`bV3h_FMj1*|`}uC|JSOkjckl>a6FCjVUDCkRU}^0j+E^d9zJ z=yiJrJvVylJa+dZ?i<{vBNX|uYl-s<=N--&&N6Y0*d&~AeCFtLGzmK_Z(FuoI;F3r zwA3b5i_hDCXE}tRr$IU;os{mBR!DyFYw<~_E@VC+A@jSZ>ZRZ*ckIc8m7|Yl%JVEq znO_816V4uG+MSuX@xXjZ3d#D7PhC~xV8&L7AVm?wiKcEhncwzw>OFUA)D;%ng@T)D z7lfqDQ(d}}4{y(q^s!fiVpT&(&dGEcH|{VJgh=b?K~O^GsV-g3+&jSV^rCjLY+Ou| z&XTnhS8HpIr{^T38JEcPq7}>`eSAm8ksqu`$UG~@2jo&70G;w$)$#JC2l3)LnF8bN zL2``XwiJam@E7N)0)0fbJs8o!^=+axT~*sI7B$(C#glU~1KBxLslqYshL*gJ8x=eT zY)^T%1G%rq;6KBf|J*kjyaqTB$<)yJv?a5^Co1TEzmhHXPlcLP#H+dT8cnh zvDfa(m{$z#JUO7hH`>Ge=!CuN2?F)KjnJI0T768*cO+$=9MBm~zZ(ooI-wj4dPYEH zFk)Vf0q>#O=L(r;2h`!is&2d+bHB0VcsU4iG6IGZ(+IHZ)HzMcJSCtH;g1-C^4iZc z6?x8(tY7@qAt){PWO%5Yl=;P9?}G-StbIIVRVp6O6c>Epc6G_xr5{tr{LNkWAlTUX zcxG}bvl-ztt2HU}`@YTriKw!)_;_*iLpT}aoNK-j%JZ>DqA*d$la%>AU+4G-8^=}q zGv&=c_Pax}e$7`$ZzNtc4x+t(Lgv?d#(_JGvOJ9?@Lt?5mKKz-f#|n;J_UQz+7u*Y zevQ|=p&_i?TX?*@5uQcA$B(2K`zxw>7Jh-(86IBRuPJU;73uHphW&9yK`*e{x&aabOzG1rfL9{o>xfuh+2K`lfR)ra_76AaCL(63+EnZdiUfSu>sD2 zU-R{qn6w+V_}X~`j`Sa2Y7_Gdg+sNWq|BofUB-4BWR|WUs6uakN!MErpd~%=i8irp zyfn=!t(LXGh5KNm(!pJJ45xHVxn|=)RQ1StR`_h~PQ>MvT|dZ_D9?SFiJ(?ttChev z-LMUF(n{kC!?`F2zRW}o@wi#s$n|n@crygMFKyD)d)a}|m}5+9`>>~`bTFTqkn2rN zE~QGri{_olo2D2|Y$v{xRjl@L*(nPGWPU>TD23?`~yUnoV+|VqpV#dudRt2#lAT)F!7Ouo{zsu@#kQRi#yO18-|(WlX1XurE`p z?D%IUT#W0xWrsB(#|&Clojj^i)l^yxb5q7p&MI>*0WKQA#(wfI+H3rZh#o zRJ<6yyJRgH#!nB25jCjYkamkTC^L2qM3qy!GuA>&OObb_%yU2L{=qT$Aa-AR>9IDk zJTxaMYbi9aA@DNlS$q&+Lz(ci3&BQ?4%+Z>MVG8a|6IFmnD!_(+xmg1dX&{%URIWn zd3uT|XA^P`HgDg$fyxCTIVVA9WK*-6Rja_#XvsIG(Nsh=TrYZ1f#%qBlK=qN=CRx4yJADf3*7E(AA5(mU^DUQsydvUkZX zS?e-YTs5VUczvUC2HX;lIS$O9pO9x5rp*@WBk(EdO#>{{n`X2yqo4P~A)3zkHcjxC z;h~zQ8hlJs27Icf0da3%-fb<`8?4(0fCqtxfFA=713v+N3OoV~0(*gdz@xy=fX9HJ z1CImyfhT|`fnNYm0Z#+Z0M7ykfP=tuz#-sy;4tt4@Jrwb@FMUM@G|fU@G9^t;3#km zcnx?R_%(1G_zmy|@FwtE;4R?4fVYAF27U*e0Nw$954;PU1pWZL2mBHE6Yyu?e}KOL z?*ktI9|9i%9|NBNp8}_V&w#%Ie*-=Tz5u=i{to;D_$Tl$;NQUi0$&0D0lvP)x?S7_ zu!qDR7WS~RhmAe#?BQS!CwsWq!_6KZ_VBWYk3IbC5nzuz_Q+?C0`@3mk0SOcW{(o~ zC}oc__9$nM3ihaEk1F;U#U9n-0qgeB;#bEK&WnYK;(dax#rg+lVc2?sN{qnkHC| zS~gpXrN^XI;=lGkSMcUMF|bn3UMZ0oRW3O)Xg)0iXUWFZlHI43qeliy(y8(7Vx&wv z5QBvY*>s`gGW-f1cu?w=vzbOoIsL`K^7NGN;#-dNwxc+q_>tll?iBVUWmAR`yJ7N+ zgZxM-9q$x2Z(c2D6OGs`T6EK-em#V0ElN}kO4sL^ige%@c-0tZn$4Dwvk6DG#cp|V z&~Qdt$}z_o>3sD~piIl4g2#Vp5a-9aB_&DOlz#--U)TtlJF=y(D{B`E#`!KNEnO|I zd+cfZ3<)E9d)rA4;e-wk+^@(Q&J_o<@t!yfpRcMf5aiiF?o zbKn+~&0{jZ)AkZx;+?f>M7YWDAgrbJWsHc$Q1*1*%`Bko*l9t0u)!qcQoe$a8j$!C zayH?J66o3M3YGU?863@n2e>;^J`m))UWHl&t5vWWo|IEvE)LRLeub8nn+7TnQ>`|1 zvH5O0I;fsSNo2}SPi}O-z94~TfS)8=QD;F z^2D)pMOC|~dGo`}o12pI7|{J}?o@9+Ue@yRT8xLeGh)0>`C@gnPAnob`ukoUELG0j zF%V{Pk{d^yDap>8`HTiS<&8NVG1t7~RJCJB&dDZg5g;Ew`*2Cib0_3%I+=x#SHCi- z$G@v&Bc;ru&)%NLij;p|KM{$H48c9(+YE`6d{?m>&+I1dVPP3&XDyU*9c0`DV5MP2eLa|cC6bq`s&N$WJ3 zGydG&os>Is)G--l-Lb(@Ea<8Vtdf&D1*@{NJy4=_-!c%fV>QGEi5>d{$Nu74_ftaP zfj}y-CU8ce-2bKjH~xM8o&F8}x&B(e1St&CEJ22cpm*fLC~s= z>MTSM^5RB9f+-fAYuyAv`}#^3sb>x?m}0luZxM+!22(6pH$*D9T`Ch?qAwgwv9R1| z(=LvDDUf7EmOm0qvFKbE>3#kDVho*Kll8v@?Q2rr3k3v(t{6It(G3yjkj=ny`N0&6 z%D+(r=2Ra{vH08&(c*LEp`(L741(Rrhqm!Sz=}{EijGbpYrQ;FYc>}ak8A&Zlew^@f*~@pxv+r!TSS<-u%O)#8DcIhJ~!G% zHW!w;&_y(JVX?dZ*Eg68i{A~Ak@xSPJ!f}@UYyU;|x9r#16SlAX?Y@s} zZ`iJ}E%)uV)mh)LUT&@MZSVywPg%~k*o8NQtEHbv9pc}iPWvTONOri{rv>~lNl$)e z76MZ%h8Zqo2TmeYhxc~Xvx5r9t!|+^E}S7To`6o} zCZNZ@FgVxA%4!(KEakHs#hHObDkp`LzWnJ~qNb<-ijpZKjl(84{W_X+$C;B9Q`@!f zFA{H6JkZ5$c!V<@gg@Y2!r;9w#@h>XgWe zRIR~*f>e4LM2Yg#{Nv?;P%6YZ_&TKbqNym(7BG|=Yi=>D5{8TO4Hd<3GhgR1S#E?0 z*B?X}%{~<PSYAI>VeYI|HdMpHup{`kR~`21R2dh;QouYzrt^)9k;ZiJMJ$DS z8(LZ()m_yO=JYJbqf*PHnRDqd7 zD@GX#ml8w^-T<&ozRld)pd{7@>-OrX`n3_-kIkJrR-cKw^!veXVx}sMkf}Lvlivk z7Y3`<3VkD1$!hMYRmct9_}Y+HjZ(Q)qgB7e{lTb^ccvsx!skyb?HVLV!!@d<-C%kb z%JSUQZON2qjEud`enzR4?NfqzP+DEq;A5Y+sdg`Ci%0C;ijIK6R3tAzU3Rs6U3O@Y zK{BFLSyp*GtXeRUq|VetV;Hzyr~jflr_Pcp*Q%YVa-$cU zpZFmxH>)xhpvss`73Va5&Kg-MH2phl&oZzU<*;H?>8ydAj-JDCOsw>a&qiNjY4&8w zlOyl~1e8-xXPoI1v)e^S`aiH-Z?-rm=Iv}5BkpV&3)BE1pcWVhj0YwF6M;#F*>bcM;zo@c8iM!}*=> zS%2sJ&hL9ed+zd4L~cl+RuG~q1wmdV2!SnkjA{{UcP;z(VaoZfYbsR4rBz7kwEHzi~^DXQV(P_kTF2U0vQKn zJP@2u1CR+o8i6E%Gy!P_(gLIv$V4D*Kqdh>1;}I|Q-Dkb(hj5p$TT1+AZZ{OAe}%? z1u`AT3?MUs%mOkS$Q&T20htSA9+3G!P6u)ZkOe@_1hNpwSwPMPvIxjxAm;#C0^}z^ zmI65!$TA?yft&|q1(5TBTma-kAQu6-7|2Q>tAMNqvIfW{KrRJx8IZL=)&aR3$Q3}= z1K9xNN+4GOxf;kdK&}OH9gvMcHUYUF$YvlEhzcYN%G&xo2xuWJo`L% zdM@>(JWZZD_X+nA_dfS;-8pyGz21GH`)v12_fU74TXG$Dz36(}wZo;m*10mSN>|YN z7w4PK=beu^yPVW{rZeo=@3_U0l1@l_qz%$VQjHW9UlpGcH;OH|M*b(YPAUAaqeBoy zaRSl9ug1luU}~+^tZm+yONJ#$ns%9CmSkCLxGL9Nsa#sLVK-i{H+_l85JgSAF4tPA ztg$jus|^N@sHxZGS}JklN)y69QY) zFr9U9j+Iom*d)o`31mWYENK}_lJN0y6(Al2cz5(~7L6SyIvzTRz3&_N_ z#-_H02JV8^+A$pl6R)85`t;sdRQRLNFZ3tcjT+YV+(a%GjyN3kQw=k2;?~zPv55F_ zIGmVbm>D}fG8dcdLH1d?La}Gp5&_VsJ%S)(_p75o1*mWfHd* zuRxo@kwjVhv^yUQJJ)#XCK_gJy0u|$tj(2ZHEfb-p5vx?18E}@+8RpdSp2<|Xt7Xb zZPx7GSXlf-8q;i=ac|ziHAST>#7*Kg^-X3Ct~Sn&qVwvgzb6N$a4_RW47atbH8{qNsId~1YHKMGH|T8s>_&v~MkI#W%*JUR zYM5=Oxymq?PIH)@=1RlN*lK4iQaHFn472s%#<;cM%DTZ;4m+VCSE9nOvBS)c-6TAA z${=e)WuUDzz*OttFV=&lvzt7Lq;3ZS3$#O^;N*uPBA!!ul!bm z;cI}yJ?+=bYN zxEpZ~Vmm@d^dRm<+=uuNL@(n0-D|t^cf7b66R=4_uMpZ3S|1u4{4m%PTpTP9><>U- zD(1HjV-h#Qf6#xYf3Dx_`^NW(Z;fw+{Ac+dd7&Kg?(<&nZT5WT`IYB9&k*;U?k@MK zZl|l?b+xO}`48v)&ZW)@#~&PBj_D3bIwtLqE|zM<_rz}TX0b!~R(J%56|`qBe{1+q zMO|<8U0S`OSj6;{-Hg36dMov=Z5dJ4w-4zQL*>HTvOBF_ZYAll%1$v^E<8JPW=geX zw|aw~5#!pYv~S=rr=s!{m*=gjCAGzoI#%@6>SbFpVz_*MmqVJS@+`NIC(GL08~T!A zpEIRe(_L8$zr3?bUv*nX92A++=O#P^4d%nXAAxhF%nUnj;@LU4mnk3ooBBKdr8?ZGHHqj99Ca581W}{Ue5_ zRh~2#{ih%LG$S@e-Y)k~Q+e9VDln4>zmXA>VOLFBLTQz!*1WTZ0~xVKYg*nn5UY5e&8v1`vt+sQU^JRid8&;yY(I{y>T5wd z&$u~VIhWB-`Ecpdw8~TLVvR<&<}0<&SND1JF60cr?d7RQaQ zfo|-^Xg3_=e=;NbwfBzYgW=(L=Xw5*44kWe&K>N`kHde%kuoP%b1lHDkDP(( z6*rh`g2HfkD=+CwmP?Bwk+hmJwvk1+o=f^#adKQ76#LNnzsxt6;ncVg;C&duu%cWz z%)%g71H5|GWjKV;&!QmL1I6AfQLqeW%SFKtM&g>_{~C$QfRjh!O5jIF;<|v>?L+Ml z*IK&sQHLT)efUV10qq0&OQ633`kz1_0Qzg7JAgh2bSKb(6@m85$M}M{~PE#K;H%WC!p^EeIMvSpof4S2Kr~9 ze*yYGKtBNbA<(}9{TtAafPM`0??C?n^b??;0{sl==Rl7DJqq*~(BnYA0Qx1+uYi6H z^c$ex0{sr?{{sCU=s$s;*h0lFDk3C=1K~ut5N?DA;YG*@AHt7N5CKFG5kiz9!iWf> z91%qfKnz3-LR26IBVvdlh)P5iVyO5e9VUvevvgp(@F4pXwj-p67KLVoT)}68n}U-9 zUj!ZrToM?qe4yNipCx81ZvR*QUH<$0OZ;>EZr@kFhkPr2HNJ9rzx+G-8hMpmD@VPr zc(-`dUcs}+^ML09&q7bgBe)-PU*R6>9_o4QBstJ)Gw`a)yNMj zKI-gpam`pTdNuk@z6Fh3E*6X@zsV)hSm-vTa%srP=9|o=A@8~VLX0VqP52~pk%)E{ z-7HUXN+p)-FKdTc6Df#IxJ=|te&ki|RJmYeqhZvWdJ#r?@kVjAxH_$J^;npQsJ&5j zpjO-PeLh;YHl=bY$vgfITScnz%bvgnNqpdERL-A%)vA*(!_@5va>j}cGHP` zG8~VkRIWS=earfZQOK`${!&s@oA^wrwZ;~*-JQQH-^|iGn*dFzT!Hez_rAoUa!Tb& zlv@w3&JbSOVsTz?ix+Wc+#WKC$MgtIFtP<~dp^0W?&o!OLU zO6AhCNGdF$%-NxG@yV?CxoYwwd6uqm>B*_P1|4WFbF`~mf}*wX7$-3yK<{uVidOcv zsMqG$I!cO^uY)VmB1eC!B_oc|$G2ugx4sSkFV{%Vc`zw0Dbzxv4wVa2HVK^B8+YZJ zu;a3#A4&-|7pQ#jdD{wumlP@=oJ&*AIk>2|Qd{edxHs;V6-%R7))iF(k#W4G^W{R{?2t5|MJTwLcs~%h!EDOAhg0)@wR{6Wqt1MGu zir@d7|55*C{$>8bexL6d-zHy+d`$i|3fD#QOxfW*?%mCz$FG-qus2qI+i;?2+6BYoB$>3Imr+9G{AYessu zwS{1WN5vB9(fZH=_B5X1aHO+lv}g1}l&g{Y@CTtRkNEt8W$mHfh@`qQB_n#Zo&g5} z)dzjDlC?+wtS$N%0uip=9_+I}7VNY3V4t-G`@$8$Mq_(9^7?~h~mj<+%(mE60lyUb$HUk@$!^veHD1J6~SbyQ#GFk zc4x<2jbB#uxuZL=@wcbP#qa=tN?av5m*!jB7vw!j7E$>j75w?j7Jbe17ZTA5s^eRA({~_h*rcz iL>pof;uOSW#1zC-L_4AbF%6MIq!AhM!S+sZ-oF5^C^>5Y diff --git a/data/wikibase-test-data.db b/data/wikibase-test-data.db index c2c2892c5d8e0339207d94ceb4d17024178c7316..d5fb96543a82fd6db5570e27fefd4c0e36aa7c0f 100644 GIT binary patch delta 236 zcmZoTz}0YoYl5_(BLf4277(ifF(VLJOw=)EbljLQMV=?sAUQF`%-F=#$TWGA!WVuN zfhL1x28;oV0u+`lU|g_Vuz*ouQGmdv21Wx(&SnMe+ z%ZZJR>sVMgo0>Y<$Pa%-PH|(NUbMIoXq4+|!e>&2hRvGo$SE3s)K0rcGy(W0age z@fxG}blp3QqSL*nGez*GWfm2eC^-51`?xy#O+PT5$!hw8=}a8cV^|m^rhmD~C^&ik b8?ouSw-_C!AK+!=o-Fs+cKfawOm6G|A_Pgl delta 235 zcmZoTz}0YoYl5_(2?GOz77$AUF(VLZPSi1GG})LiMV`mp#M02*$k5z4&C+C(!WVuN zfhL1x28;pQ1ss?@@K2l|uqZ%a*#gD|iyRa-H82{8b2c+@uw7>0-^;7dbBoKF^8?2& z&ZTUZH#V}cv2f027Mj@T$-+5_Y0tz)M}E#`CLW;WAOK`CPIMIGYEE%s7x(mJY;&IO z&&()0{lZm7w&@#L8RZyNr%$}bC_Y{H4x{LFpXp2y(=SYC;+!7C!YDU=_jD$M>0fU$ VR!l#@%g8-h?y>FmT{D>6*Z~3%LV^GQ From 6d2c4046f969d1a09f3cec21c567a90afc416ba7 Mon Sep 17 00:00:00 2001 From: Ricki Jay Date: Mon, 18 Nov 2024 13:53:55 +0100 Subject: [PATCH 05/29] Remove Table --- alembic/versions/0de663bb9c13_only_month.py | 64 +++++++++++++++ .../e0cad634525c_populate_month_fields.py | 63 ++++++++++++++- data/wikibase-data.db | Bin 21401600 -> 21401600 bytes data/wikibase-test-data.db | Bin 172032 -> 172032 bytes model/database/__init__.py | 1 - model/database/wikibase_model.py | 10 ++- .../database/wikibase_observation/__init__.py | 1 - .../wikibase_observation/log/__init__.py | 3 - .../wikibase_log_month_observation_model.py | 33 ++------ .../log/wikibase_log_observation_model.py | 75 ------------------ 10 files changed, 136 insertions(+), 114 deletions(-) create mode 100644 alembic/versions/0de663bb9c13_only_month.py delete mode 100644 model/database/wikibase_observation/log/wikibase_log_observation_model.py diff --git a/alembic/versions/0de663bb9c13_only_month.py b/alembic/versions/0de663bb9c13_only_month.py new file mode 100644 index 0000000..360b27d --- /dev/null +++ b/alembic/versions/0de663bb9c13_only_month.py @@ -0,0 +1,64 @@ +"""Only Month + +Revision ID: 0de663bb9c13 +Revises: e0cad634525c +Create Date: 2024-11-18 13:50:09.959941 + +""" + +from typing import Sequence, Union + +from alembic import op +import sqlalchemy as sa + + +# revision identifiers, used by Alembic. +revision: str = "0de663bb9c13" +down_revision: Union[str, None] = "e0cad634525c" +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + # ### commands auto generated by Alembic - please adjust! ### + op.drop_table("wikibase_log_observation") + with op.batch_alter_table("wikibase_log_observation_month") as batch_op: + batch_op.alter_column("first", existing_type=sa.BOOLEAN(), nullable=False) + batch_op.alter_column("wikibase_id", existing_type=sa.INTEGER(), nullable=False) + batch_op.alter_column("anything", existing_type=sa.BOOLEAN(), nullable=False) + batch_op.alter_column("date", existing_type=sa.DATETIME(), nullable=False) + # ### end Alembic commands ### + + +def downgrade() -> None: + # ### commands auto generated by Alembic - please adjust! ### + with op.batch_alter_table("wikibase_log_observation_month") as batch_op: + batch_op.alter_column("date", existing_type=sa.DATETIME(), nullable=True) + batch_op.alter_column("anything", existing_type=sa.BOOLEAN(), nullable=True) + batch_op.alter_column("wikibase_id", existing_type=sa.INTEGER(), nullable=True) + batch_op.alter_column("first", existing_type=sa.BOOLEAN(), nullable=True) + op.create_table( + "wikibase_log_observation", + sa.Column("id", sa.INTEGER(), nullable=False), + sa.Column("wikibase_id", sa.INTEGER(), nullable=False), + sa.Column("anything", sa.BOOLEAN(), nullable=False), + sa.Column("date", sa.DATETIME(), nullable=False), + sa.Column("first_log_date", sa.DATETIME(), nullable=True), + sa.Column("last_log_date", sa.DATETIME(), nullable=True), + sa.Column("last_log_user_type", sa.VARCHAR(length=7), nullable=True), + sa.Column("first_month_id", sa.INTEGER(), nullable=True), + sa.Column("last_month_id", sa.INTEGER(), nullable=True), + sa.ForeignKeyConstraint( + ["first_month_id"], + ["wikibase_log_observation_month.id"], + name="first_month", + ), + sa.ForeignKeyConstraint( + ["last_month_id"], ["wikibase_log_observation_month.id"], name="last_month" + ), + sa.ForeignKeyConstraint( + ["wikibase_id"], ["wikibase.id"], name="observation_wikibase" + ), + sa.PrimaryKeyConstraint("id"), + ) + # ### end Alembic commands ### diff --git a/alembic/versions/e0cad634525c_populate_month_fields.py b/alembic/versions/e0cad634525c_populate_month_fields.py index 2877b51..82581ac 100644 --- a/alembic/versions/e0cad634525c_populate_month_fields.py +++ b/alembic/versions/e0cad634525c_populate_month_fields.py @@ -6,7 +6,8 @@ """ -from typing import Sequence, Union +from datetime import datetime +from typing import Optional, Sequence, Union from alembic import op import sqlalchemy as sa @@ -14,9 +15,65 @@ from model.database.wikibase_observation.log.wikibase_log_month_observation_model import ( WikibaseLogMonthObservationModel, ) -from model.database.wikibase_observation.log.wikibase_log_observation_model import ( - WikibaseLogObservationModel, +from model.database.wikibase_observation.wikibase_observation_model import ( + WikibaseObservationModel, ) +from model.enum import WikibaseUserType + + +class WikibaseLogObservationModel(WikibaseObservationModel): + """Wikibase Log Observation Table""" + + __tablename__ = "wikibase_log_observation" + + first_log_date: sa.orm.Mapped[Optional[datetime]] = sa.orm.mapped_column( + "first_log_date", sa.DateTime(timezone=True), nullable=True + ) + """Oldest Log Date""" + + last_log_date: sa.orm.Mapped[Optional[datetime]] = sa.orm.mapped_column( + "last_log_date", sa.DateTime(timezone=True), nullable=True + ) + """Most Recent Log Date""" + + last_log_user_type: sa.orm.Mapped[Optional[WikibaseUserType]] = ( + sa.orm.mapped_column( + "last_log_user_type", sa.Enum(WikibaseUserType), nullable=True + ) + ) + """Most Recent Log User Type - User or Bot?""" + + first_month_id: sa.orm.Mapped[Optional[int]] = sa.orm.mapped_column( + "first_month_id", + sa.ForeignKey(column="wikibase_log_observation_month.id", name="first_month"), + nullable=True, + ) + """First Month ID""" + + first_month: sa.orm.Mapped[Optional[WikibaseLogMonthObservationModel]] = ( + sa.orm.relationship( + "WikibaseLogMonthObservationModel", + lazy="selectin", + primaryjoin=first_month_id == WikibaseLogMonthObservationModel.id, + ) + ) + """First Month Log Record""" + + last_month_id: sa.orm.Mapped[Optional[int]] = sa.orm.mapped_column( + "last_month_id", + sa.ForeignKey(column="wikibase_log_observation_month.id", name="last_month"), + nullable=True, + ) + """Last Month ID""" + + last_month: sa.orm.Mapped[Optional[WikibaseLogMonthObservationModel]] = ( + sa.orm.relationship( + "WikibaseLogMonthObservationModel", + lazy="selectin", + primaryjoin=last_month_id == WikibaseLogMonthObservationModel.id, + ) + ) + """Last Month Log Record""" # revision identifiers, used by Alembic. diff --git a/data/wikibase-data.db b/data/wikibase-data.db index a5991ce7933fc44b76f80fca5a96b9570e0f0b19..a9f2bd0e55e50714665d686f7d6c928154a6a062 100644 GIT binary patch delta 3273 zcmdtk?L!pR9tZF_^SC>^vr9aw__PQpB8G(dlpxQgrDB?bnDL+`Mv9tx%ZLT_=+zsJ zX`Iw@FoP|7teonR?R7v^31@W@&20?UqmXJHERw`Umcd&v$?4oEdgz_RKl> z8`tW7FgxFoCy2l0>qJ)svQLT^ZdK9CH3;Z8__evk@x zK^pXjyI}wfgh4PEhQK{A6w)CBhCwD|K{n(-F62Qz+za=?a2Nsi!vio9M!{$(fHCkO zjD>OV5R8WjPzXgZ5sG0FJPecJ5h#IDD1&mC0#o5pcnqdN1x$y>p%R{e888#7U>3}V zIq)RRg?TU^{svFM)35-ZfogabYQPV*PzMWP5iAA@OJFJd9hSjz_y;@(^+EaG71!U@ z3{1y7VJp*o=I~6dUE8j$&?>Y9^_Kd9+MrgcBUDZKNU2eZlqmT#*_21f4r!mXSjv%N z#FOF+;)9}x93qQJrf@?z!OVg!vBB86W(*OtGFeJ*vlCmGVE!r&8_Lp?(5M{os?L6j zH}lx4SaV-+eoxa)SaiI(g%$QNH;eIoSwW2%#n$#R*R%XMlQSvK%n{?`S@mqQQ*dme zxm63wVdizO^-IB>BqkWv)^sK1|0f;cx_6u)kd0Uaa)WNOK~=@bp)9hYp{$!z!eZb6 z9fR|+Jg9(QFC1;BykvDl12A|I*1%d=2QR_PupTUEgbnaYaCO7R$~6J3KVi>j$~8IU zJHdO>`>c0@=elRTC&$yp*ki0SvWysat9y-mn7gBEr>nu0p`X?l>JRJ0`HJ&C$0f%q z#~|&DwovP>9#J=|OVz1rw%T2}p`22J%4%hnGD_*AIOOy49@&zgk&EO3a)fk6IxKCH zmP%8lY^l3=9TVvtQC}rii6g}%Q6r30ZaT~Si^O@M9cz6gPmw0nrLximW`t$z@sVrR zGp#=IR!H6JBay`N9`=#CP|SxuGK5$w(I~dYedHsrSr<{);?%C}>dR&XTe7<)i8V$X z?_`bN=OeFMPaLriy@k5XI)J*a!hRQ0g~fum$(!%F=vnC*Xq+yqnv z*EE+$Z_&r;eVxagbDW(VI~*n2Ev-q*Q!lFZYJb&GwkhM~>+zXj}T&wIEw2CojqXRj8s_XVx>>xMt7h$&q~>C-&^}Z zABmAd8^-uZmSnXQ`baNp-9z@(e}Vd(R9HKc#g+#mEcbYv6nd}7N5Y9U291f0RD zzq?;_r@Ov!EpjF3hx8}(aOZaCM8_YF^^Wjr6eh|bCQpAAXW)3tSI!^!#)y4qzSdf?9=JEsiWx44PBUg z`;Z9SaIy96I3IbBwNDF_vKLAM#lfPAKm=h|P_miR18wa7QgjwP9(aOPG9$2)m0>h^aYi7LuvV0;X-Z%b z>sB5pZfIi#QvVy&|K>aX7CE!Bzd!|Vv$w{Z;JN8} z)l=>1VO%pBjV$-~?vQ&59@=@=E?2p0sD4(bda{!{=Q&3^+8sw6a~uz7SG85zU@csI zS6!(NR8A<>m`&I4=HyF1NY6>h;xTcq*qQ7iQ%JgSPM|BZqFL{$fivu>^1#HP_NaZ) z$L+GqV}TgL`c4THv9BAI6 z@-HbrN%<+tzoPtW%1={%hVpMHZ=?Jy<=;|%j`Hs)KTrAhl>dwJA1J>-`Hz(UMEOO^ zFH!z8<(DbHLitt7f1&(W%73H0o$`NE{yXLWq5KcZ|D^mH<<}{{LHSL}Z!P1bo)Zv3 z0vQxgK?4UkK?fJO!2l0y(Aa2H8$*mp_s{MF?mBmd>!#~V*9)$huHmjY zy-j~#pRad!?r~0c8jkIbLaa+CHNQ4d6Vy{`otmJ0qBJWbm0t2;IUtXeewEfp!^HFA zaxs;hAl0OYupeK|j^|Z(4IW%?4kSUc+MGs$W1cdf6N5$b%z7fmSK9k?H2bQ`)LE#? zoE-dSf!QDhBj=bNk&I=#pT*{#Hp{%5U7Bt7WYcD0qn}%2Ca_aAXzrhF_F`|$GMBRr z_UNG5W`&rM&q@;UkHK~v70hlwFiQN(-Mn98!!JgCW-O`C>;Z8Q4++o{dO;%DmD!u9 zJaUuZ{lG|ef1zhMe{{UevEv>) vhT4&CM}{54?8vku%Z_XkQaa|4YwY%1YN&clULm*U5JWBs$*p}EGrkOW8ofrOe#25Eu?1yR9X$l!DAy3|V%27K#c zU%(R3CtX0X45B`tihw@fdtw26VtbbVZ+4TJO_uWC@B7_vv-g~H&)j?G+*59v)~~iV z0im4K?u4YzCd76%A+CD~F|S#38fg5&_<+%G_{gx((1UxMTfpVe-E=lhVWI0D?8(;OPlFtH zpt&F=vX}Gn-Ry+g0##KRgyt0aih_l{(n4RE*YB$+F01hQ+2$>gyn*;NSm-bH7U9>T zilU}#$s&r`xXr4FKuqHzudl44sI($bnjoTp_v#`tlSKsCv7Ezs ztP2h06nOd0I&Ft!+5nrLe>hjZd2u!En4(TcXFgwJV0YR=9ohH!hqKvoTPVkw&*$lV zyKJE>cG+W*Y4xgmF4qaBiZbB zPbk;bslH=NX9q&rX*(Y1?%&W^Xbfc+bmX&i&OXWN`4}A$&MS}YtfDfnue>5yR1pj$ ztWy@BsY~eHUIIHdI_wq5Kj`(B zRAAEw6ZduopRSAOl`Nt}uDaSwbLCYwtYCL_j??)LI!BLWj$--b+!~tKGrq$EUSIr$ zDo@b61D~deU@m7UNB*fxHO-dSVVjucylcHQ&7H=lYBbc#JkC(2yr&dY>YEy0%QAIB zihK!zQ~4B~sdKV!KDIgQaBgq)?JF)UD)ANtFrgK`V1kGg-lL0P>ztu1d1jWX+$SBd z%VIBphj(iv>j81p~e%T!9n5xc@n|Epblohx_=@&FHF5aoLv^O!O75*}&n;0kW z&}oxRj6ZN;6XW3Rx`<>GQK9+GG>sD^D;nu{~zxEjmM5 z(mMJ|*vWj@ub9cP@Mevc!Q^z{d_gw-u}E%$)p446lTLKhjU@|A4rj-$jEOgDeKgt1 z_=*y&OiVi?Z_pW>?X)XVZTzucBe^jHQedd(T4-TF?8&sntm(H%UfiB=^;DBE+T(9$%vGkPDq!4Sfsgjl(?Du&3Sh5BU;_e^7cJ;zI5BjN%ZR;NKaQ0!c znB9kF$*!zE)Wf#+h|FhC^^Ro8$MCm{ecCVLihb?g^NQIw%wI5V?!0RkTs3X>EbqwL ziPgia$M^HvZF6QMbjVd5`_SvzJwfW}n!w_%sT*HCa(JzGRCQyogx}fmIvfdW3nP6B zbTMs+^`AA`9J9T*HH!wDcZE0~z1KKg1qT=aBVYo|fCaDuHoy)z04LxA+<*s20aAf9 zpaYN&WB{2!7SIvs1Y`p_KrWC6JugMf>H!9XQY1q=a(0>gl6pavKYi~vRgmjI)H z(ZCp>7N`Tp0^@-3zyx3-FbSv!c%T7j1O#9*a49eamd0OkV=fE$6EfSZ9^fLnpvfZKsPfIES^fQ3L4AOeek zyMey|_W%-bFK{1lKkxwXAn*|IFz`sTarmRZZDqStwih?2jD7E*XT_<{?9KL+(2pLA z%j6@Kbm?+SXygQg!SJDBli>kFjWx~kuH`Aq-P{pwHP>KlG782_%MkM)=B?(NxJ>#1 zU23uz_nC`KCr!_ot};D^_jo_qV7l8hpCo=vJ{H=vqp#|pzRrdkLi^Z0Q|M`S#u6IJ z4e2ePp6jKZSfwTO63Z}!HgSUovO)GxC-wsus$-8vuP)=ND%k)Ve)$&-{kPd_47n(~ z&K!D)>(-4OIuOlhuCs<-vZR|mM<^j_cF+GjKYG6Kob-I-^s!*REZ zeL!Fq?Ic-(M@q1@2HQ|syF{?fg}eTTc7 zYrkurE6w?ebBfdG_>Al!Yb_63uC#c}Tg;7)t&VAqa{E#HHv2TY$@YwGob@NGcdd1( z@63z}Vr-#pDj&K#)antdseEv|(wx{dhA-NfWEzGR$iq@IXpl>3zX z4fj0v2-kO*wYjbu=hx1?&e=}8BkH)%QRpz+pSRy@?}wRs+E!;hV_jt(Z0$&=(K2$x za@w-OQbz7Jd~9ekG;)7(GFQ#z(>>--4ZE=>CEO|QIJbtI#JTC8bQ^|_KT-MqsKDkQ zh<<7}%_jEv95?@OZ}ei0Yq(3U!c*qTb(dkDclHg@k z=DIhco6S4cPxsPNs_wTxcc?q*)c>QMxykOqE~IO)D=p2=&AShmb~h|1W+Ptp@y1__ z>v8&gXISpp=&40KE4yo4-@2B$E^>bE+>JF$b!>A?cUbMu+9%k4wQaDCu%5K8uvS=( zTkf+AFn?%%**wKCko$srfb-Lj=tA0qyiFEhj~Nd=n8KO9QbX80LubC9<`~fSVM1Aj zU-n{;=ASQ8=*RcfCA4?*At9LL=J)0M=p60c{1`{f%~!o~1hQv!^ZW3sq=m0Uo{pFxFCG)be8L`lcby|QnIp)ibvm3IixFb<-T9F^eVeRAk(z&i)2U`9 zx?*2z~*GK$6t?T$CDB@e2w1QCSt%~{)*y~3r7sq ze6=p3O~eov5x8)~FpM9ji)a%u#6=Y85rb=}Ftmj>2XS{TMes*^*U;=rZ9gEI4is1T zTN5W^MoyS8vUd0oeuzFsdg3y~$MCU3c?cYyRudmpe3eeyCMt>1220yVC6#=o&d??* zi8BNeqLRV-NPm&L&sYn*JY5fOfeLu!P7 zWN9_ElprzaSGQImF)L8a2Q?OIP8!TtD*F$sgZ+%byg@#o6PuEW+2EB3#mY@s<^W%$ zleP)@@W4o%lw3H2Q^fmq5r%f7sks*>e~)PNuj+9AdS44I)|X47Eh?A4dkuDtoiK&6 zy}n5U?8K%fPC=YZTPE+?t!2lKVPT|XrD`i zfZ|v#ZDGtFo;{aVsGiLIh)kUQ9UnOEaCEi5X}{jy(YC{Oh0SGs(K^}6!G#}d{>i)! zXLX(XNB3IyFxS_vC9ZPkG3UMJVJ1Aasl@oHagni?;T^*QLpHaQyMlAlEtDt!gJ2xX z%kHNGox+W+s&X8A>D{Qm?|BYExORUL7U&G}hwzX5>U)S|6Dhes1rlU|#FQKn$)d*j ztsIn-{qVW6nB`!!B?%QvGA$%c4L7tB$*19*rOV^cc1}oC-Fa8T^=(K3@y=MT^!7U% zu50b2R2~B-FH7Ezwq=wU(XF>N%u67mvs9w?`LLUBX}GqPNd5;%0&4Ovemp0y^o=() z%+;tH;V{RjgN0{b)%gn==4jL(#i(N@?Cc5A>u+edMx*v$;iVnr2-@s;`ZAWhn|EEq z)f$Ntp89!}oqKJ=Y>oP%R_b#pd(E7Nt2Al{e6m3A^C&y}>V_*@sj&ciK<|qy8k_nn zuWGm=q5GEET6gR@F2ADT^44zo9dve-haywk@~ANhGiUKaD+!@;2;ovDn~8LI+|m`p z(gk7WDgucq=?1=0=V|jM!y@6(xZs=2^9{OyHg9rVK>SUP*IUlM=JhB0ovz=$UB2;v50y-x|s5RbJ@#sj2iLe+9DX+WD!b zeJAh}bqVrj#FXjrRQUOPI5piU^xyo7B07`|3#Zxas0Rt0g(1;c^S{1{zCn{Xg5 z0-pATY67G9(YgrvPt08#BN%h&FX2b&GMLcdCxnAB(vkcnI%%6&A*KHggMyuc9az+sr-Bvzce`muV8@Z(&%n63t8O z1e(sz&`7B4zee3r$3}${VzXr9KaHQR^JvBaioxf?Q5cMu%+6;Vrt;Ht37T=xBm~MX zY#gTWQ*{xE^BGqHA`*;4>`^b}r|2B)&|KO<@D=&|WwA0+(CbqxGzg+wHZB%FX7=G` zzo3CpRA7?Ori~z%WobXhB$Pl zN=+N}qzm;*l~24}88MEXT1f6Ap0H!*Rm#u)}9RY`@7q z$@a5trR^f?XI9bL&2qpp2Z^~C%o9vMnpUCMeiSZIcf$e09D|43!c9W@ZwBzEIpm6-j&9DpX$nr>95jjlW&llBNwKbrMBG$ITe5mD>2%=V-h9c8e zL^9d?ki$;=5V5mWD^d&J1-Gk*0Y^5Vi6&NS^~~p}Lsos?Cst=1w+1uRu0)?oc9GqJsFjvMW?lUX$g_f|)nw84{N89L^Qsw~d;f^E ztnO3fX+hMgt{{5obv%fA?Ie|KBu`EeTdS<<)Ya@mU*^7O=T!0pi3p-rP(|PRebIFG z$-XGu3QR5%OlQ4BLjFp^`J!4}Mch2PCfvy-h*}jDQVRA%%BB0GdHrHh+1c;TkPYN< zLDUMY9Jk^C^ioUZ@f|6X^@6CCQz3QHf#`^M*IKep5L>IK>TWl_5%sH)#kn$KWDR*N zUqtOiRd`-pVPv!8cYnZt&l^!+zw5sq5p=1@c>bElDs%3+TU3qOUW|fjs%7u4n}=(hp0=kLqryn zCBp3q)brnhcvbefqRB@|NVqM5;mlj=EMOYX<@Aw<$s@w82^?eJ#x{Hdjhz1u_B?z3 z<%m1}tPhfhgj*6Q{qN|q&+4j@`^f{s%?TVYymNu@`Cf9La8m+95b<{B^EfW#9wJ57 zM|Mtg870z(EVmj9EgxC#wB#BNo41>1nJtD}O~*`n?M4(ZQlzGhQqz-C(^FE@(^AtjQq!|i({obO^Dj3=jgnP_k05Im4o@B8jZj@yoEF% z%$XKkV1Th>pwf9Z7R{sif~dt3;5ejm;6*xD{R1vpt{`fm131RlhT}d0-o3NFAG8zA z7DO$Q0Lyc=h}UD+qnIt@aC*!%(k$9h5VarzR1eqbg3lVs(F~d?h+547EX&41`|}tM z+JUAEqFQ5!-JJwS=D2W+Vd|P}m}n|Z6GW{F0rKVJ;?r|3!$&h1UOEO3n!S{ z_@lb0TM)I{1Bi?h!hZJm@!_0Q>Yz?R)QSq|lO}}YhX=J$yCAmK5Y)rt_=Ipe%bkd; zh1%gmEz~NATEPHBr>(RDTRSnFf4*aeny5Kntcq`>lkttD6$T*0F)7?~DpLb4yNX&N z0DZgEzU7m`xv7*=P7t*c0Q!d3t80kaI_H7>CnbWY)dj$^wmzK2zQ{S8Ex*#ehPF6) zzmq?vh*~iKR4?)v^c88Nojt;bGg6Vu{7n#B6aDJk%xl1u&d595BH$PDs~~D=e)RWh zgn(Zgpf&l4{49vAY5mx|HirH3RT(UhMfswdy2lHQpE|jwh^<+9)#KSPIebB9 z@gzCbD7I$jRadUzQarkC(Z;x^_bvIZQ7mfN@@ltxN=tXK>VJPtzR4H;F$H4Q%@vP8 zVkSnto{3o8Cy2fle)ZuWor0NDZ<3~xugHl;aeyv+Y3RjtCQJ?IvFxeg>{Rk^@}EYr zzuwQ*PsQTKZkv8}Pm6wEl79(eVS@Y@r^S~xelr&V*+28ee(1*x-CiB;WX%`*#{XJ} zzhLnCB=G(mf=_^94DU(V(*0haf{*F?^}%MNhdl=jtOF~1fp3>AS50< z9U-3yV$X!W9ZMh}xD*1OTpDq)-Vj({1qJ$^GTcCYhX(U%~0OHfTj2+vL|MHvRFVR8OJK5i7dwh*Wb|LC%aU;bmK z>h3IjaAT@PUH-q5k0d23DVro^v!ra1lourBMM-%{QeKvnW=Yv9DcdCF6-jwjQvN0> zuSv>wN!cMOJ0)e8r0kZIJ(99lQeKyoeUh?YQVvMU8FJ4rbyDW@dmw4{75DgTv}A0*|Br2Hr; zKS|2ZlJbkB{3LsI^HK%tK)6yN{@U<6En8L$9Wzy{a>2jB!;fE(}t zDL^Wa26O<@feauM$O1Y7oq%j02gn8TfPDJA(wR0jxUAQb-gE;oRT@9HcJTZasf@2I z+uRSh2e{t1On2Sr%5}bGF*&E=UeQzLXW+YhZ$D@rXP;-!ux&H_XnWLFZ2i!*#u_!= z8>{PFXUO1QcdqA_Q|ryHa1ZRPi9LdqxtjFu)s>EhA(9sr-d8Y zhQ*Oa=2;pMSkp2%bKX&hs#u{A?#ZG;_>1OmCx_E0Yn&4Pta)barz5=s4`ygpAGAj^U1O4r>3*{;GOCWVXHepH<;%Q_HO) z+_>42hg(IPNs!!Ta&C6=P9N%Fw>mo8f3~l)-)zsdouTz~faeJtvkkHy_grFq#d?|b zBFiyPSN8$SR?8IgpYFNlkIZ-B&g}2F&GL=u1JmuMiLS+_LgNR<1;%XGH7<|gRl^K} z8R?zdxM#U>^d}70@@ZJq-$~fv8XUBZBf`tr*a_i9>LsJ0syjQxz%TN%|{;Y z#Oj8JzivJ`0@K%g_a)&9$~uk?KS`a&W$S9g$MDBJW5bj-A0MaA?0XZ!*#<^0 z2`^%=jlw$Z932+f&{}-Es}8?Ob2Ug^4~N2(0tBjOt?}zjGgKcN$;XwlbtnV!R6jzRTHZxjvQ01{t&wc z>m5>8H@dp2_R(i>;TBC_wdtoeg=*7ZZ3d`~Pi_2aQ=~QlwF#|h0Y9=WL0{TW5Va~32sV#|B0VpGA_DTEaW{tcrhOX4)=H9U)0bQV zPb8LD_0XQQSG}m#h~mf4>8fyMJeR8Go^iC4dVOkU9sfg-P>rY;)f!QZgOHYVY-@ZG zRR6akE<*iplXHm>M6E~!jmC_^@f|x|uSq?e8h3cR(;kAT6@@_b)~K+*GepMy=;WL_ V?Mk~nvU4bXWalvYJWed-{{VsOVRZli diff --git a/data/wikibase-test-data.db b/data/wikibase-test-data.db index d5fb96543a82fd6db5570e27fefd4c0e36aa7c0f..f80195b4e51fd6827b928c5d030ab1af565b6ed1 100644 GIT binary patch delta 247 zcmZoTz}0YoYl5_(3j+g#77(ifF%ts=qs>GeV@8*a2~*^G3{q0f%#4$gERzk5Hz|DK zM-?z&yuiZj%D}#rmC=Ejg+q$@$i~J4%-cEH8NXL-7jR(uz&~+29wG}M=`Dzb}x2uPfx~n&guTljIz@&TxDdNzLAwtj!|m*#A}S=({=AKica^L i&ZIwm0ym=zYg%Seamn=e%s@do7RIpc+%uTIvI79fphYX#%Y<)ltmxXCCn`R+r`>6$kg`5C3APrSw` zK3(??BmZ>!D~zg>^IrN*UwacMbY?n}y?$C|QE^FpZhl@#Mto+9f~Q}ItGjEEjsh2F zP9heG>4tY1m6ee+DY*Fuxq7<$DR{d^Y9NVEf5^`$F@64BMy=@wcp14T%RRQ99Qu@B z5n(EdPJ}RAqcj_rCWjQG@y14d#_gQ!jNdC31t@G=z{tK}yF>xwPyR�-G8b4J0|6 z893OkF!1l?)#tgz<;?kkV;ARAwks?rHa4zfVc~3M{Z{rZeeJZhR@C1PX|h#FA767sn9S5KmuMkO0`53Qqq1 PKCX^_+c!>U^5*~mOl_A) diff --git a/model/database/__init__.py b/model/database/__init__.py index 1d6c57d..579498a 100644 --- a/model/database/__init__.py +++ b/model/database/__init__.py @@ -10,7 +10,6 @@ WikibaseLogMonthLogTypeObservationModel, WikibaseLogMonthObservationModel, WikibaseLogMonthUserTypeObservationModel, - WikibaseLogObservationModel, WikibasePropertyPopularityCountModel, WikibasePropertyPopularityObservationModel, WikibaseQuantityObservationModel, diff --git a/model/database/wikibase_model.py b/model/database/wikibase_model.py index 9478c5d..a9ff5d9 100644 --- a/model/database/wikibase_model.py +++ b/model/database/wikibase_model.py @@ -8,7 +8,7 @@ from model.database.wikibase_category_model import WikibaseCategoryModel from model.database.wikibase_observation import ( WikibaseConnectivityObservationModel, - WikibaseLogObservationModel, + WikibaseLogMonthObservationModel, WikibasePropertyPopularityObservationModel, WikibaseQuantityObservationModel, WikibaseSoftwareVersionObservationModel, @@ -221,10 +221,12 @@ class WikibaseModel(ModelBase): ) """Connectivity Observations""" - log_observations: Mapped[list[WikibaseLogObservationModel]] = relationship( - "WikibaseLogObservationModel", back_populates="wikibase", lazy="select" + log_month_observations: Mapped[list[WikibaseLogMonthObservationModel]] = ( + relationship( + "WikibaseLogMonthObservationModel", back_populates="wikibase", lazy="select" + ) ) - """Log Observations""" + """Log Month Observations""" property_popularity_observations: Mapped[ list[WikibasePropertyPopularityObservationModel] diff --git a/model/database/wikibase_observation/__init__.py b/model/database/wikibase_observation/__init__.py index d16a84d..ba4d10e 100644 --- a/model/database/wikibase_observation/__init__.py +++ b/model/database/wikibase_observation/__init__.py @@ -9,7 +9,6 @@ WikibaseLogMonthLogTypeObservationModel, WikibaseLogMonthObservationModel, WikibaseLogMonthUserTypeObservationModel, - WikibaseLogObservationModel, ) from model.database.wikibase_observation.property import ( WikibasePropertyPopularityObservationModel, diff --git a/model/database/wikibase_observation/log/__init__.py b/model/database/wikibase_observation/log/__init__.py index 3fb5a5b..1264ab8 100644 --- a/model/database/wikibase_observation/log/__init__.py +++ b/model/database/wikibase_observation/log/__init__.py @@ -9,6 +9,3 @@ from model.database.wikibase_observation.log.wikibase_log_month_user_type_observation_model import ( WikibaseLogMonthUserTypeObservationModel, ) -from model.database.wikibase_observation.log.wikibase_log_observation_model import ( - WikibaseLogObservationModel, -) diff --git a/model/database/wikibase_observation/log/wikibase_log_month_observation_model.py b/model/database/wikibase_observation/log/wikibase_log_month_observation_model.py index 145cfa7..bb85b96 100644 --- a/model/database/wikibase_observation/log/wikibase_log_month_observation_model.py +++ b/model/database/wikibase_observation/log/wikibase_log_month_observation_model.py @@ -2,7 +2,7 @@ from datetime import datetime from typing import Optional -from sqlalchemy import Boolean, DateTime, Enum, ForeignKey, Integer, func +from sqlalchemy import Boolean, DateTime, Enum, Integer from sqlalchemy.orm import Mapped, mapped_column, relationship from model.database.base import ModelBase @@ -12,39 +12,18 @@ from model.database.wikibase_observation.log.wikibase_log_month_user_type_observation_model import ( WikibaseLogMonthUserTypeObservationModel, ) +from model.database.wikibase_observation.wikibase_observation_model import ( + WikibaseObservationModel, +) from model.enum import WikibaseUserType -class WikibaseLogMonthObservationModel(ModelBase): +class WikibaseLogMonthObservationModel(ModelBase, WikibaseObservationModel): """Wikibase Log Month Observation Table""" __tablename__ = "wikibase_log_observation_month" - id: Mapped[int] = mapped_column("id", Integer, primary_key=True, autoincrement=True) - """ID""" - - wikibase_id: Mapped[Optional[int]] = mapped_column( - "wikibase_id", - ForeignKey(column="wikibase.id", name="observation_wikibase"), - nullable=True, - ) - """Wikibase ID""" - - returned_data: Mapped[Optional[bool]] = mapped_column( - "anything", Boolean, nullable=True - ) - """Returned Data?""" - - observation_date: Mapped[Optional[datetime]] = mapped_column( - "date", - DateTime(timezone=True), - nullable=True, - # pylint: disable=not-callable - default=func.now(), - ) - """Date""" - - first_month: Mapped[Optional[bool]] = mapped_column("first", Boolean, nullable=True) + first_month: Mapped[bool] = mapped_column("first", Boolean, nullable=False) first_log_date: Mapped[Optional[datetime]] = mapped_column( "first_log_date", DateTime(timezone=True), nullable=True diff --git a/model/database/wikibase_observation/log/wikibase_log_observation_model.py b/model/database/wikibase_observation/log/wikibase_log_observation_model.py deleted file mode 100644 index 44f52b8..0000000 --- a/model/database/wikibase_observation/log/wikibase_log_observation_model.py +++ /dev/null @@ -1,75 +0,0 @@ -"""Wikibase Log Observation Table""" - -from datetime import datetime -from typing import Optional -from sqlalchemy import DateTime, Enum, ForeignKey -from sqlalchemy.orm import Mapped, mapped_column, relationship - -from model.database.base import ModelBase -from model.database.wikibase_observation.log.wikibase_log_month_observation_model import ( - WikibaseLogMonthObservationModel, -) -from model.database.wikibase_observation.wikibase_observation_model import ( - WikibaseObservationModel, -) -from model.enum import WikibaseUserType - - -class WikibaseLogObservationModel(ModelBase, WikibaseObservationModel): - """Wikibase Log Observation Table""" - - __tablename__ = "wikibase_log_observation" - - first_log_date: Mapped[Optional[datetime]] = mapped_column( - "first_log_date", DateTime(timezone=True), nullable=True - ) - """Oldest Log Date""" - - last_log_date: Mapped[Optional[datetime]] = mapped_column( - "last_log_date", DateTime(timezone=True), nullable=True - ) - """Most Recent Log Date""" - - last_log_user_type: Mapped[Optional[WikibaseUserType]] = mapped_column( - "last_log_user_type", Enum(WikibaseUserType), nullable=True - ) - """Most Recent Log User Type - User or Bot?""" - - first_month_id: Mapped[Optional[int]] = mapped_column( - "first_month_id", - ForeignKey(column="wikibase_log_observation_month.id", name="first_month"), - nullable=True, - ) - """First Month ID""" - - first_month: Mapped[Optional[WikibaseLogMonthObservationModel]] = relationship( - "WikibaseLogMonthObservationModel", - lazy="selectin", - primaryjoin=first_month_id == WikibaseLogMonthObservationModel.id, - ) - """First Month Log Record""" - - last_month_id: Mapped[Optional[int]] = mapped_column( - "last_month_id", - ForeignKey(column="wikibase_log_observation_month.id", name="last_month"), - nullable=True, - ) - """Last Month ID""" - - last_month: Mapped[Optional[WikibaseLogMonthObservationModel]] = relationship( - "WikibaseLogMonthObservationModel", - lazy="selectin", - primaryjoin=last_month_id == WikibaseLogMonthObservationModel.id, - ) - """Last Month Log Record""" - - def __str__(self) -> str: - return ( - "WikibaseLogObservationModel(" - + f"id={self.id}, " - + f"observation_date={self.observation_date}, " - + f"first_log_date={self.first_log_date}, " - + f"last_log_date={self.last_log_date}, " - + f"last_log_by={self.last_log_user_type}" - + ")" - ) From 8729c70c5440b2a67384b8c12f30144f4e018188 Mon Sep 17 00:00:00 2001 From: Ricki Jay Date: Mon, 18 Nov 2024 13:59:59 +0100 Subject: [PATCH 06/29] Create Month Log --- .../log_data/create_log_observation.py | 69 +++++++++---------- .../wikibase_log_month_observation_model.py | 7 +- 2 files changed, 37 insertions(+), 39 deletions(-) diff --git a/fetch_data/api_data/log_data/create_log_observation.py b/fetch_data/api_data/log_data/create_log_observation.py index e3ffd0c..370850a 100644 --- a/fetch_data/api_data/log_data/create_log_observation.py +++ b/fetch_data/api_data/log_data/create_log_observation.py @@ -13,7 +13,6 @@ from fetch_data.api_data.log_data.wikibase_log_record import WikibaseLogRecord from fetch_data.api_data.user_data import ( get_multiple_user_data, - get_user_type_from_wikibase, get_user_type_from_user_data, ) from fetch_data.utils import get_wikibase_from_database @@ -21,13 +20,12 @@ WikibaseLogMonthLogTypeObservationModel, WikibaseLogMonthObservationModel, WikibaseLogMonthUserTypeObservationModel, - WikibaseLogObservationModel, WikibaseModel, ) from model.enum import WikibaseUserType -async def create_log_observation(wikibase_id: int) -> bool: +async def create_log_observation(wikibase_id: int, first_month: bool) -> bool: """Create Log Observation""" async with get_async_session() as async_session: @@ -38,58 +36,49 @@ async def create_log_observation(wikibase_id: int) -> bool: require_action_api=True, ) - observation = WikibaseLogObservationModel() + observation = WikibaseLogMonthObservationModel( + wikibase_id=wikibase.id, first_month=first_month + ) try: - print("FETCHING OLDEST LOG") - oldest_log = get_log_list_from_url( - wikibase.action_api_url.url + get_log_param_string(limit=1, oldest=True) - )[0] - observation.first_log_date = oldest_log.log_date - - print("FETCHING NEWEST LOG") - most_recent_log = get_log_list_from_url( - wikibase.action_api_url.url + get_log_param_string(limit=1) - )[0] - observation.last_log_date = most_recent_log.log_date - observation.last_log_user_type = get_user_type_from_wikibase( - wikibase, most_recent_log.user - ) - - print("FETCHING FIRST MONTH'S LOGS") - first_month_log_list = get_month_log_list( + print("FETCHING LOGS") + log_list = get_month_log_list( wikibase.action_api_url.url, - comparison_date=oldest_log.log_date, - oldest=True, - ) - observation.first_month = await create_log_month( - wikibase, first_month_log_list - ) - - print("FETCHING LAST MONTH'S LOGS") - last_month_log_list = get_month_log_list( - wikibase.action_api_url.url, comparison_date=datetime.today() - ) - observation.last_month = await create_log_month( - wikibase, last_month_log_list + comparison_date=get_log_list_comparison_date(wikibase, first_month), + oldest=first_month, ) - + observation = await create_log_month(wikibase, log_list, observation) observation.returned_data = True except (ConnectionError, JSONDecodeError, ReadTimeout, SSLError): observation.returned_data = False - wikibase.log_observations.append(observation) + wikibase.log_month_observations.append(observation) await async_session.commit() return observation.returned_data +def get_log_list_comparison_date(wikibase: WikibaseModel, first: bool) -> datetime: + """Return either date of first log or today""" + + if first: + print("FETCHING OLDEST LOG") + oldest_log = get_log_list_from_url( + wikibase.action_api_url.url + get_log_param_string(limit=1, oldest=True) + )[0] + return oldest_log.log_date + + return datetime.today() + + async def create_log_month( - wikibase: WikibaseModel, log_list: Iterable[WikibaseLogRecord] + wikibase: WikibaseModel, + log_list: Iterable[WikibaseLogRecord], + result: WikibaseLogMonthObservationModel, ) -> WikibaseLogMonthObservationModel: """Create Log Month""" - result = WikibaseLogMonthObservationModel(log_count=len(log_list)) + result.log_count = len(log_list) if len(log_list) > 0: result.first_log_date = min(log.log_date for log in log_list) @@ -111,6 +100,10 @@ async def create_log_month( for u in user_data: user_type_dict[u["name"]] = get_user_type_from_user_data(u) + result.last_log_user_type = user_type_dict.get( + max(log_list, key=lambda log: log.log_date).user + ) + result.human_user_count = len( [u for u in users if user_type_dict.get(u) == WikibaseUserType.USER] ) diff --git a/model/database/wikibase_observation/log/wikibase_log_month_observation_model.py b/model/database/wikibase_observation/log/wikibase_log_month_observation_model.py index bb85b96..f7fa364 100644 --- a/model/database/wikibase_observation/log/wikibase_log_month_observation_model.py +++ b/model/database/wikibase_observation/log/wikibase_log_month_observation_model.py @@ -1,6 +1,6 @@ """Wikibase Log Month Observation Table""" -from datetime import datetime +from datetime import datetime, timezone from typing import Optional from sqlalchemy import Boolean, DateTime, Enum, Integer from sqlalchemy.orm import Mapped, mapped_column, relationship @@ -65,6 +65,11 @@ class WikibaseLogMonthObservationModel(ModelBase, WikibaseObservationModel): ) """User Type Observations""" + def __init__(self, wikibase_id: int, first_month: bool): + self.wikibase_id = wikibase_id + self.first_month = first_month + self.observation_date = datetime.now(tz=timezone.utc) + def __str__(self) -> str: return ( "WikibaseLogMonthObservationModel(" From 5b8a3d4af4b626eb9d13f257b20b21d10cb5d25f Mon Sep 17 00:00:00 2001 From: Ricki Jay Date: Mon, 18 Nov 2024 14:02:05 +0100 Subject: [PATCH 07/29] Unused Code --- fetch_data/api_data/__init__.py | 2 -- fetch_data/api_data/user_data/__init__.py | 6 +----- .../api_data/user_data/fetch_single_user_data.py | 14 -------------- fetch_data/api_data/user_data/fetch_user_type.py | 15 --------------- 4 files changed, 1 insertion(+), 36 deletions(-) delete mode 100644 fetch_data/api_data/user_data/fetch_single_user_data.py diff --git a/fetch_data/api_data/__init__.py b/fetch_data/api_data/__init__.py index 91e0391..46af6e4 100644 --- a/fetch_data/api_data/__init__.py +++ b/fetch_data/api_data/__init__.py @@ -4,7 +4,5 @@ from fetch_data.api_data.user_data import ( create_user_observation, get_multiple_user_data, - get_single_user_data, - get_user_type_from_wikibase, get_user_type_from_user_data, ) diff --git a/fetch_data/api_data/user_data/__init__.py b/fetch_data/api_data/user_data/__init__.py index 6d689ac..d0d741f 100644 --- a/fetch_data/api_data/user_data/__init__.py +++ b/fetch_data/api_data/user_data/__init__.py @@ -6,8 +6,4 @@ from fetch_data.api_data.user_data.fetch_multiple_user_data import ( get_multiple_user_data, ) -from fetch_data.api_data.user_data.fetch_single_user_data import get_single_user_data -from fetch_data.api_data.user_data.fetch_user_type import ( - get_user_type_from_wikibase, - get_user_type_from_user_data, -) +from fetch_data.api_data.user_data.fetch_user_type import get_user_type_from_user_data diff --git a/fetch_data/api_data/user_data/fetch_single_user_data.py b/fetch_data/api_data/user_data/fetch_single_user_data.py deleted file mode 100644 index ac021ee..0000000 --- a/fetch_data/api_data/user_data/fetch_single_user_data.py +++ /dev/null @@ -1,14 +0,0 @@ -"""Fetch Single User Data""" - -import json -import requests -from fetch_data.api_data.user_data.user_data_url import user_url -from model.database import WikibaseModel - - -def get_single_user_data(wikibase: WikibaseModel, user: str) -> dict: - """Fetch Single User Data""" - - result = requests.get(wikibase.action_api_url.url + user_url(user), timeout=10) - data = json.loads(result.content) - return data["query"]["users"][0] diff --git a/fetch_data/api_data/user_data/fetch_user_type.py b/fetch_data/api_data/user_data/fetch_user_type.py index 2dfae30..ce48f10 100644 --- a/fetch_data/api_data/user_data/fetch_user_type.py +++ b/fetch_data/api_data/user_data/fetch_user_type.py @@ -1,24 +1,9 @@ """Fetch User Type""" import re -from typing import Optional -from fetch_data.api_data.user_data.fetch_single_user_data import get_single_user_data -from model.database import WikibaseModel from model.enum import WikibaseUserType -def get_user_type_from_wikibase( - wikibase: WikibaseModel, user: Optional[str] -) -> WikibaseUserType: - """User Type from Wikibase""" - - if user is None: - return WikibaseUserType.NONE - - user_data = get_single_user_data(wikibase, user) - return get_user_type_from_user_data(user_data) - - def get_user_type_from_user_data(user_data: dict) -> WikibaseUserType: """User Type from User Data""" From f0b77c2ae6cee29423914f9cc84dcc13a9904741 Mon Sep 17 00:00:00 2001 From: Ricki Jay Date: Mon, 18 Nov 2024 14:07:58 +0100 Subject: [PATCH 08/29] Month Output Fields --- .../log/wikibase_log_collection.py | 41 ++++++++++++++++--- 1 file changed, 35 insertions(+), 6 deletions(-) diff --git a/model/strawberry/output/observation/log/wikibase_log_collection.py b/model/strawberry/output/observation/log/wikibase_log_collection.py index 6f88044..00b18e8 100644 --- a/model/strawberry/output/observation/log/wikibase_log_collection.py +++ b/model/strawberry/output/observation/log/wikibase_log_collection.py @@ -10,6 +10,8 @@ WikibaseLogMonthUserTypeObservationModel, ) from model.enum import WikibaseLogType, WikibaseUserType +from model.strawberry.output.observation.log.wikibase_log import WikibaseLogStrawberryModel, WikibaseLogUserStrawberryModel +from model.strawberry.output.observation.wikibase_observation import WikibaseObservationStrawberryModel from model.strawberry.scalars import BigInt @@ -78,18 +80,33 @@ def marshal( @strawberry.type -class WikibaseLogMonthStrawberryModel(WikibaseLogCollectionStrawberryModel): +class WikibaseLogMonthStrawberryModel(WikibaseObservationStrawberryModel): """Wikibase Log Month""" + + all_users: int = strawberry.field( + description="Distinct User Count", graphql_type=BigInt + ) + log_count: Optional[int] = strawberry.field( + description="Log Count", graphql_type=Optional[BigInt] + ) + human_users: int = strawberry.field( + description="Distinct (Probably) Human User Count", graphql_type=BigInt + ) + + first_log: Optional[WikibaseLogStrawberryModel] = strawberry.field( + description="First Log" + ) + last_log: Optional[WikibaseLogUserStrawberryModel] = strawberry.field( + description="Last Log" + ) + log_type_records: list[WikibaseLogMonthLogTypeStrawberryModel] = strawberry.field( description="Records of Each Type" ) user_type_records: list[WikibaseLogMonthUserTypeStrawberryModel] = strawberry.field( description="Records of Each Type" ) - human_users: int = strawberry.field( - description="Distinct (Probably) Human User Count", graphql_type=BigInt - ) @classmethod def marshal( @@ -99,11 +116,23 @@ def marshal( return cls( id=strawberry.ID(model.id), + observation_date=model.observation_date, + returned_data=model.returned_data, log_count=model.log_count, all_users=model.user_count, human_users=model.human_user_count, - first_log_date=model.first_log_date, - last_log_date=model.last_log_date, + first_log=( + WikibaseLogStrawberryModel(date=model.first_log_date) + if model.returned_data + else None + ), + last_log=( + WikibaseLogUserStrawberryModel( + date=model.last_log_date, user_type=model.last_log_user_type.name + ) + if model.returned_data + else None + ), log_type_records=sorted( [ WikibaseLogMonthLogTypeStrawberryModel.marshal(r) From fff07db8c4fbcbd65d671fd1280543c3dedfef42 Mon Sep 17 00:00:00 2001 From: Ricki Jay Date: Mon, 18 Nov 2024 14:14:14 +0100 Subject: [PATCH 09/29] Output --- .../log/wikibase_log_collection.py | 74 ------------- .../log/wikibase_log_month_observation.py | 88 +++++++++++++++ .../log/wikibase_log_observation.py | 101 +++++------------- model/strawberry/output/wikibase.py | 13 +-- 4 files changed, 117 insertions(+), 159 deletions(-) create mode 100644 model/strawberry/output/observation/log/wikibase_log_month_observation.py diff --git a/model/strawberry/output/observation/log/wikibase_log_collection.py b/model/strawberry/output/observation/log/wikibase_log_collection.py index 00b18e8..6a81720 100644 --- a/model/strawberry/output/observation/log/wikibase_log_collection.py +++ b/model/strawberry/output/observation/log/wikibase_log_collection.py @@ -6,12 +6,9 @@ from model.database import ( WikibaseLogMonthLogTypeObservationModel, - WikibaseLogMonthObservationModel, WikibaseLogMonthUserTypeObservationModel, ) from model.enum import WikibaseLogType, WikibaseUserType -from model.strawberry.output.observation.log.wikibase_log import WikibaseLogStrawberryModel, WikibaseLogUserStrawberryModel -from model.strawberry.output.observation.wikibase_observation import WikibaseObservationStrawberryModel from model.strawberry.scalars import BigInt @@ -77,74 +74,3 @@ def marshal( first_log_date=model.first_log_date, last_log_date=model.last_log_date, ) - - -@strawberry.type -class WikibaseLogMonthStrawberryModel(WikibaseObservationStrawberryModel): - """Wikibase Log Month""" - - - all_users: int = strawberry.field( - description="Distinct User Count", graphql_type=BigInt - ) - log_count: Optional[int] = strawberry.field( - description="Log Count", graphql_type=Optional[BigInt] - ) - human_users: int = strawberry.field( - description="Distinct (Probably) Human User Count", graphql_type=BigInt - ) - - first_log: Optional[WikibaseLogStrawberryModel] = strawberry.field( - description="First Log" - ) - last_log: Optional[WikibaseLogUserStrawberryModel] = strawberry.field( - description="Last Log" - ) - - log_type_records: list[WikibaseLogMonthLogTypeStrawberryModel] = strawberry.field( - description="Records of Each Type" - ) - user_type_records: list[WikibaseLogMonthUserTypeStrawberryModel] = strawberry.field( - description="Records of Each Type" - ) - - @classmethod - def marshal( - cls, model: WikibaseLogMonthObservationModel - ) -> "WikibaseLogMonthStrawberryModel": - """Coerce Database Model to Strawberry Model""" - - return cls( - id=strawberry.ID(model.id), - observation_date=model.observation_date, - returned_data=model.returned_data, - log_count=model.log_count, - all_users=model.user_count, - human_users=model.human_user_count, - first_log=( - WikibaseLogStrawberryModel(date=model.first_log_date) - if model.returned_data - else None - ), - last_log=( - WikibaseLogUserStrawberryModel( - date=model.last_log_date, user_type=model.last_log_user_type.name - ) - if model.returned_data - else None - ), - log_type_records=sorted( - [ - WikibaseLogMonthLogTypeStrawberryModel.marshal(r) - for r in model.log_type_records - ], - key=lambda x: x.log_type.value, - ), - user_type_records=sorted( - [ - WikibaseLogMonthUserTypeStrawberryModel.marshal(r) - for r in model.user_type_records - ], - key=lambda x: x.user_type.value, - ), - ) diff --git a/model/strawberry/output/observation/log/wikibase_log_month_observation.py b/model/strawberry/output/observation/log/wikibase_log_month_observation.py new file mode 100644 index 0000000..76a5f1b --- /dev/null +++ b/model/strawberry/output/observation/log/wikibase_log_month_observation.py @@ -0,0 +1,88 @@ +"""Wikibase Log Collection Strawberry Models""" + +from typing import Optional +import strawberry + +from model.database import WikibaseLogMonthObservationModel +from model.strawberry.output.observation.log.wikibase_log import ( + WikibaseLogStrawberryModel, + WikibaseLogUserStrawberryModel, +) +from model.strawberry.output.observation.log.wikibase_log_collection import ( + WikibaseLogMonthLogTypeStrawberryModel, + WikibaseLogMonthUserTypeStrawberryModel, +) +from model.strawberry.output.observation.wikibase_observation import ( + WikibaseObservationStrawberryModel, +) +from model.strawberry.scalars import BigInt + + +@strawberry.type +class WikibaseLogMonthStrawberryModel(WikibaseObservationStrawberryModel): + """Wikibase Log Month""" + + all_users: int = strawberry.field( + description="Distinct User Count", graphql_type=BigInt + ) + log_count: Optional[int] = strawberry.field( + description="Log Count", graphql_type=Optional[BigInt] + ) + human_users: int = strawberry.field( + description="Distinct (Probably) Human User Count", graphql_type=BigInt + ) + + first_log: Optional[WikibaseLogStrawberryModel] = strawberry.field( + description="First Log" + ) + last_log: Optional[WikibaseLogUserStrawberryModel] = strawberry.field( + description="Last Log" + ) + + log_type_records: list[WikibaseLogMonthLogTypeStrawberryModel] = strawberry.field( + description="Records of Each Type" + ) + user_type_records: list[WikibaseLogMonthUserTypeStrawberryModel] = strawberry.field( + description="Records of Each Type" + ) + + @classmethod + def marshal( + cls, model: WikibaseLogMonthObservationModel + ) -> "WikibaseLogMonthStrawberryModel": + """Coerce Database Model to Strawberry Model""" + + return cls( + id=strawberry.ID(model.id), + observation_date=model.observation_date, + returned_data=model.returned_data, + log_count=model.log_count, + all_users=model.user_count, + human_users=model.human_user_count, + first_log=( + WikibaseLogStrawberryModel(date=model.first_log_date) + if model.returned_data + else None + ), + last_log=( + WikibaseLogUserStrawberryModel( + date=model.last_log_date, user_type=model.last_log_user_type.name + ) + if model.returned_data + else None + ), + log_type_records=sorted( + [ + WikibaseLogMonthLogTypeStrawberryModel.marshal(r) + for r in model.log_type_records + ], + key=lambda x: x.log_type.value, + ), + user_type_records=sorted( + [ + WikibaseLogMonthUserTypeStrawberryModel.marshal(r) + for r in model.user_type_records + ], + key=lambda x: x.user_type.value, + ), + ) diff --git a/model/strawberry/output/observation/log/wikibase_log_observation.py b/model/strawberry/output/observation/log/wikibase_log_observation.py index 93886d8..77063ad 100644 --- a/model/strawberry/output/observation/log/wikibase_log_observation.py +++ b/model/strawberry/output/observation/log/wikibase_log_observation.py @@ -1,97 +1,48 @@ """Wikibase Log Data Observation Strawberry Model""" -from datetime import datetime -from typing import Optional import strawberry -from model.database import WikibaseLogObservationModel -from model.strawberry.output.observation.log.wikibase_log import ( - WikibaseLogStrawberryModel, - WikibaseLogUserStrawberryModel, -) -from model.strawberry.output.observation.log.wikibase_log_collection import ( +from model.database import WikibaseModel +from model.strawberry.output.observation.log.wikibase_log_month_observation import ( WikibaseLogMonthStrawberryModel, ) -from model.strawberry.output.observation.wikibase_observation import ( - WikibaseObservationStrawberryModel, +from model.strawberry.output.observation.wikibase_observation_set import ( + WikibaseObservationSetStrawberryModel, ) -from model.strawberry.scalars import BigInt @strawberry.type -class WikibaseLogObservationStrawberryModel(WikibaseObservationStrawberryModel): +class WikibaseLogObservationStrawberryModel: """Wikibase Log Data Observation""" - id: strawberry.ID - - instance_age: Optional[int] = strawberry.field( - description="Age of Instance in Days (Estimated from First Log Date)", - graphql_type=Optional[BigInt], - ) - first_log: Optional[WikibaseLogStrawberryModel] = strawberry.field( - description="First Log" - ) - last_log: Optional[WikibaseLogUserStrawberryModel] = strawberry.field( - description="Last Log" - ) - - first_month: Optional[WikibaseLogMonthStrawberryModel] = strawberry.field( - description="First Month's Logs" - ) - last_month: Optional[WikibaseLogMonthStrawberryModel] = strawberry.field( - description="Last Month's Logs" - ) + first_month: WikibaseObservationSetStrawberryModel[ + WikibaseLogMonthStrawberryModel + ] = strawberry.field(description="First Month's Logs") + last_month: WikibaseObservationSetStrawberryModel[ + WikibaseLogMonthStrawberryModel + ] = strawberry.field(description="Last Month's Logs") @classmethod - def marshal( - cls, model: WikibaseLogObservationModel - ) -> "WikibaseLogObservationStrawberryModel": + def marshal(cls, model: WikibaseModel) -> "WikibaseLogObservationStrawberryModel": """Coerce Database Model to Strawberry Model""" - if model.returned_data and ( - model.first_log_date is None - or model.last_log_date is None - or model.last_log_user_type is None - or model.last_month is None - # or model.first_month is None - ): - raise ValueError( - f"Log Observation {model.id}: Expected data when observation returned data" - ) - return cls( - id=strawberry.ID(model.id), - observation_date=model.observation_date, - returned_data=model.returned_data, - instance_age=( - (datetime.now() - model.first_log_date).days - if model.first_log_date is not None - else None - ), - first_log=( - WikibaseLogStrawberryModel(date=model.first_log_date) - if model.returned_data - else None - ), - last_log=( - WikibaseLogUserStrawberryModel( - date=model.last_log_date, user_type=model.last_log_user_type.name - ) - if model.returned_data - else None - ), first_month=( - WikibaseLogMonthStrawberryModel.marshal(model.first_month) - if model.returned_data - and model.first_month is not None - and model.first_month.log_count > 0 - else None + WikibaseObservationSetStrawberryModel.marshal( + [ + WikibaseLogMonthStrawberryModel.marshal(m) + for m in model.log_month_observations + if m.first_month + ] + ) ), last_month=( - WikibaseLogMonthStrawberryModel.marshal(model.last_month) - if model.returned_data - and model.last_month is not None - and model.last_month.log_count > 0 - else None + WikibaseObservationSetStrawberryModel.marshal( + [ + WikibaseLogMonthStrawberryModel.marshal(m) + for m in model.log_month_observations + if not m.first_month + ] + ) ), ) diff --git a/model/strawberry/output/wikibase.py b/model/strawberry/output/wikibase.py index cbe502b..7e69e88 100644 --- a/model/strawberry/output/wikibase.py +++ b/model/strawberry/output/wikibase.py @@ -68,9 +68,7 @@ async def connectivity_observations( ) @strawberry.field(description="Log Data") - async def log_observations( - self, - ) -> WikibaseObservationSetStrawberryModel[WikibaseLogObservationStrawberryModel]: + async def log_observations(self) -> WikibaseLogObservationStrawberryModel: """Summon Log Data on Specific Request""" async with get_async_session() as async_session: @@ -78,19 +76,14 @@ async def log_observations( ( await async_session.scalars( select(WikibaseModel) - .options(joinedload(WikibaseModel.log_observations)) + .options(joinedload(WikibaseModel.log_month_observations)) .where(WikibaseModel.id == int(self.id)) ) ) .unique() .one() ) - return WikibaseObservationSetStrawberryModel.marshal( - [ - WikibaseLogObservationStrawberryModel.marshal(o) - for o in model.log_observations - ] - ) + return WikibaseLogObservationStrawberryModel.marshal(model) @strawberry.field(description="Property Popularity Data") async def property_popularity_observations( From 69571bf8481d9434d209271a3672a2bcdd3fca3a Mon Sep 17 00:00:00 2001 From: Ricki Jay Date: Mon, 18 Nov 2024 14:16:36 +0100 Subject: [PATCH 10/29] Created --- resolvers/get_aggregate_created.py | 19 +++++++++++-------- 1 file changed, 11 insertions(+), 8 deletions(-) diff --git a/resolvers/get_aggregate_created.py b/resolvers/get_aggregate_created.py index 1933e68..d45bee1 100644 --- a/resolvers/get_aggregate_created.py +++ b/resolvers/get_aggregate_created.py @@ -3,7 +3,7 @@ from sqlalchemy import Select, and_, select, func from data import get_async_session -from model.database import WikibaseLogObservationModel, WikibaseModel +from model.database import WikibaseLogMonthObservationModel, WikibaseModel from model.strawberry.output import WikibaseYearCreatedAggregateStrawberryModel @@ -27,33 +27,36 @@ def get_created_query() -> Select[tuple[int, int]]: rank_subquery = ( select( - WikibaseLogObservationModel.id, + WikibaseLogMonthObservationModel.id, # pylint: disable=not-callable func.rank() .over( - partition_by=WikibaseLogObservationModel.wikibase_id, - order_by=WikibaseLogObservationModel.observation_date.desc(), + partition_by=WikibaseLogMonthObservationModel.wikibase_id, + order_by=WikibaseLogMonthObservationModel.observation_date.desc(), ) .label("rank"), ) .where( and_( - WikibaseLogObservationModel.returned_data, - WikibaseLogObservationModel.wikibase.has(WikibaseModel.checked), + WikibaseLogMonthObservationModel.returned_data, + WikibaseLogMonthObservationModel.first_month, + WikibaseLogMonthObservationModel.wikibase.has(WikibaseModel.checked), ) ) .subquery() ) query = ( select( - func.substr(WikibaseLogObservationModel.first_log_date, 1, 4).label("year"), + func.substr(WikibaseLogMonthObservationModel.first_log_date, 1, 4).label( + "year" + ), # pylint: disable=not-callable func.count().label("wikibase_count"), ) .join( rank_subquery, onclause=and_( - WikibaseLogObservationModel.id == rank_subquery.c.id, + WikibaseLogMonthObservationModel.id == rank_subquery.c.id, rank_subquery.c.rank == 1, ), ) From 561ef710b2442aedc5e293a7f745e635c93ede11 Mon Sep 17 00:00:00 2001 From: Ricki Jay Date: Mon, 18 Nov 2024 14:17:43 +0100 Subject: [PATCH 11/29] Most Passing --- fetch_data/utils/get_wikibase.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fetch_data/utils/get_wikibase.py b/fetch_data/utils/get_wikibase.py index 4fd7597..4d6aada 100644 --- a/fetch_data/utils/get_wikibase.py +++ b/fetch_data/utils/get_wikibase.py @@ -22,7 +22,7 @@ async def get_wikibase_from_database( if include_observations: query = query.options( joinedload(WikibaseModel.connectivity_observations), - joinedload(WikibaseModel.log_observations), + joinedload(WikibaseModel.log_month_observations), joinedload(WikibaseModel.property_popularity_observations), joinedload(WikibaseModel.quantity_observations), joinedload(WikibaseModel.software_version_observations), From 1479631acaa79e3507d6eacd76271faad6f1a1c8 Mon Sep 17 00:00:00 2001 From: Ricki Jay Date: Mon, 18 Nov 2024 14:58:31 +0100 Subject: [PATCH 12/29] Fix Test Create --- .../test_create_log_observation.py | 237 ++++++++++++------ 1 file changed, 161 insertions(+), 76 deletions(-) diff --git a/tests/test_create_observation/test_create_log_observation/test_create_log_observation.py b/tests/test_create_observation/test_create_log_observation/test_create_log_observation.py index dc95f3d..acab9ef 100644 --- a/tests/test_create_observation/test_create_log_observation/test_create_log_observation.py +++ b/tests/test_create_observation/test_create_log_observation/test_create_log_observation.py @@ -13,9 +13,11 @@ @freeze_time("2024-03-01") @pytest.mark.asyncio -@pytest.mark.dependency(name="log-success-1", depends=["add-wikibase"], scope="session") +@pytest.mark.dependency( + name="log-first-success-1", depends=["add-wikibase"], scope="session" +) @pytest.mark.log -async def test_create_log_observation(mocker): +async def test_create_log_observation_first_success(mocker): """Test One-Pull Per Month, Data Returned Scenario""" mock_logs: list[dict] = [] @@ -32,51 +34,156 @@ async def test_create_log_observation(mocker): } ) oldest_mock_log = min(mock_logs, key=lambda x: x.get("timestamp")) - newest_mock_log = max(mock_logs, key=lambda x: x.get("timestamp")) + + def mockery(*args, **kwargs) -> MockResponse: + assert kwargs.get("timeout") == 10 + + query = ParsedUrl(args[0]) + + assert query.base_url == "example.com/w/api.php" + assert query.params.get("action") == "query" + assert query.params.get("format") == "json" + assert query.params.get("list") == "logevents" + assert query.params.get("formatversion") == 2 + + match (query.params.get("ledir"), query.params.get("lelimit")): + # oldest + case ("newer", 1): + return MockResponse( + query, 200, json.dumps({"query": {"logevents": [oldest_mock_log]}}) + ) + # first month + case ("newer", 500): + return MockResponse( + query, + 200, + json.dumps( + { + "query": { + "logevents": sorted( + mock_logs, key=lambda x: x.get("timestamp") + ) + } + } + ), + ) + raise NotImplementedError(query.raw_url) mocker.patch( - "fetch_data.api_data.log_data.fetch_log_data.fetch_api_data", - side_effect=[ - {"query": {"logevents": [oldest_mock_log]}}, # oldest - {"query": {"logevents": [newest_mock_log]}}, # newest - { - "query": { - "logevents": sorted(mock_logs, key=lambda x: x.get("timestamp")) - } - }, # first month + "fetch_data.utils.fetch_data_from_api.requests.get", side_effect=mockery + ) + success = await create_log_observation(1, first_month=True) + assert success + + +@freeze_time("2024-03-01") +@pytest.mark.asyncio +@pytest.mark.dependency( + name="log-last-success-1", depends=["add-wikibase"], scope="session" +) +@pytest.mark.log +async def test_create_log_observation_last_success(mocker): + """Test One-Pull Per Month, Data Returned Scenario""" + + mock_logs: list[dict] = [] + for i in range(70): + mock_logs.append( { - "query": { - "logevents": sorted( - mock_logs, key=lambda x: x.get("timestamp"), reverse=True - ) - } - }, # last month - ], + "logid": i + 1, + "timestamp": (datetime(2024, 3, 1) - timedelta(hours=i * 24)).strftime( + "%Y-%m-%dT%H:%M:%SZ" + ), + "user": None, + "type": "thanks", + "action": "thank", + } + ) + + def mockery(*args, **kwargs) -> MockResponse: + assert kwargs.get("timeout") == 10 + + query = ParsedUrl(args[0]) + + assert query.base_url == "example.com/w/api.php" + assert query.params.get("action") == "query" + assert query.params.get("format") == "json" + assert query.params.get("list") == "logevents" + assert query.params.get("formatversion") == 2 + + match (query.params.get("ledir"), query.params.get("lelimit")): + # last month + case ("older", 500): + return MockResponse( + query, + 200, + json.dumps( + { + "query": { + "logevents": sorted( + mock_logs, + key=lambda x: x.get("timestamp"), + reverse=True, + ) + } + } + ), + ) + raise NotImplementedError(query.raw_url) + + mocker.patch( + "fetch_data.utils.fetch_data_from_api.requests.get", side_effect=mockery ) - success = await create_log_observation(1) + success = await create_log_observation(1, first_month=False) assert success @freeze_time("2024-03-01") @pytest.mark.asyncio -@pytest.mark.dependency(name="log-failure", depends=["add-wikibase"], scope="session") +@pytest.mark.dependency( + name="log-first-failure", depends=["add-wikibase"], scope="session" +) +@pytest.mark.log +async def test_create_log_first_observation_error(mocker): + """Test One-Pull Per Month, Error Returned Scenario""" + + mocker.patch( + "fetch_data.api_data.log_data.fetch_log_data.fetch_api_data", + side_effect=[ReadTimeout()], + ) + success = await create_log_observation(1, first_month=True) + assert success is False + + +@freeze_time("2024-03-01") +@pytest.mark.asyncio +@pytest.mark.dependency( + name="log-last-failure", depends=["add-wikibase"], scope="session" +) @pytest.mark.log -async def test_create_log_observation_error(mocker): +async def test_create_log_last_observation_error(mocker): """Test One-Pull Per Month, Error Returned Scenario""" mocker.patch( "fetch_data.api_data.log_data.fetch_log_data.fetch_api_data", side_effect=[ReadTimeout()], ) - success = await create_log_observation(1) + success = await create_log_observation(1, first_month=False) assert success is False @freeze_time("2024-03-01") @pytest.mark.asyncio -@pytest.mark.dependency(name="log-success-2", depends=["log-success-1", "log-failure"]) +@pytest.mark.dependency( + name="log-success-2", + depends=[ + "log-first-success-1", + "log-last-success-1", + "log-first-failure", + "log-last-failure", + ], +) @pytest.mark.log -async def test_create_log_observation_no_last_month(mocker): +async def test_create_log_last_observation_no_last_month(mocker): """Test One-Pull Per Month, No Data In Range Returned Scenario""" time.sleep(1) @@ -96,58 +203,36 @@ async def test_create_log_observation_no_last_month(mocker): "action": "thank", } ) - oldest_mock_log = min(mock_logs, key=lambda x: x.get("timestamp")) - newest_mock_log = max(mock_logs, key=lambda x: x.get("timestamp")) - # pylint: disable=unused-argument def mockery(*args, **kwargs): + assert kwargs.get("timeout") == 10 + query = ParsedUrl(args[0]) - match (query.params.get("action"), query.params.get("list")): - case ("query", "logevents"): - match (query.params.get("ledir"), query.params.get("lelimit")): - case ("newer", 1): - return MockResponse( - query.raw_url, - 200, - json.dumps({"query": {"logevents": [oldest_mock_log]}}), - ) - case ("newer", 500): - return MockResponse( - query.raw_url, - 200, - json.dumps( - { - "query": { - "logevents": sorted( - mock_logs, key=lambda x: x.get("timestamp") - ) - } - } - ), - ) - case ("older", 1): - return MockResponse( - query.raw_url, - 200, - json.dumps({"query": {"logevents": [newest_mock_log]}}), - ) - case ("older", 500): - return MockResponse( - query.raw_url, - 200, - json.dumps( - { - "query": { - "logevents": sorted( - mock_logs, - key=lambda x: x.get("timestamp"), - reverse=True, - ) - } - } - ), - ) - case ("query", "users"): + assert query.base_url == "example.com/w/api.php" + assert query.params.get("format") == "json" + assert query.params.get("formatversion") == 2 + assert query.params.get("action") == "query" + + match query.params.get("list"): + case "logevents": + assert query.params.get("ledir") == "older" + assert query.params.get("lelimit") == 500 + return MockResponse( + query.raw_url, + 200, + json.dumps( + { + "query": { + "logevents": sorted( + mock_logs, + key=lambda x: x.get("timestamp"), + reverse=True, + ) + } + } + ), + ) + case "users": users = [] if "User:A" in query.params.get("ususers"): users.append({"name": "User:A", "groups": ["*", "users", "admin"]}) @@ -158,10 +243,10 @@ def mockery(*args, **kwargs): return MockResponse( query.raw_url, 200, json.dumps({"query": {"users": users}}) ) - raise NotImplementedError(query) + raise NotImplementedError(query.raw_url) mocker.patch( "fetch_data.utils.fetch_data_from_api.requests.get", side_effect=mockery ) - success = await create_log_observation(1) + success = await create_log_observation(1, first_month=False) assert success From 6de80565db1162b094690ac74f586893f5919cb7 Mon Sep 17 00:00:00 2001 From: Ricki Jay Date: Mon, 18 Nov 2024 15:00:48 +0100 Subject: [PATCH 13/29] Fix Test Agg --- tests/test_query/aggregation/test_aggregate_created_query.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/tests/test_query/aggregation/test_aggregate_created_query.py b/tests/test_query/aggregation/test_aggregate_created_query.py index 850776d..9984124 100644 --- a/tests/test_query/aggregation/test_aggregate_created_query.py +++ b/tests/test_query/aggregation/test_aggregate_created_query.py @@ -17,7 +17,10 @@ @pytest.mark.asyncio @pytest.mark.agg -@pytest.mark.dependency(depends=["log-success-1", "log-success-2"], scope="session") +@pytest.mark.dependency( + depends=["log-first-success-1", "log-last-success-1", "log-success-2"], + scope="session", +) @pytest.mark.log @pytest.mark.query async def test_aggregate_created_query(): From b4138a17459ae994a286f90d67126ba559b18980 Mon Sep 17 00:00:00 2001 From: Ricki Jay Date: Mon, 18 Nov 2024 15:07:33 +0100 Subject: [PATCH 14/29] Fix test queries --- tests/test_query/test_wikibase_list_query.py | 11 ++++++++-- .../wikibase/log_obs/log_fragment.py | 21 ++++--------------- ...est_wikibase_log_all_observations_query.py | 19 ++++++++++++++--- ...ibase_log_most_recent_observation_query.py | 16 +++++++++++--- 4 files changed, 42 insertions(+), 25 deletions(-) diff --git a/tests/test_query/test_wikibase_list_query.py b/tests/test_query/test_wikibase_list_query.py index 1e1d939..c48912b 100644 --- a/tests/test_query/test_wikibase_list_query.py +++ b/tests/test_query/test_wikibase_list_query.py @@ -37,8 +37,15 @@ } } logObservations { - mostRecent { - id + firstMonth { + mostRecent { + id + } + } + lastMonth { + mostRecent { + id + } } } propertyPopularityObservations { diff --git a/tests/test_query/wikibase/log_obs/log_fragment.py b/tests/test_query/wikibase/log_obs/log_fragment.py index e90aa13..0ee5292 100644 --- a/tests/test_query/wikibase/log_obs/log_fragment.py +++ b/tests/test_query/wikibase/log_obs/log_fragment.py @@ -1,33 +1,20 @@ """Log Observation Fragment""" WIKIBASE_LOG_OBSERVATION_FRAGMENT = """ -fragment WikibaseLogObservationStrawberryModelFragment on WikibaseLogObservationStrawberryModel { +fragment WikibaseLogMonthStrawberryModelFragment on WikibaseLogMonthStrawberryModel { id observationDate returnedData - instanceAge + logCount + allUsers + humanUsers firstLog { date } - firstMonth { - ...WikibaseLogMonthStrawberryModelFragment - } lastLog { date userType } - lastMonth { - ...WikibaseLogMonthStrawberryModelFragment - } -} - -fragment WikibaseLogMonthStrawberryModelFragment on WikibaseLogMonthStrawberryModel { - id - firstLogDate - lastLogDate - logCount - allUsers - humanUsers logTypeRecords { id logType diff --git a/tests/test_query/wikibase/log_obs/test_wikibase_log_all_observations_query.py b/tests/test_query/wikibase/log_obs/test_wikibase_log_all_observations_query.py index acc6755..f1c6176 100644 --- a/tests/test_query/wikibase/log_obs/test_wikibase_log_all_observations_query.py +++ b/tests/test_query/wikibase/log_obs/test_wikibase_log_all_observations_query.py @@ -24,8 +24,15 @@ wikibase(wikibaseId: $wikibaseId) { id logObservations { - allObservations { - ...WikibaseLogObservationStrawberryModelFragment + firstMonth { + allObservations { + ...WikibaseLogMonthStrawberryModelFragment + } + } + lastMonth { + allObservations { + ...WikibaseLogMonthStrawberryModelFragment + } } } } @@ -39,7 +46,13 @@ @freeze_time("2024-04-01") @pytest.mark.asyncio @pytest.mark.dependency( - depends=["log-success-1", "log-success-2", "log-failure"], scope="session" + depends=[ + "log-first-success-1", + "log-last-success-1", + "log-success-2", + "log-failure", + ], + scope="session", ) @pytest.mark.log @pytest.mark.query diff --git a/tests/test_query/wikibase/log_obs/test_wikibase_log_most_recent_observation_query.py b/tests/test_query/wikibase/log_obs/test_wikibase_log_most_recent_observation_query.py index ca6e963..c9e363b 100644 --- a/tests/test_query/wikibase/log_obs/test_wikibase_log_most_recent_observation_query.py +++ b/tests/test_query/wikibase/log_obs/test_wikibase_log_most_recent_observation_query.py @@ -22,8 +22,15 @@ wikibase(wikibaseId: $wikibaseId) { id logObservations { - mostRecent { - ...WikibaseLogObservationStrawberryModelFragment + firstMonth { + mostRecent { + ...WikibaseLogMonthStrawberryModelFragment + } + } + lastMonth { + mostRecent { + ...WikibaseLogMonthStrawberryModelFragment + } } } } @@ -36,7 +43,10 @@ @freeze_time("2024-04-01") @pytest.mark.asyncio -@pytest.mark.dependency(depends=["log-success-1", "log-success-2"], scope="session") +@pytest.mark.dependency( + depends=["log-first-success-1", "log-last-success-1", "log-success-2"], + scope="session", +) @pytest.mark.log @pytest.mark.query async def test_wikibase_log_most_recent_observation_query(): From 3a64c66e9681084b128e997a02591b18a4260dee Mon Sep 17 00:00:00 2001 From: Ricki Jay Date: Mon, 18 Nov 2024 16:53:09 +0100 Subject: [PATCH 15/29] No Reason --- .../test_update_software_data.py | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/test_create_observation/test_create_software_version_observation/test_update_software_data.py b/tests/test_create_observation/test_create_software_version_observation/test_update_software_data.py index cd17dcd..c5cd1e3 100644 --- a/tests/test_create_observation/test_create_software_version_observation/test_update_software_data.py +++ b/tests/test_create_observation/test_create_software_version_observation/test_update_software_data.py @@ -23,7 +23,6 @@ async def test_update_software_data(mocker): # pylint: disable=unused-argument,too-many-return-statements def mockery(*args, **kwargs): - print(args) query = args[0] match query: # Babel From 978b2740d18735e935071b76698ccb9d25f3bcfc Mon Sep 17 00:00:00 2001 From: Ricki Jay Date: Mon, 18 Nov 2024 17:07:51 +0100 Subject: [PATCH 16/29] Check --- .../test_update_software_data.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/test_create_observation/test_create_software_version_observation/test_update_software_data.py b/tests/test_create_observation/test_create_software_version_observation/test_update_software_data.py index c5cd1e3..8b51ae7 100644 --- a/tests/test_create_observation/test_create_software_version_observation/test_update_software_data.py +++ b/tests/test_create_observation/test_create_software_version_observation/test_update_software_data.py @@ -23,6 +23,7 @@ async def test_update_software_data(mocker): # pylint: disable=unused-argument,too-many-return-statements def mockery(*args, **kwargs): + assert kwargs.get("timeout") == 10 query = args[0] match query: # Babel From f24e3ef6966da5ddc9e2518adf8d1eedafc2edde Mon Sep 17 00:00:00 2001 From: Ricki Jay Date: Mon, 18 Nov 2024 17:08:38 +0100 Subject: [PATCH 17/29] Fix list query --- model/strawberry/output/observation/log/wikibase_log.py | 5 ++++- .../observation/log/wikibase_log_month_observation.py | 7 ++++++- 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/model/strawberry/output/observation/log/wikibase_log.py b/model/strawberry/output/observation/log/wikibase_log.py index 4e29f35..bb6d256 100644 --- a/model/strawberry/output/observation/log/wikibase_log.py +++ b/model/strawberry/output/observation/log/wikibase_log.py @@ -1,6 +1,7 @@ """Wikibase Log Strawberry Models""" from datetime import datetime +from typing import Optional import strawberry @@ -15,4 +16,6 @@ class WikibaseLogStrawberryModel: class WikibaseLogUserStrawberryModel(WikibaseLogStrawberryModel): """Wikibase Log""" - user_type: str = strawberry.field(description="User Type - Bot, User, or Missing?") + user_type: Optional[str] = strawberry.field( + description="User Type - Bot, User, or Missing?" + ) diff --git a/model/strawberry/output/observation/log/wikibase_log_month_observation.py b/model/strawberry/output/observation/log/wikibase_log_month_observation.py index 76a5f1b..88c5546 100644 --- a/model/strawberry/output/observation/log/wikibase_log_month_observation.py +++ b/model/strawberry/output/observation/log/wikibase_log_month_observation.py @@ -66,7 +66,12 @@ def marshal( ), last_log=( WikibaseLogUserStrawberryModel( - date=model.last_log_date, user_type=model.last_log_user_type.name + date=model.last_log_date, + user_type=( + model.last_log_user_type.name + if model.last_log_user_type is not None + else None + ), ) if model.returned_data else None From 0d892e82a297e9ba1e21dbd02cf178a28baf9648 Mon Sep 17 00:00:00 2001 From: Ricki Jay Date: Mon, 18 Nov 2024 22:02:47 +0100 Subject: [PATCH 18/29] Fix Assert --- tests/test_query/wikibase/log_obs/assert_log.py | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/tests/test_query/wikibase/log_obs/assert_log.py b/tests/test_query/wikibase/log_obs/assert_log.py index c0551f1..4fff6b5 100644 --- a/tests/test_query/wikibase/log_obs/assert_log.py +++ b/tests/test_query/wikibase/log_obs/assert_log.py @@ -4,6 +4,7 @@ from typing import Optional from tests.utils import ( assert_layered_property_count, + assert_layered_property_value, assert_property_value, DATETIME_FORMAT, ) @@ -24,11 +25,15 @@ def assert_month_record( """Assert Month Record""" assert_property_value(month_record, "id", expected_id) - assert_property_value( - month_record, "firstLogDate", expected_first_log_date.strftime(DATETIME_FORMAT) + assert_layered_property_value( + month_record, + ["firstLog", "date"], + expected_first_log_date.strftime(DATETIME_FORMAT), ) - assert_property_value( - month_record, "lastLogDate", expected_last_log_date.strftime(DATETIME_FORMAT) + assert_layered_property_value( + month_record, + ["lastLog", "date"], + expected_last_log_date.strftime(DATETIME_FORMAT), ) assert_property_value(month_record, "logCount", expected_log_count) assert_property_value(month_record, "allUsers", expected_user_count) From e882d1880f0cc14498683cbe4a7c14fca9020d72 Mon Sep 17 00:00:00 2001 From: Ricki Jay Date: Mon, 18 Nov 2024 23:15:54 +0100 Subject: [PATCH 19/29] Optionals --- .../log/wikibase_log_month_observation.py | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/model/strawberry/output/observation/log/wikibase_log_month_observation.py b/model/strawberry/output/observation/log/wikibase_log_month_observation.py index 88c5546..1924b2e 100644 --- a/model/strawberry/output/observation/log/wikibase_log_month_observation.py +++ b/model/strawberry/output/observation/log/wikibase_log_month_observation.py @@ -22,14 +22,15 @@ class WikibaseLogMonthStrawberryModel(WikibaseObservationStrawberryModel): """Wikibase Log Month""" - all_users: int = strawberry.field( - description="Distinct User Count", graphql_type=BigInt + all_users: Optional[int] = strawberry.field( + description="Distinct User Count", graphql_type=Optional[BigInt] ) log_count: Optional[int] = strawberry.field( description="Log Count", graphql_type=Optional[BigInt] ) - human_users: int = strawberry.field( - description="Distinct (Probably) Human User Count", graphql_type=BigInt + human_users: Optional[int] = strawberry.field( + description="Distinct (Probably) Human User Count", + graphql_type=Optional[BigInt], ) first_log: Optional[WikibaseLogStrawberryModel] = strawberry.field( @@ -61,7 +62,7 @@ def marshal( human_users=model.human_user_count, first_log=( WikibaseLogStrawberryModel(date=model.first_log_date) - if model.returned_data + if model.returned_data and model.first_log_date is not None else None ), last_log=( @@ -73,7 +74,7 @@ def marshal( else None ), ) - if model.returned_data + if model.returned_data and model.last_log_date is not None else None ), log_type_records=sorted( From 080e810487bf66434dd2566482c5f128ce55cc58 Mon Sep 17 00:00:00 2001 From: Ricki Jay Date: Mon, 18 Nov 2024 23:47:15 +0100 Subject: [PATCH 20/29] Shuffle --- .../test_create_log_observation.py | 149 ++++++++++-------- 1 file changed, 84 insertions(+), 65 deletions(-) diff --git a/tests/test_create_observation/test_create_log_observation/test_create_log_observation.py b/tests/test_create_observation/test_create_log_observation/test_create_log_observation.py index acab9ef..0dead76 100644 --- a/tests/test_create_observation/test_create_log_observation/test_create_log_observation.py +++ b/tests/test_create_observation/test_create_log_observation/test_create_log_observation.py @@ -2,7 +2,6 @@ from datetime import datetime, timedelta import json -import time from freezegun import freeze_time import pytest @@ -18,17 +17,23 @@ ) @pytest.mark.log async def test_create_log_observation_first_success(mocker): - """Test One-Pull Per Month, Data Returned Scenario""" + """ + Test One-Pull Per Month, Data Returned Scenario + + log_month_id 1, first month, users, 'thanks/thank' + """ mock_logs: list[dict] = [] for i in range(70): mock_logs.append( { "logid": i + 1, - "timestamp": (datetime(2024, 3, 1) - timedelta(hours=i * 24)).strftime( + "timestamp": (datetime(2024, 1, 1) - timedelta(hours=i * 24)).strftime( "%Y-%m-%dT%H:%M:%SZ" ), - "user": None, + "user": ( + "User:A" if i % 3 == 0 else "User:B" if i % 2 == 0 else "User:C" + ), "type": "thanks", "action": "thank", } @@ -43,30 +48,45 @@ def mockery(*args, **kwargs) -> MockResponse: assert query.base_url == "example.com/w/api.php" assert query.params.get("action") == "query" assert query.params.get("format") == "json" - assert query.params.get("list") == "logevents" - assert query.params.get("formatversion") == 2 - match (query.params.get("ledir"), query.params.get("lelimit")): - # oldest - case ("newer", 1): - return MockResponse( - query, 200, json.dumps({"query": {"logevents": [oldest_mock_log]}}) - ) - # first month - case ("newer", 500): + match query.params.get("list"): + case "logevents": + assert query.params.get("formatversion") == 2 + match (query.params.get("ledir"), query.params.get("lelimit")): + # oldest + case ("newer", 1): + return MockResponse( + query, + 200, + json.dumps({"query": {"logevents": [oldest_mock_log]}}), + ) + # first month + case ("newer", 500): + return MockResponse( + query, + 200, + json.dumps( + { + "query": { + "logevents": sorted( + mock_logs, key=lambda x: x.get("timestamp") + ) + } + } + ), + ) + case "users": + users = [] + if "User:A" in query.params.get("ususers"): + users.append({"name": "User:A", "groups": ["*", "users", "admin"]}) + if "User:B" in query.params.get("ususers"): + users.append({"name": "User:B", "invalid": True}) + if "User:C" in query.params.get("ususers"): + users.append({"name": "User:C", "groups": ["*", "users", "bot"]}) return MockResponse( - query, - 200, - json.dumps( - { - "query": { - "logevents": sorted( - mock_logs, key=lambda x: x.get("timestamp") - ) - } - } - ), + query.raw_url, 200, json.dumps({"query": {"users": users}}) ) + raise NotImplementedError(query.raw_url) mocker.patch( @@ -83,7 +103,10 @@ def mockery(*args, **kwargs) -> MockResponse: ) @pytest.mark.log async def test_create_log_observation_last_success(mocker): - """Test One-Pull Per Month, Data Returned Scenario""" + """ + Test One-Pull Per Month, Data Returned Scenario + + log_month_id 2, last month, None user, thanks/thank""" mock_logs: list[dict] = [] for i in range(70): @@ -137,14 +160,18 @@ def mockery(*args, **kwargs) -> MockResponse: assert success -@freeze_time("2024-03-01") +@freeze_time("2024-03-02") @pytest.mark.asyncio @pytest.mark.dependency( name="log-first-failure", depends=["add-wikibase"], scope="session" ) @pytest.mark.log async def test_create_log_first_observation_error(mocker): - """Test One-Pull Per Month, Error Returned Scenario""" + """ + Test One-Pull Per Month, Error Returned Scenario + + log_month_id 3, first month, fail + """ mocker.patch( "fetch_data.api_data.log_data.fetch_log_data.fetch_api_data", @@ -154,14 +181,17 @@ async def test_create_log_first_observation_error(mocker): assert success is False -@freeze_time("2024-03-01") +@freeze_time("2024-03-02") @pytest.mark.asyncio @pytest.mark.dependency( name="log-last-failure", depends=["add-wikibase"], scope="session" ) @pytest.mark.log async def test_create_log_last_observation_error(mocker): - """Test One-Pull Per Month, Error Returned Scenario""" + """ + Test One-Pull Per Month, Error Returned Scenario + + log_month_id 4, last month, fail""" mocker.patch( "fetch_data.api_data.log_data.fetch_log_data.fetch_api_data", @@ -171,7 +201,7 @@ async def test_create_log_last_observation_error(mocker): assert success is False -@freeze_time("2024-03-01") +@freeze_time("2024-03-03") @pytest.mark.asyncio @pytest.mark.dependency( name="log-success-2", @@ -184,9 +214,10 @@ async def test_create_log_last_observation_error(mocker): ) @pytest.mark.log async def test_create_log_last_observation_no_last_month(mocker): - """Test One-Pull Per Month, No Data In Range Returned Scenario""" + """ + Test One-Pull Per Month, No Data In Range Returned Scenario - time.sleep(1) + log_month_id 5, last month, success, no data""" mock_logs: list[dict] = [] for i in range(70): @@ -208,42 +239,30 @@ def mockery(*args, **kwargs): assert kwargs.get("timeout") == 10 query = ParsedUrl(args[0]) + assert query.base_url == "example.com/w/api.php" assert query.params.get("format") == "json" assert query.params.get("formatversion") == 2 assert query.params.get("action") == "query" - - match query.params.get("list"): - case "logevents": - assert query.params.get("ledir") == "older" - assert query.params.get("lelimit") == 500 - return MockResponse( - query.raw_url, - 200, - json.dumps( - { - "query": { - "logevents": sorted( - mock_logs, - key=lambda x: x.get("timestamp"), - reverse=True, - ) - } - } - ), - ) - case "users": - users = [] - if "User:A" in query.params.get("ususers"): - users.append({"name": "User:A", "groups": ["*", "users", "admin"]}) - if "User:B" in query.params.get("ususers"): - users.append({"name": "User:B", "invalid": True}) - if "User:C" in query.params.get("ususers"): - users.append({"name": "User:C", "groups": ["*", "users", "bot"]}) - return MockResponse( - query.raw_url, 200, json.dumps({"query": {"users": users}}) - ) - raise NotImplementedError(query.raw_url) + assert query.params.get("list") == "logevents" + assert query.params.get("ledir") == "older" + assert query.params.get("lelimit") == 500 + + return MockResponse( + query.raw_url, + 200, + json.dumps( + { + "query": { + "logevents": sorted( + mock_logs, + key=lambda x: x.get("timestamp"), + reverse=True, + ) + } + } + ), + ) mocker.patch( "fetch_data.utils.fetch_data_from_api.requests.get", side_effect=mockery From f3326a22194c8c523bad6beca776b5b6bd487ab7 Mon Sep 17 00:00:00 2001 From: Ricki Jay Date: Mon, 18 Nov 2024 23:49:22 +0100 Subject: [PATCH 21/29] Rename --- ...est_wikibase_log_all_observations_query.py | 52 +++++++++---------- 1 file changed, 26 insertions(+), 26 deletions(-) diff --git a/tests/test_query/wikibase/log_obs/test_wikibase_log_all_observations_query.py b/tests/test_query/wikibase/log_obs/test_wikibase_log_all_observations_query.py index f1c6176..cef0de1 100644 --- a/tests/test_query/wikibase/log_obs/test_wikibase_log_all_observations_query.py +++ b/tests/test_query/wikibase/log_obs/test_wikibase_log_all_observations_query.py @@ -73,24 +73,24 @@ async def test_wikibase_log_all_observations_query(): assert "allObservations" in result_wikibase["logObservations"] assert ( len( - user_observation_list := result_wikibase["logObservations"][ + log_observation_list := result_wikibase["logObservations"][ "allObservations" ] ) == 3 ) - assert_layered_property_value(user_observation_list, [0, "id"], "1") - assert "observationDate" in user_observation_list[0] - assert_layered_property_value(user_observation_list, [0, "returnedData"], True) - assert_layered_property_value(user_observation_list, [0, "instanceAge"], 100) + assert_layered_property_value(log_observation_list, [0, "id"], "1") + assert "observationDate" in log_observation_list[0] + assert_layered_property_value(log_observation_list, [0, "returnedData"], True) + assert_layered_property_value(log_observation_list, [0, "instanceAge"], 100) assert_layered_property_value( - user_observation_list, + log_observation_list, [0, "firstLog", "date"], datetime(2023, 12, 23).strftime(DATETIME_FORMAT), ) assert_month_record( - user_observation_list[0]["firstMonth"], + log_observation_list[0]["firstMonth"], "1", datetime(2023, 12, 23), datetime(2024, 1, 22), @@ -101,7 +101,7 @@ async def test_wikibase_log_all_observations_query(): 0, ) assert_month_type_record( - user_observation_list[0]["firstMonth"]["logTypeRecords"][0], + log_observation_list[0]["firstMonth"]["logTypeRecords"][0], expected_id="1", expected_log_type="THANK", expected_first_log_date=datetime(2023, 12, 23), @@ -112,15 +112,15 @@ async def test_wikibase_log_all_observations_query(): ) assert_layered_property_value( - user_observation_list, + log_observation_list, [0, "lastLog", "date"], datetime(2024, 3, 1).strftime(DATETIME_FORMAT), ) assert_layered_property_value( - user_observation_list, [0, "lastLog", "userType"], "NONE" + log_observation_list, [0, "lastLog", "userType"], "NONE" ) assert_month_record( - user_observation_list[0]["lastMonth"], + log_observation_list[0]["lastMonth"], "2", datetime(2024, 1, 31), datetime(2024, 3, 1), @@ -131,7 +131,7 @@ async def test_wikibase_log_all_observations_query(): 0, ) assert_month_type_record( - user_observation_list[0]["lastMonth"]["logTypeRecords"][0], + log_observation_list[0]["lastMonth"]["logTypeRecords"][0], expected_id="2", expected_log_type="THANK", expected_first_log_date=datetime(2024, 1, 31), @@ -141,21 +141,21 @@ async def test_wikibase_log_all_observations_query(): expected_human_count=0, ) - assert_layered_property_value(user_observation_list, [1, "id"], "2") - assert "observationDate" in user_observation_list[1] - assert_layered_property_value(user_observation_list, [1, "returnedData"], False) + assert_layered_property_value(log_observation_list, [1, "id"], "2") + assert "observationDate" in log_observation_list[1] + assert_layered_property_value(log_observation_list, [1, "returnedData"], False) - assert_layered_property_value(user_observation_list, [2, "id"], "3") - assert "observationDate" in user_observation_list[2] - assert_layered_property_value(user_observation_list, [2, "returnedData"], True) - assert_layered_property_value(user_observation_list, [2, "instanceAge"], 160) + assert_layered_property_value(log_observation_list, [2, "id"], "3") + assert "observationDate" in log_observation_list[2] + assert_layered_property_value(log_observation_list, [2, "returnedData"], True) + assert_layered_property_value(log_observation_list, [2, "instanceAge"], 160) assert_layered_property_value( - user_observation_list, + log_observation_list, [2, "firstLog", "date"], datetime(2023, 10, 24).strftime(DATETIME_FORMAT), ) assert_month_record( - user_observation_list[2]["firstMonth"], + log_observation_list[2]["firstMonth"], "3", datetime(2023, 10, 24), datetime(2023, 11, 23), @@ -166,7 +166,7 @@ async def test_wikibase_log_all_observations_query(): 3, ) assert_month_type_record( - user_observation_list[2]["firstMonth"]["logTypeRecords"][0], + log_observation_list[2]["firstMonth"]["logTypeRecords"][0], expected_id="3", expected_log_type="THANK", expected_first_log_date=datetime(2023, 10, 24), @@ -190,7 +190,7 @@ async def test_wikibase_log_all_observations_query(): ] ): assert_month_type_record( - user_observation_list[2]["firstMonth"]["userTypeRecords"][index], + log_observation_list[2]["firstMonth"]["userTypeRecords"][index], expected_id=expected_id, expected_user_type=expected_user_type, expected_first_log_date=expected_first_log_date, @@ -200,11 +200,11 @@ async def test_wikibase_log_all_observations_query(): ) assert_layered_property_value( - user_observation_list, + log_observation_list, [2, "lastLog", "date"], datetime(2024, 1, 1).strftime(DATETIME_FORMAT), ) assert_layered_property_value( - user_observation_list, [2, "lastLog", "userType"], "USER" + log_observation_list, [2, "lastLog", "userType"], "USER" ) - assert_layered_property_value(user_observation_list, [2, "lastMonth"], None) + assert_layered_property_value(log_observation_list, [2, "lastMonth"], None) From 9947bd122054772b3bb2f43e8083557c05684927 Mon Sep 17 00:00:00 2001 From: Ricki Jay Date: Mon, 18 Nov 2024 23:52:12 +0100 Subject: [PATCH 22/29] Split --- ...est_wikibase_log_all_observations_query.py | 211 ++++++++++++++++++ ...ibase_log_most_recent_observation_query.py | 0 ...est_wikibase_log_all_observations_query.py | 0 ...ibase_log_most_recent_observation_query.py | 131 +++++++++++ 4 files changed, 342 insertions(+) create mode 100644 tests/test_query/wikibase/log_obs/first_month/test_wikibase_log_all_observations_query.py rename tests/test_query/wikibase/log_obs/{ => first_month}/test_wikibase_log_most_recent_observation_query.py (100%) rename tests/test_query/wikibase/log_obs/{ => last_month}/test_wikibase_log_all_observations_query.py (100%) create mode 100644 tests/test_query/wikibase/log_obs/last_month/test_wikibase_log_most_recent_observation_query.py diff --git a/tests/test_query/wikibase/log_obs/first_month/test_wikibase_log_all_observations_query.py b/tests/test_query/wikibase/log_obs/first_month/test_wikibase_log_all_observations_query.py new file mode 100644 index 0000000..c5e0855 --- /dev/null +++ b/tests/test_query/wikibase/log_obs/first_month/test_wikibase_log_all_observations_query.py @@ -0,0 +1,211 @@ +"""Test Wikibase All Log Observations Query""" + +from datetime import datetime +from freezegun import freeze_time +import pytest +from tests.test_query.wikibase.log_obs.assert_log import ( + assert_month_record, + assert_month_type_record, +) +from tests.test_query.wikibase.log_obs.log_fragment import ( + WIKIBASE_LOG_OBSERVATION_FRAGMENT, +) +from tests.test_schema import test_schema +from tests.utils import ( + assert_layered_property_value, + assert_property_value, + DATETIME_FORMAT, +) + + +WIKIBASE_LOG_ALL_OBSERVATIONS_QUERY = ( + """ +query MyQuery($wikibaseId: Int!) { + wikibase(wikibaseId: $wikibaseId) { + id + logObservations { + firstMonth { + allObservations { + ...WikibaseLogMonthStrawberryModelFragment + } + } + lastMonth { + allObservations { + ...WikibaseLogMonthStrawberryModelFragment + } + } + } + } +} + +""" + + WIKIBASE_LOG_OBSERVATION_FRAGMENT +) + + +@freeze_time("2024-04-01") +@pytest.mark.asyncio +@pytest.mark.dependency( + depends=[ + "log-first-success-1", + "log-last-success-1", + "log-success-2", + "log-failure", + ], + scope="session", +) +@pytest.mark.log +@pytest.mark.query +async def test_wikibase_log_first_month_all_observations_query(): + """Test Wikibase All Log Observations Query""" + + result = await test_schema.execute( + WIKIBASE_LOG_ALL_OBSERVATIONS_QUERY, variable_values={"wikibaseId": 1} + ) + + assert result.errors is None + assert result.data is not None + assert "wikibase" in result.data + result_wikibase = result.data["wikibase"] + assert_property_value(result_wikibase, "id", "1") + assert "logObservations" in result_wikibase + assert "firstMonth" in result_wikibase["logObservations"] + assert "allObservations" in result_wikibase["logObservations"]['firstMonth'] + + assert ( + len( + log_observation_list := result_wikibase["logObservations"]['firstMonth'][ + "allObservations" + ] + ) + == 3 + ) + + assert_layered_property_value(log_observation_list, [0, "id"], "1") + assert "observationDate" in log_observation_list[0] + assert_layered_property_value(log_observation_list, [0, "returnedData"], True) + assert_layered_property_value(log_observation_list, [0, "instanceAge"], 100) + assert_layered_property_value( + log_observation_list, + [0, "firstLog", "date"], + datetime(2023, 12, 23).strftime(DATETIME_FORMAT), + ) + assert_month_record( + log_observation_list[0]["firstMonth"], + "1", + datetime(2023, 12, 23), + datetime(2024, 1, 22), + 31, + 0, + 0, + 1, + 0, + ) + assert_month_type_record( + log_observation_list[0]["firstMonth"]["logTypeRecords"][0], + expected_id="1", + expected_log_type="THANK", + expected_first_log_date=datetime(2023, 12, 23), + expected_last_log_date=datetime(2024, 1, 22), + expected_log_count=31, + expected_user_count=0, + expected_human_count=0, + ) + + assert_layered_property_value( + log_observation_list, + [0, "lastLog", "date"], + datetime(2024, 3, 1).strftime(DATETIME_FORMAT), + ) + assert_layered_property_value( + log_observation_list, [0, "lastLog", "userType"], "NONE" + ) + assert_month_record( + log_observation_list[0]["lastMonth"], + "2", + datetime(2024, 1, 31), + datetime(2024, 3, 1), + 31, + 0, + 0, + 1, + 0, + ) + assert_month_type_record( + log_observation_list[0]["lastMonth"]["logTypeRecords"][0], + expected_id="2", + expected_log_type="THANK", + expected_first_log_date=datetime(2024, 1, 31), + expected_last_log_date=datetime(2024, 3, 1), + expected_log_count=31, + expected_user_count=0, + expected_human_count=0, + ) + + assert_layered_property_value(log_observation_list, [1, "id"], "2") + assert "observationDate" in log_observation_list[1] + assert_layered_property_value(log_observation_list, [1, "returnedData"], False) + + assert_layered_property_value(log_observation_list, [2, "id"], "3") + assert "observationDate" in log_observation_list[2] + assert_layered_property_value(log_observation_list, [2, "returnedData"], True) + assert_layered_property_value(log_observation_list, [2, "instanceAge"], 160) + assert_layered_property_value( + log_observation_list, + [2, "firstLog", "date"], + datetime(2023, 10, 24).strftime(DATETIME_FORMAT), + ) + assert_month_record( + log_observation_list[2]["firstMonth"], + "3", + datetime(2023, 10, 24), + datetime(2023, 11, 23), + 31, + 3, + 1, + 1, + 3, + ) + assert_month_type_record( + log_observation_list[2]["firstMonth"]["logTypeRecords"][0], + expected_id="3", + expected_log_type="THANK", + expected_first_log_date=datetime(2023, 10, 24), + expected_last_log_date=datetime(2023, 11, 23), + expected_log_count=31, + expected_user_count=3, + expected_human_count=1, + ) + for index, ( + expected_id, + expected_user_type, + expected_first_log_date, + expected_last_log_date, + expected_log_count, + expected_user_count, + ) in enumerate( + [ + ("1", "BOT", datetime(2023, 10, 26), datetime(2023, 11, 21), 10, 1), + ("2", "MISSING", datetime(2023, 10, 25), datetime(2023, 11, 22), 10, 1), + ("3", "USER", datetime(2023, 10, 24), datetime(2023, 11, 23), 11, 1), + ] + ): + assert_month_type_record( + log_observation_list[2]["firstMonth"]["userTypeRecords"][index], + expected_id=expected_id, + expected_user_type=expected_user_type, + expected_first_log_date=expected_first_log_date, + expected_last_log_date=expected_last_log_date, + expected_log_count=expected_log_count, + expected_user_count=expected_user_count, + ) + + assert_layered_property_value( + log_observation_list, + [2, "lastLog", "date"], + datetime(2024, 1, 1).strftime(DATETIME_FORMAT), + ) + assert_layered_property_value( + log_observation_list, [2, "lastLog", "userType"], "USER" + ) + assert_layered_property_value(log_observation_list, [2, "lastMonth"], None) diff --git a/tests/test_query/wikibase/log_obs/test_wikibase_log_most_recent_observation_query.py b/tests/test_query/wikibase/log_obs/first_month/test_wikibase_log_most_recent_observation_query.py similarity index 100% rename from tests/test_query/wikibase/log_obs/test_wikibase_log_most_recent_observation_query.py rename to tests/test_query/wikibase/log_obs/first_month/test_wikibase_log_most_recent_observation_query.py diff --git a/tests/test_query/wikibase/log_obs/test_wikibase_log_all_observations_query.py b/tests/test_query/wikibase/log_obs/last_month/test_wikibase_log_all_observations_query.py similarity index 100% rename from tests/test_query/wikibase/log_obs/test_wikibase_log_all_observations_query.py rename to tests/test_query/wikibase/log_obs/last_month/test_wikibase_log_all_observations_query.py diff --git a/tests/test_query/wikibase/log_obs/last_month/test_wikibase_log_most_recent_observation_query.py b/tests/test_query/wikibase/log_obs/last_month/test_wikibase_log_most_recent_observation_query.py new file mode 100644 index 0000000..c9e363b --- /dev/null +++ b/tests/test_query/wikibase/log_obs/last_month/test_wikibase_log_most_recent_observation_query.py @@ -0,0 +1,131 @@ +"""Test Wikibase Most Recent Log Observation""" + +from datetime import datetime +from freezegun import freeze_time +import pytest +from tests.test_query.wikibase.log_obs.assert_log import assert_month_type_record +from tests.test_query.wikibase.log_obs.log_fragment import ( + WIKIBASE_LOG_OBSERVATION_FRAGMENT, +) +from tests.test_schema import test_schema +from tests.utils import ( + assert_layered_property_count, + assert_layered_property_value, + assert_property_value, + DATETIME_FORMAT, +) + + +WIKIBASE_LOG_MOST_RECENT_OBSERVATION_QUERY = ( + """ +query MyQuery($wikibaseId: Int!) { + wikibase(wikibaseId: $wikibaseId) { + id + logObservations { + firstMonth { + mostRecent { + ...WikibaseLogMonthStrawberryModelFragment + } + } + lastMonth { + mostRecent { + ...WikibaseLogMonthStrawberryModelFragment + } + } + } + } +} + +""" + + WIKIBASE_LOG_OBSERVATION_FRAGMENT +) + + +@freeze_time("2024-04-01") +@pytest.mark.asyncio +@pytest.mark.dependency( + depends=["log-first-success-1", "log-last-success-1", "log-success-2"], + scope="session", +) +@pytest.mark.log +@pytest.mark.query +async def test_wikibase_log_most_recent_observation_query(): + """Test Wikibase Most Recent Log Observation""" + + result = await test_schema.execute( + WIKIBASE_LOG_MOST_RECENT_OBSERVATION_QUERY, variable_values={"wikibaseId": 1} + ) + + assert result.errors is None + assert result.data is not None + assert "wikibase" in result.data + result_wikibase = result.data["wikibase"] + assert_property_value(result_wikibase, "id", "1") + assert "logObservations" in result_wikibase + assert "mostRecent" in result_wikibase["logObservations"] + most_recent = result_wikibase["logObservations"]["mostRecent"] + + assert_property_value(most_recent, "id", "3") + assert "observationDate" in most_recent + assert_property_value(most_recent, "returnedData", True) + assert_property_value(most_recent, "instanceAge", 160) + assert_layered_property_value( + most_recent, + ["firstLog", "date"], + datetime(2023, 10, 24).strftime(DATETIME_FORMAT), + ) + assert_layered_property_value(most_recent, ["firstMonth", "id"], "3") + assert_layered_property_value( + most_recent, + ["firstMonth", "firstLogDate"], + datetime(2023, 10, 24).strftime(DATETIME_FORMAT), + ) + assert_layered_property_value( + most_recent, + ["firstMonth", "lastLogDate"], + datetime(2023, 11, 23).strftime(DATETIME_FORMAT), + ) + assert_layered_property_value(most_recent, ["firstMonth", "logCount"], 31) + assert_layered_property_value(most_recent, ["firstMonth", "allUsers"], 3) + assert_layered_property_value(most_recent, ["firstMonth", "humanUsers"], 1) + assert_layered_property_count(most_recent, ["firstMonth", "logTypeRecords"], 1) + assert_month_type_record( + most_recent["firstMonth"]["logTypeRecords"][0], + expected_id="3", + expected_log_type="THANK", + expected_first_log_date=datetime(2023, 10, 24), + expected_last_log_date=datetime(2023, 11, 23), + expected_log_count=31, + expected_user_count=3, + expected_human_count=1, + ) + assert_layered_property_count(most_recent, ["firstMonth", "userTypeRecords"], 3) + for index, ( + expected_id, + expected_user_type, + expected_first_log_date, + expected_last_log_date, + expected_log_count, + expected_user_count, + ) in enumerate( + [ + ("1", "BOT", datetime(2023, 10, 26), datetime(2023, 11, 21), 10, 1), + ("2", "MISSING", datetime(2023, 10, 25), datetime(2023, 11, 22), 10, 1), + ("3", "USER", datetime(2023, 10, 24), datetime(2023, 11, 23), 11, 1), + ] + ): + assert_month_type_record( + most_recent["firstMonth"]["userTypeRecords"][index], + expected_id=expected_id, + expected_user_type=expected_user_type, + expected_first_log_date=expected_first_log_date, + expected_last_log_date=expected_last_log_date, + expected_log_count=expected_log_count, + expected_user_count=expected_user_count, + ) + + assert_layered_property_value( + most_recent, ["lastLog", "date"], datetime(2024, 1, 1).strftime(DATETIME_FORMAT) + ) + assert_layered_property_value(most_recent, ["lastLog", "userType"], "USER") + assert_property_value(most_recent, "lastMonth", None) From aaa23d912d8df42486c7af2d1b669da3672206ac Mon Sep 17 00:00:00 2001 From: Ricki Jay Date: Mon, 18 Nov 2024 23:54:19 +0100 Subject: [PATCH 23/29] Rename --- ...est_wikibase_log_first_month_all_observations_query.py} | 5 ----- ...ibase_log_first_month_most_recent_observation_query.py} | 7 +------ ...test_wikibase_log_last_month_all_observations_query.py} | 7 +------ ...kibase_log_last_month_most_recent_observation_query.py} | 7 +------ 4 files changed, 3 insertions(+), 23 deletions(-) rename tests/test_query/wikibase/log_obs/first_month/{test_wikibase_log_all_observations_query.py => test_wikibase_log_first_month_all_observations_query.py} (98%) rename tests/test_query/wikibase/log_obs/first_month/{test_wikibase_log_most_recent_observation_query.py => test_wikibase_log_first_month_most_recent_observation_query.py} (96%) rename tests/test_query/wikibase/log_obs/last_month/{test_wikibase_log_all_observations_query.py => test_wikibase_log_last_month_all_observations_query.py} (97%) rename tests/test_query/wikibase/log_obs/last_month/{test_wikibase_log_most_recent_observation_query.py => test_wikibase_log_last_month_most_recent_observation_query.py} (96%) diff --git a/tests/test_query/wikibase/log_obs/first_month/test_wikibase_log_all_observations_query.py b/tests/test_query/wikibase/log_obs/first_month/test_wikibase_log_first_month_all_observations_query.py similarity index 98% rename from tests/test_query/wikibase/log_obs/first_month/test_wikibase_log_all_observations_query.py rename to tests/test_query/wikibase/log_obs/first_month/test_wikibase_log_first_month_all_observations_query.py index c5e0855..f90a89f 100644 --- a/tests/test_query/wikibase/log_obs/first_month/test_wikibase_log_all_observations_query.py +++ b/tests/test_query/wikibase/log_obs/first_month/test_wikibase_log_first_month_all_observations_query.py @@ -29,11 +29,6 @@ ...WikibaseLogMonthStrawberryModelFragment } } - lastMonth { - allObservations { - ...WikibaseLogMonthStrawberryModelFragment - } - } } } } diff --git a/tests/test_query/wikibase/log_obs/first_month/test_wikibase_log_most_recent_observation_query.py b/tests/test_query/wikibase/log_obs/first_month/test_wikibase_log_first_month_most_recent_observation_query.py similarity index 96% rename from tests/test_query/wikibase/log_obs/first_month/test_wikibase_log_most_recent_observation_query.py rename to tests/test_query/wikibase/log_obs/first_month/test_wikibase_log_first_month_most_recent_observation_query.py index c9e363b..8ef3b41 100644 --- a/tests/test_query/wikibase/log_obs/first_month/test_wikibase_log_most_recent_observation_query.py +++ b/tests/test_query/wikibase/log_obs/first_month/test_wikibase_log_first_month_most_recent_observation_query.py @@ -27,11 +27,6 @@ ...WikibaseLogMonthStrawberryModelFragment } } - lastMonth { - mostRecent { - ...WikibaseLogMonthStrawberryModelFragment - } - } } } } @@ -49,7 +44,7 @@ ) @pytest.mark.log @pytest.mark.query -async def test_wikibase_log_most_recent_observation_query(): +async def test_wikibase_log_first_month_most_recent_observation_query(): """Test Wikibase Most Recent Log Observation""" result = await test_schema.execute( diff --git a/tests/test_query/wikibase/log_obs/last_month/test_wikibase_log_all_observations_query.py b/tests/test_query/wikibase/log_obs/last_month/test_wikibase_log_last_month_all_observations_query.py similarity index 97% rename from tests/test_query/wikibase/log_obs/last_month/test_wikibase_log_all_observations_query.py rename to tests/test_query/wikibase/log_obs/last_month/test_wikibase_log_last_month_all_observations_query.py index cef0de1..5d87bb4 100644 --- a/tests/test_query/wikibase/log_obs/last_month/test_wikibase_log_all_observations_query.py +++ b/tests/test_query/wikibase/log_obs/last_month/test_wikibase_log_last_month_all_observations_query.py @@ -24,11 +24,6 @@ wikibase(wikibaseId: $wikibaseId) { id logObservations { - firstMonth { - allObservations { - ...WikibaseLogMonthStrawberryModelFragment - } - } lastMonth { allObservations { ...WikibaseLogMonthStrawberryModelFragment @@ -56,7 +51,7 @@ ) @pytest.mark.log @pytest.mark.query -async def test_wikibase_log_all_observations_query(): +async def test_wikibase_log_last_month_all_observations_query(): """Test Wikibase All Log Observations Query""" result = await test_schema.execute( diff --git a/tests/test_query/wikibase/log_obs/last_month/test_wikibase_log_most_recent_observation_query.py b/tests/test_query/wikibase/log_obs/last_month/test_wikibase_log_last_month_most_recent_observation_query.py similarity index 96% rename from tests/test_query/wikibase/log_obs/last_month/test_wikibase_log_most_recent_observation_query.py rename to tests/test_query/wikibase/log_obs/last_month/test_wikibase_log_last_month_most_recent_observation_query.py index c9e363b..3653c8b 100644 --- a/tests/test_query/wikibase/log_obs/last_month/test_wikibase_log_most_recent_observation_query.py +++ b/tests/test_query/wikibase/log_obs/last_month/test_wikibase_log_last_month_most_recent_observation_query.py @@ -22,11 +22,6 @@ wikibase(wikibaseId: $wikibaseId) { id logObservations { - firstMonth { - mostRecent { - ...WikibaseLogMonthStrawberryModelFragment - } - } lastMonth { mostRecent { ...WikibaseLogMonthStrawberryModelFragment @@ -49,7 +44,7 @@ ) @pytest.mark.log @pytest.mark.query -async def test_wikibase_log_most_recent_observation_query(): +async def test_wikibase_log_last_month_most_recent_observation_query(): """Test Wikibase Most Recent Log Observation""" result = await test_schema.execute( From 42617138c3dc9b8fc7a91c6e924e96f135233cef Mon Sep 17 00:00:00 2001 From: Ricki Jay Date: Tue, 19 Nov 2024 00:10:37 +0100 Subject: [PATCH 24/29] Specify --- .../test_create_log_observation/test_create_log_observation.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_create_observation/test_create_log_observation/test_create_log_observation.py b/tests/test_create_observation/test_create_log_observation/test_create_log_observation.py index 0dead76..3b25c6b 100644 --- a/tests/test_create_observation/test_create_log_observation/test_create_log_observation.py +++ b/tests/test_create_observation/test_create_log_observation/test_create_log_observation.py @@ -204,7 +204,7 @@ async def test_create_log_last_observation_error(mocker): @freeze_time("2024-03-03") @pytest.mark.asyncio @pytest.mark.dependency( - name="log-success-2", + name="log-last-success-2", depends=[ "log-first-success-1", "log-last-success-1", From b44a8177388fc2e901218f599696de0e8dcff41e Mon Sep 17 00:00:00 2001 From: Ricki Jay Date: Tue, 19 Nov 2024 00:11:28 +0100 Subject: [PATCH 25/29] Fix Dependency --- tests/test_query/aggregation/test_aggregate_created_query.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/tests/test_query/aggregation/test_aggregate_created_query.py b/tests/test_query/aggregation/test_aggregate_created_query.py index 9984124..b25e0cf 100644 --- a/tests/test_query/aggregation/test_aggregate_created_query.py +++ b/tests/test_query/aggregation/test_aggregate_created_query.py @@ -17,10 +17,7 @@ @pytest.mark.asyncio @pytest.mark.agg -@pytest.mark.dependency( - depends=["log-first-success-1", "log-last-success-1", "log-success-2"], - scope="session", -) +@pytest.mark.dependency(depends=["log-first-success-1"], scope="session") @pytest.mark.log @pytest.mark.query async def test_aggregate_created_query(): From ee30d61a7afeb107718fe1a94cf69bd068b2a95f Mon Sep 17 00:00:00 2001 From: Ricki Jay Date: Tue, 19 Nov 2024 00:13:12 +0100 Subject: [PATCH 26/29] Successful ID Check --- ..._log_first_month_all_observations_query.py | 20 +++++++--------- ...rst_month_most_recent_observation_query.py | 14 +++++------ ...e_log_last_month_all_observations_query.py | 24 ++++++++++--------- ...ast_month_most_recent_observation_query.py | 11 +++++---- 4 files changed, 36 insertions(+), 33 deletions(-) diff --git a/tests/test_query/wikibase/log_obs/first_month/test_wikibase_log_first_month_all_observations_query.py b/tests/test_query/wikibase/log_obs/first_month/test_wikibase_log_first_month_all_observations_query.py index f90a89f..314247e 100644 --- a/tests/test_query/wikibase/log_obs/first_month/test_wikibase_log_first_month_all_observations_query.py +++ b/tests/test_query/wikibase/log_obs/first_month/test_wikibase_log_first_month_all_observations_query.py @@ -41,13 +41,7 @@ @freeze_time("2024-04-01") @pytest.mark.asyncio @pytest.mark.dependency( - depends=[ - "log-first-success-1", - "log-last-success-1", - "log-success-2", - "log-failure", - ], - scope="session", + depends=["log-first-success-1", "log-first-failure"], scope="session" ) @pytest.mark.log @pytest.mark.query @@ -65,18 +59,19 @@ async def test_wikibase_log_first_month_all_observations_query(): assert_property_value(result_wikibase, "id", "1") assert "logObservations" in result_wikibase assert "firstMonth" in result_wikibase["logObservations"] - assert "allObservations" in result_wikibase["logObservations"]['firstMonth'] + assert "allObservations" in result_wikibase["logObservations"]["firstMonth"] assert ( len( - log_observation_list := result_wikibase["logObservations"]['firstMonth'][ + log_observation_list := result_wikibase["logObservations"]["firstMonth"][ "allObservations" ] ) - == 3 + == 2 ) assert_layered_property_value(log_observation_list, [0, "id"], "1") + """ assert "observationDate" in log_observation_list[0] assert_layered_property_value(log_observation_list, [0, "returnedData"], True) assert_layered_property_value(log_observation_list, [0, "instanceAge"], 100) @@ -136,8 +131,10 @@ async def test_wikibase_log_first_month_all_observations_query(): expected_user_count=0, expected_human_count=0, ) + """ - assert_layered_property_value(log_observation_list, [1, "id"], "2") + assert_layered_property_value(log_observation_list, [1, "id"], "3") + """ assert "observationDate" in log_observation_list[1] assert_layered_property_value(log_observation_list, [1, "returnedData"], False) @@ -204,3 +201,4 @@ async def test_wikibase_log_first_month_all_observations_query(): log_observation_list, [2, "lastLog", "userType"], "USER" ) assert_layered_property_value(log_observation_list, [2, "lastMonth"], None) + """ diff --git a/tests/test_query/wikibase/log_obs/first_month/test_wikibase_log_first_month_most_recent_observation_query.py b/tests/test_query/wikibase/log_obs/first_month/test_wikibase_log_first_month_most_recent_observation_query.py index 8ef3b41..07e66e0 100644 --- a/tests/test_query/wikibase/log_obs/first_month/test_wikibase_log_first_month_most_recent_observation_query.py +++ b/tests/test_query/wikibase/log_obs/first_month/test_wikibase_log_first_month_most_recent_observation_query.py @@ -38,10 +38,7 @@ @freeze_time("2024-04-01") @pytest.mark.asyncio -@pytest.mark.dependency( - depends=["log-first-success-1", "log-last-success-1", "log-success-2"], - scope="session", -) +@pytest.mark.dependency(depends=["log-first-success-1"], scope="session") @pytest.mark.log @pytest.mark.query async def test_wikibase_log_first_month_most_recent_observation_query(): @@ -57,10 +54,12 @@ async def test_wikibase_log_first_month_most_recent_observation_query(): result_wikibase = result.data["wikibase"] assert_property_value(result_wikibase, "id", "1") assert "logObservations" in result_wikibase - assert "mostRecent" in result_wikibase["logObservations"] - most_recent = result_wikibase["logObservations"]["mostRecent"] + assert "firstMonth" in result_wikibase["logObservations"] + assert "mostRecent" in result_wikibase["logObservations"]["firstMonth"] + most_recent = result_wikibase["logObservations"]["firstMonth"]["mostRecent"] - assert_property_value(most_recent, "id", "3") + assert_property_value(most_recent, "id", "1") + """ assert "observationDate" in most_recent assert_property_value(most_recent, "returnedData", True) assert_property_value(most_recent, "instanceAge", 160) @@ -124,3 +123,4 @@ async def test_wikibase_log_first_month_most_recent_observation_query(): ) assert_layered_property_value(most_recent, ["lastLog", "userType"], "USER") assert_property_value(most_recent, "lastMonth", None) + """ diff --git a/tests/test_query/wikibase/log_obs/last_month/test_wikibase_log_last_month_all_observations_query.py b/tests/test_query/wikibase/log_obs/last_month/test_wikibase_log_last_month_all_observations_query.py index 5d87bb4..63f4f9e 100644 --- a/tests/test_query/wikibase/log_obs/last_month/test_wikibase_log_last_month_all_observations_query.py +++ b/tests/test_query/wikibase/log_obs/last_month/test_wikibase_log_last_month_all_observations_query.py @@ -41,12 +41,7 @@ @freeze_time("2024-04-01") @pytest.mark.asyncio @pytest.mark.dependency( - depends=[ - "log-first-success-1", - "log-last-success-1", - "log-success-2", - "log-failure", - ], + depends=["log-last-success-1", "log-last-success-2", "log-last-failure"], scope="session", ) @pytest.mark.log @@ -64,18 +59,20 @@ async def test_wikibase_log_last_month_all_observations_query(): result_wikibase = result.data["wikibase"] assert_property_value(result_wikibase, "id", "1") assert "logObservations" in result_wikibase + assert "lastMonth" in result_wikibase["logObservations"] + assert "allObservations" in result_wikibase["logObservations"]["lastMonth"] - assert "allObservations" in result_wikibase["logObservations"] assert ( len( - log_observation_list := result_wikibase["logObservations"][ + log_observation_list := result_wikibase["logObservations"]["lastMonth"][ "allObservations" ] ) == 3 ) - assert_layered_property_value(log_observation_list, [0, "id"], "1") + assert_layered_property_value(log_observation_list, [0, "id"], "2") + """ assert "observationDate" in log_observation_list[0] assert_layered_property_value(log_observation_list, [0, "returnedData"], True) assert_layered_property_value(log_observation_list, [0, "instanceAge"], 100) @@ -135,12 +132,16 @@ async def test_wikibase_log_last_month_all_observations_query(): expected_user_count=0, expected_human_count=0, ) + """ - assert_layered_property_value(log_observation_list, [1, "id"], "2") + assert_layered_property_value(log_observation_list, [1, "id"], "4") + """ assert "observationDate" in log_observation_list[1] assert_layered_property_value(log_observation_list, [1, "returnedData"], False) + """ - assert_layered_property_value(log_observation_list, [2, "id"], "3") + assert_layered_property_value(log_observation_list, [2, "id"], "5") + """ assert "observationDate" in log_observation_list[2] assert_layered_property_value(log_observation_list, [2, "returnedData"], True) assert_layered_property_value(log_observation_list, [2, "instanceAge"], 160) @@ -203,3 +204,4 @@ async def test_wikibase_log_last_month_all_observations_query(): log_observation_list, [2, "lastLog", "userType"], "USER" ) assert_layered_property_value(log_observation_list, [2, "lastMonth"], None) + """ diff --git a/tests/test_query/wikibase/log_obs/last_month/test_wikibase_log_last_month_most_recent_observation_query.py b/tests/test_query/wikibase/log_obs/last_month/test_wikibase_log_last_month_most_recent_observation_query.py index 3653c8b..ca18a8f 100644 --- a/tests/test_query/wikibase/log_obs/last_month/test_wikibase_log_last_month_most_recent_observation_query.py +++ b/tests/test_query/wikibase/log_obs/last_month/test_wikibase_log_last_month_most_recent_observation_query.py @@ -39,7 +39,7 @@ @freeze_time("2024-04-01") @pytest.mark.asyncio @pytest.mark.dependency( - depends=["log-first-success-1", "log-last-success-1", "log-success-2"], + depends=["log-last-success-1", "log-last-success-2"], scope="session", ) @pytest.mark.log @@ -57,10 +57,12 @@ async def test_wikibase_log_last_month_most_recent_observation_query(): result_wikibase = result.data["wikibase"] assert_property_value(result_wikibase, "id", "1") assert "logObservations" in result_wikibase - assert "mostRecent" in result_wikibase["logObservations"] - most_recent = result_wikibase["logObservations"]["mostRecent"] + assert "lastMonth" in result_wikibase["logObservations"] + assert "mostRecent" in result_wikibase["logObservations"]["lastMonth"] + most_recent = result_wikibase["logObservations"]["lastMonth"]["mostRecent"] - assert_property_value(most_recent, "id", "3") + assert_property_value(most_recent, "id", "5") + """ assert "observationDate" in most_recent assert_property_value(most_recent, "returnedData", True) assert_property_value(most_recent, "instanceAge", 160) @@ -124,3 +126,4 @@ async def test_wikibase_log_last_month_most_recent_observation_query(): ) assert_layered_property_value(most_recent, ["lastLog", "userType"], "USER") assert_property_value(most_recent, "lastMonth", None) + """ From 882b0b475fe16134a75ce40302fb64ff94b9f3d0 Mon Sep 17 00:00:00 2001 From: Ricki Jay Date: Tue, 19 Nov 2024 00:59:51 +0100 Subject: [PATCH 27/29] Fix Tests --- ..._log_first_month_all_observations_query.py | 113 +++----------- ...rst_month_most_recent_observation_query.py | 43 +++--- ...e_log_last_month_all_observations_query.py | 138 ++++-------------- ...ast_month_most_recent_observation_query.py | 77 ++-------- 4 files changed, 84 insertions(+), 287 deletions(-) diff --git a/tests/test_query/wikibase/log_obs/first_month/test_wikibase_log_first_month_all_observations_query.py b/tests/test_query/wikibase/log_obs/first_month/test_wikibase_log_first_month_all_observations_query.py index 314247e..e0da2fd 100644 --- a/tests/test_query/wikibase/log_obs/first_month/test_wikibase_log_first_month_all_observations_query.py +++ b/tests/test_query/wikibase/log_obs/first_month/test_wikibase_log_first_month_all_observations_query.py @@ -3,15 +3,13 @@ from datetime import datetime from freezegun import freeze_time import pytest -from tests.test_query.wikibase.log_obs.assert_log import ( - assert_month_record, - assert_month_type_record, -) +from tests.test_query.wikibase.log_obs.assert_log import assert_month_type_record from tests.test_query.wikibase.log_obs.log_fragment import ( WIKIBASE_LOG_OBSERVATION_FRAGMENT, ) from tests.test_schema import test_schema from tests.utils import ( + assert_layered_property_count, assert_layered_property_value, assert_property_value, DATETIME_FORMAT, @@ -71,96 +69,34 @@ async def test_wikibase_log_first_month_all_observations_query(): ) assert_layered_property_value(log_observation_list, [0, "id"], "1") - """ - assert "observationDate" in log_observation_list[0] + assert_layered_property_value( + log_observation_list, + [0, "observationDate"], + datetime(2024, 3, 1).strftime(DATETIME_FORMAT), + ) assert_layered_property_value(log_observation_list, [0, "returnedData"], True) - assert_layered_property_value(log_observation_list, [0, "instanceAge"], 100) + # assert_layered_property_value(log_observation_list, [0, "instanceAge"], 100) assert_layered_property_value( log_observation_list, [0, "firstLog", "date"], - datetime(2023, 12, 23).strftime(DATETIME_FORMAT), - ) - assert_month_record( - log_observation_list[0]["firstMonth"], - "1", - datetime(2023, 12, 23), - datetime(2024, 1, 22), - 31, - 0, - 0, - 1, - 0, - ) - assert_month_type_record( - log_observation_list[0]["firstMonth"]["logTypeRecords"][0], - expected_id="1", - expected_log_type="THANK", - expected_first_log_date=datetime(2023, 12, 23), - expected_last_log_date=datetime(2024, 1, 22), - expected_log_count=31, - expected_user_count=0, - expected_human_count=0, + datetime(2023, 10, 24).strftime(DATETIME_FORMAT), ) - assert_layered_property_value( log_observation_list, [0, "lastLog", "date"], - datetime(2024, 3, 1).strftime(DATETIME_FORMAT), + datetime(2023, 11, 23).strftime(DATETIME_FORMAT), ) assert_layered_property_value( - log_observation_list, [0, "lastLog", "userType"], "NONE" + log_observation_list, [0, "lastLog", "userType"], "USER" ) - assert_month_record( - log_observation_list[0]["lastMonth"], - "2", - datetime(2024, 1, 31), - datetime(2024, 3, 1), - 31, - 0, - 0, - 1, - 0, - ) - assert_month_type_record( - log_observation_list[0]["lastMonth"]["logTypeRecords"][0], - expected_id="2", - expected_log_type="THANK", - expected_first_log_date=datetime(2024, 1, 31), - expected_last_log_date=datetime(2024, 3, 1), - expected_log_count=31, - expected_user_count=0, - expected_human_count=0, - ) - """ + assert_layered_property_value(log_observation_list, [0, "logCount"], 31) + assert_layered_property_value(log_observation_list, [0, "allUsers"], 3) + assert_layered_property_value(log_observation_list, [0, "humanUsers"], 1) - assert_layered_property_value(log_observation_list, [1, "id"], "3") - """ - assert "observationDate" in log_observation_list[1] - assert_layered_property_value(log_observation_list, [1, "returnedData"], False) - - assert_layered_property_value(log_observation_list, [2, "id"], "3") - assert "observationDate" in log_observation_list[2] - assert_layered_property_value(log_observation_list, [2, "returnedData"], True) - assert_layered_property_value(log_observation_list, [2, "instanceAge"], 160) - assert_layered_property_value( - log_observation_list, - [2, "firstLog", "date"], - datetime(2023, 10, 24).strftime(DATETIME_FORMAT), - ) - assert_month_record( - log_observation_list[2]["firstMonth"], - "3", - datetime(2023, 10, 24), - datetime(2023, 11, 23), - 31, - 3, - 1, - 1, - 3, - ) + assert_layered_property_count(log_observation_list, [0, "logTypeRecords"], 1) assert_month_type_record( - log_observation_list[2]["firstMonth"]["logTypeRecords"][0], - expected_id="3", + log_observation_list[0]["logTypeRecords"][0], + expected_id="1", expected_log_type="THANK", expected_first_log_date=datetime(2023, 10, 24), expected_last_log_date=datetime(2023, 11, 23), @@ -168,6 +104,8 @@ async def test_wikibase_log_first_month_all_observations_query(): expected_user_count=3, expected_human_count=1, ) + + assert_layered_property_count(log_observation_list, [0, "userTypeRecords"], 3) for index, ( expected_id, expected_user_type, @@ -183,7 +121,7 @@ async def test_wikibase_log_first_month_all_observations_query(): ] ): assert_month_type_record( - log_observation_list[2]["firstMonth"]["userTypeRecords"][index], + log_observation_list[0]["userTypeRecords"][index], expected_id=expected_id, expected_user_type=expected_user_type, expected_first_log_date=expected_first_log_date, @@ -192,13 +130,10 @@ async def test_wikibase_log_first_month_all_observations_query(): expected_user_count=expected_user_count, ) + assert_layered_property_value(log_observation_list, [1, "id"], "3") assert_layered_property_value( log_observation_list, - [2, "lastLog", "date"], - datetime(2024, 1, 1).strftime(DATETIME_FORMAT), + [1, "observationDate"], + datetime(2024, 3, 2).strftime(DATETIME_FORMAT), ) - assert_layered_property_value( - log_observation_list, [2, "lastLog", "userType"], "USER" - ) - assert_layered_property_value(log_observation_list, [2, "lastMonth"], None) - """ + assert_layered_property_value(log_observation_list, [1, "returnedData"], False) diff --git a/tests/test_query/wikibase/log_obs/first_month/test_wikibase_log_first_month_most_recent_observation_query.py b/tests/test_query/wikibase/log_obs/first_month/test_wikibase_log_first_month_most_recent_observation_query.py index 07e66e0..c03f437 100644 --- a/tests/test_query/wikibase/log_obs/first_month/test_wikibase_log_first_month_most_recent_observation_query.py +++ b/tests/test_query/wikibase/log_obs/first_month/test_wikibase_log_first_month_most_recent_observation_query.py @@ -59,33 +59,32 @@ async def test_wikibase_log_first_month_most_recent_observation_query(): most_recent = result_wikibase["logObservations"]["firstMonth"]["mostRecent"] assert_property_value(most_recent, "id", "1") - """ - assert "observationDate" in most_recent - assert_property_value(most_recent, "returnedData", True) - assert_property_value(most_recent, "instanceAge", 160) - assert_layered_property_value( + assert_property_value( most_recent, - ["firstLog", "date"], - datetime(2023, 10, 24).strftime(DATETIME_FORMAT), + "observationDate", + datetime(2024, 3, 1).strftime(DATETIME_FORMAT), ) - assert_layered_property_value(most_recent, ["firstMonth", "id"], "3") + assert_property_value(most_recent, "returnedData", True) + # assert_layered_property_value(log_observation_list, [0, "instanceAge"], 100) assert_layered_property_value( most_recent, - ["firstMonth", "firstLogDate"], + ["firstLog", "date"], datetime(2023, 10, 24).strftime(DATETIME_FORMAT), ) assert_layered_property_value( most_recent, - ["firstMonth", "lastLogDate"], + ["lastLog", "date"], datetime(2023, 11, 23).strftime(DATETIME_FORMAT), ) - assert_layered_property_value(most_recent, ["firstMonth", "logCount"], 31) - assert_layered_property_value(most_recent, ["firstMonth", "allUsers"], 3) - assert_layered_property_value(most_recent, ["firstMonth", "humanUsers"], 1) - assert_layered_property_count(most_recent, ["firstMonth", "logTypeRecords"], 1) + assert_layered_property_value(most_recent, ["lastLog", "userType"], "USER") + assert_property_value(most_recent, "logCount", 31) + assert_property_value(most_recent, "allUsers", 3) + assert_property_value(most_recent, "humanUsers", 1) + + assert_layered_property_count(most_recent, ["logTypeRecords"], 1) assert_month_type_record( - most_recent["firstMonth"]["logTypeRecords"][0], - expected_id="3", + most_recent["logTypeRecords"][0], + expected_id="1", expected_log_type="THANK", expected_first_log_date=datetime(2023, 10, 24), expected_last_log_date=datetime(2023, 11, 23), @@ -93,7 +92,8 @@ async def test_wikibase_log_first_month_most_recent_observation_query(): expected_user_count=3, expected_human_count=1, ) - assert_layered_property_count(most_recent, ["firstMonth", "userTypeRecords"], 3) + + assert_layered_property_count(most_recent, ["userTypeRecords"], 3) for index, ( expected_id, expected_user_type, @@ -109,7 +109,7 @@ async def test_wikibase_log_first_month_most_recent_observation_query(): ] ): assert_month_type_record( - most_recent["firstMonth"]["userTypeRecords"][index], + most_recent["userTypeRecords"][index], expected_id=expected_id, expected_user_type=expected_user_type, expected_first_log_date=expected_first_log_date, @@ -117,10 +117,3 @@ async def test_wikibase_log_first_month_most_recent_observation_query(): expected_log_count=expected_log_count, expected_user_count=expected_user_count, ) - - assert_layered_property_value( - most_recent, ["lastLog", "date"], datetime(2024, 1, 1).strftime(DATETIME_FORMAT) - ) - assert_layered_property_value(most_recent, ["lastLog", "userType"], "USER") - assert_property_value(most_recent, "lastMonth", None) - """ diff --git a/tests/test_query/wikibase/log_obs/last_month/test_wikibase_log_last_month_all_observations_query.py b/tests/test_query/wikibase/log_obs/last_month/test_wikibase_log_last_month_all_observations_query.py index 63f4f9e..2174596 100644 --- a/tests/test_query/wikibase/log_obs/last_month/test_wikibase_log_last_month_all_observations_query.py +++ b/tests/test_query/wikibase/log_obs/last_month/test_wikibase_log_last_month_all_observations_query.py @@ -3,15 +3,13 @@ from datetime import datetime from freezegun import freeze_time import pytest -from tests.test_query.wikibase.log_obs.assert_log import ( - assert_month_record, - assert_month_type_record, -) +from tests.test_query.wikibase.log_obs.assert_log import assert_month_type_record from tests.test_query.wikibase.log_obs.log_fragment import ( WIKIBASE_LOG_OBSERVATION_FRAGMENT, ) from tests.test_schema import test_schema from tests.utils import ( + assert_layered_property_count, assert_layered_property_value, assert_property_value, DATETIME_FORMAT, @@ -72,58 +70,30 @@ async def test_wikibase_log_last_month_all_observations_query(): ) assert_layered_property_value(log_observation_list, [0, "id"], "2") - """ - assert "observationDate" in log_observation_list[0] + assert_layered_property_value( + log_observation_list, + [0, "observationDate"], + datetime(2024, 3, 1).strftime(DATETIME_FORMAT), + ) assert_layered_property_value(log_observation_list, [0, "returnedData"], True) - assert_layered_property_value(log_observation_list, [0, "instanceAge"], 100) + # assert_layered_property_value(log_observation_list, [0, "instanceAge"], 100) assert_layered_property_value( log_observation_list, [0, "firstLog", "date"], - datetime(2023, 12, 23).strftime(DATETIME_FORMAT), - ) - assert_month_record( - log_observation_list[0]["firstMonth"], - "1", - datetime(2023, 12, 23), - datetime(2024, 1, 22), - 31, - 0, - 0, - 1, - 0, - ) - assert_month_type_record( - log_observation_list[0]["firstMonth"]["logTypeRecords"][0], - expected_id="1", - expected_log_type="THANK", - expected_first_log_date=datetime(2023, 12, 23), - expected_last_log_date=datetime(2024, 1, 22), - expected_log_count=31, - expected_user_count=0, - expected_human_count=0, + datetime(2024, 1, 31).strftime(DATETIME_FORMAT), ) - assert_layered_property_value( log_observation_list, [0, "lastLog", "date"], datetime(2024, 3, 1).strftime(DATETIME_FORMAT), ) assert_layered_property_value( - log_observation_list, [0, "lastLog", "userType"], "NONE" - ) - assert_month_record( - log_observation_list[0]["lastMonth"], - "2", - datetime(2024, 1, 31), - datetime(2024, 3, 1), - 31, - 0, - 0, - 1, - 0, + log_observation_list, [0, "lastLog", "userType"], None ) + assert_layered_property_value(log_observation_list, [0, "logCount"], 31) + assert_layered_property_count(log_observation_list, [0, "logTypeRecords"], 1) assert_month_type_record( - log_observation_list[0]["lastMonth"]["logTypeRecords"][0], + log_observation_list[0]["logTypeRecords"][0], expected_id="2", expected_log_type="THANK", expected_first_log_date=datetime(2024, 1, 31), @@ -132,76 +102,30 @@ async def test_wikibase_log_last_month_all_observations_query(): expected_user_count=0, expected_human_count=0, ) - """ + assert_layered_property_value(log_observation_list, [0, "allUsers"], 0) + assert_layered_property_value(log_observation_list, [0, "humanUsers"], 0) + assert_layered_property_count(log_observation_list, [0, "userTypeRecords"], 0) assert_layered_property_value(log_observation_list, [1, "id"], "4") - """ - assert "observationDate" in log_observation_list[1] - assert_layered_property_value(log_observation_list, [1, "returnedData"], False) - """ - - assert_layered_property_value(log_observation_list, [2, "id"], "5") - """ - assert "observationDate" in log_observation_list[2] - assert_layered_property_value(log_observation_list, [2, "returnedData"], True) - assert_layered_property_value(log_observation_list, [2, "instanceAge"], 160) assert_layered_property_value( log_observation_list, - [2, "firstLog", "date"], - datetime(2023, 10, 24).strftime(DATETIME_FORMAT), + [1, "observationDate"], + datetime(2024, 3, 2).strftime(DATETIME_FORMAT), ) - assert_month_record( - log_observation_list[2]["firstMonth"], - "3", - datetime(2023, 10, 24), - datetime(2023, 11, 23), - 31, - 3, - 1, - 1, - 3, - ) - assert_month_type_record( - log_observation_list[2]["firstMonth"]["logTypeRecords"][0], - expected_id="3", - expected_log_type="THANK", - expected_first_log_date=datetime(2023, 10, 24), - expected_last_log_date=datetime(2023, 11, 23), - expected_log_count=31, - expected_user_count=3, - expected_human_count=1, - ) - for index, ( - expected_id, - expected_user_type, - expected_first_log_date, - expected_last_log_date, - expected_log_count, - expected_user_count, - ) in enumerate( - [ - ("1", "BOT", datetime(2023, 10, 26), datetime(2023, 11, 21), 10, 1), - ("2", "MISSING", datetime(2023, 10, 25), datetime(2023, 11, 22), 10, 1), - ("3", "USER", datetime(2023, 10, 24), datetime(2023, 11, 23), 11, 1), - ] - ): - assert_month_type_record( - log_observation_list[2]["firstMonth"]["userTypeRecords"][index], - expected_id=expected_id, - expected_user_type=expected_user_type, - expected_first_log_date=expected_first_log_date, - expected_last_log_date=expected_last_log_date, - expected_log_count=expected_log_count, - expected_user_count=expected_user_count, - ) + assert_layered_property_value(log_observation_list, [1, "returnedData"], False) + assert_layered_property_value(log_observation_list, [2, "id"], "5") assert_layered_property_value( log_observation_list, - [2, "lastLog", "date"], - datetime(2024, 1, 1).strftime(DATETIME_FORMAT), + [2, "observationDate"], + datetime(2024, 3, 3).strftime(DATETIME_FORMAT), ) - assert_layered_property_value( - log_observation_list, [2, "lastLog", "userType"], "USER" - ) - assert_layered_property_value(log_observation_list, [2, "lastMonth"], None) - """ + assert_layered_property_value(log_observation_list, [2, "returnedData"], True) + # assert_layered_property_value(log_observation_list, [2, "instanceAge"], 160) + assert_layered_property_value(log_observation_list, [2, "firstLog"], None) + assert_layered_property_value(log_observation_list, [2, "lastLog"], None) + assert_layered_property_value(log_observation_list, [2, "logCount"], 0) + assert_layered_property_count(log_observation_list, [2, "logTypeRecords"], 0) + assert_layered_property_value(log_observation_list, [2, "allUsers"], 0) + assert_layered_property_value(log_observation_list, [2, "humanUsers"], 0) + assert_layered_property_count(log_observation_list, [2, "userTypeRecords"], 0) diff --git a/tests/test_query/wikibase/log_obs/last_month/test_wikibase_log_last_month_most_recent_observation_query.py b/tests/test_query/wikibase/log_obs/last_month/test_wikibase_log_last_month_most_recent_observation_query.py index ca18a8f..a692016 100644 --- a/tests/test_query/wikibase/log_obs/last_month/test_wikibase_log_last_month_most_recent_observation_query.py +++ b/tests/test_query/wikibase/log_obs/last_month/test_wikibase_log_last_month_most_recent_observation_query.py @@ -3,14 +3,12 @@ from datetime import datetime from freezegun import freeze_time import pytest -from tests.test_query.wikibase.log_obs.assert_log import assert_month_type_record from tests.test_query.wikibase.log_obs.log_fragment import ( WIKIBASE_LOG_OBSERVATION_FRAGMENT, ) from tests.test_schema import test_schema from tests.utils import ( assert_layered_property_count, - assert_layered_property_value, assert_property_value, DATETIME_FORMAT, ) @@ -62,68 +60,15 @@ async def test_wikibase_log_last_month_most_recent_observation_query(): most_recent = result_wikibase["logObservations"]["lastMonth"]["mostRecent"] assert_property_value(most_recent, "id", "5") - """ - assert "observationDate" in most_recent - assert_property_value(most_recent, "returnedData", True) - assert_property_value(most_recent, "instanceAge", 160) - assert_layered_property_value( - most_recent, - ["firstLog", "date"], - datetime(2023, 10, 24).strftime(DATETIME_FORMAT), - ) - assert_layered_property_value(most_recent, ["firstMonth", "id"], "3") - assert_layered_property_value( - most_recent, - ["firstMonth", "firstLogDate"], - datetime(2023, 10, 24).strftime(DATETIME_FORMAT), - ) - assert_layered_property_value( - most_recent, - ["firstMonth", "lastLogDate"], - datetime(2023, 11, 23).strftime(DATETIME_FORMAT), - ) - assert_layered_property_value(most_recent, ["firstMonth", "logCount"], 31) - assert_layered_property_value(most_recent, ["firstMonth", "allUsers"], 3) - assert_layered_property_value(most_recent, ["firstMonth", "humanUsers"], 1) - assert_layered_property_count(most_recent, ["firstMonth", "logTypeRecords"], 1) - assert_month_type_record( - most_recent["firstMonth"]["logTypeRecords"][0], - expected_id="3", - expected_log_type="THANK", - expected_first_log_date=datetime(2023, 10, 24), - expected_last_log_date=datetime(2023, 11, 23), - expected_log_count=31, - expected_user_count=3, - expected_human_count=1, + assert_property_value( + most_recent, "observationDate", datetime(2024, 3, 3).strftime(DATETIME_FORMAT) ) - assert_layered_property_count(most_recent, ["firstMonth", "userTypeRecords"], 3) - for index, ( - expected_id, - expected_user_type, - expected_first_log_date, - expected_last_log_date, - expected_log_count, - expected_user_count, - ) in enumerate( - [ - ("1", "BOT", datetime(2023, 10, 26), datetime(2023, 11, 21), 10, 1), - ("2", "MISSING", datetime(2023, 10, 25), datetime(2023, 11, 22), 10, 1), - ("3", "USER", datetime(2023, 10, 24), datetime(2023, 11, 23), 11, 1), - ] - ): - assert_month_type_record( - most_recent["firstMonth"]["userTypeRecords"][index], - expected_id=expected_id, - expected_user_type=expected_user_type, - expected_first_log_date=expected_first_log_date, - expected_last_log_date=expected_last_log_date, - expected_log_count=expected_log_count, - expected_user_count=expected_user_count, - ) - - assert_layered_property_value( - most_recent, ["lastLog", "date"], datetime(2024, 1, 1).strftime(DATETIME_FORMAT) - ) - assert_layered_property_value(most_recent, ["lastLog", "userType"], "USER") - assert_property_value(most_recent, "lastMonth", None) - """ + assert_property_value(most_recent, "returnedData", True) + # assert_property_value(most_recent, "instanceAge", 160) + assert_property_value(most_recent, "firstLog", None) + assert_property_value(most_recent, "lastLog", None) + assert_property_value(most_recent, "logCount", 0) + assert_layered_property_count(most_recent, ["logTypeRecords"], 0) + assert_property_value(most_recent, "allUsers", 0) + assert_property_value(most_recent, "humanUsers", 0) + assert_layered_property_count(most_recent, ["userTypeRecords"], 0) From 8a9275cb3e0aa7ad3f6fe48e4ec6fa68bdfc6a4f Mon Sep 17 00:00:00 2001 From: Ricki Jay Date: Tue, 19 Nov 2024 01:03:16 +0100 Subject: [PATCH 28/29] Unused --- .../test_query/wikibase/log_obs/assert_log.py | 43 +------------------ 1 file changed, 1 insertion(+), 42 deletions(-) diff --git a/tests/test_query/wikibase/log_obs/assert_log.py b/tests/test_query/wikibase/log_obs/assert_log.py index 4fff6b5..70b120f 100644 --- a/tests/test_query/wikibase/log_obs/assert_log.py +++ b/tests/test_query/wikibase/log_obs/assert_log.py @@ -2,48 +2,7 @@ from datetime import datetime from typing import Optional -from tests.utils import ( - assert_layered_property_count, - assert_layered_property_value, - assert_property_value, - DATETIME_FORMAT, -) - - -# pylint: disable=too-many-arguments,too-many-positional-arguments -def assert_month_record( - month_record: dict, - expected_id: str, - expected_first_log_date: datetime, - expected_last_log_date: datetime, - expected_log_count: int, - expected_user_count: int, - expected_human_count: int, - expected_log_type_record_count: int, - expected_user_type_record_count: int, -): - """Assert Month Record""" - - assert_property_value(month_record, "id", expected_id) - assert_layered_property_value( - month_record, - ["firstLog", "date"], - expected_first_log_date.strftime(DATETIME_FORMAT), - ) - assert_layered_property_value( - month_record, - ["lastLog", "date"], - expected_last_log_date.strftime(DATETIME_FORMAT), - ) - assert_property_value(month_record, "logCount", expected_log_count) - assert_property_value(month_record, "allUsers", expected_user_count) - assert_property_value(month_record, "humanUsers", expected_human_count) - assert_layered_property_count( - month_record, ["logTypeRecords"], expected_log_type_record_count - ) - assert_layered_property_count( - month_record, ["userTypeRecords"], expected_user_type_record_count - ) +from tests.utils import assert_property_value, DATETIME_FORMAT # pylint: disable=too-many-arguments,too-many-positional-arguments From 02d8570ccbb8d4943c2e75185730807d70274c03 Mon Sep 17 00:00:00 2001 From: Ricki Jay Date: Tue, 19 Nov 2024 01:43:23 +0100 Subject: [PATCH 29/29] Data Checks --- .../wikibase_log_observation_checkpoint.json | 16 +++---- ...ase_log_observation_expectation_suite.json | 32 ------------- ...og_observation_month_expetation_suite.json | 26 ++++++++++ ...ase_log_observation_expectation_suite.json | 47 ------------------- ...g_observation_month_expectation_suite.json | 35 +++++++------- data/gx/great_expectations.yml | 18 ++----- ...ervation_month_validation_definition.json} | 10 ++-- ...servation_month_validation_definition.json | 22 +++++++++ ...log_observation_validation_definition.json | 22 --------- ...log_observation_validation_definition.json | 22 --------- 10 files changed, 82 insertions(+), 168 deletions(-) delete mode 100644 data/gx/expectations/valid_wikibase_log_observation_expectation_suite.json create mode 100644 data/gx/expectations/valid_wikibase_log_observation_month_expetation_suite.json delete mode 100644 data/gx/expectations/wikibase_log_observation_expectation_suite.json rename data/gx/validation_definitions/{obs_wikibase_log_observation_validation_definition.json => obs_wikibase_log_observation_month_validation_definition.json} (53%) create mode 100644 data/gx/validation_definitions/valid_wikibase_log_observation_month_validation_definition.json delete mode 100644 data/gx/validation_definitions/valid_wikibase_log_observation_validation_definition.json delete mode 100644 data/gx/validation_definitions/wikibase_log_observation_validation_definition.json diff --git a/data/gx/checkpoints/wikibase_log_observation_checkpoint.json b/data/gx/checkpoints/wikibase_log_observation_checkpoint.json index 553b381..32ed03b 100644 --- a/data/gx/checkpoints/wikibase_log_observation_checkpoint.json +++ b/data/gx/checkpoints/wikibase_log_observation_checkpoint.json @@ -11,21 +11,17 @@ "result_format": { "result_format": "COMPLETE" }, "validation_definitions": [ { - "id": "93e6d1cf-7d1a-4b77-bfe0-73b3b6b2835b", - "name": "obs_wikibase_log_observation_validation_definition" - }, - { - "id": "4b10a864-7fa1-4a53-9c3d-9d83170a4f88", - "name": "wikibase_log_observation_validation_definition" - }, - { - "id": "3b4b23fa-ae5d-4e56-826c-aff3d8692ea3", - "name": "valid_wikibase_log_observation_validation_definition" + "id": "5493f4ed-c5a0-4ef1-8c6f-27a213dbc067", + "name": "obs_wikibase_log_observation_month_validation_definition" }, { "id": "636b350a-c392-4fde-b855-4da9dae3378a", "name": "wikibase_log_observation_month_validation_definition" }, + { + "id": "aaf9cfe1-62f8-4383-a467-ac16567c5ac1", + "name": "valid_wikibase_log_observation_month_validation_definition" + }, { "id": "d7831039-f3ab-4726-a84a-ae6258d00931", "name": "nonzero_wikibase_log_observation_month_validation_definition" diff --git a/data/gx/expectations/valid_wikibase_log_observation_expectation_suite.json b/data/gx/expectations/valid_wikibase_log_observation_expectation_suite.json deleted file mode 100644 index 46af8d7..0000000 --- a/data/gx/expectations/valid_wikibase_log_observation_expectation_suite.json +++ /dev/null @@ -1,32 +0,0 @@ -{ - "expectations": [ - { - "id": "a51a3257-666b-4269-a2fb-64a5af988d21", - "kwargs": { "column": "first_log_date" }, - "meta": {}, - "type": "expect_column_values_to_not_be_null" - }, - { - "id": "d7757fa4-8dc8-4291-9a9a-8aa16bcf4933", - "kwargs": { "column": "last_log_date" }, - "meta": {}, - "type": "expect_column_values_to_not_be_null" - }, - { - "id": "b7abe3e6-d269-4767-bf9e-1ae6e4c8f654", - "kwargs": { "column": "last_log_user_type" }, - "meta": {}, - "type": "expect_column_values_to_not_be_null" - }, - { - "id": "e1cbaf19-fac4-4c62-a425-2da1fb3e4402", - "kwargs": { "column": "last_month_id" }, - "meta": {}, - "type": "expect_column_values_to_not_be_null" - } - ], - "id": "cfaf8d67-eb62-4596-86ea-0d49d9efd676", - "meta": { "great_expectations_version": "1.0.0" }, - "name": "valid_wikibase_log_observation_expectation_suite", - "notes": null -} diff --git a/data/gx/expectations/valid_wikibase_log_observation_month_expetation_suite.json b/data/gx/expectations/valid_wikibase_log_observation_month_expetation_suite.json new file mode 100644 index 0000000..3d02ddc --- /dev/null +++ b/data/gx/expectations/valid_wikibase_log_observation_month_expetation_suite.json @@ -0,0 +1,26 @@ +{ + "expectations": [ + { + "id": "eed8d4dd-5c7f-4c5a-ad33-d988e95f41ac", + "kwargs": { "column": "log_count" }, + "meta": {}, + "type": "expect_column_values_to_not_be_null" + }, + { + "id": "2d60689d-96de-45fb-b9ad-e750676ce331", + "kwargs": { "column": "user_count" }, + "meta": {}, + "type": "expect_column_values_to_not_be_null" + }, + { + "id": "9a9e385e-6b2d-44dd-9b83-186806d5c17c", + "kwargs": { "column": "user_count_no_bot" }, + "meta": {}, + "type": "expect_column_values_to_not_be_null" + } + ], + "id": "e7cf9760-5c98-4050-be83-e93c5db55fda", + "meta": { "great_expectations_version": "1.2.0" }, + "name": "valid_wikibase_log_observation_month_expetation_suite", + "notes": null +} diff --git a/data/gx/expectations/wikibase_log_observation_expectation_suite.json b/data/gx/expectations/wikibase_log_observation_expectation_suite.json deleted file mode 100644 index 5a03eb5..0000000 --- a/data/gx/expectations/wikibase_log_observation_expectation_suite.json +++ /dev/null @@ -1,47 +0,0 @@ -{ - "expectations": [ - { - "id": "0647e63f-3ccd-43fd-8f2e-dd10330b6140", - "kwargs": { "column": "first_log_date" }, - "meta": {}, - "type": "expect_column_to_exist" - }, - { - "id": "672ba557-6cd6-4f51-ab70-9e85eae1d93a", - "kwargs": { "column": "last_log_date" }, - "meta": {}, - "type": "expect_column_to_exist" - }, - { - "id": "dcc722eb-d0b9-44a5-be41-c450cc23dd8b", - "kwargs": { "column": "last_log_user_type" }, - "meta": {}, - "type": "expect_column_to_exist" - }, - { - "id": "9263c0a2-3145-40d5-950d-b620ea46b92e", - "kwargs": { - "column": "last_log_user_type", - "value_set": ["BOT", "MISSING", "USER", "NONE"] - }, - "meta": {}, - "type": "expect_column_distinct_values_to_be_in_set" - }, - { - "id": "52d2eefb-e5af-45ed-b03c-223dc201211f", - "kwargs": { "column": "first_month_id" }, - "meta": {}, - "type": "expect_column_to_exist" - }, - { - "id": "6d7c20aa-1ec4-4ba9-8687-1ae277b525b1", - "kwargs": { "column": "last_month_id" }, - "meta": {}, - "type": "expect_column_to_exist" - } - ], - "id": "78e8c824-a077-4257-92b7-c0fed94709df", - "meta": { "great_expectations_version": "1.0.0" }, - "name": "wikibase_log_observation_expectation_suite", - "notes": null -} diff --git a/data/gx/expectations/wikibase_log_observation_month_expectation_suite.json b/data/gx/expectations/wikibase_log_observation_month_expectation_suite.json index c757383..bfa9a5a 100644 --- a/data/gx/expectations/wikibase_log_observation_month_expectation_suite.json +++ b/data/gx/expectations/wikibase_log_observation_month_expectation_suite.json @@ -1,22 +1,22 @@ { "expectations": [ { - "id": "1ce3d321-968f-409f-9d30-51b8262cdff4", - "kwargs": { "column": "id" }, + "id": "92b373dc-4ca4-4e3b-82e8-240787026104", + "kwargs": { "column": "first" }, "meta": {}, "type": "expect_column_to_exist" }, { - "id": "702ce133-2ebc-46d4-a834-0662d54ef247", - "kwargs": { "column": "id" }, + "id": "cd5da7b8-4d1f-4d83-b379-d8db3d4a9f57", + "kwargs": { "column": "first" }, "meta": {}, "type": "expect_column_values_to_not_be_null" }, { - "id": "b914888a-8125-4fbd-ab45-3c30cae5a687", - "kwargs": { "column": "id" }, + "id": "8525b570-4478-455c-8567-01d84dd5c2ab", + "kwargs": { "column": "first", "value_set": [false, true] }, "meta": {}, - "type": "expect_column_values_to_be_unique" + "type": "expect_column_values_to_be_in_set" }, { "id": "1fde4329-71b4-40a0-8f46-a9d5d71e6cc0", @@ -31,28 +31,31 @@ "type": "expect_column_to_exist" }, { - "id": "7e4800d9-4ab5-4dad-93fe-0ad77459c5b4", - "kwargs": { "column": "log_count" }, + "id": "ca504c90-79db-46a8-8b81-c925e28b2d78", + "kwargs": { "column": "last_log_user_type" }, "meta": {}, "type": "expect_column_to_exist" }, { - "id": "e8a7b3ca-333b-4176-93c2-ab29b0d65a41", - "kwargs": { "column": "log_count" }, + "id": "7f067007-4d86-4b6a-a1aa-046dea398761", + "kwargs": { + "column": "last_log_user_type", + "value_set": ["BOT", "MISSING", "USER", "NONE"] + }, "meta": {}, - "type": "expect_column_values_to_not_be_null" + "type": "expect_column_values_to_be_in_set" }, { - "id": "92fc91e2-1682-424b-9cd6-44b72ea728d7", - "kwargs": { "column": "user_count" }, + "id": "7e4800d9-4ab5-4dad-93fe-0ad77459c5b4", + "kwargs": { "column": "log_count" }, "meta": {}, "type": "expect_column_to_exist" }, { - "id": "7ff581fa-7fe4-4ba8-aee7-1b8b1703a391", + "id": "92fc91e2-1682-424b-9cd6-44b72ea728d7", "kwargs": { "column": "user_count" }, "meta": {}, - "type": "expect_column_values_to_not_be_null" + "type": "expect_column_to_exist" }, { "id": "5a554d7e-cef1-4c5d-bba8-4d26d733137a", diff --git a/data/gx/great_expectations.yml b/data/gx/great_expectations.yml index e2977c1..427a9dc 100644 --- a/data/gx/great_expectations.yml +++ b/data/gx/great_expectations.yml @@ -146,16 +146,6 @@ fluent_datasources: partitioner: table_name: wikibase_connectivity_observation_object_relationship_count schema_name: - wikibase_log_observation_table: - type: table - id: 66094242-25ea-4a8a-be00-0b21cdea786c - batch_metadata: {} - batch_definitions: - FULL_TABLE: - id: 09db6e09-83a0-4906-a910-94a4f40218bc - partitioner: - table_name: wikibase_log_observation - schema_name: wikibase_log_observation_month_table: type: table id: 254c7785-cfcf-4f90-b564-42cb429eea6e @@ -304,15 +294,15 @@ fluent_datasources: id: b2549e79-5908-4abe-8fb4-544dde87cded partitioner: query: SELECT * FROM wikibase_connectivity_observation WHERE anything - valid_wikibase_log_observations: + valid_wikibase_log_observation_months: type: query - id: 0f77d1c3-5227-49a5-b347-020f5cd85a28 + id: c9935b87-4498-49b6-a9fe-c64705045d8a batch_metadata: {} batch_definitions: FULL_TABLE: - id: f46678c3-344e-4c04-9869-52f8692c7e03 + id: a119863d-469e-49eb-8f10-4a0303545a76 partitioner: - query: SELECT * FROM wikibase_log_observation WHERE anything + query: SELECT * FROM wikibase_log_observation_month WHERE anything valid_wikibase_quantity_observations: type: query id: 0b2995ed-52af-4e48-8b4e-27379c8e9f83 diff --git a/data/gx/validation_definitions/obs_wikibase_log_observation_validation_definition.json b/data/gx/validation_definitions/obs_wikibase_log_observation_month_validation_definition.json similarity index 53% rename from data/gx/validation_definitions/obs_wikibase_log_observation_validation_definition.json rename to data/gx/validation_definitions/obs_wikibase_log_observation_month_validation_definition.json index 31930d4..76b48a4 100644 --- a/data/gx/validation_definitions/obs_wikibase_log_observation_validation_definition.json +++ b/data/gx/validation_definitions/obs_wikibase_log_observation_month_validation_definition.json @@ -1,11 +1,11 @@ { "data": { "asset": { - "id": "66094242-25ea-4a8a-be00-0b21cdea786c", - "name": "wikibase_log_observation_table" + "id": "254c7785-cfcf-4f90-b564-42cb429eea6e", + "name": "wikibase_log_observation_month_table" }, "batch_definition": { - "id": "09db6e09-83a0-4906-a910-94a4f40218bc", + "id": "e00ad34a-5a21-4fa2-8f71-e2a524063cd2", "name": "FULL_TABLE" }, "datasource": { @@ -13,8 +13,8 @@ "name": "wikibase_datasource" } }, - "id": "93e6d1cf-7d1a-4b77-bfe0-73b3b6b2835b", - "name": "obs_wikibase_log_observation_validation_definition", + "id": "5493f4ed-c5a0-4ef1-8c6f-27a213dbc067", + "name": "obs_wikibase_log_observation_month_validation_definition", "suite": { "id": "a5f967b2-e9e0-4b2c-8dd3-46acbbb2401a", "name": "wikibase_observation_expectation_suite" diff --git a/data/gx/validation_definitions/valid_wikibase_log_observation_month_validation_definition.json b/data/gx/validation_definitions/valid_wikibase_log_observation_month_validation_definition.json new file mode 100644 index 0000000..e87590e --- /dev/null +++ b/data/gx/validation_definitions/valid_wikibase_log_observation_month_validation_definition.json @@ -0,0 +1,22 @@ +{ + "data": { + "asset": { + "id": "c9935b87-4498-49b6-a9fe-c64705045d8a", + "name": "valid_wikibase_log_observation_months" + }, + "batch_definition": { + "id": "a119863d-469e-49eb-8f10-4a0303545a76", + "name": "FULL_TABLE" + }, + "datasource": { + "id": "4a58b404-1ee3-4e91-a373-78d1b365f987", + "name": "wikibase_datasource" + } + }, + "id": "aaf9cfe1-62f8-4383-a467-ac16567c5ac1", + "name": "valid_wikibase_log_observation_month_validation_definition", + "suite": { + "id": "e7cf9760-5c98-4050-be83-e93c5db55fda", + "name": "valid_wikibase_log_observation_month_expetation_suite" + } +} diff --git a/data/gx/validation_definitions/valid_wikibase_log_observation_validation_definition.json b/data/gx/validation_definitions/valid_wikibase_log_observation_validation_definition.json deleted file mode 100644 index 0b8a167..0000000 --- a/data/gx/validation_definitions/valid_wikibase_log_observation_validation_definition.json +++ /dev/null @@ -1,22 +0,0 @@ -{ - "data": { - "asset": { - "id": "0f77d1c3-5227-49a5-b347-020f5cd85a28", - "name": "valid_wikibase_log_observations" - }, - "batch_definition": { - "id": "f46678c3-344e-4c04-9869-52f8692c7e03", - "name": "FULL_TABLE" - }, - "datasource": { - "id": "4a58b404-1ee3-4e91-a373-78d1b365f987", - "name": "wikibase_datasource" - } - }, - "id": "3b4b23fa-ae5d-4e56-826c-aff3d8692ea3", - "name": "valid_wikibase_log_observation_validation_definition", - "suite": { - "id": "cfaf8d67-eb62-4596-86ea-0d49d9efd676", - "name": "valid_wikibase_log_observation_expectation_suite" - } -} diff --git a/data/gx/validation_definitions/wikibase_log_observation_validation_definition.json b/data/gx/validation_definitions/wikibase_log_observation_validation_definition.json deleted file mode 100644 index 58b13d2..0000000 --- a/data/gx/validation_definitions/wikibase_log_observation_validation_definition.json +++ /dev/null @@ -1,22 +0,0 @@ -{ - "data": { - "asset": { - "id": "66094242-25ea-4a8a-be00-0b21cdea786c", - "name": "wikibase_log_observation_table" - }, - "batch_definition": { - "id": "09db6e09-83a0-4906-a910-94a4f40218bc", - "name": "FULL_TABLE" - }, - "datasource": { - "id": "4a58b404-1ee3-4e91-a373-78d1b365f987", - "name": "wikibase_datasource" - } - }, - "id": "4b10a864-7fa1-4a53-9c3d-9d83170a4f88", - "name": "wikibase_log_observation_validation_definition", - "suite": { - "id": "78e8c824-a077-4257-92b7-c0fed94709df", - "name": "wikibase_log_observation_expectation_suite" - } -}