From 66d9c7c6b51c48b52fc5a3e9837d199e18803eb1 Mon Sep 17 00:00:00 2001 From: Keith Edmunds Date: Tue, 17 Jan 2023 18:25:55 +0000 Subject: [PATCH] First pass processing tests run OK --- app/helpers.py | 1 - app/urma.py | 93 +++++++++++++++++--------- tests/boosted.pickle | Bin 0 -> 3157 bytes tests/boosting.pickle | Bin 0 -> 6872 bytes tests/favourited.pickle | Bin 0 -> 5813 bytes tests/test_processing_posts.py | 119 +++++++++++++++++++++++++++++++++ 6 files changed, 179 insertions(+), 34 deletions(-) create mode 100644 tests/boosted.pickle create mode 100644 tests/boosting.pickle create mode 100644 tests/favourited.pickle create mode 100644 tests/test_processing_posts.py diff --git a/app/helpers.py b/app/helpers.py index 5166a44..7f1d6ef 100644 --- a/app/helpers.py +++ b/app/helpers.py @@ -15,7 +15,6 @@ def index_ojects_by_parameter(object_list: List, param: Any): """ Create a dictionary from passed list where each list entry is keyed by passed param - n """ results = {} diff --git a/app/urma.py b/app/urma.py index 3d99906..413dabc 100755 --- a/app/urma.py +++ b/app/urma.py @@ -94,7 +94,7 @@ class MastodonAPI: # Have we reached minimum id? if datum.id < since: break - # Is this our post that we boosted? + # Is this a post that we boosted? if datum.account.id == self.me.id and datum.reblog: # Add in new data results.append(datum) @@ -154,13 +154,14 @@ def main() -> None: update_followed_hashtags(session, mastapi) favourited = mastapi.get_favourited(since) - process_favourited_posts(session, favourited) + process_favourited_posts(session, favourited, mastapi.me.id) - boosted = mastapi.get_boosted(since) - process_boosted_posts(session, boosted) + boosted_by_me = mastapi.get_boosted(since) + process_posts_boosted_by_me(session, boosted_by_me, mastapi.me.id) bookmarked = mastapi.get_bookmarked(since) - process_bookmarked_posts(session, mastapi, bookmarked) + process_bookmarked_posts(session, bookmarked, mastapi.me.id) + # TODO: mastapi.unbookmark(int(post.id)) def get_since_id(session: Session) -> int: @@ -186,53 +187,44 @@ def get_since_id(session: Session) -> int: return max(max_days_ago_id, newest_db_id) -def process_bookmarked_posts(session, mastapi, posts) -> None: +def process_bookmarked_posts(session: Session, + posts: List[Posts], me_id: int) -> None: """ Process bookmarked posts """ for post in posts: - record = _process_post(session, post) + record = _process_post(session, post, me_id) # Posts that are favourited and bookmarked are genuine bookmark # posts: ignore. if record.favourited: continue record.bookmarked = True return - # TODO: mastapi.unbookmark(int(post.id)) -def process_boosted_posts(session, posts) -> None: - """ - Process boosted posts - """ - - for post in posts: - record = _process_post(session, post) - record.boosted = True - - -def process_favourited_posts(session, posts) -> None: +def process_favourited_posts(session: Session, + posts: List[Posts], me_id: int) -> None: """ Process favourited posts """ for post in posts: - record = _process_post(session, post) - record.favourited = True + if post.favourited: + record = _process_post(session, post, me_id) + record.favourited = True + else: + log.debug( + f"process_favourited_posts({post.id=}) not favourited" + ) -def _process_post(session: Session, post) -> Posts: +def _process_post(session: Session, post: Posts, me_id) -> Posts: """ Add passsed post to database """ log.debug(f"{post.id=} processing") - rec = Posts.get_or_create(session, str(post.id)) - if rec.account_id is not None: - # We already have this post - log.debug(f"{post.id=} already in db") - return rec # Create account record if needed log.debug(f"{post.id=} processing {post.account.id=}") @@ -244,7 +236,25 @@ def _process_post(session: Session, post) -> Posts: account_rec.display_name = post.account.display_name account_rec.bot = post.account.bot account_rec.url = post.account.url - rec.account_id = account_rec.id + + if post.reblog: + # We're only interesting the boosted post, not this onej + log.debug(f"{post.id=} {post.reblog.id=}") + boosted_record = _process_post(session, post.reblog, me_id) + # Record who bosed the post unless it was us + if post.account.id == me_id: + boosted_record.boosted_by_id = None + else: + boosted_record.boosted_by_id = account_rec.id + return boosted_record + + rec = Posts.get_or_create(session, str(post.id)) + if rec.account_id is not None: + # We already have this post + log.debug(f"{post.id=} already in db") + return rec + else: + rec.account_id = account_rec.id # Create hashtag records as needed for tag in post.tags: @@ -255,14 +265,31 @@ def _process_post(session: Session, post) -> Posts: rec.created_at = post.created_at rec.uri = post.uri - if post.reblog: - log.debug(f"{post.id=} {post.reblog.id=}") - boosted_rec = _process_post(session, post.reblog) - rec.boosted_by_id = boosted_rec.account_id - return rec +def process_posts_boosted_by_me(session: Session, + posts: List[Posts], me_id: int) -> None: + """ + Process boosted posts + """ + + for post in posts: + if post.account.id != me_id: + log.debug( + f"{post.id=} in process_posts_boosted_by_me but not " + f"boosted by me" + ) + if post.reblog: + record = _process_post(session, post, me_id) + record.boosted = True + record.boosted_by = None + else: + log.debug( + f"{post.id=} in process_boosted_posts but not boosted" + ) + + def update_followed_accounts(session: Session, mastapi: MastodonAPI) -> None: """ Retrieve list of followed accounts and update accounts diff --git a/tests/boosted.pickle b/tests/boosted.pickle new file mode 100644 index 0000000000000000000000000000000000000000..b1c43da69847fd0dcd32fb08930d3955d57e82ae GIT binary patch literal 3157 zcmcguTW=dh6i!+vP12^c66yn`Xk=0W!r5Khxt1hqNr7Hy9-1O5A!=rK$KHu&XO@{+ zyDmZki2|)g0;$6rydfd+3wVXUz(3#t9{B}GoY}RL^eRB2Q4;Tt&wbAMzVk`tn@dN= ziXU?<#Y&4=Ths~eAy*+Mb zB1y0&aSLl}reb*1G$s2V@7c|nLw}6V9G^P{-mE$5gq$Bw--2IjCiJabN2UGG?NyMV zyd_D-1}!aG;BL#D*nfhfNaS3X-%KmQ71gvy>>YDNWr8vyTbkU2aKPD2k184tbDGTZ zkkJT&W0-ex+<`a`5N^%ETvBV!Ct7FfjPJLl;Ci0$*i#~+nEAOPQu!Ss+HFar#8(nLEg)wH$(qoOC-Lvy3^;wrWC8|{2P?4C@T zV{P0M5VPJdHcetUC@x~l%A|zGtosYQVpd#d%2HS@f$&61HCDqAeH`eF$D;ghS9= zfCnNakyWIz>=3c*3U zeU$D#H;>C@q~_xdS6rB!Hf7os(7tcYWZ70QpNEDsX_EsEB$CSAspmKE{&;%(vvD)! zriWt(3e&r-Z%qYMU`C(YJv#f~*jaNVrYd82&??nu-f|O&uEGd{b0(s)a}S0>yWt1n zNI-XB=rY&m9Vzn6-bp5sDL|m#4^Nh8XOr0iWhz`~i0M7(n0HBtpzmsE8+uQZOWP>L zlxz4SvW75^QARPRN_*(H@4o-@r>~$>wW=XEK=9!NdeBz_bs$ZY=88sMQLp5rd2w`w z52oku0Rd!#dl-7c8afh>V(#t2vXC7w??MAc3Gs)uR(`2dppf4Tn*MUV;WyxOrM~Ph zFE05)(hlW-Urhj-l|~zf%ONH~ec9W|NXMGl(m>0>nS;fr>3cFE7$(u0imO* zSE7Z+a&BR<3#^INH&6${eV~9xZm-kWcH!e=xabFYYxN2Q%JeoKu$G+Msq%>&)0)%_0=;A zD`y&YZ?U=hY7m?W0-NL998ZBLF%(k6G^N0JMb)O<69qJPC9G)NDGGSmy#VaTBVape zULW=Ae`g4{@=y&RWnOqPJ3)3aQIVvX3(a2dkB^O=Q;2XFYhWUP9;yq{3<;19W(;l^ zMp?B7j(6KaqFf;@P(igWL0}lihBX8ju83o3Si~S{Cp4nxn&Yp+fZzznJ~WR)?LO zQ-~MF1iXM_N9~&6)GZdEjwK)zNT;|_Mc;(6saUSEo3M9G}ei|^7Xg@RHU6m+snV237QrQ>Ib8K zdpsdzlJqF)w_KwtztB^D5Ulu%3xVHQ3HFGfzOvA)H{vLWf_O1(Mqb7N7pF-Y!u$;5 znhIGi6Rdz^=ia&g#(XVZUqAik_Knwq8yCe#UG}k>o4dZSeQTYFx3@c67i?}S8LT>X HEBF2eNkDnM literal 0 HcmV?d00001 diff --git a/tests/boosting.pickle b/tests/boosting.pickle new file mode 100644 index 0000000000000000000000000000000000000000..dcfa9703ad87d1b008428b5e9c276de2f3f8f552 GIT binary patch literal 6872 zcmcgx&u<&Y6_y;ymK8g7TsIAjIGxZA(xNn%q$rAVEWwVP#EE1Eh+@=eo6xhnL(Z7H zJIv0kD7gsQ0!?EX3#34{x8_!$KrcmaxwiKniv9;ZwCSmTL4kg=%avC0kB~v@ATqhj z_vU-^=Kc8g?ayX@{g<;->3{JlpDHa~8B`)IJfU}z_|#RcLt$Ta9H!JY;ppVE{X(^6)6@f%cf3WVV{=LRO6PiYSratv)QWGTFu7ta=p2*Y*7bhq|U>d-v4^dlg1;0 zsZP52WPK{e9ov(=!ktrHx+Sq6T@*XcT--x25d+ zH0-BW#~c}G7U+bZiRZe)(=5dO_mlW&rkwC2CNB8Q{kf@Eez*3g-+VmHXZc)q=FeU@ zhX_mfNpr6JRGUBB=BIc0H~AUY*%2O8ePMlypIbl5U&QBk+MMk2m+tY> zEWQ zyVCRI5XQ>^3t)60fxyC$g(aeC*kgL7)UlDH{nF}XHGrEPk1DlN527(4c=?eM#=fC~;7oQ?BwJH-+G4Npowh%{>^sr6hM@l%A%s*%2^ZAh- z-k!Celh_U2qKJVc7P+|N29?8A*`W?+D7a}YR92>w7G-Opx?nBU>Q)_}t=f{cwAi#{ zD0(758CgKix9VMLFWHn;YfF_o1J+C8li30tQ{7E}c#_yhIitwDB<2%rIMW$p!!4&# zUvd`NLfvK!w@$HPv)@dAZ@*6v`WspDqp6M>*^U29$D^C5iFtSH#LwidHox^r{0ypq zPK}O~;Tw6E8HMx5hC3W(zPuw;+Mv3Ed9L}IwI~F7ntaD@?4Zb+ke}V6UWCs-KTzfN zlCcsn{r{<4c?l^NxYFuD;D^Z+s#E?ms!$R?jf@xAaz@3@)vNV|i`Cl2TJ^o!@})-W zQoUALTv+~owR)*qO(Gh_&-!2n6pqx$CLb+6_1c{ICQ!YN3uiR#r4HOQ7bx>?Uf9mY zujTRj^*e&$enO*2#Va`P|w2& zBb8Nm$Mo5*3`wMjmLzShW`pP`ru&r%=!{~fBZHA*(?%$7$7R#6(gkEdsJlf22mKMO z7+f$db(%Rm5FbWpu>{so$g9ZHWin($c0rmRpa*FFTq|%0u%t7R<0TK4UvF44N;*W?bAMkwNP*Mq&~PGu0FE*@vTjrRo^>le%Z%F4@cY<}=+^@F$MPx{_R>cWLvjm^6&Om1xU+SijPo*97J zB<%=Z8DtF5gln7E_et{Uzy9&d@->j;Mm(24RwTSxyl53KF75JHclqUed}XhJpLqU2 z1Ls$7ojIN|(+xa-;5Z+sW{2CS$!hBjur8(F>0V8v9ibHBd+03`q^C=Zu zEvgmj&zPV$*C&9wxn2ZF$+5pPCeY>(#ZB=E&_)^uA-l*C%~*g4Brc5*lHjX`<~UqB z(gVXE)1D1#ht!o|xFat6)t=5(3Kpeg3OsEC^lnVxl!Q%yn#va-$AoEC)uc$d%H&{6 zlY`|dlY_M;2g_B8U}a(m{FvVCMFJ#vf>X;B3V3BS$2)RdJk!Z0Kr`Y8u*R!6=*;c} zC`P=1g8u4BG}rO4(!MtVSSnvc8xN+7$rwuf1VAH=B9y5oh9msQ?Oc6M@x? ziMt_!XnTT9rt$@}F=1>&M=)=k04Y}~fSHO7Mmr{E&l4`1<^+%}Y875g*eBlO$#c{fA#zdnj)ObGWqJ7uv6P!w>P%tTj*`-UhroHI2n$B{{FZml!2Il&hFvTgt zV|PPox68T<-K9pW>bR|JiooA~GB6wKxA~3rPk4L%Du3r;kNn31DFT;mW5R{URDSb) zOrx`z^sRXA#wfQE{=@iR4}_k1bj?#Kon)??z*RTQ;OaHGjR*fj^T@9rVGlbh>_k!BJ90N%TeKO29> zldR3}wxf^mGN?14W`Jha=$MBrJobV8dMs^z3Rmxh>GU{%QS>fe6rCJ&fxRn8-3D8FI{|SUwiQsK>`!p#yKDB4t

-o$2|W@nD-ciL(p%D;Yf`-> z?>(RX-Jh;si2i;S>Ua8BD(-NW4;1yN8p68yEWG)va7kaJPWb8k z^MA{mAO7LRlwPz2!HPI-tU^8Sz@wsll70Lvyt_E}uc^gr-<*Z{LVd*unR)W+8T>*$ zqn-uIj_vIH zkDomo?y`G=Q^%4DD;Sklv9MCB)vDD(qgEOZ{_i*;qKaFNyGW zHwumm2p|b8xD^NwCE>kMShwS_j!b7FN*>Xw1#9oG{{Zd6Dxq!IABP_@w8LSy(bDer z&dp}>a!8eXB(RJ=*x%k-Mt57SDE^<|0dCoX`U=Uw_qkB39RXHb1a7NBEg#ziZ8BR7 zeZ>V@YPB}O@O`*C=_qXBGTMh*lo>!v+mH+3*2v}9iDmS~^FQ8PHf94s&>kFsLoOU} zi-DAg=e&b^a9sqt>w+>^>rqCU$KFsXQp%SIiUKRvkW_-O9t7!;0r6zAGMnrPADS%} zXo54^6@(mZ;8%Dujor-dq3`3q|HFwKg2Zwr4uRdQLldk`a=XsLo5b{MI%KZvS$JdY zW?ktD-d{F(A>OsT^t^#Bh7A(Gl|h|QPlFOb<2N#cx3hQ`>dOw5zK4hH zSp8~75`FRe7vO=%?HD&Mb+|IH3mP^b5P>@$0mzx@2_E?2C+@7f0NDS}#j7#X2;D0Y zzE3mAXW`?&TzXBcIg7;~b5=K*asx?$5-m@1r}qRMV0(BL&LyU^M`KTLpECg2iF~`i z+iIPKW5(`FUiD#5#Y9eRQ5HNs3$G%>S+^d+B}jnd4pla%tci+yTbul0zubE;{Gj@% z$R0bkQ>@eu-mi8Z?2DbPpqyyj;|v=rBbz5r0gwzse)1G(q2yhacZlT@&;Jp?E3jrD zSy6IP+^6SM7(%T!bp=0!gvWU=nB)6VYvov5sk5aNiJdgU4%G~;^Jsqc;nc~zy zQavDJQHpZErN5i{lg%bpmhwX80A2!QFVt=-Y&$G}RD1y4$_EFQjcu1$Nph1`tP`V{6%W}E5!kyrk(EM%t*l;RCf7>kTrM_juS7Jwm7$9g@0I?xX z+8BNajp46lmHI@v+<|uJ`