kae/urma
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Tests run, database populates

Browse Source
This commit is contained in:
Keith Edmunds 2023-01-18 21:23:23 +00:00
parent 66d9c7c6b5
commit b15c1029ef
4 changed files with 49 additions and 142 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
app/config.py
View File

@ -18,6 +18,6 @@ class Config(object):
MAIL_SERVER = os.environ.get('MAIL_SERVER') or "woodlands.midnighthax.com"
MAIL_USERNAME = os.environ.get('MAIL_USERNAME')
MAIL_USE_TLS = os.environ.get('MAIL_USE_TLS') is not None
MAX_DAYS_TO_FETCH = 10
MAX_POSTS_TO_FETCH = 200
POINTS_BOOSTED = 1
POINTS_FAVOURITED = 1

4
app/models.py
View File

@ -175,9 +175,6 @@ class Posts(Base):
account_id = Column(Integer, ForeignKey('accounts.id'), nullable=True)
account = relationship("Accounts", foreign_keys=[account_id])
boosted_by_id = Column(Integer, ForeignKey('accounts.id'), nullable=True)
boosted_by = relationship("Accounts", foreign_keys=[boosted_by_id])
created_at = Column(DateTime, index=True, default=None)
uri = Column(String(256), index=False)
@ -185,7 +182,6 @@ class Posts(Base):
hashtags = association_proxy("posts_to_tags", "hashtag")
favourited = Column(Boolean, index=True, nullable=False, default=False)
boosted = Column(Boolean, index=True, nullable=False, default=False)
bookmarked = Column(Boolean, index=True, nullable=False, default=False)
def __repr__(self) -> str:

145
app/urma.py
View File

@ -82,45 +82,6 @@ class MastodonAPI:
return results
def get_boosted(self, since: int) -> List[dict]:
"""
Return posts boosted since id 'since'
"""
results = []
data = self.mastodon.account_statuses(self.me.id)
while data:
for datum in data:
# Have we reached minimum id?
if datum.id < since:
break
# Is this a post that we boosted?
if datum.account.id == self.me.id and datum.reblog:
# Add in new data
results.append(datum)
# Get more data
data = self.mastodon.fetch_next(data)
return results
def get_favourited(self, since: Union[int, List[dict]]) -> List[dict]:
"""
Return posts favourite since id 'since'
"""
results = []
data = self.mastodon.favourites()
while data:
# Add in new data
results.extend(data)
# Have we reached minimum id?
if min([a.id for a in data]) < since:
break
# Get more data
data = self.mastodon.fetch_next(data)
return results
def get_hashtag_following(self):
"""
Return a list of hashtag_dicts that we are following
@ -148,49 +109,47 @@ def main() -> None:
mastapi = MastodonAPI(Config.ACCESS_TOKEN)
with Session() as session:
since = get_since_id(session)
update_followed_accounts(session, mastapi)
update_followed_hashtags(session, mastapi)
favourited = mastapi.get_favourited(since)
process_favourited_posts(session, favourited, mastapi.me.id)
boosted_by_me = mastapi.get_boosted(since)
process_posts_boosted_by_me(session, boosted_by_me, mastapi.me.id)
bookmarked = mastapi.get_bookmarked(since)
process_bookmarked_posts(session, bookmarked, mastapi.me.id)
# TODO: mastapi.unbookmark(int(post.id))
get_and_process_favourited(session, mastapi)
get_and_process_bookmarked(session, mastapi)
def get_since_id(session: Session) -> int:
"""
Return id to use as 'min_id' when fetching posts.
def get_and_process_bookmarked(session, mastapi):
"""Get newly bookmarked posts and add to db"""
We don't want to fetch anything older than MAX_DAYS_TO_FETCH.
"""
posts_fetched = 0
# Build psuedo id for MAX_DAYS_TO_FETCH time ago
now = datetime.datetime.now()
max_days_ago_dt = now - datetime.timedelta(days=Config.MAX_DAYS_TO_FETCH)
# From mastodon.py package, use code from internals.py:__unpack_id
max_days_ago_id = (int(max_days_ago_dt.timestamp()) << 16) * 1000
return max_days_ago_id
bookmarked = mastapi.mastodon.bookmarks()
while bookmarked and posts_fetched <= Config.MAX_POSTS_TO_FETCH:
posts_fetched += len(bookmarked)
if process_bookmarked_posts(session, bookmarked, mastapi.me.id):
return
bookmarked = mastapi.mastodon.fetch_next(bookmarked)
# Get newest ID from database
newest_db_id = Posts.max_post_id(session)
if not newest_db_id:
return max_days_ago_id
else:
return max(max_days_ago_id, newest_db_id)
def get_and_process_favourited(session, mastapi):
"""Get newly favourited posts and add to db"""
posts_fetched = 0
favourited = mastapi.mastodon.favourites()
while favourited and posts_fetched <= Config.MAX_POSTS_TO_FETCH:
posts_fetched += len(favourited)
if process_favourited_posts(session, favourited, mastapi.me.id):
return
favourited = mastapi.mastodon.fetch_next(favourited)
def process_bookmarked_posts(session: Session,
posts: List[Posts], me_id: int) -> None:
posts: List[Posts], me_id: int) -> bool:
"""
Process bookmarked posts
Stop when we find post has already been marked bookmarked.
Return True if that's why we stopped, else False.
"""
for post in posts:
@ -199,25 +158,39 @@ def process_bookmarked_posts(session: Session,
# posts: ignore.
if record.favourited:
continue
record.bookmarked = True
return
if record.bookmarked:
return True
else:
record.bookmarked = True
# TODO: mastapi.unbookmark(int(post.id))
return False
def process_favourited_posts(session: Session,
posts: List[Posts], me_id: int) -> None:
posts: List[Posts], me_id: int) -> bool:
"""
Process favourited posts
Process favourited posts.
Stop when we find post has already been marked favourited
Return True if that's why we stopped, else False.
"""
for post in posts:
if post.favourited:
record = _process_post(session, post, me_id)
record.favourited = True
if record.favourited:
return True
else:
record.favourited = True
else:
log.debug(
f"process_favourited_posts({post.id=}) not favourited"
)
return False
def _process_post(session: Session, post: Posts, me_id) -> Posts:
"""
@ -243,9 +216,9 @@ def _process_post(session: Session, post: Posts, me_id) -> Posts:
boosted_record = _process_post(session, post.reblog, me_id)
# Record who bosed the post unless it was us
if post.account.id == me_id:
boosted_record.boosted_by_id = None
boosted_record.boosting_account_id = None
else:
boosted_record.boosted_by_id = account_rec.id
boosted_record.boosting_account_id = account_rec.id
return boosted_record
rec = Posts.get_or_create(session, str(post.id))
@ -268,28 +241,6 @@ def _process_post(session: Session, post: Posts, me_id) -> Posts:
return rec
def process_posts_boosted_by_me(session: Session,
posts: List[Posts], me_id: int) -> None:
"""
Process boosted posts
"""
for post in posts:
if post.account.id != me_id:
log.debug(
f"{post.id=} in process_posts_boosted_by_me but not "
f"boosted by me"
)
if post.reblog:
record = _process_post(session, post, me_id)
record.boosted = True
record.boosted_by = None
else:
log.debug(
f"{post.id=} in process_boosted_posts but not boosted"
)
def update_followed_accounts(session: Session, mastapi: MastodonAPI) -> None:
"""
Retrieve list of followed accounts and update accounts

40
tests/test_processing_posts.py
View File

@ -11,7 +11,6 @@ from models import (
)
from urma import (
process_bookmarked_posts,
process_posts_boosted_by_me,
process_favourited_posts,
)
@ -78,42 +77,3 @@ def test_process_post_hashtags(session):
expected = ['fdroid', 'apps', 'android', 'foss', 'free', 'AndroidAppRain']
for hashtag in all_tags:
assert hashtag.name in expected
def test_process_boosts_by_me_no_boost(session):
"""Test processing boosted posts with no post that's been boosted"""
with open(BOOSTING_POST, "rb") as inp:
post = pickle.load(inp)
post['reblog'] = None
process_posts_boosted_by_me(session, [post], ME_ID)
all_posts = session.execute(select(Posts)).scalars().all()
assert len(all_posts) == 0
def test_process_boosts_by_me(session):
"""Test processing posts boosted by me"""
with open(BOOSTING_POST, "rb") as inp:
boosting = pickle.load(inp)
with open(BOOSTED_POST, "rb") as inp:
boosted = pickle.load(inp)
process_posts_boosted_by_me(session, [boosting], ME_ID)
all_posts = session.execute(select(Posts)).scalars().all()
assert len(all_posts) == 1
stored_post = all_posts[0]
assert boosted.account.id == int(stored_post.account.account_id)
assert boosted.id == int(stored_post.post_id)
def test_process_favourite_containing_boosted_post(session):
"""Test processing a favourited post that is a boost"""
pass