diff --git a/app/config.py b/app/config.py index 7ab1dea..0937481 100644 --- a/app/config.py +++ b/app/config.py @@ -18,6 +18,6 @@ class Config(object): MAIL_SERVER = os.environ.get('MAIL_SERVER') or "woodlands.midnighthax.com" MAIL_USERNAME = os.environ.get('MAIL_USERNAME') MAIL_USE_TLS = os.environ.get('MAIL_USE_TLS') is not None - MAX_DAYS_TO_FETCH = 10 + MAX_POSTS_TO_FETCH = 200 POINTS_BOOSTED = 1 POINTS_FAVOURITED = 1 diff --git a/app/models.py b/app/models.py index d0fec68..b3e8b56 100644 --- a/app/models.py +++ b/app/models.py @@ -175,9 +175,6 @@ class Posts(Base): account_id = Column(Integer, ForeignKey('accounts.id'), nullable=True) account = relationship("Accounts", foreign_keys=[account_id]) - boosted_by_id = Column(Integer, ForeignKey('accounts.id'), nullable=True) - boosted_by = relationship("Accounts", foreign_keys=[boosted_by_id]) - created_at = Column(DateTime, index=True, default=None) uri = Column(String(256), index=False) @@ -185,7 +182,6 @@ class Posts(Base): hashtags = association_proxy("posts_to_tags", "hashtag") favourited = Column(Boolean, index=True, nullable=False, default=False) - boosted = Column(Boolean, index=True, nullable=False, default=False) bookmarked = Column(Boolean, index=True, nullable=False, default=False) def __repr__(self) -> str: diff --git a/app/urma.py b/app/urma.py index 413dabc..3ac155f 100755 --- a/app/urma.py +++ b/app/urma.py @@ -82,45 +82,6 @@ class MastodonAPI: return results - def get_boosted(self, since: int) -> List[dict]: - """ - Return posts boosted since id 'since' - """ - - results = [] - data = self.mastodon.account_statuses(self.me.id) - while data: - for datum in data: - # Have we reached minimum id? - if datum.id < since: - break - # Is this a post that we boosted? - if datum.account.id == self.me.id and datum.reblog: - # Add in new data - results.append(datum) - # Get more data - data = self.mastodon.fetch_next(data) - - return results - - def get_favourited(self, since: Union[int, List[dict]]) -> List[dict]: - """ - Return posts favourite since id 'since' - """ - - results = [] - data = self.mastodon.favourites() - while data: - # Add in new data - results.extend(data) - # Have we reached minimum id? - if min([a.id for a in data]) < since: - break - # Get more data - data = self.mastodon.fetch_next(data) - - return results - def get_hashtag_following(self): """ Return a list of hashtag_dicts that we are following @@ -148,49 +109,47 @@ def main() -> None: mastapi = MastodonAPI(Config.ACCESS_TOKEN) with Session() as session: - since = get_since_id(session) - update_followed_accounts(session, mastapi) update_followed_hashtags(session, mastapi) - favourited = mastapi.get_favourited(since) - process_favourited_posts(session, favourited, mastapi.me.id) - - boosted_by_me = mastapi.get_boosted(since) - process_posts_boosted_by_me(session, boosted_by_me, mastapi.me.id) - - bookmarked = mastapi.get_bookmarked(since) - process_bookmarked_posts(session, bookmarked, mastapi.me.id) - # TODO: mastapi.unbookmark(int(post.id)) + get_and_process_favourited(session, mastapi) + get_and_process_bookmarked(session, mastapi) -def get_since_id(session: Session) -> int: - """ - Return id to use as 'min_id' when fetching posts. +def get_and_process_bookmarked(session, mastapi): + """Get newly bookmarked posts and add to db""" - We don't want to fetch anything older than MAX_DAYS_TO_FETCH. - """ + posts_fetched = 0 - # Build psuedo id for MAX_DAYS_TO_FETCH time ago - now = datetime.datetime.now() - max_days_ago_dt = now - datetime.timedelta(days=Config.MAX_DAYS_TO_FETCH) - # From mastodon.py package, use code from internals.py:__unpack_id - max_days_ago_id = (int(max_days_ago_dt.timestamp()) << 16) * 1000 - return max_days_ago_id + bookmarked = mastapi.mastodon.bookmarks() + while bookmarked and posts_fetched <= Config.MAX_POSTS_TO_FETCH: + posts_fetched += len(bookmarked) + if process_bookmarked_posts(session, bookmarked, mastapi.me.id): + return + bookmarked = mastapi.mastodon.fetch_next(bookmarked) - # Get newest ID from database - newest_db_id = Posts.max_post_id(session) - if not newest_db_id: - return max_days_ago_id - else: - return max(max_days_ago_id, newest_db_id) +def get_and_process_favourited(session, mastapi): + """Get newly favourited posts and add to db""" + + posts_fetched = 0 + + favourited = mastapi.mastodon.favourites() + while favourited and posts_fetched <= Config.MAX_POSTS_TO_FETCH: + posts_fetched += len(favourited) + if process_favourited_posts(session, favourited, mastapi.me.id): + return + favourited = mastapi.mastodon.fetch_next(favourited) def process_bookmarked_posts(session: Session, - posts: List[Posts], me_id: int) -> None: + posts: List[Posts], me_id: int) -> bool: """ Process bookmarked posts + + Stop when we find post has already been marked bookmarked. + + Return True if that's why we stopped, else False. """ for post in posts: @@ -199,25 +158,39 @@ def process_bookmarked_posts(session: Session, # posts: ignore. if record.favourited: continue - record.bookmarked = True - return + if record.bookmarked: + return True + else: + record.bookmarked = True + # TODO: mastapi.unbookmark(int(post.id)) + + return False def process_favourited_posts(session: Session, - posts: List[Posts], me_id: int) -> None: + posts: List[Posts], me_id: int) -> bool: """ - Process favourited posts + Process favourited posts. + + Stop when we find post has already been marked favourited + + Return True if that's why we stopped, else False. """ for post in posts: if post.favourited: record = _process_post(session, post, me_id) - record.favourited = True + if record.favourited: + return True + else: + record.favourited = True else: log.debug( f"process_favourited_posts({post.id=}) not favourited" ) + return False + def _process_post(session: Session, post: Posts, me_id) -> Posts: """ @@ -243,9 +216,9 @@ def _process_post(session: Session, post: Posts, me_id) -> Posts: boosted_record = _process_post(session, post.reblog, me_id) # Record who bosed the post unless it was us if post.account.id == me_id: - boosted_record.boosted_by_id = None + boosted_record.boosting_account_id = None else: - boosted_record.boosted_by_id = account_rec.id + boosted_record.boosting_account_id = account_rec.id return boosted_record rec = Posts.get_or_create(session, str(post.id)) @@ -268,28 +241,6 @@ def _process_post(session: Session, post: Posts, me_id) -> Posts: return rec -def process_posts_boosted_by_me(session: Session, - posts: List[Posts], me_id: int) -> None: - """ - Process boosted posts - """ - - for post in posts: - if post.account.id != me_id: - log.debug( - f"{post.id=} in process_posts_boosted_by_me but not " - f"boosted by me" - ) - if post.reblog: - record = _process_post(session, post, me_id) - record.boosted = True - record.boosted_by = None - else: - log.debug( - f"{post.id=} in process_boosted_posts but not boosted" - ) - - def update_followed_accounts(session: Session, mastapi: MastodonAPI) -> None: """ Retrieve list of followed accounts and update accounts diff --git a/tests/test_processing_posts.py b/tests/test_processing_posts.py index 5bd1177..985319f 100644 --- a/tests/test_processing_posts.py +++ b/tests/test_processing_posts.py @@ -11,7 +11,6 @@ from models import ( ) from urma import ( process_bookmarked_posts, - process_posts_boosted_by_me, process_favourited_posts, ) @@ -78,42 +77,3 @@ def test_process_post_hashtags(session): expected = ['fdroid', 'apps', 'android', 'foss', 'free', 'AndroidAppRain'] for hashtag in all_tags: assert hashtag.name in expected - - -def test_process_boosts_by_me_no_boost(session): - """Test processing boosted posts with no post that's been boosted""" - - with open(BOOSTING_POST, "rb") as inp: - post = pickle.load(inp) - - post['reblog'] = None - - process_posts_boosted_by_me(session, [post], ME_ID) - - all_posts = session.execute(select(Posts)).scalars().all() - assert len(all_posts) == 0 - - -def test_process_boosts_by_me(session): - """Test processing posts boosted by me""" - - with open(BOOSTING_POST, "rb") as inp: - boosting = pickle.load(inp) - with open(BOOSTED_POST, "rb") as inp: - boosted = pickle.load(inp) - - process_posts_boosted_by_me(session, [boosting], ME_ID) - - all_posts = session.execute(select(Posts)).scalars().all() - assert len(all_posts) == 1 - - stored_post = all_posts[0] - - assert boosted.account.id == int(stored_post.account.account_id) - assert boosted.id == int(stored_post.post_id) - - -def test_process_favourite_containing_boosted_post(session): - """Test processing a favourited post that is a boost""" - - pass