Tests run, database populates

app/config.py

@@ -18,6 +18,6 @@ class Config(object):
     MAIL_SERVER = os.environ.get('MAIL_SERVER') or "woodlands.midnighthax.com"
     MAIL_USERNAME = os.environ.get('MAIL_USERNAME')
     MAIL_USE_TLS = os.environ.get('MAIL_USE_TLS') is not None
-    MAX_DAYS_TO_FETCH = 10
+    MAX_POSTS_TO_FETCH = 200
     POINTS_BOOSTED = 1
     POINTS_FAVOURITED = 1

app/models.py

@@ -175,9 +175,6 @@ class Posts(Base):
     account_id = Column(Integer, ForeignKey('accounts.id'), nullable=True)
     account = relationship("Accounts", foreign_keys=[account_id])
 
-    boosted_by_id = Column(Integer, ForeignKey('accounts.id'), nullable=True)
-    boosted_by = relationship("Accounts", foreign_keys=[boosted_by_id])
-
     created_at = Column(DateTime, index=True, default=None)
     uri = Column(String(256), index=False)
 
@@ -185,7 +182,6 @@ class Posts(Base):
     hashtags = association_proxy("posts_to_tags", "hashtag")
 
     favourited = Column(Boolean, index=True, nullable=False, default=False)
-    boosted = Column(Boolean, index=True, nullable=False, default=False)
     bookmarked = Column(Boolean, index=True, nullable=False, default=False)
 
     def __repr__(self) -> str:

app/urma.py

@@ -82,45 +82,6 @@ class MastodonAPI:
 
         return results
 
-    def get_boosted(self, since: int) -> List[dict]:
-        """
-        Return posts boosted since id 'since'
-        """
-
-        results = []
-        data = self.mastodon.account_statuses(self.me.id)
-        while data:
-            for datum in data:
-                # Have we reached minimum id?
-                if datum.id < since:
-                    break
-                # Is this a post that we boosted?
-                if datum.account.id == self.me.id and datum.reblog:
-                    # Add in new data
-                    results.append(datum)
-            # Get more data
-            data = self.mastodon.fetch_next(data)
-
-        return results
-
-    def get_favourited(self, since: Union[int, List[dict]]) -> List[dict]:
-        """
-        Return posts favourite since id 'since'
-        """
-
-        results = []
-        data = self.mastodon.favourites()
-        while data:
-            # Add in new data
-            results.extend(data)
-            # Have we reached minimum id?
-            if min([a.id for a in data]) < since:
-                break
-            # Get more data
-            data = self.mastodon.fetch_next(data)
-
-        return results
-
     def get_hashtag_following(self):
         """
         Return a list of hashtag_dicts that we are following
@@ -148,49 +109,47 @@ def main() -> None:
     mastapi = MastodonAPI(Config.ACCESS_TOKEN)
 
     with Session() as session:
-        since = get_since_id(session)
-
         update_followed_accounts(session, mastapi)
         update_followed_hashtags(session, mastapi)
 
-        favourited = mastapi.get_favourited(since)
+        get_and_process_favourited(session, mastapi)
-        process_favourited_posts(session, favourited, mastapi.me.id)
+        get_and_process_bookmarked(session, mastapi)
-
-        boosted_by_me = mastapi.get_boosted(since)
-        process_posts_boosted_by_me(session, boosted_by_me, mastapi.me.id)
-
-        bookmarked = mastapi.get_bookmarked(since)
-        process_bookmarked_posts(session, bookmarked, mastapi.me.id)
-        # TODO: mastapi.unbookmark(int(post.id))
 
 
-def get_since_id(session: Session) -> int:
+def get_and_process_bookmarked(session, mastapi):
-    """
+    """Get newly bookmarked posts and add to db"""
-    Return id to use as 'min_id' when fetching posts.
 
-    We don't want to fetch anything older than MAX_DAYS_TO_FETCH.
+    posts_fetched = 0
-    """
 
-    # Build psuedo id for MAX_DAYS_TO_FETCH time ago
+    bookmarked = mastapi.mastodon.bookmarks()
-    now = datetime.datetime.now()
+    while bookmarked and posts_fetched <= Config.MAX_POSTS_TO_FETCH:
-    max_days_ago_dt = now - datetime.timedelta(days=Config.MAX_DAYS_TO_FETCH)
+        posts_fetched += len(bookmarked)
-    # From mastodon.py package, use code from internals.py:__unpack_id
+        if process_bookmarked_posts(session, bookmarked, mastapi.me.id):
-    max_days_ago_id = (int(max_days_ago_dt.timestamp()) << 16) * 1000
+            return
-    return max_days_ago_id
+        bookmarked = mastapi.mastodon.fetch_next(bookmarked)
 
-    # Get newest ID from database
-    newest_db_id = Posts.max_post_id(session)
 
-    if not newest_db_id:
+def get_and_process_favourited(session, mastapi):
-        return max_days_ago_id
+    """Get newly favourited posts and add to db"""
-    else:
+
-        return max(max_days_ago_id, newest_db_id)
+    posts_fetched = 0
+
+    favourited = mastapi.mastodon.favourites()
+    while favourited and posts_fetched <= Config.MAX_POSTS_TO_FETCH:
+        posts_fetched += len(favourited)
+        if process_favourited_posts(session, favourited, mastapi.me.id):
+            return
+        favourited = mastapi.mastodon.fetch_next(favourited)
 
 
 def process_bookmarked_posts(session: Session,
-                             posts: List[Posts], me_id: int) -> None:
+                             posts: List[Posts], me_id: int) -> bool:
     """
     Process bookmarked posts
+
+    Stop when we find post has already been marked bookmarked.
+
+    Return True if that's why we stopped, else False.
     """
 
     for post in posts:
@@ -199,25 +158,39 @@ def process_bookmarked_posts(session: Session,
         # posts: ignore.
         if record.favourited:
             continue
-        record.bookmarked = True
+        if record.bookmarked:
-        return
+            return True
+        else:
+            record.bookmarked = True
+            # TODO: mastapi.unbookmark(int(post.id))
+
+    return False
 
 
 def process_favourited_posts(session: Session,
-                             posts: List[Posts], me_id: int) -> None:
+                             posts: List[Posts], me_id: int) -> bool:
     """
-    Process favourited posts
+    Process favourited posts.
+
+    Stop when we find post has already been marked favourited
+
+    Return True if that's why we stopped, else False.
     """
 
     for post in posts:
         if post.favourited:
             record = _process_post(session, post, me_id)
-            record.favourited = True
+            if record.favourited:
+                return True
+            else:
+                record.favourited = True
         else:
             log.debug(
                 f"process_favourited_posts({post.id=}) not favourited"
             )
 
+    return False
+
 
 def _process_post(session: Session, post: Posts, me_id) -> Posts:
     """
@@ -243,9 +216,9 @@ def _process_post(session: Session, post: Posts, me_id) -> Posts:
         boosted_record = _process_post(session, post.reblog, me_id)
         # Record who bosed the post unless it was us
         if post.account.id == me_id:
-            boosted_record.boosted_by_id = None
+            boosted_record.boosting_account_id = None
         else:
-            boosted_record.boosted_by_id = account_rec.id
+            boosted_record.boosting_account_id = account_rec.id
         return boosted_record
 
     rec = Posts.get_or_create(session, str(post.id))
@@ -268,28 +241,6 @@ def _process_post(session: Session, post: Posts, me_id) -> Posts:
     return rec
 
 
-def process_posts_boosted_by_me(session: Session,
-                                posts: List[Posts], me_id: int) -> None:
-    """
-    Process boosted posts
-    """
-
-    for post in posts:
-        if post.account.id != me_id:
-            log.debug(
-                f"{post.id=} in process_posts_boosted_by_me but not "
-                f"boosted by me"
-            )
-        if post.reblog:
-            record = _process_post(session, post, me_id)
-            record.boosted = True
-            record.boosted_by = None
-        else:
-            log.debug(
-                f"{post.id=} in process_boosted_posts but not boosted"
-            )
-
-
 def update_followed_accounts(session: Session, mastapi: MastodonAPI) -> None:
     """
     Retrieve list of followed accounts and update accounts

tests/test_processing_posts.py

@@ -11,7 +11,6 @@ from models import (
 )
 from urma import (
     process_bookmarked_posts,
-    process_posts_boosted_by_me,
     process_favourited_posts,
 )
 
@@ -78,42 +77,3 @@ def test_process_post_hashtags(session):
     expected = ['fdroid', 'apps', 'android', 'foss', 'free', 'AndroidAppRain']
     for hashtag in all_tags:
         assert hashtag.name in expected
-
-
-def test_process_boosts_by_me_no_boost(session):
-    """Test processing boosted posts with no post that's been boosted"""
-
-    with open(BOOSTING_POST, "rb") as inp:
-        post = pickle.load(inp)
-
-    post['reblog'] = None
-
-    process_posts_boosted_by_me(session, [post], ME_ID)
-
-    all_posts = session.execute(select(Posts)).scalars().all()
-    assert len(all_posts) == 0
-
-
-def test_process_boosts_by_me(session):
-    """Test processing posts boosted by me"""
-
-    with open(BOOSTING_POST, "rb") as inp:
-        boosting = pickle.load(inp)
-    with open(BOOSTED_POST, "rb") as inp:
-        boosted = pickle.load(inp)
-
-    process_posts_boosted_by_me(session, [boosting], ME_ID)
-
-    all_posts = session.execute(select(Posts)).scalars().all()
-    assert len(all_posts) == 1
-
-    stored_post = all_posts[0]
-
-    assert boosted.account.id == int(stored_post.account.account_id)
-    assert boosted.id == int(stored_post.post_id)
-
-
-def test_process_favourite_containing_boosted_post(session):
-    """Test processing a favourited post that is a boost"""
-
-    pass