#! /usr/bin/env python import ipdb import pickle import random from config import Config from dbconfig import engine, Session, scoped_session from log import log from mastodon import Mastodon from models import ( Accounts, Attachments, Base, Hashtags, Posts, PostTags, ) MAXINT = 2147483647 TESTDATA = "/home/kae/git/urma/hometl.pickle" # Mastodon.create_app( # 'urma', # api_base_url='mastodon.org.uk', # to_file='urma_clientcred.secret' # ) # API_BASE_URL = 'mastodon.org.uk' # mastodon = Mastodon(client_id = 'urma_clientcred.secret',) # mastodon.log_in('kae@midnighthax.com', '^ZUaiC8P6vLV49', # to_file='urma_usercred.secret') # hometl = Mastodon.timeline() # hometl = mastodon.timeline() # hometl # len(hometl) # hometl0=hometl[0] # hometl0 # history # mastodon.me() # following=mastodon.account_following(kaeid) # len(following) # following[0] # following[39] # following._pagination_next # following._pagination_prev # history Base.metadata.create_all(engine) # mastodon = Mastodon(access_token=Config.ACCESS_TOKEN) def process_post(post): rec = Posts.get_or_create(session, str(post.id)) if rec.account_id is not None: # We already have this post return # Create account record if needed account_rec = Accounts.get_or_create(session, str(post.account.id)) if account_rec.username is None: account_rec.username = post.account.username account_rec.acct = post.account.acct account_rec.display_name = post.account.display_name account_rec.bot = post.account.bot account_rec.url = post.account.url rec.account_id = account_rec.id # Create hashtag records as needed for tag in post.tags: hashtag = Hashtags.get_or_create(session, tag.name, tag.url) rec.hashtags.append(hashtag) # Handle media for media in post.media_attachments: media_rec = Attachments.get_or_create(session, str(media.id), rec.id) if not media_rec.type: media_rec.type = media.type media_rec.url = media.url media_rec.preview_url = media.preview_url media_rec.description = media.description rec.account_id = account_rec.id rec.created_at = post.created_at rec.uri = post.uri rec.url = post.url rec.content = post.content if post.reblog: rec.child_id = process_post(post.reblog).id return rec # Data for development with open(TESTDATA, "rb") as inp: hometl = pickle.load(inp) with Session() as session: for post in hometl: process_post(post)