103 lines
2.5 KiB
Python
Executable File
103 lines
2.5 KiB
Python
Executable File
#! /usr/bin/env python
|
|
|
|
import ipdb
|
|
import pickle
|
|
import random
|
|
|
|
from config import Config
|
|
from dbconfig import engine, Session, scoped_session
|
|
from log import log
|
|
from mastodon import Mastodon
|
|
from models import (
|
|
Accounts,
|
|
Attachments,
|
|
Base,
|
|
Hashtags,
|
|
Posts,
|
|
PostTags,
|
|
)
|
|
|
|
MAXINT = 2147483647
|
|
TESTDATA = "/home/kae/git/urma/hometl.pickle"
|
|
|
|
# Mastodon.create_app(
|
|
# 'urma',
|
|
# api_base_url='mastodon.org.uk',
|
|
# to_file='urma_clientcred.secret'
|
|
# )
|
|
|
|
# API_BASE_URL = 'mastodon.org.uk'
|
|
|
|
# mastodon = Mastodon(client_id = 'urma_clientcred.secret',)
|
|
# mastodon.log_in('kae@midnighthax.com', '^ZUaiC8P6vLV49',
|
|
# to_file='urma_usercred.secret')
|
|
# hometl = Mastodon.timeline()
|
|
# hometl = mastodon.timeline()
|
|
# hometl
|
|
# len(hometl)
|
|
# hometl0=hometl[0]
|
|
# hometl0
|
|
# history
|
|
# mastodon.me()
|
|
# following=mastodon.account_following(kaeid)
|
|
# len(following)
|
|
# following[0]
|
|
# following[39]
|
|
# following._pagination_next
|
|
# following._pagination_prev
|
|
# history
|
|
|
|
Base.metadata.create_all(engine)
|
|
# mastodon = Mastodon(access_token=Config.ACCESS_TOKEN)
|
|
|
|
|
|
def process_post(post):
|
|
rec = Posts.get_or_create(session, str(post.id))
|
|
if rec.account_id is not None:
|
|
# We already have this post
|
|
return
|
|
|
|
# Create account record if needed
|
|
account_rec = Accounts.get_or_create(session, str(post.account.id))
|
|
if account_rec.username is None:
|
|
account_rec.username = post.account.username
|
|
account_rec.acct = post.account.acct
|
|
account_rec.display_name = post.account.display_name
|
|
account_rec.bot = post.account.bot
|
|
account_rec.url = post.account.url
|
|
rec.account_id = account_rec.id
|
|
|
|
# Create hashtag records as needed
|
|
for tag in post.tags:
|
|
hashtag = Hashtags.get_or_create(session, tag.name, tag.url)
|
|
rec.hashtags.append(hashtag)
|
|
|
|
# Handle media
|
|
for media in post.media_attachments:
|
|
media_rec = Attachments.get_or_create(session, str(media.id), rec.id)
|
|
if not media_rec.type:
|
|
media_rec.type = media.type
|
|
media_rec.url = media.url
|
|
media_rec.preview_url = media.preview_url
|
|
media_rec.description = media.description
|
|
|
|
rec.account_id = account_rec.id
|
|
rec.created_at = post.created_at
|
|
rec.uri = post.uri
|
|
rec.url = post.url
|
|
rec.content = post.content
|
|
|
|
if post.reblog:
|
|
rec.child_id = process_post(post.reblog).id
|
|
|
|
return rec
|
|
|
|
|
|
# Data for development
|
|
with open(TESTDATA, "rb") as inp:
|
|
hometl = pickle.load(inp)
|
|
|
|
with Session() as session:
|
|
for post in hometl:
|
|
process_post(post)
|