Calculate recent upvote efficiency on Steem

getUpvotes.py

from steem import Steem
from steem.account import Account
import pickle
import os

userlist = 'top-users.txt'
#userlist = 'some-users.txt'
outputFile = 'rewards-shares.txt'

with open( userlist, "r" ) as f:
    users = [ l.strip() for l in f ]

users = set( users )
earliest = '2018-07-23'


class Checkpoint:
    def __init__( self ):
        self.upvoted = set()
        self.rewards = {}
        self.rshares = {}
        self.numRewards = {}
        self.numUpvotes = {}

        self.usersVisited = set()
        self.articlesVisited = set()

    def save( self ):
        with open( "checkpoint.tmp", "wb" ) as f:
            pickle.dump( self, f )
        os.rename( "checkpoint.tmp", "checkpoint" )

    def restore():
        try:
            with open( "checkpoint", "rb" ) as f:
                cp = pickle.load( f )
                print( "Checkpoint loaded." )
                print( len( cp.usersVisited ), "/", len( users ),
                       "users visited" )
                print( len( cp.articlesVisited), "/", len( cp.upvoted ),
                       "articles visited" )
                return cp
        except FileNotFoundError:
            print( "No checkpoint!" )
            return Checkpoint()
        except EOFError:
            print( "Invalid checkpoint!" )
            return Checkpoint()
    
s = Steem()
cp = Checkpoint.restore()
iterCount = 0

for u in users.difference( cp.usersVisited ):
    print( "Fetching history for", u, "...", end=" " )
    
    a = Account( u, s )
    total = 0.0
    count = 0
    for a in a.history_reverse( filter_by=['curation_reward'] ):
        if a['timestamp'] < earliest:
            break
        vests, _ = a['reward'].split( " " )
        content = ( a['comment_author'],
                    a['comment_permlink'] )
        total += float( vests )
        count += 1
        cp.upvoted.add( content )

    cp.rewards[u] = total
    cp.numRewards[u] = count
    cp.usersVisited.add( u )
    print( count, "rewards." )
    iterCount += 1

    if iterCount == 20:
        iterCount = 0
        cp.save()
    

toFetch = cp.upvoted.difference( cp.articlesVisited )

print( len( toFetch ), "articles to fetch." )

for (a,p) in toFetch:
    print( "Fetching votes for", a, p )
    c = s.get_content( a, p )
    for v in c['active_votes']:
        voter = v['voter']
        if voter in users:
            cp.rshares[voter] = cp.rshares.get( voter, 0 ) + int( v['rshares'] )
            cp.numUpvotes[voter] = cp.numUpvotes.get( voter, 0 ) + 1
            
    cp.articlesVisited.add( (a,p) )

    iterCount += 1

    if iterCount == 20:
        iterCount = 0
        cp.save()
            
with open( outputFile, "w" ) as out:
    print( "user", "numRewards", "vests", "numUpvotes", "totalRshares",
           "dustEfficiency", "vestEfficiency", 
           file=out )
    for u in sorted( users ):
        if cp.rshares.get( u, 0 ) > 0 and \
           cp.numRewards.get( u, 0 ) > 0 and \
           cp.numUpvotes.get( u, 0 ) > 0:
            dustEfficiency = cp.numRewards[u] / cp.numUpvotes[u]
            efficiency = cp.rewards[u] / ( cp.rshares[u] / 1.0e9 )
            print( u, cp.numRewards[u], cp.rewards[u],
                   cp.numUpvotes[u], cp.rshares[u],
                   dustEfficiency, efficiency, file=out )