import_transactions_bulk.rb

#!/usr/local/bin/ruby

# Used to export Blocks from a Web3 compatible endpoint
# I use:
# parity --geth --jsonrpc-server-threads=4 --mode=offline -d /data/io.parity.ethereum/ --cache-size 65536

require 'web3/eth'
require 'mongoid'

force_sync = ARGV[0]

# Lazy initialise the Transaction class

class Transaction
    include Mongoid::Document
    include Mongoid::Attributes::Dynamic
end

Mongoid.load!("./mongoid.yml", :production)


web3 = Web3::Eth::Rpc.new 

begin
    syncing = web3.eth.syncing
    last_block = syncing["currentBlock"].to_i(16)
rescue => exception
    last_block = web3.eth.blockNumber
end

puts(last_block)


def push_block(txs, working_block)
    Transaction.collection.insert_many(txs)  
    puts("Saving Block #{working_block}")  
end

all_transactions = []

until last_block == 0

    block = web3.eth.getBlockByNumber(last_block, true)
    transactions = []
    for tx in block.transactions
        transactions << {:partition => last_block % 20, :hash => tx.hash, :block_number => last_block, :block_time => block.timestamp_time, :from => tx.from, :to => tx.to, :value => tx.value_eth, :data => tx.input}
    end

    # Make a copy as this is going to get nailed at next iteration when the commit thread runs
    txs = transactions

    
    th1 = Thread.new{push_block(txs, last_block)}
    last_block-=1            

    # Give a bit of grace to CosmosDB - This will be the lever for RU/s throttling
    if Thread.list.length == 50
        sleep(1)
    end 
end