eloqua_upload_helper_functions.py

def upload(line):
    """
    Executes some operation using a dict and returns some output
    
    This is where the Eloqua POST function would go,
    I haven't implemented anything
    :param line: 
    :return: 
    """
    raise NotImplementedError


def load_row(row_file_path):
    with open(row_file_path, 'r') as row_file:
        return int(row_file.read())


def seek_csv_row(csv_file, row_n):
    """
    Seeks to the given row number of a csv DictReader by calling readline

    :param csv_file:
    :param row_n:
    :return:
    """
    for i in range(row_n):
        csv_file.readline()

eloqua_upload_loop.py

from eloqua_upload.helper_functions import upload


def loop_function(csv_file, output_file, row_n, row_file_path):

    try:
        for line in csv_file:
            output_file.append(upload(line))
            row_n += 1

    except Exception as e:
        # If you want to do some operation on the error you could do it here
        # The way I've written it you'd still want to raise the error and fail
        # the loop at this point.
        #
        # You can resolve the error in the data and start up from here,
        # since the line n will be equal to this line.
        #
        # If you want to write some way to set aside lines with errors,
        # continue to process the rest of the uploads,
        # and then handle the error lines separately you could,
        # just make sure you implement it correctly

        raise e

    finally:
        with open(row_file_path, 'w') as row_file:
            row_file.write(str(row_n))

eloqua_upload_run.py

import csv

from eloqua_upload.helper_functions import load_row, seek_csv_row
from eloqua_upload.loop import loop_function


def run_function(input_file_path, output_file_path, row_file_path, operation):
    """
    Sets up file objects and executes operation on each row from input

    :param input_file_path: The input file path
    :param output_file_path: Path to output file for executed function
    :param row_file_path: File that contains row number process ran till
    :param operation: (callable) Callable to be executed, takes file objects as argument
    :return:
    """

    row_n = load_row(row_file_path)

    # I'm using with blocks here so that even an exception in one of the file
    # paths will still close any open files without corrupting them
    # I'm opening the input file in read only mode
    with open(input_file_path, 'r') as input_file:
        csv_file = csv.DictReader(input_file)

        # Seek to the correct row
        seek_csv_row(csv_file=csv_file, row_n=row_n)

        # i'm opening the output file in append mode,
        # since we're going to append to the same file even if crashes occur
        with open(output_file_path, 'a') as output_file:
                operation(csv_file, output_file, row_n, row_file_path)
                

if __name__ == '__main__':
    run_function(
        input_file_path='/path/to/input/file.csv',
        output_file_path='/path/to/output/file.csv',
        row_file_path='/path/to/row/file.txt',
        operation=loop_function
    )