Get result data from Turk, in a way that readily makes a Pandas DataFrame

responses.py

from collections import OrderedDict
from boto.mturk.connection import MTurkConnection
from dateutil.parser import parse as dateparse

mtc = MTurkConnection(host='mechanicalturk.amazonaws.com')


def responses(hit_group_id):
    responses = []
    for hit in mtc.get_all_hits():
        if hit.HITGroupId != hit_group_id:
            continue

        for assignment in mtc.get_assignments(hit_id=hit.HITId, page_size=100, status='Submitted'):
            startTime = dateparse(assignment.AcceptTime)
            endTime = dateparse(assignment.SubmitTime)
            dur = (endTime - startTime).total_seconds()
            response = OrderedDict(
                assignmentId = assignment.AssignmentId,
                hitId = assignment.HITId,
                workerId=assignment.WorkerId,
                acceptTime = assignment.AcceptTime,
                submitTime = assignment.SubmitTime,
                dur=dur)
            assert len(assignment.answers) == 1
            for answer in assignment.answers[0]:
                assert len(answer.fields) == 1
                response['ans_'+answer.qid.replace('-','_')] = answer.fields[0]
            responses.append(response)

    return responses