igm_scrape_detail_2.py

for survey_question, table in zip(survey_questions, tables):
    rows = table.findAll('tr')#, {'class': 'parent-row'})
    for row in rows:
        if row.get('class') == ['parent-row']:
            cells = row.findAll('td')
            response = cells[2].get_text().strip()
            confidence = cells[3].get_text().strip()
            comment = cells[4].get_text().strip()
            tmp_data = {
                'survey_date': dtutil.parse(survey_date),
                'topic_name': topic,
                'topic_url': domain + handle,
                'survey_question': survey_question,
                'economist_name': cells[0].get_text().strip(),
                'economist_url':  domain + cells[0].a['href'],
                'economist_headshot': domain + cells[0].img['src'],
                'institution': cells[1].get_text().strip(),
                'response': response,
                'confidence': confidence,
                'comment': comment,
            }
            
        data += [tmp_data]