prep_scopus_pmid_scrape.py

import nose
from nose.tools import assert_equals
from tests import slow, online, notimplemented, acceptance

import urllib2
import random
import datasources
from datasources import pubmed
from datasources import urlopener

#urllib2.install_opener(opener)

def test_get_lists_of_length():
    response = get_lists_of_length(range(7), 4)
    assert_equals(response, [[0, 1, 2, 3], [4, 5, 6]])
    
def get_lists_of_length(seq, n):
    out = []
    last = 0
    while last < len(seq):
        out.append(seq[last:last + n])
        last += n
    return out

import urllib2
import datasources
from datasources import pubmed
from datasources import urlopener

def test_get_pmids_from_gist_query():
    gist_url = "http://gist.github.com/raw/377863/fba998429b36d07c304a28a07c4dfde592b0c4f7/sensorimotor_fmri_pubmed_query.txt"
    response = get_pmids_from_gist_query(gist_url)
    assert_equals(response[0:5], ['20092581', '20055536', '20053105', '20053076', '20037218'])
    assert_equals(len(response), 2851)

def get_pmids_from_gist_query(query_text_url):
    query_text = urllib2.urlopen(query_text_url).read()
    all_pmids = pubmed.search(query_text)
    return(all_pmids)
    
    
def test_get_pmid_string_for_scopus():
    #response = get_pmid_string_for_scopus(number_to_return, number_to_concat, query_text_urls)
    response = get_pmid_string_for_scopus([str(a) for a in range(5)], 2)
    assert_equals(response, ['pmid(0) OR pmid(1)', 'pmid(2) OR pmid(3)', 'pmid(4)'])

def get_pmid_string_for_scopus(active_pmids, number_to_concat):
    wrapped_pmids = ["pmid(" + pmid + ")" for pmid in active_pmids]
    print " OR ".join(wrapped_pmids)
    pmid_lists = get_lists_of_length(wrapped_pmids, number_to_concat)
    active_concats = [" OR ".join(pmid_list) for pmid_list in pmid_lists]
    for concat, count in zip(active_concats, range(len(active_concats))):
        print
        print count+1
        print concat
    return active_concats


def running(query_text_urls, additional_filter, number_to_return, number_to_concat):
    for query in query_text_urls:
        print "\n\n==>", query
        all_pmids = get_pmids_from_gist_query(query_text_urls[query])
        filtered_pmids = pubmed.filter_pmids(all_pmids, additional_filter)
        random.shuffle(filtered_pmids)
        active_pmids = filtered_pmids[0:number_to_return]
        active_concats = get_pmid_string_for_scopus(active_pmids, number_to_concat)
        


query_text_urls = dict()
query_text_urls["sensorimotor"]     = "http://gist.github.com/raw/377863/fba998429b36d07c304a28a07c4dfde592b0c4f7/sensorimotor_fmri_pubmed_query.txt"
query_text_urls["temporalvisual"]   = "http://gist.github.com/raw/377890/95fcdcbe70f940bfb2a2ceab2975236b79beac30/temporalvisual_fmri_pubmed_query.txt"
query_text_urls["highercog"]        = "http://gist.github.com/raw/377893/9177f550cf4b3c0708352af299f01c0dbc0b29f8/highercog_fmri_pubmed_query.txt"

additional_filter = '("2009"[PDAT] : "2009"[PDAT])'

number_to_return = 500
number_to_concat = 40

running(query_text_urls, additional_filter, number_to_return, number_to_concat)