inspire_institutions.py

def get_institutions(q):
    page=1
    if q:
        url=f'https://inspirehep.net/api/institutions?q={q}&size=250&page={page}'
    else:
        url=f'https://inspirehep.net/api/institutions?size=250&page={page}' 
    r=requests.get(url)
    time.sleep(sleep)
    i=r.json()['hits']['hits']
    total=r.json().get('hits').get('total')
    if not page or page==1:
        ins=[d.get('metadata') for d in i]
    j=page
    next=r.json().get('links').get('next')
    while next:
        print(j)
        r=requests.get(next)
        time.sleep(sleep)
        i=r.json()['hits']['hits']
        ins.extend([d.get('metadata') for d in i])
        next=r.json().get('links').get('next')
        j+=1
        if j > total//250:
            break
    return ins
  
# Maxium results 10 000 → use main countries
kk=[]
for q in ['','u','england','germany','france','italy','spain','korea','china','japan','united states',
    'russia','brazil']:
    print(q)
    ins=get_institutions(q)
    for d in ins:
        if d not in kk:
            kk.append(d)
    print(len(kk))
    if len(kk)>11835:
        break        

results.csv