scrolling-cmr-search-for-urls.py

from requests import Session


search_url = 'https://cmr.earthdata.nasa.gov/search/granules'
parameters = {
    'scroll': 'true',
    'provider': 'ASF',
    'page_size':  2000,
    # set any search criteria here
    #'short_name': ['SENTINEL-1A_SP_GRD_HIGH','SENTINEL-1B_SP_GRD_HIGH','SENTINEL-1A_DP_GRD_HIGH','SENTINEL-1B_DP_GRD_HIGH'],
}
output_format = 'umm_json'


def process_granules(granules):
    for granule in granules:
        # do whatever you need to do with each granule here
        urls = [item['URL'] for item in granule['umm']['RelatedUrls'] if item['Type'] in ['GET DATA', 'GET RELATED VISUALIZATION']]
        for url in urls:
            print(url)
    return len(granules)


session = Session()
response = session.get(f'{search_url}.{output_format}', params=parameters)
response.raise_for_status()

scroll_id = response.headers['CMR-Scroll-Id']
hits = int(response.headers['CMR-Hits'])

granules_processed = 0
while granules_processed < hits:
    granules = response.json()['items'] # adjust for your output format
    granules_processed += process_granules(granules)
    if granules_processed >= hits:
        break
    response = session.get(search_url, headers={'CMR-Scroll-Id': scroll_id})
    response.raise_for_status()