Sample program to stream large data from a REST or HTTP endpoint using Python. For more details refer to https://amalgjose.com/2020/08/10/program-to-stream-large-data-from-a-rest-endpoint-using-python/

stream_data.py

import requests

session = requests.Session()
authentication = {"USER":"", "PASSWORD":""}
payload = {"query":"some query"}
local_file = "data.json"

# This is a dummy URL. You can replace this with the actual URL
URL = "https://sampledatadowload.com/somedata"

# This is a POST request
with session.post(URL, stream=True, data=payload, auth=(authentication["USER"], authentication["PASSWORD"]), verify=False) as r:
    r.raise_for_status()
    with open(local_file, 'wb') as f:
        for chunk in r.iter_content(chunk_size=128):
            f.write(chunk)