download.py

import norgatedata
import logging
import pandas as pd
import os

# Setup logging configuration for tracking progress
logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(levelname)s - %(message)s')

# Define the folder to save individual ticker CSV files
output_folder = './ticker_data'
os.makedirs(output_folder, exist_ok=True)  # Create folder if it doesn't exist

def download_stock_data(symbol):
    """Download all available historical data for a given equity symbol, add security name and sector, and save as CSV."""
    logging.info(f"Downloading data for {symbol}...")
    try:
        priceadjust = norgatedata.StockPriceAdjustmentType.TOTALRETURN
        padding_setting = norgatedata.PaddingType.NONE
        timeseriesformat = 'pandas-dataframe'

        # Fetch all available data for the symbol
        pricedata = norgatedata.price_timeseries(
            symbol,
            stock_price_adjustment_setting=priceadjust,
            padding_setting=padding_setting,
            timeseriesformat=timeseriesformat,
            interval='D'
        )

        # Add Security Name and Sector Information
        if pricedata is not None:
            pricedata['Symbol'] = symbol  # Add symbol as a column
            security_name = norgatedata.security_name(symbol)
            sector = norgatedata.classification_at_level(symbol, 'GICS', 'Name', level=1)

            # Add the Security Name and Sector columns to the data
            pricedata['Security_Name'] = security_name
            pricedata['Sector'] = sector
            logging.info(f"Added Security Name for {symbol}: {security_name}")
            logging.info(f"Added Sector for {symbol}: {sector}")

            # Define the file path and save as CSV
            file_path = os.path.join(output_folder, f"{symbol}.csv")
            pricedata.to_csv(file_path, index=False)
            logging.info(f"Data for {symbol} saved to {file_path}")
    except Exception as e:
        logging.error(f"Error downloading data for {symbol}: {e}")

def get_equity_symbols():
    """Retrieve only equity symbols from active and delisted stocks."""
    logging.info("Retrieving all equity symbols from US Equities and US Equities Delisted...")
    active_symbols = [symbol for symbol in norgatedata.database_symbols('US Equities') if norgatedata.subtype1(symbol) == 'Equity']
    delisted_symbols = [symbol for symbol in norgatedata.database_symbols('US Equities Delisted') if norgatedata.subtype1(symbol) == 'Equity']
    logging.info(f"Found {len(active_symbols)} active equity symbols and {len(delisted_symbols)} delisted equity symbols.")
    return active_symbols, delisted_symbols

# Step 1: Retrieve equity symbols only
active_equities, delisted_equities = get_equity_symbols()

# Step 2: Download data for each equity symbol and save as an individual CSV
logging.info("Downloading and saving data for active equities...")
for symbol in active_equities:
    download_stock_data(symbol)
    logging.info("Download " + str(symbol))

logging.info("Downloading and saving data for delisted equities...")
for symbol in delisted_equities:
    download_stock_data(symbol)
    logging.info("Download " + str(symbol))