Real-time scanner for detecting large price movements in all US stocks and ETFs

scanner.py

"""
This script demonstrates how to build a real-time scanner that detects significant 
price movements across all US stocks and ETFs using Databento's market data APIs.

Features:
- Handles entire US equities universe of ~9,000 symbols efficiently
- Median sub-ms feed delay to NY4/5 (WAN-shaped) and ~5s to start scanning
- Monitors all US stocks and ETFs for price movements exceeding a configurable threshold
- Compares current prices against previous day's closing prices
- Displays alerts when significant moves are detected

Example output:
[2025-04-23T04:00:01.688717194-04:00] NVDA moved by 4.40% (current: 98.1900, previous: 102.7100)
[2025-04-23T04:00:02.121450448-04:00] BABA moved by 10.34% (current: 131.2750, previous: 118.9700)
[2025-04-23T06:36:23.507827237-04:00] AMZN moved by 7.36% (current: 193.8850, previous: 180.6000)
"""


from datetime import timedelta
from typing import Dict, Any

import databento as db
import pandas as pd
import pytz


TODAY = "2025-04-25"


class PriceMovementScanner:
    """Scanner for detecting large price movements in all US equities."""

    # Constants
    PX_SCALE: float = 1e-9
    PX_NULL: int = 2**63 - 1
    PCT_MOVE_THRESHOLD: float = 0.03

    def __init__(self, pct_threshold: float = None, today: str = TODAY) -> None:
        """Initialize scanner with configurable threshold and date."""
        self.pct_threshold = pct_threshold or self.PCT_MOVE_THRESHOLD
        self.today = today
        self.today_midnight_ns = int(pd.Timestamp(today).timestamp() * 1e9)
        self.symbol_directory: Dict[int, str] = {}
        self.last_day_lookup: Dict[str, float] = self.get_last_day_lookup()
        self.is_signal_lit: Dict[str, bool] = {symbol: False for symbol in self.last_day_lookup}

    def get_last_day_lookup(self) -> Dict[str, float]:
        """Get yesterday's closing prices for all symbols."""
        client = db.Historical()

        now = pd.Timestamp(self.today).date()
        yesterday = (pd.Timestamp(self.today) - timedelta(days=1)).date()

        # Get yesterday's closing prices
        data = client.timeseries.get_range(
            dataset="EQUS.SUMMARY",
            schema="ohlcv-1d",
            symbols="ALL_SYMBOLS",
            start=yesterday,
            end=now,
        )

        # Request symbology: This is required for ALL_SYMBOLS requests
        # which don't automatically map instrument ID to raw ticker symbol
        symbology_json = data.request_symbology(client)
        data.insert_symbology_json(symbology_json, clear_existing=True)

        df = data.to_df()
        # TODO: Adjust for overnight splits here, e.g., using Databento corporate actions API

        return dict(zip(df["symbol"], df["close"]))

    def scan(self, event: Any) -> None:
        """
        Scan for large price movements in market data events.
        """
        if isinstance(event, db.SymbolMappingMsg):
            self.symbol_directory[event.hd.instrument_id] = event.stype_out_symbol
            return

        if not isinstance(event, db.MBP1Msg):
            return
            
        # Skip if event is from replay before today using `.subscribe(..., start=...)` parameter
        #if event.hd.ts_event < self.today_midnight_ns:
        #    return

        symbol = self.symbol_directory[event.instrument_id]
        bid = event.levels[0].bid_px
        ask = event.levels[0].ask_px

        if bid == self.PX_NULL or ask == self.PX_NULL:
            # Handle when one side of book is empty
            return

        mid = (event.levels[0].bid_px + event.levels[0].ask_px) * self.PX_SCALE * 0.5
        last = self.last_day_lookup[symbol]
        abs_r = abs(mid - last) / last

        if abs_r > self.pct_threshold and not self.is_signal_lit[symbol]:
            ts = pd.Timestamp(event.hd.ts_event, unit='ns').tz_localize('UTC').tz_convert('US/Eastern')
            print(
                f"[{ts.isoformat()}] {symbol} moved by {abs_r * 100:.2f}% "
                f"(current: {mid:.4f}, previous: {last:.4f})"
            )
            self.is_signal_lit[symbol] = True

def main() -> None:
    scanner = PriceMovementScanner()
    live = db.Live()

    live.subscribe(
        dataset="EQUS.MINI",
        schema="mbp-1",
        stype_in="raw_symbol",
        symbols="ALL_SYMBOLS",
        start=0,
    )

    live.add_callback(scanner.scan)
    live.start()
    live.block_for_close()


if __name__ == "__main__":
    main()

scanner_out.log

[2025-04-24T04:00:00.007704938-04:00] TSLA moved by 5.43% (current: 245.4300, previous: 259.5100)
[2025-04-24T04:00:00.008339140-04:00] TQQQ moved by 11.78% (current: 46.0000, previous: 52.1400)
[2025-04-24T04:00:00.009881258-04:00] DBC moved by 5.84% (current: 22.5650, previous: 21.3200)
[2025-04-24T04:00:00.011338853-04:00] DBA moved by 4.81% (current: 25.9400, previous: 27.2500)
[2025-04-24T04:00:00.019717308-04:00] WEAT moved by 3.35% (current: 4.4650, previous: 4.6200)
[2025-04-24T04:00:00.048734686-04:00] RSST moved by 4.37% (current: 19.5950, previous: 20.4900)
[2025-04-24T04:00:00.052540506-04:00] BCI moved by 17.21% (current: 24.3800, previous: 20.8000)
[2025-04-24T04:00:00.111035469-04:00] SOYB moved by 4.26% (current: 22.8100, previous: 21.8777)
[2025-04-24T04:00:00.158550097-04:00] FTGC moved by 3.49% (current: 25.6250, previous: 24.7600)
[2025-04-24T04:00:00.158736420-04:00] CMDY moved by 3.03% (current: 51.7200, previous: 50.2000)
[2025-04-24T04:00:00.611051675-04:00] SOXL moved by 10.42% (current: 10.7500, previous: 12.0000)
[2025-04-24T04:00:00.846266718-04:00] AVGO moved by 3.27% (current: 194.3000, previous: 188.1500)
[2025-04-24T04:00:01.064255011-04:00] MSFO moved by 3.04% (current: 15.4550, previous: 15.9400)
[2025-04-24T04:00:01.072265030-04:00] TSL moved by 6.22% (current: 9.3400, previous: 9.9600)
[2025-04-24T04:00:01.082340146-04:00] TSLR moved by 8.77% (current: 12.6900, previous: 13.9100)
[2025-04-24T04:00:01.113355946-04:00] ROBN moved by 15.55% (current: 12.7600, previous: 15.1100)
[2025-04-24T04:00:01.132234337-04:00] TSLS moved by 5.46% (current: 10.8200, previous: 10.2600)
[2025-04-24T04:00:01.155620431-04:00] RGTX moved by 7.04% (current: 23.4550, previous: 25.2300)
[2025-04-24T04:00:01.168584574-04:00] NVDQ moved by 10.33% (current: 3.5250, previous: 3.1950)
[2025-04-24T04:00:01.174271455-04:00] CONI moved by 6.27% (current: 12.8050, previous: 12.0500)
[2025-04-24T04:00:01.185282707-04:00] MSTU moved by 5.02% (current: 6.7150, previous: 7.0700)
[2025-04-24T04:00:01.213264307-04:00] AMZD moved by 4.10% (current: 13.5050, previous: 12.9729)
[2025-04-24T04:00:01.214255143-04:00] MSTZ moved by 5.39% (current: 6.5500, previous: 6.2150)
[2025-04-24T04:00:01.221272147-04:00] BTCZ moved by 3.28% (current: 4.8850, previous: 4.7300)
[2025-04-24T04:00:01.277243406-04:00] NVDD moved by 4.91% (current: 7.3650, previous: 7.0200)
[2025-04-24T04:00:01.283289236-04:00] AMDL moved by 9.57% (current: 3.5900, previous: 3.9700)
[2025-04-24T04:00:01.362255512-04:00] TSLZ moved by 9.58% (current: 3.3750, previous: 3.0800)
[2025-04-24T04:00:01.410314702-04:00] SMCZ moved by 23.46% (current: 17.4200, previous: 14.1100)
[2025-04-24T04:00:01.422267022-04:00] SMST moved by 6.42% (current: 1.9050, previous: 1.7900)
[2025-04-24T04:00:01.439459787-04:00] RIOX moved by 9.81% (current: 7.2150, previous: 8.0000)
[2025-04-24T04:00:01.453292778-04:00] GGLS moved by 3.66% (current: 14.4900, previous: 13.9784)
[2025-04-24T04:00:01.485510056-04:00] MULL moved by 12.07% (current: 7.8700, previous: 8.9500)
[2025-04-24T04:00:01.518329722-04:00] SMCL moved by 15.98% (current: 9.8550, previous: 11.7300)
[2025-04-24T04:00:01.522758564-04:00] TSLL moved by 7.32% (current: 8.1000, previous: 8.7400)
[2025-04-24T04:00:01.538278074-04:00] MSFD moved by 3.85% (current: 15.1200, previous: 14.5600)
[2025-04-24T04:00:01.572529758-04:00] SOFX moved by 8.40% (current: 9.1600, previous: 10.0000)
[2025-04-24T04:00:01.576241770-04:00] TSLT moved by 8.28% (current: 10.9600, previous: 11.9500)
[2025-04-24T04:00:01.581265793-04:00] AVGX moved by 13.16% (current: 15.1100, previous: 17.4000)
[2025-04-24T04:00:01.655301692-04:00] CRWL moved by 15.09% (current: 25.0750, previous: 29.5300)
[2025-04-24T04:00:01.693439439-04:00] FAZ moved by 4.68% (current: 5.9250, previous: 5.6600)
[2025-04-24T04:00:01.703172468-04:00] PLTD moved by 9.40% (current: 13.7850, previous: 12.6000)
[2025-04-24T04:00:01.709452276-04:00] BABX moved by 3.98% (current: 26.6250, previous: 27.7300)
[2025-04-24T04:00:01.710126529-04:00] SGLY moved by 4.97% (current: 0.7280, previous: 0.6935)
[2025-04-24T04:00:01.753371336-04:00] CONY moved by 3.95% (current: 7.7900, previous: 8.1100)
[2025-04-24T04:00:01.756559353-04:00] MUU moved by 12.62% (current: 8.7900, previous: 10.0600)
[2025-04-24T04:00:01.876435951-04:00] NVDG moved by 9.82% (current: 6.4300, previous: 7.1300)
[2025-04-24T04:00:01.880268476-04:00] TSLY moved by 4.13% (current: 7.6600, previous: 7.9900)
[2025-04-24T04:00:01.881295182-04:00] TSLG moved by 8.76% (current: 3.9600, previous: 4.3400)
[2025-04-24T04:00:02.005215627-04:00] IYW moved by 4.04% (current: 134.5200, previous: 140.1800)
[2025-04-24T04:00:02.241785581-04:00] BILI moved by 3.28% (current: 18.2500, previous: 17.6700)
[2025-04-24T04:00:02.309862335-04:00] BTOG moved by 29.57% (current: 0.1915, previous: 0.1478)
[2025-04-24T04:00:02.368459867-04:00] JYD moved by 3.42% (current: 0.2327, previous: 0.2250)
[2025-04-24T04:00:02.450403063-04:00] TMF moved by 13.43% (current: 34.5000, previous: 39.8500)
[2025-04-24T04:00:02.693435950-04:00] WOLF moved by 19.11% (current: 2.5400, previous: 3.1400)
[2025-04-24T04:00:02.775154519-04:00] CLSK moved by 4.18% (current: 9.2300, previous: 8.8600)
[2025-04-24T04:00:02.836149723-04:00] STSS moved by 1703.74% (current: 0.3860, previous: 0.0214)
[2025-04-24T04:00:02.875740256-04:00] PLUG moved by 10.77% (current: 0.9400, previous: 0.8486)
[2025-04-24T04:00:02.898836195-04:00] HSAI moved by 18.56% (current: 12.2400, previous: 15.0300)
[2025-04-24T04:00:03.248963901-04:00] DTSS moved by 10.04% (current: 2.0150, previous: 2.2400)
[2025-04-24T04:00:03.425913804-04:00] GMHS moved by 28.69% (current: 1.5700, previous: 1.2200)
[2025-04-24T04:00:03.426315170-04:00] BBAI moved by 44.33% (current: 4.2000, previous: 2.9100)
[2025-04-24T04:00:03.435060579-04:00] AGRI moved by 12.36% (current: 2.0000, previous: 1.7800)
[2025-04-24T04:00:03.569186588-04:00] SOUN moved by 7.52% (current: 8.6750, previous: 9.3800)
[2025-04-24T04:00:03.620702410-04:00] ASTS moved by 4.86% (current: 22.9000, previous: 24.0700)
[2025-04-24T04:00:03.766217385-04:00] PET moved by 31.15% (current: 0.2000, previous: 0.1525)
[2025-04-24T04:00:04.003267275-04:00] DGXX moved by 26.32% (current: 0.7590, previous: 1.0300)
[2025-04-24T04:00:04.113404677-04:00] DUO moved by 3.77% (current: 0.2750, previous: 0.2650)
[2025-04-24T04:00:04.225039143-04:00] BTDR moved by 8.65% (current: 9.5000, previous: 10.4000)
[2025-04-24T04:00:04.258249229-04:00] SOXS moved by 26.70% (current: 24.1750, previous: 19.0800)
[2025-04-24T04:00:04.339755183-04:00] LI moved by 3.27% (current: 24.6000, previous: 23.8200)
[2025-04-24T04:00:04.419607976-04:00] SVIX moved by 7.15% (current: 11.3000, previous: 12.1700)
[2025-04-24T04:00:04.538939507-04:00] IQ moved by 7.85% (current: 1.7600, previous: 1.9100)
[2025-04-24T04:00:04.588406222-04:00] EH moved by 14.96% (current: 19.2450, previous: 16.7400)
[2025-04-24T04:00:04.816284997-04:00] NVDA moved by 3.26% (current: 102.9600, previous: 106.4300)
[2025-04-24T04:00:05.006456486-04:00] SPXU moved by 6.17% (current: 26.0000, previous: 24.4900)
[2025-04-24T04:00:05.031851245-04:00] TLT moved by 4.49% (current: 92.2000, previous: 88.2400)
[2025-04-24T04:00:05.130802700-04:00] DJT moved by 13.46% (current: 22.6300, previous: 26.1500)
[2025-04-24T04:00:05.312994404-04:00] RXRX moved by 5.22% (current: 6.0500, previous: 5.7500)
[2025-04-24T04:00:05.374543536-04:00] AAL moved by 4.27% (current: 9.2000, previous: 9.6100)
[2025-04-24T04:00:05.420594735-04:00] AMC moved by 20.11% (current: 3.2550, previous: 2.7100)
[2025-04-24T04:00:05.434400288-04:00] PFE moved by 9.75% (current: 25.0000, previous: 22.7800)
[2025-04-24T04:00:05.682936888-04:00] YMAX moved by 8.65% (current: 14.2000, previous: 13.0700)
[2025-04-24T04:00:05.756178899-04:00] NIO moved by 7.06% (current: 3.8850, previous: 4.1800)
[2025-04-24T04:00:06.093409156-04:00] GRAB moved by 5.98% (current: 4.4000, previous: 4.6800)
[2025-04-24T04:00:06.313516456-04:00] INTC moved by 7.89% (current: 19.7950, previous: 21.4900)
[2025-04-24T04:00:06.364224892-04:00] GME moved by 8.94% (current: 24.6600, previous: 27.0800)
[2025-04-24T04:00:06.813325148-04:00] UGA moved by 3.29% (current: 57.2350, previous: 59.1800)
[2025-04-24T04:00:06.847837175-04:00] UVIX moved by 9.72% (current: 51.3500, previous: 46.8000)
[2025-04-24T04:00:07.375620892-04:00] BILL moved by 18.25% (current: 53.0000, previous: 44.8200)
[2025-04-24T04:00:07.550693169-04:00] TIGR moved by 4.08% (current: 7.2800, previous: 7.5900)
[2025-04-24T04:00:07.665706046-04:00] SOFI moved by 8.25% (current: 11.2950, previous: 12.3100)
[2025-04-24T04:00:07.697799583-04:00] COIN moved by 32.09% (current: 269.3000, previous: 203.8700)
[2025-04-24T04:00:07.704504506-04:00] NVDX moved by 7.78% (current: 7.4800, previous: 6.9400)
[2025-04-24T04:00:08.296401339-04:00] PLTR moved by 21.28% (current: 84.8400, previous: 107.7800)
[2025-04-24T04:00:08.349791400-04:00] CHA moved by 6.12% (current: 33.2150, previous: 31.3000)
[2025-04-24T04:00:08.836882482-04:00] NVDL moved by 7.99% (current: 30.7500, previous: 33.4200)
[2025-04-24T04:00:08.855855468-04:00] FUTU moved by 15.03% (current: 108.0000, previous: 93.8900)
[2025-04-24T04:00:08.920875683-04:00] LCID moved by 8.37% (current: 2.2450, previous: 2.4500)
[2025-04-24T04:00:08.972357883-04:00] MSTX moved by 4.44% (current: 31.9650, previous: 33.4500)
[2025-04-24T04:00:09.027238071-04:00] QYLD moved by 4.20% (current: 15.5000, previous: 16.1800)