Improved NL date/time parser (smart_time.py) — deterministic rule layer + optional LLM, past dates allowed, testable. Built on jeffmylife/b2ed8ad8608c6f321ba8f1e04a53d04d with fixes for LLM-only fragility, forced-future dates, hallucinated dates, and untestability.

smart_time_improved.py

"""
smart_time.py — Natural-language date/time parser (IMPROVED).

Keeps the good ideas from jeffmylife's gist (season awareness, word
translations like "few"=3-4, vagueness detection, rich structured output)
but fixes its real flaws:

  * FLAW 1 (LLM-only, no fallback): if the API key is missing or the model
    returns garbage/empty (we saw 402s + None completions with OpenRouter),
    the original CRASHES. This version has a deterministic rule layer
    (dateutil + regex) that works with ZERO network. The LLM is an
    *optional* enhancer, not a hard dependency.
  * FLAW 2 ("never return a date in the past"): WRONG for finance logs.
    "last night" / "yesterday" MUST resolve to a past date. Past dates are
    now allowed; the resolver anchors on a reference "now".
  * FLAW 3 (no rule layer / hallucinated dates): the LLM's output is
    VALIDATED against the calendar; on garbage we fall back to the rules
    instead of trusting a made-up date.
  * FLAW 4 (fromisoformat crash): robust parsing + safe defaults.
  * FLAW 5 (no time-of-day, untestable): extracts HH:MM; reference datetime
    is injectable so it is 100% testable.
  * FLAW 6 (Anthropic-only): model/provider are configurable; works with any
    OpenAI-compatible endpoint (OpenRouter, etc.) or no LLM at all.

Output dict:
  datetime : datetime | None   (parsed, anchored to `reference`)
  is_time_related : bool
  is_vague : bool
  reason   : str
  source   : "rule" | "llm" | "llm_fallback_rule" | "none"
  raw      : str  (what the model / rules saw)

Usage:
  from smart_time import parse_natural_time
  parse_natural_time("next Tuesday")
  parse_natural_time("last night", reference=datetime(2026,7,26))
"""
from __future__ import annotations

import json
import os
import re
from datetime import datetime, timedelta, timezone

try:
    from dateutil import parser as _dtparser
    _HAVE_DATEUTIL = True
except Exception:  # pragma: no cover
    _HAVE_DATEUTIL = False


# --------------------------------------------------------------------------
# Configurable word translations (from the gist — kept, since they're good)
# --------------------------------------------------------------------------
WORD_TRANS = {
    "few": 3.5, "a few": 3.5,
    "several": 5.5,
    "a while": 6 * 30,        # ~6 months in days
    "a couple": 2.5, "a couple of": 2.5,
}

SEASONS = {
    "winter": ["december", "january", "february"],
    "spring": ["march", "april", "may"],
    "summer": ["june", "july", "august"],
    "fall": ["september", "october", "november"],
}
_MONTH_TO_SEASON = {m: s for s, ms in SEASONS.items() for m in ms}


def month_to_season(month: str) -> str:
    return _MONTH_TO_SEASON.get(month.lower().strip(), "unknown")


# --------------------------------------------------------------------------
# Deterministic rule layer (the fallback / primary resolver)
# --------------------------------------------------------------------------
_RELATIVE_RULES = [
    # "in N days/weeks/months" (future)  e.g. "in 2 weeks"
    (re.compile(r"(?i)\bin\s+(\d+)\s*(day|days|week|weeks|month|months|year|years)\b"),
     lambda m, ref: _shift(ref, -int(m.group(1)), m.group(2).lower().rstrip("s"))),
    # "N days/weeks/months ago" (past)
    (re.compile(r"(?i)(\d+)\s*(day|days|week|weeks|month|months|year|years)\s+ago"),
     lambda m, ref: _shift(ref, int(m.group(1)), m.group(2).lower().rstrip("s"))),
    (re.compile(r"(?i)\b(yesterday|last\s*night|last\s*evening|tonight|last\s*afternoon)\b"),
     lambda m, ref: ref - timedelta(days=1)),
    (re.compile(r"(?i)\b(today|this\s*morning|this\s*afternoon|this\s*evening|just\s*now|right\s*now|now)\b"),
     lambda m, ref: ref),
    (re.compile(r"(?i)\b(tomorrow|next\s*(?:day|morning))\b"),
     lambda m, ref: ref + timedelta(days=1)),
    (re.compile(r"(?i)\b(last|next|on)?\s*(monday|tuesday|wednesday|thursday|friday|saturday|sunday)\b"),
     lambda m, ref: _nearest_weekday(ref, m.group(2).lower(),
                                     "last" if m.group(1) == "last"
                                     else "next" if m.group(1) == "next"
                                     else "recent")),
    (re.compile(r"(?i)\b(this|next|last)\s*(winter|spring|summer|fall)\b"),
     lambda m, ref: _season_date(ref, m.group(2).lower(),
                                 {"this": "this", "next": "next", "last": "last"}[m.group(1).lower()])),
]

# fuzzy-quantity words -> day offsets (from the gist's translations)
_WORD_OFFSET_DAYS = {
    "few": 3.5 * 30, "a few": 3.5 * 30,
    "several": 5.5 * 30,
    "a while": 6 * 30,
    "a couple": 2.5 * 30, "a couple of": 2.5 * 30,
}

_WEEKDAYS = {"monday": 0, "tuesday": 1, "wednesday": 2, "thursday": 3,
             "friday": 4, "saturday": 5, "sunday": 6}
_DIGIT_DATE = re.compile(
    r"(?i)(\d{1,2}[/-]\d{1,2})|(20\d{2})|"
    r"(\d{1,2}(?:st|nd|rd|th)?\s+(?:of\s+)?(?:jan|feb|mar|apr|may|jun|jul|aug|sep|oct|nov|dec))")


def _shift(ref: datetime, n: float, unit: str) -> datetime:
    n = float(n)
    if unit in ("week", "weeks"):
        return ref - timedelta(weeks=n)
    if unit in ("month", "months"):
        return ref - timedelta(days=30 * n)
    if unit in ("year", "years"):
        return ref - timedelta(days=365 * n)
    return ref - timedelta(days=n)


def _nearest_weekday(ref: datetime, wd: str, mode: str) -> datetime:
    target = _WEEKDAYS[wd]
    cur = ref.weekday()
    delta = (cur - target) % 7
    if mode == "next":
        return ref + timedelta(days=(7 - delta) % 7 or 7)
    if mode == "recent":
        return ref - timedelta(days=delta)
    return ref - timedelta(days=delta + 7)


def _season_date(ref: datetime, season: str, mode: str) -> datetime:
    months = SEASONS[season]
    first_month = datetime.strptime(months[0], "%B").month
    year = ref.year
    if mode == "next" and ref.month > first_month:
        year += 1
    if mode == "last":
        year = ref.year - 1
    return datetime(year, first_month, 15, tzinfo=ref.tzinfo)


def _extract_time(text: str):
    if not _HAVE_DATEUTIL:
        return None
    m = re.search(r"(?i)\b(\d{1,2})(:\d{2})\s*(am|pm)?\b|\b(\d{1,2})\s*(am|pm)\b", text)
    if not m:
        return None
    try:
        return _dtparser.parse(m.group(0).strip(), default=datetime(2000, 1, 1)).strftime("%H:%M")
    except Exception:
        return None


def _rule_resolve(text: str, ref: datetime):
    """Return a resolved datetime via rules, or None if rules can't handle it."""
    low = (text or "").strip().lower()
    if not low or low in ("null", "none", ""):
        return None
    # fuzzy-quantity words ("a few months", "several weeks", "a while")
    for phrase, days in _WORD_OFFSET_DAYS.items():
        if phrase in low:
            return ref + timedelta(days=days)
    if _HAVE_DATEUTIL and _DIGIT_DATE.search(low):
        try:
            parsed = _dtparser.parse(low, default=ref.replace(hour=0, minute=0, second=0, microsecond=0))
            if abs((parsed.date() - ref.date()).days) < 3650:
                return parsed
        except Exception:
            pass
    for rx, fn in _RELATIVE_RULES:
        m = rx.search(low)
        if m:
            return fn(m, ref)
    return None


# --------------------------------------------------------------------------
# LLM layer (optional)
# --------------------------------------------------------------------------
_SYSTEM = """You are a time-parsing assistant. Convert a natural-language time
expression into a specific ISO datetime string (YYYY-MM-DD, optionally with
HH:MM). Return ONLY raw JSON (no markdown) with fields:
  datetime: ISO string or null
  is_time_related: boolean
  reason: short explanation
  is_vague: boolean (true only when no specific date can be determined without guessing)
Past dates ARE allowed (e.g. "yesterday" or "last night" -> a past date).
Word hints: "few"=3-4, "several"=5-6, "a while"=6 months, "a couple"=2-3.
Current reference date: {cur_date} ({cur_day}), season: {cur_season}."""


def _llm_parse(text: str, ref: datetime, api_key: str | None, model: str, base_url: str | None):
    if not api_key:
        return None
    try:
        from openai import OpenAI
    except Exception:
        return None
    try:
        client = OpenAI(api_key=api_key, base_url=base_url) if base_url else OpenAI(api_key=api_key)
        r = client.chat.completions.create(
            model=model,
            messages=[
                {"role": "system", "content": _SYSTEM.format(
                    cur_date=ref.strftime("%Y-%m-%d"), cur_day=ref.strftime("%A"),
                    cur_season=month_to_season(ref.strftime("%B")))},
                {"role": "user", "content": text},
            ],
            temperature=0.1,
        )
        content = r.choices[0].message.content.strip()
        if content.startswith("```"):
            content = re.sub(r"^```[a-zA-Z]*\s*|\s*```$", "", content)
        return json.loads(content)
    except Exception:
        return None


# --------------------------------------------------------------------------
# Public API
# --------------------------------------------------------------------------
def parse_natural_time(text: str, reference: datetime | None = None,
                       api_key: str | None = None, model: str = "gpt-4o-mini",
                       base_url: str | None = None) -> dict:
    """Parse a natural-language time expression into a structured result.

    Args:
        text: the expression ("next Tuesday", "last night", "in 2 weeks"...).
        reference: anchor "now" (injectable for testing). Defaults to now (UTC).
        api_key / model / base_url: optional LLM. If omitted or the call
            fails, the deterministic rule layer is used instead.
    """
    ref = reference or datetime.now(timezone.utc)
    raw = (text or "").strip()

    llm_out = _llm_parse(raw, ref, api_key, model, base_url) if api_key else None

    llm_dt = None
    if isinstance(llm_out, dict) and llm_out.get("datetime"):
        try:
            llm_dt = datetime.fromisoformat(llm_out["datetime"])
            if llm_dt.tzinfo is None:
                llm_dt = llm_dt.replace(tzinfo=ref.tzinfo)
        except Exception:
            llm_dt = None

    rule_dt = _rule_resolve(raw, ref)
    rule_time = _extract_time(raw) if rule_dt else None

    if llm_dt is not None:
        dt, src = llm_dt, "llm"
    elif rule_dt is not None:
        dt, src = rule_dt, "rule"
        if llm_out is not None:
            src = "llm_fallback_rule"
    else:
        dt, src = None, "none"

    is_time_related = dt is not None or (isinstance(llm_out, dict) and llm_out.get("is_time_related"))
    is_vague = bool(isinstance(llm_out, dict) and llm_out.get("is_vague"))
    if dt is None:
        is_vague = True
    reason = (llm_out or {}).get("reason") or (f"resolved via {src}" if dt else "no date could be resolved")

    return {
        "datetime": dt,
        "is_time_related": bool(is_time_related),
        "is_vague": bool(is_vague),
        "reason": reason,
        "source": src,
        "raw": raw,
    }


if __name__ == "__main__":
    REF = datetime(2026, 7, 26, 9, 0, tzinfo=timezone.utc)  # a Sunday
    # No API key -> fully deterministic rule path (proves FLAW 1 fixed)
    for t in ["next Tuesday", "last night", "yesterday", "in 2 weeks",
              "a few months", "next summer", "3 days ago", "on Friday at 6pm",
              "2026-07-20", "hey how's it going?", ""]:
        r = parse_natural_time(t, reference=REF)
        print(f"{t!r:22} -> {r['datetime']}  src={r['source']:18} vague={r['is_vague']}  {r['reason']}")