PythagorasGoal/scripts/research/blind/agents/agent_34_knn_analog.py

"""Agent 34 — kNN analog matching (family=ml, slug=knn_analog).

THE ANGLE (assigned): find the PAST windows most similar to the CURRENT window and
predict the average forward move from how those analogs played out — fully causal.

HOW IT WORKS
  * At each decision row i, build a normalized "shape" descriptor of the recent window
    (the last W bars of standardized log-returns) plus a couple of slow-context features
    (trailing momentum & realized vol). This is the QUERY.
  * The DATABASE of analogs is every past anchor j whose forward outcome is already
    realized as of close[i] (i.e. j + FWD_H <= i). Each anchor stores its descriptor and
    its realized forward log-return over j -> j+FWD_H.
  * Distance = Euclidean on the standardized descriptors. Take the K nearest analogs,
    weight them by 1/(eps+dist), and the forecast is the weighted-average forward return
    of those neighbors. "What happened next, the last K times the tape looked like this."
  * Forecast -> bounded conviction (tanh of the standardized forecast).

CAUSALITY (the whole game):
  * The query descriptor at i uses ONLY returns up to and including bar i.
  * An anchor j is admissible ONLY if its forward window is complete as of i
    (j + FWD_H <= i). We never peek at row i's own unrealized future, nor any j past i.
  * Descriptor standardization uses each window's own mean/std (self-contained), so no
    global statistics leak across the cut.
  -> Verified by causality_ok (signal on a prefix matches the full-array tail).

WHAT THE TRAIN DATA SAYS (honest): next-bar direction on these curves is a coin flip, so
analogs are matched on SHAPE and asked for a multi-bar forward move (FWD_H). Like the other
ML angles on these strongly up-trending curves, shorting destroys value (the tape only goes
up), so the analog forecast is used as a LONG-vs-FLAT conviction with vol-targeting to cap
the drawdown — the win is risk control / staying out of the froth, not return generation.
"""
import numpy as np
import blindlib as bl

# ---- tuned on split='train' only ----
W = 10              # window length (bars) of the shape descriptor; interior opt (6/14/18 worse)
FWD_H = 15          # forward horizon predicted by the analogs (bars); interior (8/12 much worse)
K = 30              # number of nearest neighbors; flat plateau 20..50, K=30 = best DD
MOM_WIN = 40        # trailing-momentum context feature window; flat 40..60
VOL_WIN = 20        # trailing realized-vol context feature window
CTX_WEIGHT = 2.0    # weight of slow-context (regime) features vs the micro shape window.
                    # The REGIME analog (where in the trend, what vol) carries most of the
                    # edge here; up-weighting it lifts PnL 0.71->1.31 AND cuts DD. Flat 1.5..2.5.
WARMUP = 200        # min anchors in the database before we trust the forecast
GAIN = 8.0          # tanh conviction gain on the standardized forecast; smooth DD/PnL dial
LONG_ONLY = True    # shorting an up-trend loses -> conviction is long-or-flat
TARGET_VOL = 0.20
VOL_WIN_DAYS = 30
LEV_CAP = 1.0


def _descriptors(c):
    """Causal feature matrix. Row i's descriptor uses ONLY data <= i.
    Columns: W standardized log-returns of the trailing window + 2 context features."""
    n = len(c)
    lr = np.zeros(n)
    lr[1:] = np.log(c[1:] / c[:-1])           # lr[i] = return of bar ending at i (causal)

    csum = np.cumsum(lr)
    # trailing momentum over MOM_WIN bars (<= i), trailing vol over VOL_WIN bars (<= i)
    mom = np.zeros(n)
    mom[MOM_WIN:] = csum[MOM_WIN:] - csum[:-MOM_WIN]
    vol = np.zeros(n)
    for i in range(VOL_WIN, n):
        vol[i] = np.std(lr[i - VOL_WIN + 1 : i + 1])

    D = W + 2
    desc = np.full((n, D), np.nan)
    for i in range(W, n):
        win = lr[i - W + 1 : i + 1]           # last W returns, all <= i
        s = np.std(win)
        if s < 1e-12:
            s = 1.0
        desc[i, :W] = (win - np.mean(win)) / s  # standardized shape (location/scale free)
        desc[i, W] = mom[i]
        desc[i, W + 1] = vol[i]
    return desc, lr


def signal(df):
    c = df["close"].values.astype(float)
    n = len(c)
    desc, lr = _descriptors(c)

    # forward log-return target[j] over bar j -> j+FWD_H (needs close[j+FWD_H]); realized
    # (admissible) only once i >= j+FWD_H.
    csum = np.cumsum(lr)
    fwd = np.full(n, np.nan)
    fwd[: n - FWD_H] = csum[FWD_H:] - csum[: n - FWD_H]

    first = W                  # earliest fully-formed descriptor
    yhat = np.zeros(n)
    scale = np.ones(n)         # CAUSAL trailing scale of the forecast (expanding std)

    # online over admissible anchors so the shape window (already unit-scale) and context
    # are comparable; computed causally.
    for i in range(first, n):
        last_anchor = i - FWD_H             # anchors j <= last_anchor have realized fwd
        if last_anchor < first + WARMUP:
            continue
        # admissible anchor descriptors & their realized forward returns
        Xj = desc[first : last_anchor + 1]
        yj = fwd[first : last_anchor + 1]
        ok = np.isfinite(Xj).all(axis=1) & np.isfinite(yj)
        if ok.sum() < WARMUP:
            continue
        Xj = Xj[ok]
        yj = yj[ok]

        q = desc[i].copy()
        if not np.isfinite(q).all():
            continue

        # scale the 2 context columns by their (causal) std across the anchor set so they
        # don't dominate / vanish vs the W unit-scale shape columns.
        ctx_sd = np.std(Xj[:, W:], axis=0)
        ctx_sd[ctx_sd < 1e-12] = 1.0
        Xs = Xj.copy()
        qs = q.copy()
        Xs[:, W:] = (Xj[:, W:] / ctx_sd) * CTX_WEIGHT
        qs[W:] = (q[W:] / ctx_sd) * CTX_WEIGHT

        d = np.sqrt(np.sum((Xs - qs) ** 2, axis=1))   # Euclidean distance to every anchor
        k = min(K, len(d))
        idx = np.argpartition(d, k - 1)[:k]            # K nearest (unordered ok)
        dk = d[idx]
        wk = 1.0 / (1e-6 + dk)                         # inverse-distance weights
        yhat[i] = np.sum(wk * yj[idx]) / np.sum(wk)    # weighted-avg forward move

        # CAUSAL forecast scale: the realized-forward-return std over the SAME admissible
        # anchor set (rows <= i-FWD_H). Self-contained, uses no future row. This is what
        # standardizes the conviction without leaking a global statistic.
        s = float(np.std(yj))
        scale[i] = s if s > 1e-9 else 1.0

    # standardize each forecast by its own causal trailing scale -> bounded conviction.
    direction = np.tanh(GAIN * yhat / scale)
    direction = np.nan_to_num(direction, nan=0.0)
    if LONG_ONLY:
        direction = np.clip(direction, 0.0, 1.0)

    pos = bl.vol_target(direction, df, target_vol=TARGET_VOL,
                        vol_win_days=VOL_WIN_DAYS, leverage_cap=LEV_CAP)
    if LONG_ONLY:
        pos = np.clip(pos, 0.0, LEV_CAP)
    return np.clip(np.nan_to_num(pos, nan=0.0), -1.0, 1.0)