PythagorasGoal/scripts/research/blind/agents/agent_29_ridge.py

"""Agent 29 — Ridge regression return forecast (family=ml, slug=ridge).

THE ANGLE (assigned): forecast the forward return with a RIDGE regression on lagged
returns + volatility features, refit on an EXPANDING window every ~20 bars, and turn the
forecast into a position. A genuine ML angle (linear model, L2 penalty), NOT a fixed
momentum sign rule — ridge *weights* the lags and lets vol modulate conviction.

WHAT THE TRAIN DATA ACTUALLY SAYS (the honest finding, not the hoped-for one):
  * NEXT-BAR return on these curves is unforecastable — the walk-forward forecast's next-bar
    hit-rate is ~0.48-0.51 (coin flip). So I forecast a multi-bar FORWARD return (horizon
    FWD_H), the autocorrelated/forecastable quantity, instead of bar-to-bar noise.
  * The expanding ridge forecast is CONSISTENTLY, mildly *negatively* correlated with the
    realized forward return (corr ~ -0.08..-0.22, same sign on BOTH series, ALL horizons).
    i.e. on these strongly up-trending curves the model's most-bullish forecasts mark froth
    that gives back, and its bearish forecasts precede the recoveries. This is a stable
    property across the grid, not one lucky cell.
  * SHORTING destroys value here (both raw-sign and inverted-sign books lose once shorts are
    allowed — the curves only go up). The only honest edge a weak forecaster has on an
    up-trend is WHEN TO HOLD vs. SIT IN CASH.

THE RULE: use the (inverted, given the negative corr) ridge forecast as a LONG-ONLY
conviction — be long when the model is bearish (post-froth recovery), flat when it is
bullish — then vol-target and clip to [0, 1]. Result on train: a book that is in-market only
~16% of the time, tiny drawdown (~0.02 vs 0.77-0.79 buy&hold), Sharpe ~0.83.

CAUSALITY (the whole game):
  * Features at row i use ONLY returns up to and including bar i (rows <= i).
  * Training TARGET for row j is the return over bar j -> j+FWD_H (needs close[j+FWD_H]).
    Sitting at decision-row i we may only train on rows j with j+FWD_H <= i (their targets
    are realized as of close[i]). We NEVER include row i's own unrealized target.
  * Refit on an EXPANDING window of those realized (X,y) pairs every REFIT_EVERY bars;
    coefficients frozen in between. No global fit, no future row touched.
  -> Verified by causality_ok (prefix tail matches full-array tail, max_diff 0.0).

TUNING (split='train' only, combined A & B): chosen cell is interior on every axis —
  FWD_H 18-25 -> Sharpe ~0.83 flat; alpha 20-100 -> Sharpe ~0.81-0.84 flat;
  refit 10-20 -> stable; gain 1.0-2.5 monotone DD/PnL dial. Picked the interior point.

HONEST READ: alpha here is THIN. The forecastability is weak and the win is risk control,
not return generation — a low-exposure, low-DD long-only sleeve, NOT a PnL engine. The
inverted-sign edge is modest and could be regime-specific; the robust, defensible part is
"never short an up-trend; let the forecast tell you when to step out of the way."
"""
import numpy as np
import blindlib as bl

# ---- tuned on split='train' only (interior of a flat plateau) ----
RIDGE_ALPHA = 50.0      # L2 penalty (strong: the lag->return edge is tiny); plateau 20..100
WARMUP = 150            # realized (X,y) pairs required before the first fit
REFIT_EVERY = 20        # expanding-window refit cadence (assigned ~20); stable 10..20
LAGS = (1, 2, 3, 5, 10) # lagged-return features
MOM_WIN = 20            # trailing momentum feature window
VOL_WIN = 20            # trailing realized-vol feature window
FWD_H = 20              # forecast HORIZON (bars). Plateau 18..25. Next-BAR is noise; a
                        # multi-bar target is the autocorrelated, forecastable quantity.
GAIN = 1.5              # tanh conviction gain on the standardized forecast (DD/PnL dial)
INVERT = True           # negative train corr (both series, all H) -> fade the forecast sign
LONG_ONLY = True        # shorting an up-trend destroys value -> conviction is long-or-flat
TARGET_VOL = 0.20       # vol-target the directional book
VOL_WIN_DAYS = 30
LEV_CAP = 1.0           # never lever past fully invested -> preserve the DD cut


def _build_features(c):
    """Causal feature matrix X (len(c) rows). Row i uses ONLY data <= i.
    Columns: lagged log-returns, trailing momentum, trailing realized vol."""
    n = len(c)
    lr = np.zeros(n)
    lr[1:] = np.log(c[1:] / c[:-1])           # lr[i] = return of bar ending at i (causal)

    cols = []
    # lagged returns: feature value at i is the return from k bars ago (all <= i)
    for k in LAGS:
        f = np.zeros(n)
        if k < n:
            f[k:] = lr[: n - k]               # lr shifted back by k -> uses past only
        cols.append(f)
    # trailing momentum: cumulative log-return over the last MOM_WIN bars (<= i)
    mom = np.zeros(n)
    csum = np.cumsum(lr)
    mom[MOM_WIN:] = csum[MOM_WIN:] - csum[:-MOM_WIN]
    cols.append(mom)
    # trailing realized vol (std of last VOL_WIN returns, <= i)
    vol = np.zeros(n)
    for i in range(VOL_WIN, n):
        vol[i] = np.std(lr[i - VOL_WIN + 1 : i + 1])
    cols.append(vol)

    X = np.column_stack(cols)
    return X, lr


def _ridge_fit(X, y, alpha):
    """Closed-form ridge with a standardized design + intercept (no sklearn needed,
    fully deterministic). Returns (mu, sd, beta0, beta) for prediction."""
    mu = X.mean(axis=0)
    sd = X.std(axis=0)
    sd[sd < 1e-12] = 1.0
    Xs = (X - mu) / sd
    p = Xs.shape[1]
    A = Xs.T @ Xs + alpha * np.eye(p)
    b = Xs.T @ (y - y.mean())
    beta = np.linalg.solve(A, b)
    beta0 = y.mean()
    return mu, sd, beta0, beta


def signal(df):
    c = df["close"].values.astype(float)
    n = len(c)
    X, lr = _build_features(c)

    # target[j] = cumulative log-return over bar j -> j+FWD_H (needs close[j+FWD_H]);
    # known (realized) only as of close[j+FWD_H].
    csum = np.cumsum(lr)
    target = np.zeros(n)
    target[: n - FWD_H] = csum[FWD_H:] - csum[: n - FWD_H]

    yhat = np.zeros(n)        # forecast of the forward return, decided at close[i]
    sig_y = np.ones(n)        # scale of recent forecast targets (for standardization)
    coef = None               # frozen (mu, sd, beta0, beta)

    for i in range(n):
        # at decision-row i we may train only on rows j whose target is realized, i.e.
        # j + FWD_H <= i  =>  j <= i - FWD_H. We NEVER include row i's own (unrealized) target.
        first = max(LAGS) + MOM_WIN  # earliest row with all features fully populated
        last_train = i - FWD_H       # target of last_train uses close[i], realized now
        ntrain = last_train - first + 1

        if ntrain >= WARMUP:
            # refit every REFIT_EVERY bars (and on the very first eligible bar)
            if coef is None or (i % REFIT_EVERY == 0):
                Xtr = X[first : last_train + 1]
                ytr = target[first : last_train + 1]
                coef = _ridge_fit(Xtr, ytr, RIDGE_ALPHA)
                s = np.std(ytr)
                sig_y[i] = s if s > 1e-9 else 1.0
            else:
                sig_y[i] = sig_y[i - 1]
            mu, sd, beta0, beta = coef
            xi = (X[i] - mu) / sd
            yhat[i] = beta0 + xi @ beta

    # forecast -> bounded conviction (de-emphasize tiny/noisy forecasts, saturate strong ones)
    s = np.where(sig_y > 1e-9, sig_y, 1.0)
    direction = np.tanh(GAIN * yhat / s)
    direction = np.nan_to_num(direction, nan=0.0)
    if INVERT:
        direction = -direction               # train corr is negative on both series/all H
    if LONG_ONLY:
        direction = np.clip(direction, 0.0, 1.0)  # never short an up-trend (shorts lose here)

    # vol-target the conviction so the DRAWDOWN is what we control
    pos = bl.vol_target(direction, df, target_vol=TARGET_VOL,
                        vol_win_days=VOL_WIN_DAYS, leverage_cap=LEV_CAP)
    if LONG_ONLY:
        pos = np.clip(pos, 0.0, LEV_CAP)
    return np.clip(np.nan_to_num(pos, nan=0.0), -1.0, 1.0)