PythagorasGoal/scripts/analysis/audit_feed.py

"""AUDIT INTEGRITA' — confronta OGNI parquet storico col prezzo REALE Binance.

Dopo la scoperta che fade/pairs/DIP01 erano edge FINTI (print fantasma del feed testnet
Cerbero), serve CERTEZZA del dato: quanto e' contaminato OGNI file? Per ogni parquet
confronta il CLOSE col Binance spot allineato (merge_asof nearest) e riporta la quota di
barre >1% e >3% fuori dalla realta', per file e per anno peggiore.

Riferimento: Binance 5m (BTC/ETH sub-orari) e 1h (tutti), provato ~ mainnet (disc <0.13%).
NON modifica nulla: solo lettura + report. Cache ref in data/raw/_ref_*.parquet.

    uv run python scripts/analysis/audit_feed.py
"""
from __future__ import annotations
import sys
from pathlib import Path
PROJECT_ROOT = Path(__file__).resolve().parents[2]
sys.path.insert(0, str(PROJECT_ROOT))
import numpy as np, pandas as pd, ccxt

RAW = PROJECT_ROOT / "data" / "raw"
SYM = {"btc": "BTC/USDT", "eth": "ETH/USDT", "ada": "ADA/USDT", "bnb": "BNB/USDT",
       "doge": "DOGE/USDT", "ltc": "LTC/USDT", "sol": "SOL/USDT", "xrp": "XRP/USDT"}
TF_MIN = {"5m": 5, "8m": 8, "13m": 13, "15m": 15, "19m": 19, "30m": 30, "1h": 60,
          "2h": 120, "3h": 180, "4h": 240, "5h": 300, "6h": 360, "8h": 480, "12h": 720,
          "13h": 780, "19h": 1140, "24h": 1440, "36h": 2160, "48h": 2880}
_EX = None


def ex():
    global _EX
    if _EX is None:
        _EX = ccxt.binance({"enableRateLimit": True})
    return _EX


def get_ref(asset: str, base: str) -> pd.DataFrame:
    """Serie Binance di riferimento (cache). Riusa _real_/_ref_ se presenti."""
    for pref in ("_real_", "_ref_"):
        c = RAW / f"{pref}{asset}_{base}.parquet"
        if c.exists():
            return pd.read_parquet(c)[["timestamp", "close"]]
    cache = RAW / f"_ref_{asset}_{base}.parquet"
    start = "2018-01-01" if base == "5m" else "2017-08-01"
    start_ms = int(pd.Timestamp(start, tz="UTC").timestamp() * 1000)
    end_ms = int(pd.Timestamp("2026-05-27", tz="UTC").timestamp() * 1000)
    tf_ms = TF_MIN[base] * 60 * 1000
    rows, since = [], start_ms
    while since <= end_ms:
        for _ in range(3):
            try:
                r = ex().fetch_ohlcv(SYM[asset], base, since=since, limit=1000); break
            except Exception:
                r = []
        if not r:
            break
        rows += r
        nxt = int(r[-1][0]) + tf_ms
        if nxt <= since:
            break
        since = nxt
    df = pd.DataFrame(rows, columns=["timestamp", "open", "high", "low", "close", "volume"])
    df = df.drop_duplicates("timestamp").sort_values("timestamp").reset_index(drop=True)
    df[["timestamp", "open", "high", "low", "close", "volume"]].to_parquet(cache, index=False)
    return df[["timestamp", "close"]]


def audit(asset: str, tf: str, ref5, ref1h):
    f = RAW / f"{asset}_{tf}.parquet"
    if not f.exists():
        return None
    df = pd.read_parquet(f)[["timestamp", "close"]].copy()
    df["timestamp"] = df["timestamp"].astype("int64")
    df = df.dropna(subset=["close"]).sort_values("timestamp").reset_index(drop=True)
    base = "5m" if (TF_MIN[tf] < 60 and asset in ("btc", "eth")) else "1h"
    ref = (ref5 if base == "5m" else ref1h).get(asset)
    if ref is None or len(ref) == 0:
        return {"file": f"{asset}_{tf}", "rows": len(df), "no_ref": True}
    ref = ref.rename(columns={"close": "cref"}).sort_values("timestamp")
    tol = TF_MIN[base] * 60 * 1000
    m = pd.merge_asof(df, ref, on="timestamp", direction="nearest", tolerance=tol)
    m = m.dropna(subset=["cref"])
    m = m[m["cref"] > 0]
    if len(m) == 0:
        return {"file": f"{asset}_{tf}", "rows": len(df), "covered": 0, "no_ref": True}
    disc = (m["close"] - m["cref"]).abs() / m["cref"]
    yr = pd.to_datetime(m["timestamp"], unit="ms", utc=True).dt.year
    worst_y, worst_p = 0, 0.0
    for y, g in disc.groupby(yr):
        p = float((g > 0.01).mean() * 100)
        if p > worst_p:
            worst_p, worst_y = p, int(y)
    return {"file": f"{asset}_{tf}", "rows": len(df), "covered": len(m) / len(df) * 100,
            "p1": float((disc > 0.01).mean() * 100), "p3": float((disc > 0.03).mean() * 100),
            "med": float(disc.median() * 100), "worst_y": worst_y, "worst_p": worst_p}


def main():
    assets = list(SYM)
    print("Fetch riferimenti Binance (1h tutti + 5m BTC/ETH)...", flush=True)
    ref1h = {a: get_ref(a, "1h") for a in assets}
    ref5 = {a: get_ref(a, "5m") for a in ("btc", "eth")}
    print("\n" + "=" * 96)
    print("  AUDIT INTEGRITA' FEED — % barre con CLOSE >1% (e >3%) fuori da Binance spot")
    print("=" * 96)
    print(f"  {'file':<10s}{'righe':>8s}{'cov%':>6s}{'med%':>7s}{'>1%':>7s}{'>3%':>7s}{'worst-anno':>13s}  giudizio")
    print("  " + "-" * 92)
    rows = []
    for a in assets:
        tfs = [tf for tf in TF_MIN if (RAW / f"{a}_{tf}.parquet").exists()]
        # ordina per minuti
        for tf in sorted(tfs, key=lambda t: TF_MIN[t]):
            r = audit(a, tf, ref5, ref1h)
            if r:
                rows.append(r)
    # ordina per contaminazione discendente
    clean = [r for r in rows if not r.get("no_ref")]
    clean.sort(key=lambda r: -r.get("p1", 0))
    poisoned = sum(1 for r in clean if r["p1"] >= 1.0)
    for r in clean:
        verdict = "PULITO" if r["p1"] < 0.5 else ("sospetto" if r["p1"] < 1.0 else "CONTAMINATO")
        print(f"  {r['file']:<10s}{r['rows']:>8d}{r['covered']:>6.0f}{r['med']:>7.2f}"
              f"{r['p1']:>7.1f}{r['p3']:>7.1f}{('%d:%.0f%%'%(r['worst_y'],r['worst_p'])):>13s}  {verdict}")
    noref = [r for r in rows if r.get("no_ref")]
    print("  " + "-" * 92)
    print(f"  Totale file audited: {len(clean)} | CONTAMINATI (>1% barre fuori): {poisoned} | "
          f"PULITI (<0.5%): {sum(1 for r in clean if r['p1']<0.5)} | senza-ref: {len(noref)}")
    if noref:
        print("  senza riferimento Binance:", ", ".join(r["file"] for r in noref))


if __name__ == "__main__":
    main()