chore(reset): v2.0.0 — storico certificato Deribit mainnet, ripartenza pulita
Reset del progetto su fondamenta verificate dopo la scoperta che l'intera libreria "validata OOS" era artefatto di feed contaminato (print fantasma del feed Cerbero TESTNET + storico Binance/USDT). - Storico ricostruito da Deribit MAINNET (ccxt pubblico, tokenless) e CERTIFICATO (certify_feed.py): BTC/ETH puliti su TUTTA la storia (mediana 2-6 bps vs Coinbase USD), integrita' OHLC + coerenza resample (maxΔ 0.00) + cross-venue OK. Alt esclusi (illiquidi/divergenti: LTC/DOGE 50-82% barre flat; XRP/BNB non certificabili). - Verdetto sul feed pulito: FADE / PAIRS / XS01 / TSM01 morti (ogni portafoglio Sharpe -2.3..-3.0, DD ~40%); solo SH01 e frammenti HONEST con segnale residuo, da ri-validare in isolamento. - Cleanup "restart pulito": strategie, stack live (src/live, src/portfolio, runner/executor, yml, docker), ~100 script ricerca/gate, waste/games/ portfolios, dati non certificati + cache e 60+ diari -> archiviati in Old/ (preservati, non cancellati). Diario consolidato in un unico documento. - Skeleton ricerca tenuto: Strategy ABC + indicatori + src/fractal + src/backtest/engine + load_data; tool dati certificati (rebuild_history, certify_feed, audit_feed, multi_source_check). - Universo dati ATTIVO: solo BTC/ETH (5m/15m/1h); guardrail fisico (load_data su alt -> FileNotFoundError). Esecuzione DISABILITATA, conto flat. Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
Adriano Dal Pastro
@@ -0,0 +1,132 @@
|
||||
"""AUDIT INTEGRITA' — confronta OGNI parquet storico col prezzo REALE Binance.
|
||||
|
||||
Dopo la scoperta che fade/pairs/DIP01 erano edge FINTI (print fantasma del feed testnet
|
||||
Cerbero), serve CERTEZZA del dato: quanto e' contaminato OGNI file? Per ogni parquet
|
||||
confronta il CLOSE col Binance spot allineato (merge_asof nearest) e riporta la quota di
|
||||
barre >1% e >3% fuori dalla realta', per file e per anno peggiore.
|
||||
|
||||
Riferimento: Binance 5m (BTC/ETH sub-orari) e 1h (tutti), provato ~ mainnet (disc <0.13%).
|
||||
NON modifica nulla: solo lettura + report. Cache ref in data/raw/_ref_*.parquet.
|
||||
|
||||
uv run python scripts/analysis/audit_feed.py
|
||||
"""
|
||||
from __future__ import annotations
|
||||
import sys
|
||||
from pathlib import Path
|
||||
PROJECT_ROOT = Path(__file__).resolve().parents[2]
|
||||
sys.path.insert(0, str(PROJECT_ROOT))
|
||||
import numpy as np, pandas as pd, ccxt
|
||||
|
||||
RAW = PROJECT_ROOT / "data" / "raw"
|
||||
SYM = {"btc": "BTC/USDT", "eth": "ETH/USDT", "ada": "ADA/USDT", "bnb": "BNB/USDT",
|
||||
"doge": "DOGE/USDT", "ltc": "LTC/USDT", "sol": "SOL/USDT", "xrp": "XRP/USDT"}
|
||||
TF_MIN = {"5m": 5, "8m": 8, "13m": 13, "15m": 15, "19m": 19, "30m": 30, "1h": 60,
|
||||
"2h": 120, "3h": 180, "4h": 240, "5h": 300, "6h": 360, "8h": 480, "12h": 720,
|
||||
"13h": 780, "19h": 1140, "24h": 1440, "36h": 2160, "48h": 2880}
|
||||
_EX = None
|
||||
|
||||
|
||||
def ex():
|
||||
global _EX
|
||||
if _EX is None:
|
||||
_EX = ccxt.binance({"enableRateLimit": True})
|
||||
return _EX
|
||||
|
||||
|
||||
def get_ref(asset: str, base: str) -> pd.DataFrame:
|
||||
"""Serie Binance di riferimento (cache). Riusa _real_/_ref_ se presenti."""
|
||||
for pref in ("_real_", "_ref_"):
|
||||
c = RAW / f"{pref}{asset}_{base}.parquet"
|
||||
if c.exists():
|
||||
return pd.read_parquet(c)[["timestamp", "close"]]
|
||||
cache = RAW / f"_ref_{asset}_{base}.parquet"
|
||||
start = "2018-01-01" if base == "5m" else "2017-08-01"
|
||||
start_ms = int(pd.Timestamp(start, tz="UTC").timestamp() * 1000)
|
||||
end_ms = int(pd.Timestamp("2026-05-27", tz="UTC").timestamp() * 1000)
|
||||
tf_ms = TF_MIN[base] * 60 * 1000
|
||||
rows, since = [], start_ms
|
||||
while since <= end_ms:
|
||||
for _ in range(3):
|
||||
try:
|
||||
r = ex().fetch_ohlcv(SYM[asset], base, since=since, limit=1000); break
|
||||
except Exception:
|
||||
r = []
|
||||
if not r:
|
||||
break
|
||||
rows += r
|
||||
nxt = int(r[-1][0]) + tf_ms
|
||||
if nxt <= since:
|
||||
break
|
||||
since = nxt
|
||||
df = pd.DataFrame(rows, columns=["timestamp", "open", "high", "low", "close", "volume"])
|
||||
df = df.drop_duplicates("timestamp").sort_values("timestamp").reset_index(drop=True)
|
||||
df[["timestamp", "open", "high", "low", "close", "volume"]].to_parquet(cache, index=False)
|
||||
return df[["timestamp", "close"]]
|
||||
|
||||
|
||||
def audit(asset: str, tf: str, ref5, ref1h):
|
||||
f = RAW / f"{asset}_{tf}.parquet"
|
||||
if not f.exists():
|
||||
return None
|
||||
df = pd.read_parquet(f)[["timestamp", "close"]].copy()
|
||||
df["timestamp"] = df["timestamp"].astype("int64")
|
||||
df = df.dropna(subset=["close"]).sort_values("timestamp").reset_index(drop=True)
|
||||
base = "5m" if (TF_MIN[tf] < 60 and asset in ("btc", "eth")) else "1h"
|
||||
ref = (ref5 if base == "5m" else ref1h).get(asset)
|
||||
if ref is None or len(ref) == 0:
|
||||
return {"file": f"{asset}_{tf}", "rows": len(df), "no_ref": True}
|
||||
ref = ref.rename(columns={"close": "cref"}).sort_values("timestamp")
|
||||
tol = TF_MIN[base] * 60 * 1000
|
||||
m = pd.merge_asof(df, ref, on="timestamp", direction="nearest", tolerance=tol)
|
||||
m = m.dropna(subset=["cref"])
|
||||
m = m[m["cref"] > 0]
|
||||
if len(m) == 0:
|
||||
return {"file": f"{asset}_{tf}", "rows": len(df), "covered": 0, "no_ref": True}
|
||||
disc = (m["close"] - m["cref"]).abs() / m["cref"]
|
||||
yr = pd.to_datetime(m["timestamp"], unit="ms", utc=True).dt.year
|
||||
worst_y, worst_p = 0, 0.0
|
||||
for y, g in disc.groupby(yr):
|
||||
p = float((g > 0.01).mean() * 100)
|
||||
if p > worst_p:
|
||||
worst_p, worst_y = p, int(y)
|
||||
return {"file": f"{asset}_{tf}", "rows": len(df), "covered": len(m) / len(df) * 100,
|
||||
"p1": float((disc > 0.01).mean() * 100), "p3": float((disc > 0.03).mean() * 100),
|
||||
"med": float(disc.median() * 100), "worst_y": worst_y, "worst_p": worst_p}
|
||||
|
||||
|
||||
def main():
|
||||
assets = list(SYM)
|
||||
print("Fetch riferimenti Binance (1h tutti + 5m BTC/ETH)...", flush=True)
|
||||
ref1h = {a: get_ref(a, "1h") for a in assets}
|
||||
ref5 = {a: get_ref(a, "5m") for a in ("btc", "eth")}
|
||||
print("\n" + "=" * 96)
|
||||
print(" AUDIT INTEGRITA' FEED — % barre con CLOSE >1% (e >3%) fuori da Binance spot")
|
||||
print("=" * 96)
|
||||
print(f" {'file':<10s}{'righe':>8s}{'cov%':>6s}{'med%':>7s}{'>1%':>7s}{'>3%':>7s}{'worst-anno':>13s} giudizio")
|
||||
print(" " + "-" * 92)
|
||||
rows = []
|
||||
for a in assets:
|
||||
tfs = [tf for tf in TF_MIN if (RAW / f"{a}_{tf}.parquet").exists()]
|
||||
# ordina per minuti
|
||||
for tf in sorted(tfs, key=lambda t: TF_MIN[t]):
|
||||
r = audit(a, tf, ref5, ref1h)
|
||||
if r:
|
||||
rows.append(r)
|
||||
# ordina per contaminazione discendente
|
||||
clean = [r for r in rows if not r.get("no_ref")]
|
||||
clean.sort(key=lambda r: -r.get("p1", 0))
|
||||
poisoned = sum(1 for r in clean if r["p1"] >= 1.0)
|
||||
for r in clean:
|
||||
verdict = "PULITO" if r["p1"] < 0.5 else ("sospetto" if r["p1"] < 1.0 else "CONTAMINATO")
|
||||
print(f" {r['file']:<10s}{r['rows']:>8d}{r['covered']:>6.0f}{r['med']:>7.2f}"
|
||||
f"{r['p1']:>7.1f}{r['p3']:>7.1f}{('%d:%.0f%%'%(r['worst_y'],r['worst_p'])):>13s} {verdict}")
|
||||
noref = [r for r in rows if r.get("no_ref")]
|
||||
print(" " + "-" * 92)
|
||||
print(f" Totale file audited: {len(clean)} | CONTAMINATI (>1% barre fuori): {poisoned} | "
|
||||
f"PULITI (<0.5%): {sum(1 for r in clean if r['p1']<0.5)} | senza-ref: {len(noref)}")
|
||||
if noref:
|
||||
print(" senza riferimento Binance:", ", ".join(r["file"] for r in noref))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Reference in New Issue
Block a user