feat(live): reconcile resting + orphan single-leg + circuit-breaker venue-lock + FEED_BOOK_GAP

Codice della tornata v1.1.27/28 (gia' in produzione, mai committato):
- reconcile_account: estensione ordini RESTING (FILLED_UNBOOKED/MISSING/STALE,
  caso MR02_BTC: TP fillato di notte scoperto ore dopo) + expected_resting in books
- strategy_worker: orphan_legs su REAL_CLOSE_PARTIAL anche single-leg, persistito
- execution: circuit-breaker su venue-lock admin (stop ordini dopo errori ripetuti)
- runner/hourly_report: alert FEED_BOOK_GAP + timestamp closed trades
- cerbero_client: get_open_orders (merge all + trigger_all)
Test: 12 nuovi, suite completa 126 passed.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
Adriano Dal Pastro
2026-06-12 20:29:02 +00:00
parent a2d581691a
commit 612f2bfced
11 changed files with 628 additions and 25 deletions
+73
View File
@@ -0,0 +1,73 @@
"""Guard FEED_BOOK_GAP (2026-06-12): il feed candele e il book d'esecuzione possono
divergere (MR02_BTC: fill reale a -443 bps dal sim; wick TP_PHANTOM il caso opposto).
Alert per episodio con isteresi a soglia/2, fail-open su errori di rete."""
import sys
from pathlib import Path
import pandas as pd
sys.path.insert(0, str(Path(__file__).resolve().parents[2]))
from src.portfolio import runner
class _Client:
def __init__(self, marks):
self._marks = marks
self.calls = 0
def get_ticker_batch(self, instruments):
self.calls += 1
return {"tickers": [{"instrument_name": i, "mark_price": self._marks[i]}
for i in instruments if i in self._marks]}
class _Boom:
def get_ticker_batch(self, instruments):
raise RuntimeError("rete giu'")
INSTR = {"BTC": "BTC_USDC-PERPETUAL"}
def _feed(close):
return {"BTC": pd.DataFrame({"close": [100.0, close]})}
def _events(monkeypatch):
sent = []
monkeypatch.setattr("src.live.telegram_notifier.notify_event",
lambda ev, data: sent.append((ev, data)))
return sent
def test_gap_alerts_once_per_episode(monkeypatch):
sent = _events(monkeypatch)
cl = _Client({"BTC_USDC-PERPETUAL": 60481.0})
alerted = set()
# feed a 63285 vs mark 60481 = il caso MR02 (~464 bps) -> alert, una volta sola
for _ in range(3):
runner._check_feed_book_gap(cl, _feed(63285.5), INSTR, 150.0, alerted)
assert len(sent) == 1 and sent[0][0] == "FEED_BOOK_GAP"
assert sent[0][1]["asset"] == "BTC" and sent[0][1]["gap_bps"] > 400
def test_recovery_with_hysteresis(monkeypatch):
sent = _events(monkeypatch)
cl = _Client({"BTC_USDC-PERPETUAL": 60000.0})
alerted = {"BTC"}
# gap fra soglia/2 e soglia: NESSUN recovery (isteresi)
runner._check_feed_book_gap(cl, _feed(60000 * 1.0090), INSTR, 150.0, alerted)
assert sent == [] and "BTC" in alerted
# gap sotto soglia/2 -> RIENTRATO
runner._check_feed_book_gap(cl, _feed(60010.0), INSTR, 150.0, alerted)
assert len(sent) == 1 and sent[0][1].get("status") == "RIENTRATO"
assert "BTC" not in alerted
def test_fail_open_and_no_instruments(monkeypatch):
sent = _events(monkeypatch)
runner._check_feed_book_gap(_Boom(), _feed(63285.5), INSTR, 150.0, set())
cl = _Client({"BTC_USDC-PERPETUAL": 60481.0})
runner._check_feed_book_gap(cl, {}, INSTR, 150.0, set()) # nessun feed
assert sent == [] and cl.calls == 0
+96
View File
@@ -0,0 +1,96 @@
"""Reconcile degli ordini RESTING (2026-06-12): TP/DSL attesi dai libri vs ordini
in book + fill non bookati (caso MR02_BTC: TP resting fillato di notte e disaster-SL
sparito, scoperti solo al close sim ore dopo). Nessuna rete: client stubbato,
PAPER puntato su tmp_path."""
import json
import sys
from pathlib import Path
sys.path.insert(0, str(Path(__file__).resolve().parents[2]))
from src.live import books
class _Client:
def __init__(self, open_orders=None, trades=None):
self._oo = open_orders or []
self._tr = trades or []
def get_open_orders(self, currency="USDC", type="all"):
# merge 'all'+'trigger_all' nel chiamante: qui si ritorna tutto due volte
return self._oo
def get_trade_history(self, limit=100, instrument_name=None):
return [t for t in self._tr
if not instrument_name or t["instrument"] == instrument_name]
def _mk_worker(root: Path, wid: str, st: dict):
d = root / wid
d.mkdir(parents=True)
(d / "status.json").write_text(json.dumps(st))
def _setup(tmp_path, monkeypatch, statuses: dict):
root = tmp_path / "portfolio_paper"
for wid, st in statuses.items():
_mk_worker(root, wid, st)
monkeypatch.setattr(books, "PAPER", root)
import importlib
ra = importlib.import_module("scripts.analysis.reconcile_account")
monkeypatch.setattr(ra, "PAPER", root)
return ra
IN_POS = {"real_in_position": True, "real_tp_order_id": "TP-1",
"real_dsl_order_id": "DSL-1"}
def test_expected_resting_reads_single_leg(tmp_path, monkeypatch):
_setup(tmp_path, monkeypatch, {
"MR01_bollinger_fade__ETH__1h": IN_POS,
# pairs e worker flat: NESSUN resting atteso
"PR01_pairs_reversion__ETH_BTC__1h": {"real_in_position": True,
"real_amount_a": 0.1},
"MR02_donchian_fade__BTC__1h": {"real_in_position": False,
"real_tp_order_id": "TP-9"},
})
exp = books.expected_resting()
assert {(e["order_id"], e["kind"]) for e in exp} == {("TP-1", "tp"), ("DSL-1", "dsl")}
assert all(e["instrument"] == "ETH_USDC-PERPETUAL" for e in exp)
def test_resting_ok_when_on_book(tmp_path, monkeypatch):
ra = _setup(tmp_path, monkeypatch, {"MR01_bollinger_fade__ETH__1h": IN_POS})
cl = _Client(open_orders=[{"order_id": "TP-1", "label": "MR01_bollinger_fade__ETH__1h"},
{"order_id": "DSL-1", "label": "MR01_bollinger_fade__ETH__1h"}])
rows = ra.compute_resting_drift(cl)
assert [r["status"] for r in rows] == ["OK", "OK"]
def test_resting_filled_unbooked_vs_missing(tmp_path, monkeypatch):
# TP sparito dal book MA con fill nel trade history -> FILLED_UNBOOKED (caso MR02);
# DSL sparito senza fill (trigger genera order_id nuovo) -> MISSING
ra = _setup(tmp_path, monkeypatch, {"MR01_bollinger_fade__ETH__1h": IN_POS})
cl = _Client(open_orders=[],
trades=[{"instrument": "ETH_USDC-PERPETUAL", "order_id": "TP-1",
"amount": 0.103, "price": 1640.6}])
by_kind = {r["kind"]: r for r in ra.compute_resting_drift(cl)}
assert by_kind["tp"]["status"] == "FILLED_UNBOOKED"
assert abs(by_kind["tp"]["filled"] - 0.103) < 1e-9
assert by_kind["dsl"]["status"] == "MISSING"
def test_resting_stale_order_from_flat_worker(tmp_path, monkeypatch):
# ordine in book con label di un NOSTRO worker flat -> STALE (fillerebbe a sorpresa);
# ordini di altri bot (label sconosciuta) ignorati
ra = _setup(tmp_path, monkeypatch, {
"MR01_bollinger_fade__ETH__1h": {"real_in_position": False}})
cl = _Client(open_orders=[
{"order_id": "OLD-7", "label": "MR01_bollinger_fade__ETH__1h",
"instrument": "ETH_USDC-PERPETUAL", "order_type": "limit"},
{"order_id": "X-1", "label": "altro_bot", "instrument": "ETH_USDC-PERPETUAL"},
])
rows = ra.compute_resting_drift(cl)
assert len(rows) == 1
assert rows[0]["status"] == "STALE" and rows[0]["order_id"] == "OLD-7"
+79
View File
@@ -0,0 +1,79 @@
"""Orfani single-leg (2026-06-12): un close fallito/cappato registra la quota
residua in orphan_legs (parita' coi pairs) cosi' il reconciler la conta come
drift SPIEGATO — prima il REAL_CLOSE_PARTIAL di MR07 (0.102 ETH nel lock testnet)
lasciava drift non spiegato. Nessuna rete: executor finto."""
import json
from types import SimpleNamespace
from src.live.execution import Fill
from src.live.strategy_worker import StrategyWorker
from src.strategies.base import Signal
from src.live import books
class FailingCloseExec:
"""Apre ok, chiude con fill ZERO (venue locked / netting negato)."""
verify_polls = 1
verify_sleep = 0.0
disaster_sl_pct = None
def open(self, instrument, side, notional, label=None):
amt = round(notional / 100.0, 6)
return Fill(instrument, side, notional, amt, 100.0, 0.0, 0.05,
"oid-open", "filled", True, filled_amount=amt)
def place_tp_limit(self, *a, **k):
return Fill("x", "sell", 0.0, 0.0, None, 0.0, 0.0, None, None, False)
def cancel_order(self, oid):
return {"state": "cancelled"}
def resting_fills(self, instrument, oid):
return 0.0, None, 0.0
def close_amount(self, instrument, side, amount, label=None):
return Fill(instrument, "sell", 0.0, amount, None, 0.0, 0.0,
None, "error", False, notes="place_order error: locked_by_admin",
filled_amount=0.0)
def _worker(tmp_path):
return StrategyWorker(
strategy=SimpleNamespace(name="MR07_return_reversal", fee_rt=0.001),
asset="ETH", tf="1h", capital=100.0, position_size=0.5, leverage=2.0,
data_dir=tmp_path, executor=FailingCloseExec(),
exec_instrument="ETH_USDC-PERPETUAL")
def test_failed_close_registers_orphan(tmp_path, monkeypatch):
monkeypatch.setattr("src.live.strategy_worker.notify_event", lambda *a, **k: None)
w = _worker(tmp_path)
w._open_position(Signal(idx=0, direction=1, entry_price=100.0,
metadata={"max_bars": 12}), 100.0)
amt = w.real_amount
assert w.real_in_position and amt > 0
w._close_position(105.0, "time_limit")
assert not w.real_in_position
assert len(w.orphan_legs) == 1
o = w.orphan_legs[0]
assert o["instrument"] == "ETH_USDC-PERPETUAL"
assert o["entry_side"] == "buy" and abs(o["amount"] - amt) < 1e-9
# persistito nello status (resume-safe) e visto da books.real_books
st = json.loads((tmp_path / w.worker_id / "status.json").read_text())
assert st["orphan_legs"] == w.orphan_legs
monkeypatch.setattr(books, "PAPER", tmp_path)
_, orphans = books.real_books()
assert abs(orphans["ETH_USDC-PERPETUAL"] - amt) < 1e-9 # buy = firmato +
def test_clean_close_no_orphan(tmp_path, monkeypatch):
monkeypatch.setattr("src.live.strategy_worker.notify_event", lambda *a, **k: None)
w = _worker(tmp_path)
w.executor.close_amount = lambda instrument, side, amount, label=None: Fill(
instrument, "sell", 0.0, amount, 105.0, 0.0, 0.05, "oid-close",
"filled", True, filled_amount=amount)
w._open_position(Signal(idx=0, direction=1, entry_price=100.0,
metadata={"max_bars": 12}), 100.0)
w._close_position(105.0, "time_limit")
assert w.orphan_legs == []
@@ -0,0 +1,86 @@
"""Circuit-breaker venue-lock (2026-06-12): dopo lock_trip reject 'locked'
consecutivi le APERTURE sono sospese senza toccare l'API (i worker seguono il
path REAL_OPEN_FAIL/sim_fallback); le chiusure si tentano sempre; il probe a
fine cooldown riarma o resetta. Nessuna rete: CerberoClient stubbato."""
import time
from src.live.execution import ExecutionClient
class _Venue:
"""Stub CerberoClient: locked finche' .locked=True, poi ordini ok."""
def __init__(self):
self.locked = True
self.place_calls = 0
def place_order(self, instrument, side, amount, order_type="market",
price=None, label=None, reduce_only=False):
self.place_calls += 1
if self.locked:
return {"state": "error", "error": "locked_by_admin"}
return {"order": {"order_id": "o1", "order_state": "filled",
"filled_amount": amount, "average_price": 100.0},
"trades": [{"amount": amount, "price": 100.0, "fee": 0.0}]}
def get_ticker(self, instrument):
return {"mark_price": 100.0}
def get_trade_history(self, limit=50, instrument_name=None):
return []
def _ec(monkeypatch, sent):
monkeypatch.setattr("src.live.telegram_notifier.notify_event",
lambda ev, data: sent.append((ev, data)))
venue = _Venue()
ec = ExecutionClient(client=venue)
return ec, venue
INST = "ETH_USDC-PERPETUAL"
def test_breaker_trips_and_blocks_opens(monkeypatch):
sent = []
ec, venue = _ec(monkeypatch, sent)
for _ in range(ec.lock_trip):
f = ec.open(INST, "buy", 50.0)
assert not f.verified
assert venue.place_calls == ec.lock_trip and ec.lock_blocked()
assert [e for e, _ in sent] == ["VENUE_LOCK"] # un alert al trip
# aperture sospese: NESSUNA chiamata API, nota dedicata
f = ec.open(INST, "buy", 50.0)
assert venue.place_calls == ec.lock_trip
assert "venue_lock_breaker" in f.notes
assert sent == sent[:1] # niente spam
def test_closes_still_attempted_and_refresh_cooldown(monkeypatch):
sent = []
ec, venue = _ec(monkeypatch, sent)
for _ in range(ec.lock_trip):
ec.open(INST, "buy", 50.0)
ec._net_close_allowance = lambda *a, **k: 0.0 # niente fallback netting
before = ec._lock_until
time.sleep(0.01)
f = ec.close_amount(INST, "buy", 0.5, label="w1") # la chiusura PASSA dall'API
assert venue.place_calls == ec.lock_trip + 1
assert not f.verified
assert ec._lock_until > before # locked dal close -> cooldown esteso
def test_probe_rearms_or_resets(monkeypatch):
sent = []
ec, venue = _ec(monkeypatch, sent)
for _ in range(ec.lock_trip):
ec.open(INST, "buy", 50.0)
# cooldown scaduto + venue ancora locked -> probe fallisce e riscatta
ec._lock_until = time.monotonic() - 1
ec.open(INST, "buy", 50.0)
assert venue.place_calls == ec.lock_trip + 1 and ec.lock_blocked()
# venue sbloccato -> il probe passa, breaker resettato, alert di rientro
ec._lock_until = time.monotonic() - 1
venue.locked = False
f = ec.open(INST, "buy", 50.0)
assert f.verified and not ec.lock_blocked() and ec._lock_streak == 0
assert sent[-1][0] == "VENUE_LOCK" and sent[-1][1].get("status") == "RIENTRATO"