PythagorasGoal/src/live/tradesdb.py

"""Libro di bordo del book live: DB dei trade ALLINEATO COL TEMPO + serie di equity.

PERCHE' ESISTE
--------------
`data/live/book_executions.jsonl` registra i fill, ma il suo `ts_utc` e' la data della
**barra di segnale** (`book_execute.py`: `pd.Timestamp(r['last_data'])`), non l'ora del fill:
19 righe su 19 a `00:00:00`. L'ora vera sta solo in `logs/cron_book.log` — che e' **gitignored,
non nel backup e ruotabile**. Quindi oggi la cronologia reale del libro live vive in un file
che una rotazione cancella, e nessuno se ne accorgerebbe.

Questo modulo MATERIALIZZA quella cronologia in `data/live/trades.db` (sqlite), che sta dentro
il perimetro gia' coperto dal backup rotativo della VPS.

FONTI, in ordine di autorita' — e nessuna delle tre e' completa da sola
----------------------------------------------------------------------
1. `logs/cron_book.log`  -> ORA VERA del giro + contesto (equity, tp_frac, skh, target, posizione)
                            e la riga di fill `-> BUY 0.0008 @ $78,094.2 fee 0.02187 (OK)`.
2. `data/live/book_executions.jsonl` -> gli stessi fill con piu' cifre, ma senza ora.
3. venue (`DeribitRead.trade_history`) -> autorevole su `order_id` e timestamp in ms,
                            **ma tronca**: al 2026-08-23 ritorna 1 trade su BTC e 0 su ETH.

Le tre si INCROCIANO, non si sovrascrivono: `reconcile()` riporta le divergenze e non ripara
niente da solo (una riparazione silenziosa fra due fonti che non concordano e' un'invenzione).

TRE STATI, non due: un fill puo' essere `ok` (le fonti concordano), `solo-log`, `solo-jsonl`.
"Non lo vedo" non e' "non c'e'".
"""
from __future__ import annotations

import hashlib
import json
import re
import sqlite3
from dataclasses import dataclass, field
from datetime import datetime, timezone
from pathlib import Path

PROJECT_ROOT = Path(__file__).resolve().parents[2]
DB_PATH = PROJECT_ROOT / "data" / "live" / "trades.db"
CRON_LOG = PROJECT_ROOT / "logs" / "cron_book.log"
EXEC_JSONL = PROJECT_ROOT / "data" / "live" / "book_executions.jsonl"

# Data di armamento dell'esecuzione reale (CLAUDE.md, 2026-06-20).
ARMING = "2026-06-20"

SCHEMA = """
CREATE TABLE IF NOT EXISTS fills (
    fill_id     TEXT PRIMARY KEY,
    ts_utc      TEXT NOT NULL,
    ts_source   TEXT NOT NULL,
    bar_ts      TEXT,
    asset       TEXT NOT NULL,
    side        TEXT NOT NULL,
    qty         REAL NOT NULL,
    price       REAL NOT NULL,
    fee         REAL NOT NULL,
    action      TEXT,
    net_target  REAL,
    pos_before  REAL,
    pos_after   REAL,
    tp_frac     REAL,
    skh_sign    INTEGER,
    skh_entry   REAL,
    equity      REAL,
    order_id    TEXT,
    verified    INTEGER,
    stato       TEXT NOT NULL DEFAULT 'ok'
);
CREATE INDEX IF NOT EXISTS ix_fills_ts ON fills(ts_utc);
CREATE TABLE IF NOT EXISTS roundtrips (
    rt_id      INTEGER PRIMARY KEY AUTOINCREMENT,
    asset      TEXT NOT NULL,
    qty        REAL NOT NULL,
    ts_in      TEXT NOT NULL,
    px_in      REAL NOT NULL,
    ts_out     TEXT NOT NULL,
    px_out     REAL NOT NULL,
    ore_tenuta REAL,
    pnl_lordo  REAL NOT NULL,
    fee_quota  REAL NOT NULL,
    pnl_netto  REAL NOT NULL
);
CREATE TABLE IF NOT EXISTS equity (
    ts_utc TEXT PRIMARY KEY,
    equity REAL NOT NULL,
    src    TEXT NOT NULL
);
CREATE TABLE IF NOT EXISTS journal (
    giorno     TEXT PRIMARY KEY,
    ts_scritto TEXT NOT NULL,
    mercato    TEXT,
    libro      TEXT,
    pnl        TEXT,
    salute     TEXT,
    nota       TEXT NOT NULL DEFAULT ''
);
CREATE TABLE IF NOT EXISTS meta (k TEXT PRIMARY KEY, v TEXT);
"""


# =============================================================================================
# parsing — puro, nessun I/O: prende testo, ritorna dati
# =============================================================================================

@dataclass
class Fill:
    ts_utc: str
    asset: str
    side: str
    qty: float
    price: float
    fee: float
    ts_source: str = "cron_log"
    bar_ts: str | None = None
    action: str | None = None
    net_target: float | None = None
    pos_before: float | None = None
    pos_after: float | None = None
    tp_frac: float | None = None
    skh_sign: int | None = None
    skh_entry: float | None = None
    equity: float | None = None
    order_id: str | None = None
    verified: int = 1

    @property
    def fill_id(self) -> str:
        raw = f"{self.ts_utc}|{self.asset}|{self.side}|{self.qty:.10f}|{self.price:.6f}"
        return hashlib.sha1(raw.encode()).hexdigest()[:16]


@dataclass
class Run:
    """Un giro orario di `book_execute`."""
    ts_utc: str
    equity: float | None = None
    last_bar: str | None = None
    feed_min: int | None = None
    fills: list[Fill] = field(default_factory=list)
    stato_asset: dict = field(default_factory=dict)


_RE_HEAD = re.compile(r"^===== (\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}Z) cron_book =====$")
_RE_EQ = re.compile(r"conto reale\s*:\s*\$([\d,\.]+)")
_RE_BAR = re.compile(r"ultima barra\s*:\s*(\S+)")
_RE_FEED = re.compile(r"feed SKH\s*:\s*\w+ \((\d+) min\)")
_RE_SIG = re.compile(
    r"^\s*(BTC|ETH) TP ([+-][\d\.]+) · SKH ([+-]\d+)\(([^)]*)\) -> net \$([+-][\d,]+) "
    r"\| pos \$([+-][\d,]+) -> (.+?)\s*$")
_RE_FILL = re.compile(
    r"^\s*-> (BUY|SELL) ([\d\.]+) @ \$([\d,\.]+) fee ([\d\.]+) \((OK|NON VERIFICATO[^)]*)\)")


def _num(s: str) -> float:
    return float(s.replace(",", "").replace("$", ""))


def parse_cron_log(text: str) -> list[Run]:
    """Estrae i giri da `cron_book.log`. L'ORA del blocco e' l'ora vera del fill."""
    runs: list[Run] = []
    cur: Run | None = None
    ultimo_asset: str | None = None
    for line in text.splitlines():
        m = _RE_HEAD.match(line)
        if m:
            cur = Run(ts_utc=m.group(1).replace("Z", "+00:00"))
            runs.append(cur)
            ultimo_asset = None
            continue
        if cur is None:
            continue
        if (m := _RE_EQ.search(line)) and cur.equity is None:
            cur.equity = _num(m.group(1)); continue
        if (m := _RE_BAR.search(line)) and cur.last_bar is None:
            cur.last_bar = m.group(1); continue
        if (m := _RE_FEED.search(line)) and cur.feed_min is None:
            cur.feed_min = int(m.group(1)); continue
        if (m := _RE_SIG.match(line)):
            asset = m.group(1)
            entry = None
            sk = m.group(4)
            if "@" in sk:
                try:
                    entry = float(sk.split("@", 1)[1])
                except ValueError:
                    entry = None
            cur.stato_asset[asset] = dict(
                tp_frac=float(m.group(2)), skh_sign=int(m.group(3)), skh_entry=entry,
                net=_num(m.group(5)), pos=_num(m.group(6)), azione=m.group(7).strip())
            ultimo_asset = asset
            continue
        if (m := _RE_FILL.match(line)) and ultimo_asset:
            st = cur.stato_asset.get(ultimo_asset, {})
            cur.fills.append(Fill(
                ts_utc=cur.ts_utc, asset=ultimo_asset, side=m.group(1).lower(),
                qty=float(m.group(2)), price=_num(m.group(3)), fee=float(m.group(4)),
                bar_ts=cur.last_bar, action=st.get("azione"), net_target=st.get("net"),
                pos_before=st.get("pos"), tp_frac=st.get("tp_frac"),
                skh_sign=st.get("skh_sign"), skh_entry=st.get("skh_entry"),
                equity=cur.equity, verified=1 if m.group(5) == "OK" else 0))
    return runs


def parse_executions_jsonl(text: str) -> list[dict]:
    return [json.loads(l) for l in text.splitlines() if l.strip()]


def reconcile(fills_log: list[Fill], righe_jsonl: list[dict], tol_px: float = 0.51) -> dict:
    """Incrocia le due fonti su (data, asset, side, qty). NON ripara: riporta.

    `tol_px`: i prezzi del log sono arrotondati a 1 decimale in stampa -> confronto con
    tolleranza, e la divergenza sopra tolleranza si REGISTRA invece di essere assorbita.
    """
    def chiave(g, a, s, q):
        return (g, a, s, round(float(q), 8))

    ix_log: dict = {}
    for f in fills_log:
        ix_log.setdefault(chiave(f.ts_utc[:10], f.asset, f.side, f.qty), []).append(f)
    ix_js: dict = {}
    for r in righe_jsonl:
        ix_js.setdefault(chiave(r["ts_utc"][:10], r["asset"], r["side"], r["filled"]), []).append(r)

    ok, solo_log, solo_js, px_diversi = [], [], [], []
    for k, gl in ix_log.items():
        gj = ix_js.get(k, [])
        for i, f in enumerate(gl):
            if i < len(gj):
                r = gj[i]
                if abs(float(r["price"]) - f.price) > tol_px:
                    px_diversi.append((f, r))
                ok.append((f, r))
            else:
                solo_log.append(f)
    for k, gj in ix_js.items():
        extra = len(gj) - len(ix_log.get(k, []))
        for r in gj[max(0, len(gj) - extra):] if extra > 0 else []:
            solo_js.append(r)
    return dict(ok=ok, solo_log=solo_log, solo_jsonl=solo_js, prezzi_divergenti=px_diversi)


def fifo_roundtrips(fills: list[Fill]) -> tuple[list[dict], dict]:
    """Round-trip chiusi per FIFO + lotti residui. Le fee si allocano PRO-QUOTA sulla qty."""
    from collections import deque, defaultdict
    lotti: dict[str, deque] = defaultdict(deque)
    rts: list[dict] = []
    for f in sorted(fills, key=lambda x: (x.ts_utc, x.asset)):
        fee_u = f.fee / f.qty if f.qty else 0.0
        if f.side == "buy":
            lotti[f.asset].append([f.qty, f.price, f.ts_utc, fee_u])
        else:
            resto = f.qty
            while resto > 1e-12 and lotti[f.asset]:
                lq, lp, lts, lfee_u = lotti[f.asset][0]
                usa = min(resto, lq)
                lordo = usa * (f.price - lp)
                quota_fee = usa * (lfee_u + fee_u)
                ore = (datetime.fromisoformat(f.ts_utc) - datetime.fromisoformat(lts)).total_seconds() / 3600
                rts.append(dict(asset=f.asset, qty=usa, ts_in=lts, px_in=lp, ts_out=f.ts_utc,
                                px_out=f.price, ore_tenuta=ore, pnl_lordo=lordo,
                                fee_quota=quota_fee, pnl_netto=lordo - quota_fee))
                lq -= usa; resto -= usa
                if lq <= 1e-12:
                    lotti[f.asset].popleft()
                else:
                    lotti[f.asset][0][0] = lq
    aperti = {a: [dict(qty=q, price=p, ts=t) for q, p, t, _ in dq] for a, dq in lotti.items() if dq}
    return rts, aperti


# =============================================================================================
# persistenza
# =============================================================================================

def connect(path: Path | str = DB_PATH) -> sqlite3.Connection:
    path = Path(path)
    path.parent.mkdir(parents=True, exist_ok=True)
    con = sqlite3.connect(path)
    con.row_factory = sqlite3.Row
    con.executescript(SCHEMA)
    return con


def upsert_fills(con: sqlite3.Connection, fills: list[Fill]) -> int:
    n = 0
    for f in fills:
        cur = con.execute(
            """INSERT INTO fills (fill_id, ts_utc, ts_source, bar_ts, asset, side, qty, price, fee,
                                  action, net_target, pos_before, pos_after, tp_frac, skh_sign,
                                  skh_entry, equity, order_id, verified, stato)
               VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)
               ON CONFLICT(fill_id) DO NOTHING""",
            (f.fill_id, f.ts_utc, f.ts_source, f.bar_ts, f.asset, f.side, f.qty, f.price, f.fee,
             f.action, f.net_target, f.pos_before, f.pos_after, f.tp_frac, f.skh_sign,
             f.skh_entry, f.equity, f.order_id, f.verified, "ok"))
        n += cur.rowcount
    con.commit()
    return n


def upsert_equity(con: sqlite3.Connection, punti: list[tuple[str, float, str]]) -> int:
    n = 0
    for ts, v, src in punti:
        cur = con.execute("INSERT INTO equity (ts_utc, equity, src) VALUES (?,?,?) "
                          "ON CONFLICT(ts_utc) DO NOTHING", (ts, v, src))
        n += cur.rowcount
    con.commit()
    return n


def rebuild_roundtrips(con: sqlite3.Connection) -> int:
    """Ricalcola da zero: i round-trip sono DERIVATI, mai inseriti a mano."""
    righe = con.execute("SELECT * FROM fills ORDER BY ts_utc, asset").fetchall()
    fills = [Fill(ts_utc=r["ts_utc"], asset=r["asset"], side=r["side"], qty=r["qty"],
                  price=r["price"], fee=r["fee"]) for r in righe]
    rts, _ = fifo_roundtrips(fills)
    con.execute("DELETE FROM roundtrips")
    con.executemany(
        """INSERT INTO roundtrips (asset, qty, ts_in, px_in, ts_out, px_out, ore_tenuta,
                                   pnl_lordo, fee_quota, pnl_netto)
           VALUES (:asset,:qty,:ts_in,:px_in,:ts_out,:px_out,:ore_tenuta,
                   :pnl_lordo,:fee_quota,:pnl_netto)""", rts)
    con.commit()
    return len(rts)


def set_meta(con: sqlite3.Connection, k: str, v: str) -> None:
    con.execute("INSERT INTO meta (k,v) VALUES (?,?) ON CONFLICT(k) DO UPDATE SET v=excluded.v", (k, v))
    con.commit()


def get_meta(con: sqlite3.Connection, k: str, default=None):
    r = con.execute("SELECT v FROM meta WHERE k=?", (k,)).fetchone()
    return r["v"] if r else default


def stato_aperto(con: sqlite3.Connection) -> dict:
    righe = con.execute("SELECT * FROM fills ORDER BY ts_utc, asset").fetchall()
    fills = [Fill(ts_utc=r["ts_utc"], asset=r["asset"], side=r["side"], qty=r["qty"],
                  price=r["price"], fee=r["fee"]) for r in righe]
    _, aperti = fifo_roundtrips(fills)
    out = {}
    for a, lotti in aperti.items():
        q = sum(x["qty"] for x in lotti)
        costo = sum(x["qty"] * x["price"] for x in lotti)
        out[a] = dict(qty=q, prezzo_medio=costo / q if q else 0.0, dal=min(x["ts"] for x in lotti))
    return out


def ora() -> str:
    return datetime.now(timezone.utc).isoformat(timespec="seconds")