diff --git a/scripts/run_phase1.py b/scripts/run_phase1.py index b709c82..6e399d8 100644 --- a/scripts/run_phase1.py +++ b/scripts/run_phase1.py @@ -37,6 +37,12 @@ def parse_args() -> argparse.Namespace: default=0.0, help="Phase 2.5: probabilità (0-1) che la mutazione invochi LLM mutator tier B", ) + p.add_argument( + "--fees-eat-alpha-threshold", + type=float, + default=0.5, + help="Adversarial gate: kill se fees/gross_pnl > soglia (default 0.5, ablation 0.7)", + ) return p.parse_args() @@ -91,6 +97,7 @@ def main() -> None: n_trials_dsr=args.n_trials_dsr, db_path=settings.db_path, prompt_mutation_weight=args.prompt_mutation_weight, + fees_eat_alpha_threshold=args.fees_eat_alpha_threshold, ) run_id = run_phase1(cfg, ohlcv=ohlcv, llm=llm) diff --git a/src/multi_swarm/agents/adversarial.py b/src/multi_swarm/agents/adversarial.py index 68f0512..0237585 100644 --- a/src/multi_swarm/agents/adversarial.py +++ b/src/multi_swarm/agents/adversarial.py @@ -59,8 +59,13 @@ class AdversarialReport: class AdversarialAgent: """Agente hand-crafted che applica check euristici a una strategia.""" - def __init__(self, fees_bp: float = 5.0) -> None: + def __init__( + self, + fees_bp: float = 5.0, + fees_eat_alpha_threshold: float = 0.5, + ) -> None: self._engine = BacktestEngine(fees_bp=fees_bp) + self._fees_eat_alpha_threshold = fees_eat_alpha_threshold def review(self, strategy: Strategy, ohlcv: pd.DataFrame) -> AdversarialReport: signal_fn = compile_strategy(strategy) @@ -163,7 +168,7 @@ class AdversarialAgent: # Se gross_pnl <= 0 il check non si applica (gia' perdente). gross_pnl = sum(t.gross_pnl for t in result.trades) total_fees = sum(t.fees for t in result.trades) - if gross_pnl > 0 and total_fees / gross_pnl > 0.5: + if gross_pnl > 0 and total_fees / gross_pnl > self._fees_eat_alpha_threshold: report.findings.append( Finding( name="fees_eat_alpha", diff --git a/src/multi_swarm/ga/loop.py b/src/multi_swarm/ga/loop.py index 9397783..893fe91 100644 --- a/src/multi_swarm/ga/loop.py +++ b/src/multi_swarm/ga/loop.py @@ -25,12 +25,16 @@ def next_generation( cfg: GAConfig, rng: random.Random, llm: Any | None = None, + cost_tracker: Any | None = None, + repo: Any | None = None, + run_id: str | None = None, ) -> list[HypothesisAgentGenome]: """Costruisce la prossima generazione: elitismo + tournament + crossover/mutate. Quando ``cfg.prompt_mutation_weight > 0`` e ``llm`` è fornito, la mutazione invoca ``weighted_random_mutate`` che con quella probabilità usa - ``mutate_prompt_llm`` (Phase 2.5). + ``mutate_prompt_llm`` (Phase 2.5). Cost tracker/repo/run_id si propagano + per registrare ``call_kind="mutation"`` sulle call mutator. """ new_pop: list[HypothesisAgentGenome] = list( elite_select(population, fitnesses, cfg.elite_k) @@ -44,7 +48,12 @@ def next_generation( else: parent = tournament_select(population, fitnesses, cfg.tournament_k, rng) child = weighted_random_mutate( - parent, rng, llm=llm, prompt_mutation_weight=cfg.prompt_mutation_weight + parent, rng, + llm=llm, + prompt_mutation_weight=cfg.prompt_mutation_weight, + cost_tracker=cost_tracker, + repo=repo, + run_id=run_id, ) new_pop.append(child) diff --git a/src/multi_swarm/genome/mutation.py b/src/multi_swarm/genome/mutation.py index cef606b..1fdd616 100644 --- a/src/multi_swarm/genome/mutation.py +++ b/src/multi_swarm/genome/mutation.py @@ -82,16 +82,24 @@ def weighted_random_mutate( rng: random.Random, llm: Any | None = None, prompt_mutation_weight: float = 0.0, + cost_tracker: Any | None = None, + repo: Any | None = None, + run_id: str | None = None, ) -> HypothesisAgentGenome: """Dispatcher pesato fra mutate_prompt_llm e random_mutate scalare. Con probabilità ``prompt_mutation_weight`` invoca ``mutate_prompt_llm``, altrimenti ``random_mutate``. Se ``llm`` è ``None`` o il peso è 0, è equivalente a ``random_mutate`` (backward-compat). + + Se ``cost_tracker``, ``repo`` e ``run_id`` sono forniti, vengono propagati a + ``mutate_prompt_llm`` per tracciare la call con ``call_kind="mutation"``. """ if llm is not None and prompt_mutation_weight > 0 and rng.random() < prompt_mutation_weight: # Import inline per evitare ciclo: mutation_prompt_llm importa da mutation. from .mutation_prompt_llm import mutate_prompt_llm - return mutate_prompt_llm(g, llm, rng) + return mutate_prompt_llm( + g, llm, rng, cost_tracker=cost_tracker, repo=repo, run_id=run_id + ) return random_mutate(g, rng) diff --git a/src/multi_swarm/genome/mutation_prompt_llm.py b/src/multi_swarm/genome/mutation_prompt_llm.py index 20213fc..8d573c0 100644 --- a/src/multi_swarm/genome/mutation_prompt_llm.py +++ b/src/multi_swarm/genome/mutation_prompt_llm.py @@ -130,6 +130,9 @@ def mutate_prompt_llm( rng: random.Random, mutator_tier: ModelTier = ModelTier.B, max_tokens: int = 2000, + cost_tracker: Any | None = None, + repo: Any | None = None, + run_id: str | None = None, ) -> HypothesisAgentGenome: """Operatore di mutazione prompt-level via LLM mutator. @@ -137,6 +140,9 @@ def mutate_prompt_llm( LLM tier B per ottenere il prompt mutato, valida l'output. Su validation fail (output troppo corto, non-strategia, troppo simile al parent), fallback silenzioso a ``random_mutate``. + + Se ``cost_tracker``, ``repo`` e ``run_id`` sono forniti, la chiamata mutator + viene registrata con ``call_kind="mutation"`` per audit budget. """ instruction_key = rng.choice(list(MUTATION_INSTRUCTIONS)) instruction = MUTATION_INSTRUCTIONS[instruction_key] @@ -160,6 +166,28 @@ def mutate_prompt_llm( except Exception: return random_mutate(g, rng) + # Cost tracking call_kind="mutation" se sink fornito. + if cost_tracker is not None and repo is not None and run_id is not None: + in_tok = getattr(result, "input_tokens", 0) + out_tok = getattr(result, "output_tokens", 0) + cr = cost_tracker.record( + input_tokens=in_tok, + output_tokens=out_tok, + tier=mutator_tier, + run_id=run_id, + agent_id=g.id, + call_kind="mutation", + ) + repo.save_cost_record( + run_id=run_id, + agent_id=g.id, + tier=mutator_tier.value, + input_tokens=in_tok, + output_tokens=out_tok, + cost_usd=cr.cost_usd, + call_kind="mutation", + ) + new_prompt = _extract_prompt(getattr(result, "text", "")) if not is_valid_prompt(new_prompt, g.system_prompt): return random_mutate(g, rng) diff --git a/src/multi_swarm/llm/cost_tracker.py b/src/multi_swarm/llm/cost_tracker.py index 9db7e54..42546d7 100644 --- a/src/multi_swarm/llm/cost_tracker.py +++ b/src/multi_swarm/llm/cost_tracker.py @@ -30,6 +30,7 @@ class CostRecord: input_tokens: int output_tokens: int cost_usd: float + call_kind: str = "hypothesis" # "hypothesis" | "mutation" @dataclass @@ -43,6 +44,7 @@ class CostTracker: tier: ModelTier, run_id: str, agent_id: str, + call_kind: str = "hypothesis", ) -> CostRecord: cost = estimate_cost(input_tokens, output_tokens, tier) rec = CostRecord( @@ -53,6 +55,7 @@ class CostTracker: input_tokens=input_tokens, output_tokens=output_tokens, cost_usd=cost, + call_kind=call_kind, ) self.records.append(rec) return rec @@ -61,16 +64,25 @@ class CostTracker: by_tier: dict[str, dict[str, float]] = defaultdict( lambda: {"calls": 0, "input_tokens": 0, "output_tokens": 0, "cost_usd": 0.0} ) + by_call_kind: dict[str, dict[str, float]] = defaultdict( + lambda: {"calls": 0, "input_tokens": 0, "output_tokens": 0, "cost_usd": 0.0} + ) for r in self.records: t = r.tier.value by_tier[t]["calls"] += 1 by_tier[t]["input_tokens"] += r.input_tokens by_tier[t]["output_tokens"] += r.output_tokens by_tier[t]["cost_usd"] += r.cost_usd + ck = r.call_kind + by_call_kind[ck]["calls"] += 1 + by_call_kind[ck]["input_tokens"] += r.input_tokens + by_call_kind[ck]["output_tokens"] += r.output_tokens + by_call_kind[ck]["cost_usd"] += r.cost_usd return { "calls": len(self.records), "input_tokens": sum(r.input_tokens for r in self.records), "output_tokens": sum(r.output_tokens for r in self.records), "cost_usd": sum(r.cost_usd for r in self.records), "by_tier": dict(by_tier), + "by_call_kind": dict(by_call_kind), } diff --git a/src/multi_swarm/orchestrator/run.py b/src/multi_swarm/orchestrator/run.py index a915c94..c6afca2 100644 --- a/src/multi_swarm/orchestrator/run.py +++ b/src/multi_swarm/orchestrator/run.py @@ -50,6 +50,7 @@ class RunConfig: n_trials_dsr: int = 50 db_path: Path = field(default_factory=lambda: Path("./runs.db")) prompt_mutation_weight: float = 0.0 # Phase 2.5: opt-in LLM mutator + fees_eat_alpha_threshold: float = 0.5 # adversarial gate, allenta verso 0.7-0.8 def run_phase1( @@ -78,7 +79,10 @@ def run_phase1( falsification_agent = FalsificationAgent( fees_bp=cfg.fees_bp, n_trials_dsr=cfg.n_trials_dsr ) - adversarial_agent = AdversarialAgent(fees_bp=cfg.fees_bp) + adversarial_agent = AdversarialAgent( + fees_bp=cfg.fees_bp, + fees_eat_alpha_threshold=cfg.fees_eat_alpha_threshold, + ) cost_tracker = CostTracker() population = build_initial_population( @@ -178,6 +182,9 @@ def run_phase1( population = next_generation( population, fitnesses, ga_cfg, rng, llm=llm if cfg.prompt_mutation_weight > 0 else None, + cost_tracker=cost_tracker if cfg.prompt_mutation_weight > 0 else None, + repo=repo if cfg.prompt_mutation_weight > 0 else None, + run_id=run_id if cfg.prompt_mutation_weight > 0 else None, ) repo.complete_run( diff --git a/src/multi_swarm/persistence/repository.py b/src/multi_swarm/persistence/repository.py index 4e46c38..09f4b39 100644 --- a/src/multi_swarm/persistence/repository.py +++ b/src/multi_swarm/persistence/repository.py @@ -26,6 +26,14 @@ class Repository: self.db_path.parent.mkdir(parents=True, exist_ok=True) with self._conn() as conn: conn.executescript(SCHEMA_SQL) + # Migration soft per DB pre-Task 6: aggiunge call_kind se manca. + try: + conn.execute( + "ALTER TABLE cost_records ADD COLUMN call_kind " + "TEXT NOT NULL DEFAULT 'hypothesis'" + ) + except sqlite3.OperationalError: + pass # colonna già presente @staticmethod def _now() -> str: @@ -184,12 +192,13 @@ class Repository: input_tokens: int, output_tokens: int, cost_usd: float, + call_kind: str = "hypothesis", ) -> None: with self._conn() as conn: conn.execute( """INSERT INTO cost_records - (run_id, agent_id, ts, tier, input_tokens, output_tokens, cost_usd) - VALUES (?,?,?,?,?,?,?)""", + (run_id, agent_id, ts, tier, input_tokens, output_tokens, cost_usd, call_kind) + VALUES (?,?,?,?,?,?,?,?)""", ( run_id, agent_id, @@ -198,6 +207,7 @@ class Repository: input_tokens, output_tokens, cost_usd, + call_kind, ), ) diff --git a/src/multi_swarm/persistence/schema.py b/src/multi_swarm/persistence/schema.py index 5b15054..c66456f 100644 --- a/src/multi_swarm/persistence/schema.py +++ b/src/multi_swarm/persistence/schema.py @@ -58,6 +58,7 @@ CREATE TABLE IF NOT EXISTS cost_records ( input_tokens INTEGER NOT NULL, output_tokens INTEGER NOT NULL, cost_usd REAL NOT NULL, + call_kind TEXT NOT NULL DEFAULT 'hypothesis', FOREIGN KEY (run_id) REFERENCES runs(id) ); diff --git a/tests/unit/test_cost_tracker.py b/tests/unit/test_cost_tracker.py index 08733d3..89fb69e 100644 --- a/tests/unit/test_cost_tracker.py +++ b/tests/unit/test_cost_tracker.py @@ -61,3 +61,28 @@ def test_tracker_summary_contains_all_five_tiers(): for tier_letter in ("S", "A", "B", "C", "D"): assert tier_letter in summary["by_tier"] assert summary["by_tier"][tier_letter]["calls"] == 1 + + +def test_tracker_default_call_kind_is_hypothesis(): + t = CostTracker() + rec = t.record(input_tokens=10, output_tokens=10, tier=ModelTier.C, run_id="r", agent_id="a") + assert rec.call_kind == "hypothesis" + summary = t.summary() + assert "hypothesis" in summary["by_call_kind"] + assert summary["by_call_kind"]["hypothesis"]["calls"] == 1 + assert "mutation" not in summary["by_call_kind"] + + +def test_tracker_by_call_kind_breakdown(): + t = CostTracker() + t.record(input_tokens=100, output_tokens=200, tier=ModelTier.C, run_id="r", agent_id="a") + t.record(input_tokens=100, output_tokens=200, tier=ModelTier.C, run_id="r", agent_id="a") + t.record( + input_tokens=50, output_tokens=80, tier=ModelTier.B, + run_id="r", agent_id="parent-x", call_kind="mutation", + ) + summary = t.summary() + assert summary["by_call_kind"]["hypothesis"]["calls"] == 2 + assert summary["by_call_kind"]["mutation"]["calls"] == 1 + assert summary["by_call_kind"]["mutation"]["input_tokens"] == 50 + assert summary["by_call_kind"]["mutation"]["output_tokens"] == 80 diff --git a/tests/unit/test_mutation_prompt_llm.py b/tests/unit/test_mutation_prompt_llm.py index bbb5784..1a0eccd 100644 --- a/tests/unit/test_mutation_prompt_llm.py +++ b/tests/unit/test_mutation_prompt_llm.py @@ -161,6 +161,69 @@ def test_mutate_prompt_llm_falls_back_on_llm_exception() -> None: assert child.generation == parent.generation + 1 +def test_mutate_prompt_llm_logs_mutation_cost_when_sink_provided() -> None: + """Quando cost_tracker+repo+run_id sono forniti, la call mutator viene loggata + con call_kind='mutation' sia in memoria sia nel repo.""" + mutated = ( + "Strategia RSI 1h evolved. Entry long quando RSI(14) < 28 e close > " + "SMA(50). Exit short quando RSI(14) > 72." + ) + + class _R: + text = f"{mutated}" + input_tokens = 350 + output_tokens = 140 + + class _FakeLLMCosted: + def complete(self, genome, system, user, max_tokens=2000): + return _R() + + tracker_calls = [] + repo_calls = [] + + class _FakeTracker: + def record(self, **kw): + tracker_calls.append(kw) + from types import SimpleNamespace + return SimpleNamespace(cost_usd=0.0042) + + class _FakeRepo: + def save_cost_record(self, **kw): + repo_calls.append(kw) + + parent = _make_genome() + child = mutate_prompt_llm( + parent, _FakeLLMCosted(), random.Random(0), + cost_tracker=_FakeTracker(), repo=_FakeRepo(), run_id="run-xyz", + ) + assert child.system_prompt == mutated + assert len(tracker_calls) == 1 + assert tracker_calls[0]["call_kind"] == "mutation" + assert tracker_calls[0]["tier"] == ModelTier.B + assert tracker_calls[0]["run_id"] == "run-xyz" + assert tracker_calls[0]["agent_id"] == parent.id + assert tracker_calls[0]["input_tokens"] == 350 + assert tracker_calls[0]["output_tokens"] == 140 + + assert len(repo_calls) == 1 + assert repo_calls[0]["call_kind"] == "mutation" + assert repo_calls[0]["tier"] == "B" + assert repo_calls[0]["cost_usd"] == 0.0042 + + +def test_mutate_prompt_llm_no_logging_without_sink() -> None: + """Senza cost_tracker+repo+run_id → niente logging cost (backward compat).""" + mutated = ( + "Strategia RSI 1h evoluta. Entry long quando RSI(14) < 25 e close > " + "SMA(60). Exit short quando RSI(14) > 75 e ATR rising." + ) + llm = _FakeLLM(response_text=f"{mutated}") + parent = _make_genome() + # Non solleva (anche se 0 sink forniti) + child = mutate_prompt_llm(parent, llm, random.Random(0)) + assert child.system_prompt == mutated + + def test_mutate_prompt_llm_picks_one_of_six_instructions() -> None: """Verifica che il system message dell'LLM includa una delle 6 istruzioni.""" mutated = (