feat(phase-2.5): Task 6 cost_kind attribution + fees_eat_alpha threshold CLI
Task 6 del piano Phase 2.5 (deferito → ora completato): - CostRecord: nuovo campo call_kind (default "hypothesis") - CostTracker.record: accetta call_kind opzionale, summary include by_call_kind breakdown (hypothesis vs mutation) - Schema cost_records: aggiunta colonna call_kind TEXT NOT NULL DEFAULT 'hypothesis' + migration soft via ALTER TABLE in init_schema (silently catched per DB pre-Task 6) - Repository.save_cost_record: nuova arg call_kind opzionale - mutate_prompt_llm: accetta cost_tracker/repo/run_id opzionali e logga la call mutator con call_kind="mutation" quando sink presente - weighted_random_mutate, next_generation: propagano cost sink - orchestrator.run_phase1: passa cost_tracker+repo+run_id a next_generation solo se prompt_mutation_weight > 0 Esposto fees_eat_alpha_threshold come parametro AdversarialAgent (default 0.5 = comportamento Phase 1.5 invariato), propagato via RunConfig.fees_eat_alpha_threshold e flag CLI --fees-eat-alpha-threshold. Abilita ablation con soglia 0.7-0.8 senza modificare codice — adversarial finding dominante in tutti i run Phase 2/2.5 (50+ HIGH per run). Tests (+4 → 186 totale): - test_cost_tracker: default call_kind="hypothesis"; breakdown by_call_kind con hypothesis+mutation - test_mutation_prompt_llm: logging mutation cost con sink completo; backward compat senza sink (no errore) Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
This commit is contained in:
@@ -37,6 +37,12 @@ def parse_args() -> argparse.Namespace:
|
|||||||
default=0.0,
|
default=0.0,
|
||||||
help="Phase 2.5: probabilità (0-1) che la mutazione invochi LLM mutator tier B",
|
help="Phase 2.5: probabilità (0-1) che la mutazione invochi LLM mutator tier B",
|
||||||
)
|
)
|
||||||
|
p.add_argument(
|
||||||
|
"--fees-eat-alpha-threshold",
|
||||||
|
type=float,
|
||||||
|
default=0.5,
|
||||||
|
help="Adversarial gate: kill se fees/gross_pnl > soglia (default 0.5, ablation 0.7)",
|
||||||
|
)
|
||||||
return p.parse_args()
|
return p.parse_args()
|
||||||
|
|
||||||
|
|
||||||
@@ -91,6 +97,7 @@ def main() -> None:
|
|||||||
n_trials_dsr=args.n_trials_dsr,
|
n_trials_dsr=args.n_trials_dsr,
|
||||||
db_path=settings.db_path,
|
db_path=settings.db_path,
|
||||||
prompt_mutation_weight=args.prompt_mutation_weight,
|
prompt_mutation_weight=args.prompt_mutation_weight,
|
||||||
|
fees_eat_alpha_threshold=args.fees_eat_alpha_threshold,
|
||||||
)
|
)
|
||||||
|
|
||||||
run_id = run_phase1(cfg, ohlcv=ohlcv, llm=llm)
|
run_id = run_phase1(cfg, ohlcv=ohlcv, llm=llm)
|
||||||
|
|||||||
@@ -59,8 +59,13 @@ class AdversarialReport:
|
|||||||
class AdversarialAgent:
|
class AdversarialAgent:
|
||||||
"""Agente hand-crafted che applica check euristici a una strategia."""
|
"""Agente hand-crafted che applica check euristici a una strategia."""
|
||||||
|
|
||||||
def __init__(self, fees_bp: float = 5.0) -> None:
|
def __init__(
|
||||||
|
self,
|
||||||
|
fees_bp: float = 5.0,
|
||||||
|
fees_eat_alpha_threshold: float = 0.5,
|
||||||
|
) -> None:
|
||||||
self._engine = BacktestEngine(fees_bp=fees_bp)
|
self._engine = BacktestEngine(fees_bp=fees_bp)
|
||||||
|
self._fees_eat_alpha_threshold = fees_eat_alpha_threshold
|
||||||
|
|
||||||
def review(self, strategy: Strategy, ohlcv: pd.DataFrame) -> AdversarialReport:
|
def review(self, strategy: Strategy, ohlcv: pd.DataFrame) -> AdversarialReport:
|
||||||
signal_fn = compile_strategy(strategy)
|
signal_fn = compile_strategy(strategy)
|
||||||
@@ -163,7 +168,7 @@ class AdversarialAgent:
|
|||||||
# Se gross_pnl <= 0 il check non si applica (gia' perdente).
|
# Se gross_pnl <= 0 il check non si applica (gia' perdente).
|
||||||
gross_pnl = sum(t.gross_pnl for t in result.trades)
|
gross_pnl = sum(t.gross_pnl for t in result.trades)
|
||||||
total_fees = sum(t.fees for t in result.trades)
|
total_fees = sum(t.fees for t in result.trades)
|
||||||
if gross_pnl > 0 and total_fees / gross_pnl > 0.5:
|
if gross_pnl > 0 and total_fees / gross_pnl > self._fees_eat_alpha_threshold:
|
||||||
report.findings.append(
|
report.findings.append(
|
||||||
Finding(
|
Finding(
|
||||||
name="fees_eat_alpha",
|
name="fees_eat_alpha",
|
||||||
|
|||||||
@@ -25,12 +25,16 @@ def next_generation(
|
|||||||
cfg: GAConfig,
|
cfg: GAConfig,
|
||||||
rng: random.Random,
|
rng: random.Random,
|
||||||
llm: Any | None = None,
|
llm: Any | None = None,
|
||||||
|
cost_tracker: Any | None = None,
|
||||||
|
repo: Any | None = None,
|
||||||
|
run_id: str | None = None,
|
||||||
) -> list[HypothesisAgentGenome]:
|
) -> list[HypothesisAgentGenome]:
|
||||||
"""Costruisce la prossima generazione: elitismo + tournament + crossover/mutate.
|
"""Costruisce la prossima generazione: elitismo + tournament + crossover/mutate.
|
||||||
|
|
||||||
Quando ``cfg.prompt_mutation_weight > 0`` e ``llm`` è fornito, la mutazione
|
Quando ``cfg.prompt_mutation_weight > 0`` e ``llm`` è fornito, la mutazione
|
||||||
invoca ``weighted_random_mutate`` che con quella probabilità usa
|
invoca ``weighted_random_mutate`` che con quella probabilità usa
|
||||||
``mutate_prompt_llm`` (Phase 2.5).
|
``mutate_prompt_llm`` (Phase 2.5). Cost tracker/repo/run_id si propagano
|
||||||
|
per registrare ``call_kind="mutation"`` sulle call mutator.
|
||||||
"""
|
"""
|
||||||
new_pop: list[HypothesisAgentGenome] = list(
|
new_pop: list[HypothesisAgentGenome] = list(
|
||||||
elite_select(population, fitnesses, cfg.elite_k)
|
elite_select(population, fitnesses, cfg.elite_k)
|
||||||
@@ -44,7 +48,12 @@ def next_generation(
|
|||||||
else:
|
else:
|
||||||
parent = tournament_select(population, fitnesses, cfg.tournament_k, rng)
|
parent = tournament_select(population, fitnesses, cfg.tournament_k, rng)
|
||||||
child = weighted_random_mutate(
|
child = weighted_random_mutate(
|
||||||
parent, rng, llm=llm, prompt_mutation_weight=cfg.prompt_mutation_weight
|
parent, rng,
|
||||||
|
llm=llm,
|
||||||
|
prompt_mutation_weight=cfg.prompt_mutation_weight,
|
||||||
|
cost_tracker=cost_tracker,
|
||||||
|
repo=repo,
|
||||||
|
run_id=run_id,
|
||||||
)
|
)
|
||||||
new_pop.append(child)
|
new_pop.append(child)
|
||||||
|
|
||||||
|
|||||||
@@ -82,16 +82,24 @@ def weighted_random_mutate(
|
|||||||
rng: random.Random,
|
rng: random.Random,
|
||||||
llm: Any | None = None,
|
llm: Any | None = None,
|
||||||
prompt_mutation_weight: float = 0.0,
|
prompt_mutation_weight: float = 0.0,
|
||||||
|
cost_tracker: Any | None = None,
|
||||||
|
repo: Any | None = None,
|
||||||
|
run_id: str | None = None,
|
||||||
) -> HypothesisAgentGenome:
|
) -> HypothesisAgentGenome:
|
||||||
"""Dispatcher pesato fra mutate_prompt_llm e random_mutate scalare.
|
"""Dispatcher pesato fra mutate_prompt_llm e random_mutate scalare.
|
||||||
|
|
||||||
Con probabilità ``prompt_mutation_weight`` invoca ``mutate_prompt_llm``,
|
Con probabilità ``prompt_mutation_weight`` invoca ``mutate_prompt_llm``,
|
||||||
altrimenti ``random_mutate``. Se ``llm`` è ``None`` o il peso è 0,
|
altrimenti ``random_mutate``. Se ``llm`` è ``None`` o il peso è 0,
|
||||||
è equivalente a ``random_mutate`` (backward-compat).
|
è equivalente a ``random_mutate`` (backward-compat).
|
||||||
|
|
||||||
|
Se ``cost_tracker``, ``repo`` e ``run_id`` sono forniti, vengono propagati a
|
||||||
|
``mutate_prompt_llm`` per tracciare la call con ``call_kind="mutation"``.
|
||||||
"""
|
"""
|
||||||
if llm is not None and prompt_mutation_weight > 0 and rng.random() < prompt_mutation_weight:
|
if llm is not None and prompt_mutation_weight > 0 and rng.random() < prompt_mutation_weight:
|
||||||
# Import inline per evitare ciclo: mutation_prompt_llm importa da mutation.
|
# Import inline per evitare ciclo: mutation_prompt_llm importa da mutation.
|
||||||
from .mutation_prompt_llm import mutate_prompt_llm
|
from .mutation_prompt_llm import mutate_prompt_llm
|
||||||
|
|
||||||
return mutate_prompt_llm(g, llm, rng)
|
return mutate_prompt_llm(
|
||||||
|
g, llm, rng, cost_tracker=cost_tracker, repo=repo, run_id=run_id
|
||||||
|
)
|
||||||
return random_mutate(g, rng)
|
return random_mutate(g, rng)
|
||||||
|
|||||||
@@ -130,6 +130,9 @@ def mutate_prompt_llm(
|
|||||||
rng: random.Random,
|
rng: random.Random,
|
||||||
mutator_tier: ModelTier = ModelTier.B,
|
mutator_tier: ModelTier = ModelTier.B,
|
||||||
max_tokens: int = 2000,
|
max_tokens: int = 2000,
|
||||||
|
cost_tracker: Any | None = None,
|
||||||
|
repo: Any | None = None,
|
||||||
|
run_id: str | None = None,
|
||||||
) -> HypothesisAgentGenome:
|
) -> HypothesisAgentGenome:
|
||||||
"""Operatore di mutazione prompt-level via LLM mutator.
|
"""Operatore di mutazione prompt-level via LLM mutator.
|
||||||
|
|
||||||
@@ -137,6 +140,9 @@ def mutate_prompt_llm(
|
|||||||
LLM tier B per ottenere il prompt mutato, valida l'output. Su validation
|
LLM tier B per ottenere il prompt mutato, valida l'output. Su validation
|
||||||
fail (output troppo corto, non-strategia, troppo simile al parent),
|
fail (output troppo corto, non-strategia, troppo simile al parent),
|
||||||
fallback silenzioso a ``random_mutate``.
|
fallback silenzioso a ``random_mutate``.
|
||||||
|
|
||||||
|
Se ``cost_tracker``, ``repo`` e ``run_id`` sono forniti, la chiamata mutator
|
||||||
|
viene registrata con ``call_kind="mutation"`` per audit budget.
|
||||||
"""
|
"""
|
||||||
instruction_key = rng.choice(list(MUTATION_INSTRUCTIONS))
|
instruction_key = rng.choice(list(MUTATION_INSTRUCTIONS))
|
||||||
instruction = MUTATION_INSTRUCTIONS[instruction_key]
|
instruction = MUTATION_INSTRUCTIONS[instruction_key]
|
||||||
@@ -160,6 +166,28 @@ def mutate_prompt_llm(
|
|||||||
except Exception:
|
except Exception:
|
||||||
return random_mutate(g, rng)
|
return random_mutate(g, rng)
|
||||||
|
|
||||||
|
# Cost tracking call_kind="mutation" se sink fornito.
|
||||||
|
if cost_tracker is not None and repo is not None and run_id is not None:
|
||||||
|
in_tok = getattr(result, "input_tokens", 0)
|
||||||
|
out_tok = getattr(result, "output_tokens", 0)
|
||||||
|
cr = cost_tracker.record(
|
||||||
|
input_tokens=in_tok,
|
||||||
|
output_tokens=out_tok,
|
||||||
|
tier=mutator_tier,
|
||||||
|
run_id=run_id,
|
||||||
|
agent_id=g.id,
|
||||||
|
call_kind="mutation",
|
||||||
|
)
|
||||||
|
repo.save_cost_record(
|
||||||
|
run_id=run_id,
|
||||||
|
agent_id=g.id,
|
||||||
|
tier=mutator_tier.value,
|
||||||
|
input_tokens=in_tok,
|
||||||
|
output_tokens=out_tok,
|
||||||
|
cost_usd=cr.cost_usd,
|
||||||
|
call_kind="mutation",
|
||||||
|
)
|
||||||
|
|
||||||
new_prompt = _extract_prompt(getattr(result, "text", ""))
|
new_prompt = _extract_prompt(getattr(result, "text", ""))
|
||||||
if not is_valid_prompt(new_prompt, g.system_prompt):
|
if not is_valid_prompt(new_prompt, g.system_prompt):
|
||||||
return random_mutate(g, rng)
|
return random_mutate(g, rng)
|
||||||
|
|||||||
@@ -30,6 +30,7 @@ class CostRecord:
|
|||||||
input_tokens: int
|
input_tokens: int
|
||||||
output_tokens: int
|
output_tokens: int
|
||||||
cost_usd: float
|
cost_usd: float
|
||||||
|
call_kind: str = "hypothesis" # "hypothesis" | "mutation"
|
||||||
|
|
||||||
|
|
||||||
@dataclass
|
@dataclass
|
||||||
@@ -43,6 +44,7 @@ class CostTracker:
|
|||||||
tier: ModelTier,
|
tier: ModelTier,
|
||||||
run_id: str,
|
run_id: str,
|
||||||
agent_id: str,
|
agent_id: str,
|
||||||
|
call_kind: str = "hypothesis",
|
||||||
) -> CostRecord:
|
) -> CostRecord:
|
||||||
cost = estimate_cost(input_tokens, output_tokens, tier)
|
cost = estimate_cost(input_tokens, output_tokens, tier)
|
||||||
rec = CostRecord(
|
rec = CostRecord(
|
||||||
@@ -53,6 +55,7 @@ class CostTracker:
|
|||||||
input_tokens=input_tokens,
|
input_tokens=input_tokens,
|
||||||
output_tokens=output_tokens,
|
output_tokens=output_tokens,
|
||||||
cost_usd=cost,
|
cost_usd=cost,
|
||||||
|
call_kind=call_kind,
|
||||||
)
|
)
|
||||||
self.records.append(rec)
|
self.records.append(rec)
|
||||||
return rec
|
return rec
|
||||||
@@ -61,16 +64,25 @@ class CostTracker:
|
|||||||
by_tier: dict[str, dict[str, float]] = defaultdict(
|
by_tier: dict[str, dict[str, float]] = defaultdict(
|
||||||
lambda: {"calls": 0, "input_tokens": 0, "output_tokens": 0, "cost_usd": 0.0}
|
lambda: {"calls": 0, "input_tokens": 0, "output_tokens": 0, "cost_usd": 0.0}
|
||||||
)
|
)
|
||||||
|
by_call_kind: dict[str, dict[str, float]] = defaultdict(
|
||||||
|
lambda: {"calls": 0, "input_tokens": 0, "output_tokens": 0, "cost_usd": 0.0}
|
||||||
|
)
|
||||||
for r in self.records:
|
for r in self.records:
|
||||||
t = r.tier.value
|
t = r.tier.value
|
||||||
by_tier[t]["calls"] += 1
|
by_tier[t]["calls"] += 1
|
||||||
by_tier[t]["input_tokens"] += r.input_tokens
|
by_tier[t]["input_tokens"] += r.input_tokens
|
||||||
by_tier[t]["output_tokens"] += r.output_tokens
|
by_tier[t]["output_tokens"] += r.output_tokens
|
||||||
by_tier[t]["cost_usd"] += r.cost_usd
|
by_tier[t]["cost_usd"] += r.cost_usd
|
||||||
|
ck = r.call_kind
|
||||||
|
by_call_kind[ck]["calls"] += 1
|
||||||
|
by_call_kind[ck]["input_tokens"] += r.input_tokens
|
||||||
|
by_call_kind[ck]["output_tokens"] += r.output_tokens
|
||||||
|
by_call_kind[ck]["cost_usd"] += r.cost_usd
|
||||||
return {
|
return {
|
||||||
"calls": len(self.records),
|
"calls": len(self.records),
|
||||||
"input_tokens": sum(r.input_tokens for r in self.records),
|
"input_tokens": sum(r.input_tokens for r in self.records),
|
||||||
"output_tokens": sum(r.output_tokens for r in self.records),
|
"output_tokens": sum(r.output_tokens for r in self.records),
|
||||||
"cost_usd": sum(r.cost_usd for r in self.records),
|
"cost_usd": sum(r.cost_usd for r in self.records),
|
||||||
"by_tier": dict(by_tier),
|
"by_tier": dict(by_tier),
|
||||||
|
"by_call_kind": dict(by_call_kind),
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -50,6 +50,7 @@ class RunConfig:
|
|||||||
n_trials_dsr: int = 50
|
n_trials_dsr: int = 50
|
||||||
db_path: Path = field(default_factory=lambda: Path("./runs.db"))
|
db_path: Path = field(default_factory=lambda: Path("./runs.db"))
|
||||||
prompt_mutation_weight: float = 0.0 # Phase 2.5: opt-in LLM mutator
|
prompt_mutation_weight: float = 0.0 # Phase 2.5: opt-in LLM mutator
|
||||||
|
fees_eat_alpha_threshold: float = 0.5 # adversarial gate, allenta verso 0.7-0.8
|
||||||
|
|
||||||
|
|
||||||
def run_phase1(
|
def run_phase1(
|
||||||
@@ -78,7 +79,10 @@ def run_phase1(
|
|||||||
falsification_agent = FalsificationAgent(
|
falsification_agent = FalsificationAgent(
|
||||||
fees_bp=cfg.fees_bp, n_trials_dsr=cfg.n_trials_dsr
|
fees_bp=cfg.fees_bp, n_trials_dsr=cfg.n_trials_dsr
|
||||||
)
|
)
|
||||||
adversarial_agent = AdversarialAgent(fees_bp=cfg.fees_bp)
|
adversarial_agent = AdversarialAgent(
|
||||||
|
fees_bp=cfg.fees_bp,
|
||||||
|
fees_eat_alpha_threshold=cfg.fees_eat_alpha_threshold,
|
||||||
|
)
|
||||||
cost_tracker = CostTracker()
|
cost_tracker = CostTracker()
|
||||||
|
|
||||||
population = build_initial_population(
|
population = build_initial_population(
|
||||||
@@ -178,6 +182,9 @@ def run_phase1(
|
|||||||
population = next_generation(
|
population = next_generation(
|
||||||
population, fitnesses, ga_cfg, rng,
|
population, fitnesses, ga_cfg, rng,
|
||||||
llm=llm if cfg.prompt_mutation_weight > 0 else None,
|
llm=llm if cfg.prompt_mutation_weight > 0 else None,
|
||||||
|
cost_tracker=cost_tracker if cfg.prompt_mutation_weight > 0 else None,
|
||||||
|
repo=repo if cfg.prompt_mutation_weight > 0 else None,
|
||||||
|
run_id=run_id if cfg.prompt_mutation_weight > 0 else None,
|
||||||
)
|
)
|
||||||
|
|
||||||
repo.complete_run(
|
repo.complete_run(
|
||||||
|
|||||||
@@ -26,6 +26,14 @@ class Repository:
|
|||||||
self.db_path.parent.mkdir(parents=True, exist_ok=True)
|
self.db_path.parent.mkdir(parents=True, exist_ok=True)
|
||||||
with self._conn() as conn:
|
with self._conn() as conn:
|
||||||
conn.executescript(SCHEMA_SQL)
|
conn.executescript(SCHEMA_SQL)
|
||||||
|
# Migration soft per DB pre-Task 6: aggiunge call_kind se manca.
|
||||||
|
try:
|
||||||
|
conn.execute(
|
||||||
|
"ALTER TABLE cost_records ADD COLUMN call_kind "
|
||||||
|
"TEXT NOT NULL DEFAULT 'hypothesis'"
|
||||||
|
)
|
||||||
|
except sqlite3.OperationalError:
|
||||||
|
pass # colonna già presente
|
||||||
|
|
||||||
@staticmethod
|
@staticmethod
|
||||||
def _now() -> str:
|
def _now() -> str:
|
||||||
@@ -184,12 +192,13 @@ class Repository:
|
|||||||
input_tokens: int,
|
input_tokens: int,
|
||||||
output_tokens: int,
|
output_tokens: int,
|
||||||
cost_usd: float,
|
cost_usd: float,
|
||||||
|
call_kind: str = "hypothesis",
|
||||||
) -> None:
|
) -> None:
|
||||||
with self._conn() as conn:
|
with self._conn() as conn:
|
||||||
conn.execute(
|
conn.execute(
|
||||||
"""INSERT INTO cost_records
|
"""INSERT INTO cost_records
|
||||||
(run_id, agent_id, ts, tier, input_tokens, output_tokens, cost_usd)
|
(run_id, agent_id, ts, tier, input_tokens, output_tokens, cost_usd, call_kind)
|
||||||
VALUES (?,?,?,?,?,?,?)""",
|
VALUES (?,?,?,?,?,?,?,?)""",
|
||||||
(
|
(
|
||||||
run_id,
|
run_id,
|
||||||
agent_id,
|
agent_id,
|
||||||
@@ -198,6 +207,7 @@ class Repository:
|
|||||||
input_tokens,
|
input_tokens,
|
||||||
output_tokens,
|
output_tokens,
|
||||||
cost_usd,
|
cost_usd,
|
||||||
|
call_kind,
|
||||||
),
|
),
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|||||||
@@ -58,6 +58,7 @@ CREATE TABLE IF NOT EXISTS cost_records (
|
|||||||
input_tokens INTEGER NOT NULL,
|
input_tokens INTEGER NOT NULL,
|
||||||
output_tokens INTEGER NOT NULL,
|
output_tokens INTEGER NOT NULL,
|
||||||
cost_usd REAL NOT NULL,
|
cost_usd REAL NOT NULL,
|
||||||
|
call_kind TEXT NOT NULL DEFAULT 'hypothesis',
|
||||||
FOREIGN KEY (run_id) REFERENCES runs(id)
|
FOREIGN KEY (run_id) REFERENCES runs(id)
|
||||||
);
|
);
|
||||||
|
|
||||||
|
|||||||
@@ -61,3 +61,28 @@ def test_tracker_summary_contains_all_five_tiers():
|
|||||||
for tier_letter in ("S", "A", "B", "C", "D"):
|
for tier_letter in ("S", "A", "B", "C", "D"):
|
||||||
assert tier_letter in summary["by_tier"]
|
assert tier_letter in summary["by_tier"]
|
||||||
assert summary["by_tier"][tier_letter]["calls"] == 1
|
assert summary["by_tier"][tier_letter]["calls"] == 1
|
||||||
|
|
||||||
|
|
||||||
|
def test_tracker_default_call_kind_is_hypothesis():
|
||||||
|
t = CostTracker()
|
||||||
|
rec = t.record(input_tokens=10, output_tokens=10, tier=ModelTier.C, run_id="r", agent_id="a")
|
||||||
|
assert rec.call_kind == "hypothesis"
|
||||||
|
summary = t.summary()
|
||||||
|
assert "hypothesis" in summary["by_call_kind"]
|
||||||
|
assert summary["by_call_kind"]["hypothesis"]["calls"] == 1
|
||||||
|
assert "mutation" not in summary["by_call_kind"]
|
||||||
|
|
||||||
|
|
||||||
|
def test_tracker_by_call_kind_breakdown():
|
||||||
|
t = CostTracker()
|
||||||
|
t.record(input_tokens=100, output_tokens=200, tier=ModelTier.C, run_id="r", agent_id="a")
|
||||||
|
t.record(input_tokens=100, output_tokens=200, tier=ModelTier.C, run_id="r", agent_id="a")
|
||||||
|
t.record(
|
||||||
|
input_tokens=50, output_tokens=80, tier=ModelTier.B,
|
||||||
|
run_id="r", agent_id="parent-x", call_kind="mutation",
|
||||||
|
)
|
||||||
|
summary = t.summary()
|
||||||
|
assert summary["by_call_kind"]["hypothesis"]["calls"] == 2
|
||||||
|
assert summary["by_call_kind"]["mutation"]["calls"] == 1
|
||||||
|
assert summary["by_call_kind"]["mutation"]["input_tokens"] == 50
|
||||||
|
assert summary["by_call_kind"]["mutation"]["output_tokens"] == 80
|
||||||
|
|||||||
@@ -161,6 +161,69 @@ def test_mutate_prompt_llm_falls_back_on_llm_exception() -> None:
|
|||||||
assert child.generation == parent.generation + 1
|
assert child.generation == parent.generation + 1
|
||||||
|
|
||||||
|
|
||||||
|
def test_mutate_prompt_llm_logs_mutation_cost_when_sink_provided() -> None:
|
||||||
|
"""Quando cost_tracker+repo+run_id sono forniti, la call mutator viene loggata
|
||||||
|
con call_kind='mutation' sia in memoria sia nel repo."""
|
||||||
|
mutated = (
|
||||||
|
"Strategia RSI 1h evolved. Entry long quando RSI(14) < 28 e close > "
|
||||||
|
"SMA(50). Exit short quando RSI(14) > 72."
|
||||||
|
)
|
||||||
|
|
||||||
|
class _R:
|
||||||
|
text = f"<prompt>{mutated}</prompt>"
|
||||||
|
input_tokens = 350
|
||||||
|
output_tokens = 140
|
||||||
|
|
||||||
|
class _FakeLLMCosted:
|
||||||
|
def complete(self, genome, system, user, max_tokens=2000):
|
||||||
|
return _R()
|
||||||
|
|
||||||
|
tracker_calls = []
|
||||||
|
repo_calls = []
|
||||||
|
|
||||||
|
class _FakeTracker:
|
||||||
|
def record(self, **kw):
|
||||||
|
tracker_calls.append(kw)
|
||||||
|
from types import SimpleNamespace
|
||||||
|
return SimpleNamespace(cost_usd=0.0042)
|
||||||
|
|
||||||
|
class _FakeRepo:
|
||||||
|
def save_cost_record(self, **kw):
|
||||||
|
repo_calls.append(kw)
|
||||||
|
|
||||||
|
parent = _make_genome()
|
||||||
|
child = mutate_prompt_llm(
|
||||||
|
parent, _FakeLLMCosted(), random.Random(0),
|
||||||
|
cost_tracker=_FakeTracker(), repo=_FakeRepo(), run_id="run-xyz",
|
||||||
|
)
|
||||||
|
assert child.system_prompt == mutated
|
||||||
|
assert len(tracker_calls) == 1
|
||||||
|
assert tracker_calls[0]["call_kind"] == "mutation"
|
||||||
|
assert tracker_calls[0]["tier"] == ModelTier.B
|
||||||
|
assert tracker_calls[0]["run_id"] == "run-xyz"
|
||||||
|
assert tracker_calls[0]["agent_id"] == parent.id
|
||||||
|
assert tracker_calls[0]["input_tokens"] == 350
|
||||||
|
assert tracker_calls[0]["output_tokens"] == 140
|
||||||
|
|
||||||
|
assert len(repo_calls) == 1
|
||||||
|
assert repo_calls[0]["call_kind"] == "mutation"
|
||||||
|
assert repo_calls[0]["tier"] == "B"
|
||||||
|
assert repo_calls[0]["cost_usd"] == 0.0042
|
||||||
|
|
||||||
|
|
||||||
|
def test_mutate_prompt_llm_no_logging_without_sink() -> None:
|
||||||
|
"""Senza cost_tracker+repo+run_id → niente logging cost (backward compat)."""
|
||||||
|
mutated = (
|
||||||
|
"Strategia RSI 1h evoluta. Entry long quando RSI(14) < 25 e close > "
|
||||||
|
"SMA(60). Exit short quando RSI(14) > 75 e ATR rising."
|
||||||
|
)
|
||||||
|
llm = _FakeLLM(response_text=f"<prompt>{mutated}</prompt>")
|
||||||
|
parent = _make_genome()
|
||||||
|
# Non solleva (anche se 0 sink forniti)
|
||||||
|
child = mutate_prompt_llm(parent, llm, random.Random(0))
|
||||||
|
assert child.system_prompt == mutated
|
||||||
|
|
||||||
|
|
||||||
def test_mutate_prompt_llm_picks_one_of_six_instructions() -> None:
|
def test_mutate_prompt_llm_picks_one_of_six_instructions() -> None:
|
||||||
"""Verifica che il system message dell'LLM includa una delle 6 istruzioni."""
|
"""Verifica che il system message dell'LLM includa una delle 6 istruzioni."""
|
||||||
mutated = (
|
mutated = (
|
||||||
|
|||||||
Reference in New Issue
Block a user