Adriano
8ec45c5c1b
Run controllo phase2-qwen25-control-001 (seed 42, stessa pipeline Phase 2, solo tier C switched) ha dimostrato che qwen-2.5-72b è qualitativamente SUPERIORE a qwen3-235b sul nostro workload: | metrica | qwen3-235b | qwen-2.5-72b | delta | | ----------------- | ---------- | ------------ | ----- | | max fitness | 0.0238 | 0.0311 | +30% | | median > 0 in gen | mai | 4 gen su 10 | -- | | entropy media | 0.199 | 0.85 | 4.3x | | genomi fit > 0 | 5 | 10 | 2x | | parse success | 97.7% | 100% | + | | durata | 50 min | 28 min | 0.56x | | LLM calls | 148 | 90 | 0.61x | | cost USD | 0.0223 | 0.0122 | 0.55x | Controintuitivo: 235B con context 262k era atteso superiore al 72B legacy. In pratica qwen3-235b in tier C produce strategie meno diverse, meno parsabili e meno ottimizzabili dal GA. Ripristinati prezzi cost_tracker tier C a 0.40/0.40 (qwen-2.5-72b). Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
64 lines
2.2 KiB
Python
64 lines
2.2 KiB
Python
from multi_swarm.genome.hypothesis import ModelTier
|
|
from multi_swarm.llm.cost_tracker import CostTracker, estimate_cost
|
|
|
|
|
|
def test_estimate_cost_tier_c():
|
|
cost = estimate_cost(input_tokens=1_000_000, output_tokens=1_000_000, tier=ModelTier.C)
|
|
assert cost == 0.40 + 0.40
|
|
|
|
|
|
def test_estimate_cost_tier_b():
|
|
cost = estimate_cost(input_tokens=1_000_000, output_tokens=1_000_000, tier=ModelTier.B)
|
|
assert cost == 0.14 + 0.28
|
|
|
|
|
|
def test_tracker_accumulates():
|
|
t = CostTracker()
|
|
t.record(input_tokens=10_000, output_tokens=20_000, tier=ModelTier.C, run_id="r", agent_id="a")
|
|
t.record(input_tokens=5_000, output_tokens=15_000, tier=ModelTier.C, run_id="r", agent_id="b")
|
|
summary = t.summary()
|
|
assert summary["calls"] == 2
|
|
assert summary["input_tokens"] == 15_000
|
|
assert summary["output_tokens"] == 35_000
|
|
assert summary["cost_usd"] > 0
|
|
|
|
|
|
def test_tracker_per_tier_breakdown():
|
|
t = CostTracker()
|
|
t.record(input_tokens=10_000, output_tokens=10_000, tier=ModelTier.C, run_id="r", agent_id="a")
|
|
t.record(input_tokens=10_000, output_tokens=10_000, tier=ModelTier.B, run_id="r", agent_id="b")
|
|
summary = t.summary()
|
|
assert "C" in summary["by_tier"]
|
|
assert "B" in summary["by_tier"]
|
|
|
|
|
|
def test_estimate_cost_tier_s():
|
|
cost = estimate_cost(input_tokens=1_000_000, output_tokens=1_000_000, tier=ModelTier.S)
|
|
assert cost == 0.50 + 3.00
|
|
|
|
|
|
def test_estimate_cost_tier_a():
|
|
cost = estimate_cost(input_tokens=1_000_000, output_tokens=1_000_000, tier=ModelTier.A)
|
|
assert cost == 0.14 + 0.28
|
|
|
|
|
|
def test_estimate_cost_tier_d():
|
|
cost = estimate_cost(input_tokens=1_000_000, output_tokens=1_000_000, tier=ModelTier.D)
|
|
assert cost == 0.03 + 0.14
|
|
|
|
|
|
def test_tracker_summary_contains_all_five_tiers():
|
|
t = CostTracker()
|
|
for tier in (ModelTier.S, ModelTier.A, ModelTier.B, ModelTier.C, ModelTier.D):
|
|
t.record(
|
|
input_tokens=1_000,
|
|
output_tokens=1_000,
|
|
tier=tier,
|
|
run_id="r",
|
|
agent_id=f"a-{tier.value}",
|
|
)
|
|
summary = t.summary()
|
|
for tier_letter in ("S", "A", "B", "C", "D"):
|
|
assert tier_letter in summary["by_tier"]
|
|
assert summary["by_tier"][tier_letter]["calls"] == 1
|