Adriano
8ec45c5c1b
Run controllo phase2-qwen25-control-001 (seed 42, stessa pipeline Phase 2, solo tier C switched) ha dimostrato che qwen-2.5-72b è qualitativamente SUPERIORE a qwen3-235b sul nostro workload: | metrica | qwen3-235b | qwen-2.5-72b | delta | | ----------------- | ---------- | ------------ | ----- | | max fitness | 0.0238 | 0.0311 | +30% | | median > 0 in gen | mai | 4 gen su 10 | -- | | entropy media | 0.199 | 0.85 | 4.3x | | genomi fit > 0 | 5 | 10 | 2x | | parse success | 97.7% | 100% | + | | durata | 50 min | 28 min | 0.56x | | LLM calls | 148 | 90 | 0.61x | | cost USD | 0.0223 | 0.0122 | 0.55x | Controintuitivo: 235B con context 262k era atteso superiore al 72B legacy. In pratica qwen3-235b in tier C produce strategie meno diverse, meno parsabili e meno ottimizzabili dal GA. Ripristinati prezzi cost_tracker tier C a 0.40/0.40 (qwen-2.5-72b). Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
82 lines
3.5 KiB
Python
82 lines
3.5 KiB
Python
"""Tests for multi_swarm.config.Settings.
|
|
|
|
Note on .env isolation:
|
|
The happy-path test relies on monkeypatch.setenv to provide values.
|
|
The "requires tokens" test forces _env_file=None when constructing Settings,
|
|
so that a developer's local .env (if present and populated) cannot mask the
|
|
absence of required env vars. This keeps the test deterministic both in CI
|
|
(no .env) and in local dev (.env may exist).
|
|
"""
|
|
|
|
import pytest
|
|
|
|
from multi_swarm.config import Settings
|
|
|
|
|
|
def test_settings_loads_from_env(monkeypatch: pytest.MonkeyPatch) -> None:
|
|
monkeypatch.setenv("CERBERO_BASE_URL", "http://test:9000")
|
|
monkeypatch.setenv("CERBERO_TESTNET_TOKEN", "tok-test")
|
|
monkeypatch.setenv("CERBERO_MAINNET_TOKEN", "tok-main")
|
|
monkeypatch.setenv("CERBERO_BOT_TAG", "swarm-poc-phase1")
|
|
monkeypatch.setenv("OPENROUTER_API_KEY", "or-key")
|
|
monkeypatch.setenv("RUN_NAME", "test-run")
|
|
|
|
s = Settings() # type: ignore[call-arg]
|
|
|
|
assert s.cerbero_base_url == "http://test:9000"
|
|
assert s.cerbero_testnet_token.get_secret_value() == "tok-test"
|
|
assert s.run_name == "test-run"
|
|
assert s.data_dir.name == "data"
|
|
assert s.db_path.name == "runs.db"
|
|
|
|
|
|
def test_settings_requires_tokens(monkeypatch: pytest.MonkeyPatch) -> None:
|
|
monkeypatch.delenv("CERBERO_TESTNET_TOKEN", raising=False)
|
|
monkeypatch.delenv("OPENROUTER_API_KEY", raising=False)
|
|
from pydantic import ValidationError
|
|
|
|
with pytest.raises(ValidationError):
|
|
# Disable .env loading to keep the test deterministic regardless of
|
|
# whether a developer's local .env exists and is populated.
|
|
Settings(_env_file=None) # type: ignore[call-arg]
|
|
|
|
|
|
def test_settings_loads_llm_model_overrides(monkeypatch: pytest.MonkeyPatch) -> None:
|
|
monkeypatch.setenv("CERBERO_TESTNET_TOKEN", "tok-test")
|
|
monkeypatch.setenv("OPENROUTER_API_KEY", "or-key")
|
|
monkeypatch.setenv("LLM_MODEL_TIER_S", "claude-mega-x")
|
|
monkeypatch.setenv("LLM_MODEL_TIER_A", "claude-premium-y")
|
|
monkeypatch.setenv("LLM_MODEL_TIER_B", "claude-opus-4-7")
|
|
monkeypatch.setenv("LLM_MODEL_TIER_C", "deepseek/deepseek-chat")
|
|
monkeypatch.setenv("LLM_MODEL_TIER_D", "mistralai/mistral-7b")
|
|
monkeypatch.setenv("OPENROUTER_BASE_URL", "https://example.com/api/v1")
|
|
|
|
s = Settings(_env_file=None) # type: ignore[call-arg]
|
|
|
|
assert s.llm_model_tier_s == "claude-mega-x"
|
|
assert s.llm_model_tier_a == "claude-premium-y"
|
|
assert s.llm_model_tier_b == "claude-opus-4-7"
|
|
assert s.llm_model_tier_c == "deepseek/deepseek-chat"
|
|
assert s.llm_model_tier_d == "mistralai/mistral-7b"
|
|
assert s.openrouter_base_url == "https://example.com/api/v1"
|
|
|
|
|
|
def test_settings_llm_model_defaults(monkeypatch: pytest.MonkeyPatch) -> None:
|
|
monkeypatch.setenv("CERBERO_TESTNET_TOKEN", "tok-test")
|
|
monkeypatch.setenv("OPENROUTER_API_KEY", "or-key")
|
|
monkeypatch.delenv("LLM_MODEL_TIER_S", raising=False)
|
|
monkeypatch.delenv("LLM_MODEL_TIER_A", raising=False)
|
|
monkeypatch.delenv("LLM_MODEL_TIER_B", raising=False)
|
|
monkeypatch.delenv("LLM_MODEL_TIER_C", raising=False)
|
|
monkeypatch.delenv("LLM_MODEL_TIER_D", raising=False)
|
|
monkeypatch.delenv("OPENROUTER_BASE_URL", raising=False)
|
|
|
|
s = Settings(_env_file=None) # type: ignore[call-arg]
|
|
|
|
assert s.llm_model_tier_s == "google/gemini-3-flash-preview"
|
|
assert s.llm_model_tier_a == "deepseek/deepseek-v4-flash"
|
|
assert s.llm_model_tier_b == "deepseek/deepseek-v4-flash"
|
|
assert s.llm_model_tier_c == "qwen/qwen-2.5-72b-instruct"
|
|
assert s.llm_model_tier_d == "openai/gpt-oss-20b"
|
|
assert s.openrouter_base_url == "https://openrouter.ai/api/v1"
|