Adriano
0e01de156f
Run phase2-5-qwen25-prompt-mut-003 fallito a gen 5 (76 evals done, $0.061 spesi) per HTTP 429 RateLimit upstream da DeepInfra su qwen-2.5-72b. RateLimitError NON era in _RETRYABLE_EXCEPTIONS quindi tenacity falliva subito, propagando il 429 a propose() e all'orchestrator (run failed). Tre fix: 1) Aggiunto openai.RateLimitError a _RETRYABLE_EXCEPTIONS. 2) Bumpato stop_after_attempt(3) → 5 e wait_exponential max 10s → 30s. Più tempo per il rate limit upstream di sbloccare prima di rinunciare. 3) hypothesis.py: try/except RateLimitError in propose() come per EmptyCompletionError — anche se tenacity esaurisce i 5 retry, il genome viene marcato fitness=0 e il loop esterno continua senza crash totale. Test: aggiornato test_completion_retries_on_connection_error per assert call_count == 5 (era 3). Tutti 182 verdi. Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
233 lines
9.1 KiB
Python
233 lines
9.1 KiB
Python
import pytest
|
|
|
|
from multi_swarm.genome.hypothesis import HypothesisAgentGenome, ModelTier
|
|
from multi_swarm.llm.client import CompletionResult, LLMClient
|
|
|
|
|
|
def make_genome(tier: ModelTier) -> HypothesisAgentGenome:
|
|
return HypothesisAgentGenome(
|
|
system_prompt="x",
|
|
feature_access=["close"],
|
|
temperature=0.9,
|
|
top_p=0.95,
|
|
model_tier=tier,
|
|
lookback_window=200,
|
|
cognitive_style="physicist",
|
|
)
|
|
|
|
|
|
def test_completion_tier_c_uses_openrouter(mocker):
|
|
fake_openai = mocker.MagicMock()
|
|
fake_response = mocker.MagicMock()
|
|
fake_response.choices = [mocker.MagicMock(message=mocker.MagicMock(content="(strategy ...)"))]
|
|
fake_response.usage = mocker.MagicMock(prompt_tokens=100, completion_tokens=200)
|
|
fake_openai.chat.completions.create.return_value = fake_response
|
|
|
|
mocker.patch("multi_swarm.llm.client.OpenAI", return_value=fake_openai)
|
|
|
|
client = LLMClient(openrouter_api_key="or-x")
|
|
g = make_genome(ModelTier.C)
|
|
out = client.complete(g, system="sys", user="usr")
|
|
|
|
assert isinstance(out, CompletionResult)
|
|
assert out.text == "(strategy ...)"
|
|
assert out.input_tokens == 100
|
|
assert out.output_tokens == 200
|
|
assert out.tier == ModelTier.C
|
|
fake_openai.chat.completions.create.assert_called_once()
|
|
|
|
|
|
def test_completion_tier_b_uses_openrouter_with_anthropic_model(mocker):
|
|
fake_openai = mocker.MagicMock()
|
|
fake_response = mocker.MagicMock()
|
|
fake_response.choices = [mocker.MagicMock(message=mocker.MagicMock(content="(strategy ...)"))]
|
|
fake_response.usage = mocker.MagicMock(prompt_tokens=80, completion_tokens=150)
|
|
fake_openai.chat.completions.create.return_value = fake_response
|
|
mocker.patch("multi_swarm.llm.client.OpenAI", return_value=fake_openai)
|
|
|
|
client = LLMClient(openrouter_api_key="or-x")
|
|
g = make_genome(ModelTier.B)
|
|
out = client.complete(g, system="sys", user="usr")
|
|
|
|
assert out.text == "(strategy ...)"
|
|
assert out.input_tokens == 80
|
|
assert out.output_tokens == 150
|
|
assert out.tier == ModelTier.B
|
|
call_kwargs = fake_openai.chat.completions.create.call_args.kwargs
|
|
assert call_kwargs["model"] == "deepseek/deepseek-v4-flash"
|
|
assert out.model == "deepseek/deepseek-v4-flash"
|
|
|
|
|
|
@pytest.mark.slow
|
|
def test_completion_retries_on_connection_error(mocker):
|
|
"""Retry esegue 3 tentativi su APIConnectionError, poi rilancia."""
|
|
import openai
|
|
|
|
fake_openai = mocker.MagicMock()
|
|
fake_openai.chat.completions.create.side_effect = openai.APIConnectionError(
|
|
request=mocker.MagicMock()
|
|
)
|
|
mocker.patch("multi_swarm.llm.client.OpenAI", return_value=fake_openai)
|
|
|
|
client = LLMClient(openrouter_api_key="or-x")
|
|
g = make_genome(ModelTier.C)
|
|
|
|
with pytest.raises(openai.APIConnectionError):
|
|
client.complete(g, system="sys", user="usr")
|
|
|
|
assert fake_openai.chat.completions.create.call_count == 5
|
|
|
|
|
|
def test_completion_uses_custom_model_tier_c(mocker):
|
|
fake_openai = mocker.MagicMock()
|
|
fake_response = mocker.MagicMock()
|
|
fake_response.choices = [
|
|
mocker.MagicMock(message=mocker.MagicMock(content="(strategy ...)"))
|
|
]
|
|
fake_response.usage = mocker.MagicMock(prompt_tokens=10, completion_tokens=20)
|
|
fake_openai.chat.completions.create.return_value = fake_response
|
|
mocker.patch("multi_swarm.llm.client.OpenAI", return_value=fake_openai)
|
|
|
|
client = LLMClient(
|
|
openrouter_api_key="or-x",
|
|
model_tier_c="deepseek/deepseek-chat",
|
|
)
|
|
g = make_genome(ModelTier.C)
|
|
out = client.complete(g, system="sys", user="usr")
|
|
|
|
fake_openai.chat.completions.create.assert_called_once()
|
|
call_kwargs = fake_openai.chat.completions.create.call_args.kwargs
|
|
assert call_kwargs["model"] == "deepseek/deepseek-chat"
|
|
assert out.model == "deepseek/deepseek-chat"
|
|
|
|
|
|
def test_completion_uses_custom_model_tier_b(mocker):
|
|
fake_openai = mocker.MagicMock()
|
|
fake_response = mocker.MagicMock()
|
|
fake_response.choices = [
|
|
mocker.MagicMock(message=mocker.MagicMock(content="(strategy ...)"))
|
|
]
|
|
fake_response.usage = mocker.MagicMock(prompt_tokens=10, completion_tokens=20)
|
|
fake_openai.chat.completions.create.return_value = fake_response
|
|
mocker.patch("multi_swarm.llm.client.OpenAI", return_value=fake_openai)
|
|
|
|
client = LLMClient(
|
|
openrouter_api_key="or-x",
|
|
model_tier_b="anthropic/claude-opus-4-7",
|
|
)
|
|
g = make_genome(ModelTier.B)
|
|
out = client.complete(g, system="sys", user="usr")
|
|
|
|
fake_openai.chat.completions.create.assert_called_once()
|
|
call_kwargs = fake_openai.chat.completions.create.call_args.kwargs
|
|
assert call_kwargs["model"] == "anthropic/claude-opus-4-7"
|
|
assert out.model == "anthropic/claude-opus-4-7"
|
|
|
|
|
|
def test_completion_tier_s_uses_openrouter_with_anthropic_model(mocker):
|
|
fake_openai = mocker.MagicMock()
|
|
fake_response = mocker.MagicMock()
|
|
fake_response.choices = [mocker.MagicMock(message=mocker.MagicMock(content="(strategy s)"))]
|
|
fake_response.usage = mocker.MagicMock(prompt_tokens=50, completion_tokens=100)
|
|
fake_openai.chat.completions.create.return_value = fake_response
|
|
mocker.patch("multi_swarm.llm.client.OpenAI", return_value=fake_openai)
|
|
|
|
client = LLMClient(openrouter_api_key="or-x")
|
|
g = make_genome(ModelTier.S)
|
|
out = client.complete(g, system="sys", user="usr")
|
|
|
|
fake_openai.chat.completions.create.assert_called_once()
|
|
call_kwargs = fake_openai.chat.completions.create.call_args.kwargs
|
|
assert call_kwargs["model"] == "google/gemini-3-flash-preview"
|
|
assert out.tier == ModelTier.S
|
|
assert out.model == "google/gemini-3-flash-preview"
|
|
|
|
|
|
def test_completion_tier_a_uses_openrouter_with_anthropic_model(mocker):
|
|
fake_openai = mocker.MagicMock()
|
|
fake_response = mocker.MagicMock()
|
|
fake_response.choices = [mocker.MagicMock(message=mocker.MagicMock(content="(strategy a)"))]
|
|
fake_response.usage = mocker.MagicMock(prompt_tokens=40, completion_tokens=80)
|
|
fake_openai.chat.completions.create.return_value = fake_response
|
|
mocker.patch("multi_swarm.llm.client.OpenAI", return_value=fake_openai)
|
|
|
|
client = LLMClient(openrouter_api_key="or-x")
|
|
g = make_genome(ModelTier.A)
|
|
out = client.complete(g, system="sys", user="usr")
|
|
|
|
fake_openai.chat.completions.create.assert_called_once()
|
|
call_kwargs = fake_openai.chat.completions.create.call_args.kwargs
|
|
assert call_kwargs["model"] == "deepseek/deepseek-v4-flash"
|
|
assert out.tier == ModelTier.A
|
|
assert out.model == "deepseek/deepseek-v4-flash"
|
|
|
|
|
|
def test_completion_tier_d_uses_openrouter_with_llama(mocker):
|
|
fake_openai = mocker.MagicMock()
|
|
fake_response = mocker.MagicMock()
|
|
fake_response.choices = [
|
|
mocker.MagicMock(message=mocker.MagicMock(content="(strategy d)"))
|
|
]
|
|
fake_response.usage = mocker.MagicMock(prompt_tokens=30, completion_tokens=70)
|
|
fake_openai.chat.completions.create.return_value = fake_response
|
|
mocker.patch("multi_swarm.llm.client.OpenAI", return_value=fake_openai)
|
|
|
|
client = LLMClient(openrouter_api_key="or-x")
|
|
g = make_genome(ModelTier.D)
|
|
out = client.complete(g, system="sys", user="usr")
|
|
|
|
fake_openai.chat.completions.create.assert_called_once()
|
|
call_kwargs = fake_openai.chat.completions.create.call_args.kwargs
|
|
assert call_kwargs["model"] == "openai/gpt-oss-20b"
|
|
assert out.tier == ModelTier.D
|
|
assert out.model == "openai/gpt-oss-20b"
|
|
|
|
|
|
def test_completion_uses_custom_model_tier_s(mocker):
|
|
fake_openai = mocker.MagicMock()
|
|
fake_response = mocker.MagicMock()
|
|
fake_response.choices = [
|
|
mocker.MagicMock(message=mocker.MagicMock(content="(strategy custom-s)"))
|
|
]
|
|
fake_response.usage = mocker.MagicMock(prompt_tokens=10, completion_tokens=20)
|
|
fake_openai.chat.completions.create.return_value = fake_response
|
|
mocker.patch("multi_swarm.llm.client.OpenAI", return_value=fake_openai)
|
|
|
|
client = LLMClient(
|
|
openrouter_api_key="or-x",
|
|
model_tier_s="anthropic/claude-future-mega",
|
|
)
|
|
g = make_genome(ModelTier.S)
|
|
out = client.complete(g, system="sys", user="usr")
|
|
|
|
call_kwargs = fake_openai.chat.completions.create.call_args.kwargs
|
|
assert call_kwargs["model"] == "anthropic/claude-future-mega"
|
|
assert out.model == "anthropic/claude-future-mega"
|
|
|
|
|
|
@pytest.mark.slow
|
|
def test_completion_succeeds_after_one_retry(mocker):
|
|
"""Dopo 1 fallimento transient, il retry riesce al 2 tentativo."""
|
|
import openai
|
|
|
|
fake_response = mocker.MagicMock()
|
|
fake_response.choices = [
|
|
mocker.MagicMock(message=mocker.MagicMock(content="(strategy ...)"))
|
|
]
|
|
fake_response.usage = mocker.MagicMock(prompt_tokens=100, completion_tokens=200)
|
|
|
|
fake_openai = mocker.MagicMock()
|
|
fake_openai.chat.completions.create.side_effect = [
|
|
openai.APITimeoutError(request=mocker.MagicMock()),
|
|
fake_response,
|
|
]
|
|
mocker.patch("multi_swarm.llm.client.OpenAI", return_value=fake_openai)
|
|
|
|
client = LLMClient(openrouter_api_key="or-x")
|
|
g = make_genome(ModelTier.C)
|
|
out = client.complete(g, system="sys", user="usr")
|
|
|
|
assert isinstance(out, CompletionResult)
|
|
assert out.text == "(strategy ...)"
|
|
assert fake_openai.chat.completions.create.call_count == 2
|