feat(llm): migrate from OpenRouter to OpusAgent
Replace OpenRouter (openai SDK) with OpusAgent REST API (httpx + polling). LLMClient now creates topics per system prompt, submits async requests, and polls for completion. Model tiers map to Claude model IDs (opus-4-7, sonnet-4-6, haiku-4-5) configurable via env vars. Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
This commit is contained in:
Adriano Dal Pastro
+1
-1
@@ -23,7 +23,7 @@
|
||||
# ./state contiene runs.db (GA) + strategy_crypto.db (paper) + WAL/SHM
|
||||
# ./src/strategy_crypto/strategy_crypto/strategies JSON freezate (ro)
|
||||
#
|
||||
# Secrets (token Cerbero + OpenRouter): caricati da .env via env_file.
|
||||
# Secrets (token Cerbero + OpusAgent): caricati da .env via env_file.
|
||||
|
||||
networks:
|
||||
traefik:
|
||||
|
||||
@@ -159,8 +159,8 @@ def parse_args() -> argparse.Namespace:
|
||||
default=1,
|
||||
help=(
|
||||
"Numero di propose() LLM concorrenti per generazione (default 1 = "
|
||||
"serial). 6-10 tipicamente accettati da OpenRouter qwen-2.5 senza "
|
||||
"rate-limit; riduce wall time GA loop di 5-8x."
|
||||
"serial). OpusAgent processa in coda FIFO; concurrency > 1 accoda "
|
||||
"piu' richieste in parallelo."
|
||||
),
|
||||
)
|
||||
return p.parse_args()
|
||||
@@ -200,13 +200,13 @@ def main() -> None:
|
||||
print(f"OHLCV loaded: {len(ohlcv)} bars from {ohlcv.index[0]} to {ohlcv.index[-1]}")
|
||||
|
||||
llm = LLMClient(
|
||||
openrouter_api_key=settings.openrouter_api_key.get_secret_value(),
|
||||
opus_agent_api_key=settings.opus_agent_api_key.get_secret_value(),
|
||||
opus_agent_base_url=settings.opus_agent_base_url,
|
||||
model_tier_s=settings.llm_model_tier_s,
|
||||
model_tier_a=settings.llm_model_tier_a,
|
||||
model_tier_b=settings.llm_model_tier_b,
|
||||
model_tier_c=settings.llm_model_tier_c,
|
||||
model_tier_d=settings.llm_model_tier_d,
|
||||
openrouter_base_url=settings.openrouter_base_url,
|
||||
)
|
||||
|
||||
cfg = RunConfig(
|
||||
|
||||
@@ -410,16 +410,14 @@ def main() -> None:
|
||||
)
|
||||
loader = CerberoOHLCVLoader(client=cerbero, cache_dir=settings.series_dir)
|
||||
|
||||
# LLM client (qwen-2.5-72b tier C come da spec progetto - vedi MEMORY:
|
||||
# cambiare modello senza ricalibrare = regressione dimostrata).
|
||||
llm = LLMClient(
|
||||
openrouter_api_key=settings.openrouter_api_key.get_secret_value(),
|
||||
opus_agent_api_key=settings.opus_agent_api_key.get_secret_value(),
|
||||
opus_agent_base_url=settings.opus_agent_base_url,
|
||||
model_tier_s=settings.llm_model_tier_s,
|
||||
model_tier_a=settings.llm_model_tier_a,
|
||||
model_tier_b=settings.llm_model_tier_b,
|
||||
model_tier_c=settings.llm_model_tier_c,
|
||||
model_tier_d=settings.llm_model_tier_d,
|
||||
openrouter_base_url=settings.openrouter_base_url,
|
||||
)
|
||||
|
||||
# Setup DB winners (separato dal GA core DB).
|
||||
|
||||
@@ -4,11 +4,9 @@ import re
|
||||
from dataclasses import dataclass, field
|
||||
from typing import Any
|
||||
|
||||
import openai
|
||||
|
||||
from ..genome.hypothesis import HypothesisAgentGenome
|
||||
from ..genome.prompt_library import PromptLibrary
|
||||
from ..llm.client import CompletionResult, EmptyCompletionError, LLMClient
|
||||
from ..llm.client import CompletionResult, EmptyCompletionError, LLMClient, OpusAgentError, OpusAgentTransientError
|
||||
from ..protocol.parser import ParseError, Strategy, parse_strategy
|
||||
from ..protocol.validator import ValidationError, validate_strategy
|
||||
|
||||
@@ -446,15 +444,15 @@ class HypothesisAgent:
|
||||
try:
|
||||
completion = self._llm.complete(genome, system=system, user=user)
|
||||
except EmptyCompletionError as e:
|
||||
# LLM esaurito retry tenacity senza una risposta. Tratta come
|
||||
# parse-fail "empty" e ritenta nel loop esterno (max_attempts).
|
||||
errors.append(f"empty_completion: {e}")
|
||||
last_raw = ""
|
||||
continue
|
||||
except openai.RateLimitError as e:
|
||||
# Provider upstream rate limited oltre i retry tenacity.
|
||||
# Marca genome come fallito senza propagare l'eccezione al run.
|
||||
errors.append(f"rate_limit: {e}")
|
||||
except OpusAgentTransientError as e:
|
||||
errors.append(f"transient_error: {e}")
|
||||
last_raw = ""
|
||||
continue
|
||||
except OpusAgentError as e:
|
||||
errors.append(f"opus_agent_error: {e}")
|
||||
last_raw = ""
|
||||
continue
|
||||
completions.append(completion)
|
||||
|
||||
@@ -23,14 +23,14 @@ class Settings(BaseSettings):
|
||||
cerbero_mainnet_token: SecretStr | None = None
|
||||
cerbero_bot_tag: str = "swarm-poc-phase1"
|
||||
|
||||
openrouter_api_key: SecretStr
|
||||
opus_agent_api_key: SecretStr
|
||||
opus_agent_base_url: str = "https://opus-agent.tielogic.xyz"
|
||||
|
||||
llm_model_tier_s: str = "google/gemini-3-flash-preview"
|
||||
llm_model_tier_a: str = "deepseek/deepseek-v4-flash"
|
||||
llm_model_tier_b: str = "deepseek/deepseek-v4-flash"
|
||||
llm_model_tier_c: str = "qwen/qwen-2.5-72b-instruct"
|
||||
llm_model_tier_d: str = "openai/gpt-oss-20b"
|
||||
openrouter_base_url: str = "https://openrouter.ai/api/v1"
|
||||
llm_model_tier_s: str = "claude-opus-4-7"
|
||||
llm_model_tier_a: str = "claude-opus-4-7"
|
||||
llm_model_tier_b: str = "claude-sonnet-4-6"
|
||||
llm_model_tier_c: str = "claude-sonnet-4-6"
|
||||
llm_model_tier_d: str = "claude-haiku-4-5-20251001"
|
||||
|
||||
run_name: str = "phase1-spike-001"
|
||||
data_dir: Path = Field(default=Path("./data"))
|
||||
|
||||
@@ -8,11 +8,11 @@ from typing import Any
|
||||
|
||||
|
||||
class ModelTier(StrEnum):
|
||||
S = "S" # top-tier reasoning (Opus / equivalent) via Anthropic
|
||||
A = "A" # premium override via Anthropic
|
||||
B = "B" # Sonnet 4.6 via Anthropic
|
||||
C = "C" # Qwen 2.5 72B via OpenRouter
|
||||
D = "D" # ultra-economic (Llama / cheap models) via OpenRouter
|
||||
S = "S" # top-tier reasoning → opus via OpusAgent
|
||||
A = "A" # premium → opus via OpusAgent
|
||||
B = "B" # standard → sonnet via OpusAgent
|
||||
C = "C" # default GA → sonnet via OpusAgent
|
||||
D = "D" # economic → haiku via OpusAgent
|
||||
|
||||
|
||||
@dataclass
|
||||
|
||||
@@ -1,10 +1,12 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
import logging
|
||||
import threading
|
||||
import time
|
||||
from dataclasses import dataclass
|
||||
from typing import Any
|
||||
|
||||
import openai
|
||||
from openai import OpenAI
|
||||
import httpx
|
||||
from tenacity import (
|
||||
retry,
|
||||
retry_if_exception_type,
|
||||
@@ -14,26 +16,33 @@ from tenacity import (
|
||||
|
||||
from ..genome.hypothesis import HypothesisAgentGenome, ModelTier
|
||||
|
||||
# Modelli configurati per Phase 1 — tutti via OpenRouter
|
||||
MODEL_TIER_S = "google/gemini-3-flash-preview"
|
||||
MODEL_TIER_A = "deepseek/deepseek-v4-flash"
|
||||
MODEL_TIER_B = "deepseek/deepseek-v4-flash"
|
||||
MODEL_TIER_C = "qwen/qwen-2.5-72b-instruct"
|
||||
MODEL_TIER_D = "openai/gpt-oss-20b"
|
||||
OPENROUTER_BASE_URL = "https://openrouter.ai/api/v1"
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
MODEL_TIER_MAP: dict[ModelTier, str] = {
|
||||
ModelTier.S: "claude-opus-4-7",
|
||||
ModelTier.A: "claude-opus-4-7",
|
||||
ModelTier.B: "claude-sonnet-4-6",
|
||||
ModelTier.C: "claude-sonnet-4-6",
|
||||
ModelTier.D: "claude-haiku-4-5-20251001",
|
||||
}
|
||||
|
||||
|
||||
class EmptyCompletionError(RuntimeError):
|
||||
pass
|
||||
|
||||
|
||||
# Errori transient: retry. Auth/InvalidRequest: NO retry.
|
||||
# RateLimitError (HTTP 429) ora retryable: provider OpenRouter come DeepInfra
|
||||
# applicano rate limit upstream temporaneo, recuperabile con backoff.
|
||||
class OpusAgentError(RuntimeError):
|
||||
pass
|
||||
|
||||
|
||||
class OpusAgentTransientError(RuntimeError):
|
||||
pass
|
||||
|
||||
|
||||
_RETRYABLE_EXCEPTIONS: tuple[type[BaseException], ...] = (
|
||||
openai.APIConnectionError,
|
||||
openai.APITimeoutError,
|
||||
openai.InternalServerError,
|
||||
openai.RateLimitError,
|
||||
httpx.ConnectError,
|
||||
httpx.TimeoutException,
|
||||
OpusAgentTransientError,
|
||||
EmptyCompletionError,
|
||||
)
|
||||
|
||||
@@ -48,47 +57,89 @@ class CompletionResult:
|
||||
|
||||
|
||||
class LLMClient:
|
||||
# Provider OpenRouter da escludere di default. Novita rifiuta /completions
|
||||
# endpoint per modelli Qwen 2.x — vedi bug 2026-05-12.
|
||||
DEFAULT_PROVIDER_IGNORE: tuple[str, ...] = ("Novita",)
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
openrouter_api_key: str,
|
||||
model_tier_s: str = MODEL_TIER_S,
|
||||
model_tier_a: str = MODEL_TIER_A,
|
||||
model_tier_b: str = MODEL_TIER_B,
|
||||
model_tier_c: str = MODEL_TIER_C,
|
||||
model_tier_d: str = MODEL_TIER_D,
|
||||
openrouter_base_url: str = OPENROUTER_BASE_URL,
|
||||
provider_ignore: tuple[str, ...] | None = None,
|
||||
opus_agent_api_key: str,
|
||||
opus_agent_base_url: str = "https://opus-agent.tielogic.xyz",
|
||||
model_tier_s: str = "claude-opus-4-7",
|
||||
model_tier_a: str = "claude-opus-4-7",
|
||||
model_tier_b: str = "claude-sonnet-4-6",
|
||||
model_tier_c: str = "claude-sonnet-4-6",
|
||||
model_tier_d: str = "claude-haiku-4-5-20251001",
|
||||
poll_interval: float = 3.0,
|
||||
poll_timeout: float = 180.0,
|
||||
) -> None:
|
||||
self.model_tier_s = model_tier_s
|
||||
self.model_tier_a = model_tier_a
|
||||
self.model_tier_b = model_tier_b
|
||||
self.model_tier_c = model_tier_c
|
||||
self.model_tier_d = model_tier_d
|
||||
self.openrouter_base_url = openrouter_base_url
|
||||
self._provider_ignore = (
|
||||
provider_ignore if provider_ignore is not None else self.DEFAULT_PROVIDER_IGNORE
|
||||
)
|
||||
self._tier_models: dict[ModelTier, str] = {
|
||||
self._base_url = opus_agent_base_url.rstrip("/")
|
||||
self._api_key = opus_agent_api_key
|
||||
self._tier_map: dict[ModelTier, str] = {
|
||||
ModelTier.S: model_tier_s,
|
||||
ModelTier.A: model_tier_a,
|
||||
ModelTier.B: model_tier_b,
|
||||
ModelTier.C: model_tier_c,
|
||||
ModelTier.D: model_tier_d,
|
||||
}
|
||||
# Timeout esplicito (60s) per prevenire hang infinito su connessioni
|
||||
# stallate. Tenacity retry su APITimeoutError gestisce il recovery.
|
||||
self._client = OpenAI(
|
||||
api_key=openrouter_api_key,
|
||||
base_url=openrouter_base_url,
|
||||
timeout=60.0,
|
||||
self._poll_interval = poll_interval
|
||||
self._poll_timeout = poll_timeout
|
||||
self._topic_cache: dict[str, str] = {}
|
||||
self._topic_lock = threading.Lock()
|
||||
self._client = httpx.Client(
|
||||
base_url=self._base_url,
|
||||
headers={"X-Api-Key": self._api_key, "Content-Type": "application/json"},
|
||||
timeout=30.0,
|
||||
)
|
||||
|
||||
def _get_or_create_topic(self, system_prompt: str) -> str:
|
||||
prompt_hash = hashlib.sha256(system_prompt.encode()).hexdigest()[:16]
|
||||
if prompt_hash in self._topic_cache:
|
||||
return self._topic_cache[prompt_hash]
|
||||
|
||||
with self._topic_lock:
|
||||
if prompt_hash in self._topic_cache:
|
||||
return self._topic_cache[prompt_hash]
|
||||
|
||||
topic_name = f"swarm-{prompt_hash}"
|
||||
resp = self._client.post("/api/topics", json={
|
||||
"name": topic_name,
|
||||
"system_prompt": system_prompt,
|
||||
})
|
||||
|
||||
if resp.status_code == 409:
|
||||
list_resp = self._client.get("/api/topics")
|
||||
list_resp.raise_for_status()
|
||||
for topic in list_resp.json()["data"]:
|
||||
if topic["name"] == topic_name:
|
||||
self._topic_cache[prompt_hash] = topic["id"]
|
||||
return topic["id"]
|
||||
raise OpusAgentError(f"Topic {topic_name} conflict but not found")
|
||||
|
||||
if resp.status_code >= 500:
|
||||
raise OpusAgentTransientError(f"Server error {resp.status_code}")
|
||||
resp.raise_for_status()
|
||||
|
||||
topic_id = resp.json()["data"]["id"]
|
||||
self._topic_cache[prompt_hash] = topic_id
|
||||
logger.debug("Created topic %s -> %s", topic_name, topic_id)
|
||||
return topic_id
|
||||
|
||||
def _poll_result(self, request_id: str) -> dict:
|
||||
deadline = time.monotonic() + self._poll_timeout
|
||||
while time.monotonic() < deadline:
|
||||
resp = self._client.get(f"/api/requests/{request_id}")
|
||||
resp.raise_for_status()
|
||||
data = resp.json()["data"]
|
||||
status = data["status"]
|
||||
if status == "completed":
|
||||
return data
|
||||
if status == "failed":
|
||||
error = data.get("error") or "unknown error"
|
||||
raise OpusAgentError(f"Request {request_id} failed: {error}")
|
||||
time.sleep(self._poll_interval)
|
||||
raise OpusAgentTransientError(
|
||||
f"Request {request_id} timed out after {self._poll_timeout}s"
|
||||
)
|
||||
|
||||
@retry(
|
||||
stop=stop_after_attempt(5),
|
||||
stop=stop_after_attempt(3),
|
||||
wait=wait_exponential(multiplier=2.0, min=2.0, max=30.0),
|
||||
retry=retry_if_exception_type(_RETRYABLE_EXCEPTIONS),
|
||||
reraise=True,
|
||||
@@ -100,28 +151,33 @@ class LLMClient:
|
||||
user: str,
|
||||
max_tokens: int = 2000,
|
||||
) -> CompletionResult:
|
||||
model = self._tier_models[genome.model_tier]
|
||||
extra_body: dict[str, Any] = {}
|
||||
if self._provider_ignore:
|
||||
extra_body["provider"] = {"ignore": list(self._provider_ignore)}
|
||||
resp = self._client.chat.completions.create(
|
||||
model=model,
|
||||
messages=[
|
||||
{"role": "system", "content": system},
|
||||
{"role": "user", "content": user},
|
||||
],
|
||||
temperature=genome.temperature,
|
||||
top_p=genome.top_p,
|
||||
max_tokens=max_tokens,
|
||||
extra_body=extra_body or None,
|
||||
)
|
||||
if not resp.choices or resp.choices[0].message.content is None:
|
||||
raise EmptyCompletionError(f"empty response from {model}")
|
||||
usage = resp.usage
|
||||
model = self._tier_map[genome.model_tier]
|
||||
topic_id = self._get_or_create_topic(system)
|
||||
|
||||
resp = self._client.post("/api/requests", json={
|
||||
"topic_id": topic_id,
|
||||
"prompt": user,
|
||||
"model": model,
|
||||
})
|
||||
|
||||
if resp.status_code == 429:
|
||||
raise OpusAgentTransientError("Rate limited")
|
||||
if resp.status_code >= 500:
|
||||
raise OpusAgentTransientError(f"Server error {resp.status_code}")
|
||||
if resp.status_code != 202:
|
||||
raise OpusAgentError(f"Unexpected status {resp.status_code}: {resp.text}")
|
||||
|
||||
request_id = resp.json()["data"]["id"]
|
||||
result = self._poll_result(request_id)
|
||||
|
||||
text = result.get("result") or ""
|
||||
if not text:
|
||||
raise EmptyCompletionError(f"empty response from OpusAgent ({model})")
|
||||
|
||||
return CompletionResult(
|
||||
text=resp.choices[0].message.content,
|
||||
input_tokens=usage.prompt_tokens if usage else 0,
|
||||
output_tokens=usage.completion_tokens if usage else 0,
|
||||
text=text,
|
||||
input_tokens=0,
|
||||
output_tokens=0,
|
||||
tier=genome.model_tier,
|
||||
model=model,
|
||||
)
|
||||
|
||||
@@ -75,8 +75,8 @@ class RunConfig:
|
||||
# strategy_crypto/prompts.json via PromptLibrary.from_json().
|
||||
prompt_library: PromptLibrary | None = None
|
||||
# Numero di propose() LLM concorrenti per generazione. 1 = sequenziale (default,
|
||||
# backward compat). 6-10 tipicamente accettati da OpenRouter qwen-2.5 senza
|
||||
# rate-limit. Riduce wall time GA loop di 5-8x su tier C.
|
||||
# backward compat). OpusAgent processa in coda FIFO; concurrency > 1
|
||||
# accoda piu' richieste in parallelo ma il throughput dipende dal server.
|
||||
llm_concurrency: int = 1
|
||||
|
||||
|
||||
@@ -90,7 +90,7 @@ def _parallel_propose(
|
||||
|
||||
``n_workers <= 1`` mantiene il comportamento serial originale (ordine fisso,
|
||||
determinismo data un seed). ``n_workers > 1`` usa un thread pool: l'order
|
||||
dei risultati e' preservato (1:1 con ``genomes``). OpenAI/openrouter client
|
||||
dei risultati e' preservato (1:1 con ``genomes``). LLMClient (OpusAgent)
|
||||
e' thread-safe; ``PromptLibrary`` e ``HypothesisAgent`` non hanno stato mutabile.
|
||||
"""
|
||||
if n_workers <= 1 or len(genomes) <= 1:
|
||||
|
||||
@@ -18,13 +18,14 @@ def test_settings_loads_from_env(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
monkeypatch.setenv("CERBERO_TESTNET_TOKEN", "tok-test")
|
||||
monkeypatch.setenv("CERBERO_MAINNET_TOKEN", "tok-main")
|
||||
monkeypatch.setenv("CERBERO_BOT_TAG", "swarm-poc-phase1")
|
||||
monkeypatch.setenv("OPENROUTER_API_KEY", "or-key")
|
||||
monkeypatch.setenv("OPUS_AGENT_API_KEY", "oa-key")
|
||||
monkeypatch.setenv("RUN_NAME", "test-run")
|
||||
|
||||
s = Settings() # type: ignore[call-arg]
|
||||
|
||||
assert s.cerbero_base_url == "http://test:9000"
|
||||
assert s.cerbero_testnet_token.get_secret_value() == "tok-test"
|
||||
assert s.opus_agent_api_key.get_secret_value() == "oa-key"
|
||||
assert s.run_name == "test-run"
|
||||
assert s.data_dir.name == "data"
|
||||
assert s.db_path.name == "runs.db"
|
||||
@@ -32,50 +33,33 @@ def test_settings_loads_from_env(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
|
||||
def test_settings_requires_tokens(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
monkeypatch.delenv("CERBERO_TESTNET_TOKEN", raising=False)
|
||||
monkeypatch.delenv("OPENROUTER_API_KEY", raising=False)
|
||||
monkeypatch.delenv("OPUS_AGENT_API_KEY", raising=False)
|
||||
from pydantic import ValidationError
|
||||
|
||||
with pytest.raises(ValidationError):
|
||||
# Disable .env loading to keep the test deterministic regardless of
|
||||
# whether a developer's local .env exists and is populated.
|
||||
Settings(_env_file=None) # type: ignore[call-arg]
|
||||
|
||||
|
||||
def test_settings_loads_llm_model_overrides(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
def test_settings_opus_agent_defaults(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
monkeypatch.setenv("CERBERO_TESTNET_TOKEN", "tok-test")
|
||||
monkeypatch.setenv("OPENROUTER_API_KEY", "or-key")
|
||||
monkeypatch.setenv("LLM_MODEL_TIER_S", "claude-mega-x")
|
||||
monkeypatch.setenv("LLM_MODEL_TIER_A", "claude-premium-y")
|
||||
monkeypatch.setenv("LLM_MODEL_TIER_B", "claude-opus-4-7")
|
||||
monkeypatch.setenv("LLM_MODEL_TIER_C", "deepseek/deepseek-chat")
|
||||
monkeypatch.setenv("LLM_MODEL_TIER_D", "mistralai/mistral-7b")
|
||||
monkeypatch.setenv("OPENROUTER_BASE_URL", "https://example.com/api/v1")
|
||||
monkeypatch.setenv("OPUS_AGENT_API_KEY", "oa-key")
|
||||
monkeypatch.delenv("OPUS_AGENT_BASE_URL", raising=False)
|
||||
|
||||
s = Settings(_env_file=None) # type: ignore[call-arg]
|
||||
|
||||
assert s.llm_model_tier_s == "claude-mega-x"
|
||||
assert s.llm_model_tier_a == "claude-premium-y"
|
||||
assert s.llm_model_tier_b == "claude-opus-4-7"
|
||||
assert s.llm_model_tier_c == "deepseek/deepseek-chat"
|
||||
assert s.llm_model_tier_d == "mistralai/mistral-7b"
|
||||
assert s.openrouter_base_url == "https://example.com/api/v1"
|
||||
assert s.opus_agent_base_url == "https://opus-agent.tielogic.xyz"
|
||||
assert s.llm_model_tier_s == "claude-opus-4-7"
|
||||
assert s.llm_model_tier_a == "claude-opus-4-7"
|
||||
assert s.llm_model_tier_b == "claude-sonnet-4-6"
|
||||
assert s.llm_model_tier_c == "claude-sonnet-4-6"
|
||||
assert s.llm_model_tier_d == "claude-haiku-4-5-20251001"
|
||||
|
||||
|
||||
def test_settings_llm_model_defaults(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
def test_settings_opus_agent_base_url_override(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
monkeypatch.setenv("CERBERO_TESTNET_TOKEN", "tok-test")
|
||||
monkeypatch.setenv("OPENROUTER_API_KEY", "or-key")
|
||||
monkeypatch.delenv("LLM_MODEL_TIER_S", raising=False)
|
||||
monkeypatch.delenv("LLM_MODEL_TIER_A", raising=False)
|
||||
monkeypatch.delenv("LLM_MODEL_TIER_B", raising=False)
|
||||
monkeypatch.delenv("LLM_MODEL_TIER_C", raising=False)
|
||||
monkeypatch.delenv("LLM_MODEL_TIER_D", raising=False)
|
||||
monkeypatch.delenv("OPENROUTER_BASE_URL", raising=False)
|
||||
monkeypatch.setenv("OPUS_AGENT_API_KEY", "oa-key")
|
||||
monkeypatch.setenv("OPUS_AGENT_BASE_URL", "https://custom.example.com")
|
||||
|
||||
s = Settings(_env_file=None) # type: ignore[call-arg]
|
||||
|
||||
assert s.llm_model_tier_s == "google/gemini-3-flash-preview"
|
||||
assert s.llm_model_tier_a == "deepseek/deepseek-v4-flash"
|
||||
assert s.llm_model_tier_b == "deepseek/deepseek-v4-flash"
|
||||
assert s.llm_model_tier_c == "qwen/qwen-2.5-72b-instruct"
|
||||
assert s.llm_model_tier_d == "openai/gpt-oss-20b"
|
||||
assert s.openrouter_base_url == "https://openrouter.ai/api/v1"
|
||||
assert s.opus_agent_base_url == "https://custom.example.com"
|
||||
|
||||
@@ -1,7 +1,14 @@
|
||||
import httpx
|
||||
import pytest
|
||||
|
||||
from multi_swarm_core.genome.hypothesis import HypothesisAgentGenome, ModelTier
|
||||
from multi_swarm_core.llm.client import CompletionResult, LLMClient
|
||||
from multi_swarm_core.llm.client import (
|
||||
CompletionResult,
|
||||
EmptyCompletionError,
|
||||
LLMClient,
|
||||
OpusAgentError,
|
||||
OpusAgentTransientError,
|
||||
)
|
||||
|
||||
|
||||
def make_genome(tier: ModelTier) -> HypothesisAgentGenome:
|
||||
@@ -16,217 +23,192 @@ def make_genome(tier: ModelTier) -> HypothesisAgentGenome:
|
||||
)
|
||||
|
||||
|
||||
def test_completion_tier_c_uses_openrouter(mocker):
|
||||
fake_openai = mocker.MagicMock()
|
||||
fake_response = mocker.MagicMock()
|
||||
fake_response.choices = [mocker.MagicMock(message=mocker.MagicMock(content="(strategy ...)"))]
|
||||
fake_response.usage = mocker.MagicMock(prompt_tokens=100, completion_tokens=200)
|
||||
fake_openai.chat.completions.create.return_value = fake_response
|
||||
TOPIC_RESPONSE = {
|
||||
"success": True,
|
||||
"data": {"id": "topic-123", "name": "swarm-test", "system_prompt": "sys"},
|
||||
}
|
||||
|
||||
mocker.patch("multi_swarm_core.llm.client.OpenAI", return_value=fake_openai)
|
||||
REQUEST_ACCEPTED = {
|
||||
"success": True,
|
||||
"data": {"id": "req-456", "session_id": None, "status": "pending"},
|
||||
}
|
||||
|
||||
client = LLMClient(openrouter_api_key="or-x")
|
||||
g = make_genome(ModelTier.C)
|
||||
out = client.complete(g, system="sys", user="usr")
|
||||
|
||||
def _completed_response(text: str = "(strategy ...)") -> dict:
|
||||
return {
|
||||
"success": True,
|
||||
"data": {
|
||||
"id": "req-456",
|
||||
"status": "completed",
|
||||
"result": text,
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
def _mock_transport(responses: list[httpx.Response]) -> httpx.MockTransport:
|
||||
call_idx = {"i": 0}
|
||||
|
||||
def handler(request: httpx.Request) -> httpx.Response:
|
||||
idx = call_idx["i"]
|
||||
call_idx["i"] += 1
|
||||
if idx < len(responses):
|
||||
return responses[idx]
|
||||
return responses[-1]
|
||||
|
||||
return httpx.MockTransport(handler)
|
||||
|
||||
|
||||
def _make_client(transport: httpx.MockTransport) -> LLMClient:
|
||||
client = LLMClient(opus_agent_api_key="test-key", poll_interval=0.01, poll_timeout=5.0)
|
||||
client._client = httpx.Client(
|
||||
base_url="https://opus-agent.tielogic.xyz",
|
||||
headers={"X-Api-Key": "test-key", "Content-Type": "application/json"},
|
||||
transport=transport,
|
||||
)
|
||||
return client
|
||||
|
||||
|
||||
def test_complete_tier_c_uses_sonnet():
|
||||
transport = _mock_transport([
|
||||
httpx.Response(201, json=TOPIC_RESPONSE),
|
||||
httpx.Response(202, json=REQUEST_ACCEPTED),
|
||||
httpx.Response(200, json=_completed_response()),
|
||||
])
|
||||
client = _make_client(transport)
|
||||
out = client.complete(make_genome(ModelTier.C), system="sys", user="usr")
|
||||
|
||||
assert isinstance(out, CompletionResult)
|
||||
assert out.text == "(strategy ...)"
|
||||
assert out.input_tokens == 100
|
||||
assert out.output_tokens == 200
|
||||
assert out.model == "claude-sonnet-4-6"
|
||||
assert out.tier == ModelTier.C
|
||||
fake_openai.chat.completions.create.assert_called_once()
|
||||
|
||||
|
||||
def test_completion_tier_b_uses_openrouter_with_anthropic_model(mocker):
|
||||
fake_openai = mocker.MagicMock()
|
||||
fake_response = mocker.MagicMock()
|
||||
fake_response.choices = [mocker.MagicMock(message=mocker.MagicMock(content="(strategy ...)"))]
|
||||
fake_response.usage = mocker.MagicMock(prompt_tokens=80, completion_tokens=150)
|
||||
fake_openai.chat.completions.create.return_value = fake_response
|
||||
mocker.patch("multi_swarm_core.llm.client.OpenAI", return_value=fake_openai)
|
||||
def test_complete_tier_s_uses_opus():
|
||||
transport = _mock_transport([
|
||||
httpx.Response(201, json=TOPIC_RESPONSE),
|
||||
httpx.Response(202, json=REQUEST_ACCEPTED),
|
||||
httpx.Response(200, json=_completed_response("(strategy s)")),
|
||||
])
|
||||
client = _make_client(transport)
|
||||
out = client.complete(make_genome(ModelTier.S), system="sys", user="usr")
|
||||
|
||||
client = LLMClient(openrouter_api_key="or-x")
|
||||
g = make_genome(ModelTier.B)
|
||||
out = client.complete(g, system="sys", user="usr")
|
||||
|
||||
assert out.text == "(strategy ...)"
|
||||
assert out.input_tokens == 80
|
||||
assert out.output_tokens == 150
|
||||
assert out.tier == ModelTier.B
|
||||
call_kwargs = fake_openai.chat.completions.create.call_args.kwargs
|
||||
assert call_kwargs["model"] == "deepseek/deepseek-v4-flash"
|
||||
assert out.model == "deepseek/deepseek-v4-flash"
|
||||
|
||||
|
||||
@pytest.mark.slow
|
||||
def test_completion_retries_on_connection_error(mocker):
|
||||
"""Retry esegue 3 tentativi su APIConnectionError, poi rilancia."""
|
||||
import openai
|
||||
|
||||
fake_openai = mocker.MagicMock()
|
||||
fake_openai.chat.completions.create.side_effect = openai.APIConnectionError(
|
||||
request=mocker.MagicMock()
|
||||
)
|
||||
mocker.patch("multi_swarm_core.llm.client.OpenAI", return_value=fake_openai)
|
||||
|
||||
client = LLMClient(openrouter_api_key="or-x")
|
||||
g = make_genome(ModelTier.C)
|
||||
|
||||
with pytest.raises(openai.APIConnectionError):
|
||||
client.complete(g, system="sys", user="usr")
|
||||
|
||||
assert fake_openai.chat.completions.create.call_count == 5
|
||||
|
||||
|
||||
def test_completion_uses_custom_model_tier_c(mocker):
|
||||
fake_openai = mocker.MagicMock()
|
||||
fake_response = mocker.MagicMock()
|
||||
fake_response.choices = [
|
||||
mocker.MagicMock(message=mocker.MagicMock(content="(strategy ...)"))
|
||||
]
|
||||
fake_response.usage = mocker.MagicMock(prompt_tokens=10, completion_tokens=20)
|
||||
fake_openai.chat.completions.create.return_value = fake_response
|
||||
mocker.patch("multi_swarm_core.llm.client.OpenAI", return_value=fake_openai)
|
||||
|
||||
client = LLMClient(
|
||||
openrouter_api_key="or-x",
|
||||
model_tier_c="deepseek/deepseek-chat",
|
||||
)
|
||||
g = make_genome(ModelTier.C)
|
||||
out = client.complete(g, system="sys", user="usr")
|
||||
|
||||
fake_openai.chat.completions.create.assert_called_once()
|
||||
call_kwargs = fake_openai.chat.completions.create.call_args.kwargs
|
||||
assert call_kwargs["model"] == "deepseek/deepseek-chat"
|
||||
assert out.model == "deepseek/deepseek-chat"
|
||||
|
||||
|
||||
def test_completion_uses_custom_model_tier_b(mocker):
|
||||
fake_openai = mocker.MagicMock()
|
||||
fake_response = mocker.MagicMock()
|
||||
fake_response.choices = [
|
||||
mocker.MagicMock(message=mocker.MagicMock(content="(strategy ...)"))
|
||||
]
|
||||
fake_response.usage = mocker.MagicMock(prompt_tokens=10, completion_tokens=20)
|
||||
fake_openai.chat.completions.create.return_value = fake_response
|
||||
mocker.patch("multi_swarm_core.llm.client.OpenAI", return_value=fake_openai)
|
||||
|
||||
client = LLMClient(
|
||||
openrouter_api_key="or-x",
|
||||
model_tier_b="anthropic/claude-opus-4-7",
|
||||
)
|
||||
g = make_genome(ModelTier.B)
|
||||
out = client.complete(g, system="sys", user="usr")
|
||||
|
||||
fake_openai.chat.completions.create.assert_called_once()
|
||||
call_kwargs = fake_openai.chat.completions.create.call_args.kwargs
|
||||
assert call_kwargs["model"] == "anthropic/claude-opus-4-7"
|
||||
assert out.model == "anthropic/claude-opus-4-7"
|
||||
|
||||
|
||||
def test_completion_tier_s_uses_openrouter_with_anthropic_model(mocker):
|
||||
fake_openai = mocker.MagicMock()
|
||||
fake_response = mocker.MagicMock()
|
||||
fake_response.choices = [mocker.MagicMock(message=mocker.MagicMock(content="(strategy s)"))]
|
||||
fake_response.usage = mocker.MagicMock(prompt_tokens=50, completion_tokens=100)
|
||||
fake_openai.chat.completions.create.return_value = fake_response
|
||||
mocker.patch("multi_swarm_core.llm.client.OpenAI", return_value=fake_openai)
|
||||
|
||||
client = LLMClient(openrouter_api_key="or-x")
|
||||
g = make_genome(ModelTier.S)
|
||||
out = client.complete(g, system="sys", user="usr")
|
||||
|
||||
fake_openai.chat.completions.create.assert_called_once()
|
||||
call_kwargs = fake_openai.chat.completions.create.call_args.kwargs
|
||||
assert call_kwargs["model"] == "google/gemini-3-flash-preview"
|
||||
assert out.model == "claude-opus-4-7"
|
||||
assert out.tier == ModelTier.S
|
||||
assert out.model == "google/gemini-3-flash-preview"
|
||||
assert out.text == "(strategy s)"
|
||||
|
||||
|
||||
def test_completion_tier_a_uses_openrouter_with_anthropic_model(mocker):
|
||||
fake_openai = mocker.MagicMock()
|
||||
fake_response = mocker.MagicMock()
|
||||
fake_response.choices = [mocker.MagicMock(message=mocker.MagicMock(content="(strategy a)"))]
|
||||
fake_response.usage = mocker.MagicMock(prompt_tokens=40, completion_tokens=80)
|
||||
fake_openai.chat.completions.create.return_value = fake_response
|
||||
mocker.patch("multi_swarm_core.llm.client.OpenAI", return_value=fake_openai)
|
||||
def test_complete_tier_d_uses_haiku():
|
||||
transport = _mock_transport([
|
||||
httpx.Response(201, json=TOPIC_RESPONSE),
|
||||
httpx.Response(202, json=REQUEST_ACCEPTED),
|
||||
httpx.Response(200, json=_completed_response("(strategy d)")),
|
||||
])
|
||||
client = _make_client(transport)
|
||||
out = client.complete(make_genome(ModelTier.D), system="sys", user="usr")
|
||||
|
||||
client = LLMClient(openrouter_api_key="or-x")
|
||||
g = make_genome(ModelTier.A)
|
||||
out = client.complete(g, system="sys", user="usr")
|
||||
|
||||
fake_openai.chat.completions.create.assert_called_once()
|
||||
call_kwargs = fake_openai.chat.completions.create.call_args.kwargs
|
||||
assert call_kwargs["model"] == "deepseek/deepseek-v4-flash"
|
||||
assert out.tier == ModelTier.A
|
||||
assert out.model == "deepseek/deepseek-v4-flash"
|
||||
|
||||
|
||||
def test_completion_tier_d_uses_openrouter_with_llama(mocker):
|
||||
fake_openai = mocker.MagicMock()
|
||||
fake_response = mocker.MagicMock()
|
||||
fake_response.choices = [
|
||||
mocker.MagicMock(message=mocker.MagicMock(content="(strategy d)"))
|
||||
]
|
||||
fake_response.usage = mocker.MagicMock(prompt_tokens=30, completion_tokens=70)
|
||||
fake_openai.chat.completions.create.return_value = fake_response
|
||||
mocker.patch("multi_swarm_core.llm.client.OpenAI", return_value=fake_openai)
|
||||
|
||||
client = LLMClient(openrouter_api_key="or-x")
|
||||
g = make_genome(ModelTier.D)
|
||||
out = client.complete(g, system="sys", user="usr")
|
||||
|
||||
fake_openai.chat.completions.create.assert_called_once()
|
||||
call_kwargs = fake_openai.chat.completions.create.call_args.kwargs
|
||||
assert call_kwargs["model"] == "openai/gpt-oss-20b"
|
||||
assert out.model == "claude-haiku-4-5-20251001"
|
||||
assert out.tier == ModelTier.D
|
||||
assert out.model == "openai/gpt-oss-20b"
|
||||
|
||||
|
||||
def test_completion_uses_custom_model_tier_s(mocker):
|
||||
fake_openai = mocker.MagicMock()
|
||||
fake_response = mocker.MagicMock()
|
||||
fake_response.choices = [
|
||||
mocker.MagicMock(message=mocker.MagicMock(content="(strategy custom-s)"))
|
||||
]
|
||||
fake_response.usage = mocker.MagicMock(prompt_tokens=10, completion_tokens=20)
|
||||
fake_openai.chat.completions.create.return_value = fake_response
|
||||
mocker.patch("multi_swarm_core.llm.client.OpenAI", return_value=fake_openai)
|
||||
def test_topic_cached_on_second_call():
|
||||
transport = _mock_transport([
|
||||
httpx.Response(201, json=TOPIC_RESPONSE),
|
||||
httpx.Response(202, json=REQUEST_ACCEPTED),
|
||||
httpx.Response(200, json=_completed_response()),
|
||||
httpx.Response(202, json=REQUEST_ACCEPTED),
|
||||
httpx.Response(200, json=_completed_response("second")),
|
||||
])
|
||||
client = _make_client(transport)
|
||||
client.complete(make_genome(ModelTier.C), system="sys", user="usr1")
|
||||
out2 = client.complete(make_genome(ModelTier.C), system="sys", user="usr2")
|
||||
|
||||
client = LLMClient(
|
||||
openrouter_api_key="or-x",
|
||||
model_tier_s="anthropic/claude-future-mega",
|
||||
)
|
||||
g = make_genome(ModelTier.S)
|
||||
out = client.complete(g, system="sys", user="usr")
|
||||
|
||||
call_kwargs = fake_openai.chat.completions.create.call_args.kwargs
|
||||
assert call_kwargs["model"] == "anthropic/claude-future-mega"
|
||||
assert out.model == "anthropic/claude-future-mega"
|
||||
assert out2.text == "second"
|
||||
|
||||
|
||||
@pytest.mark.slow
|
||||
def test_completion_succeeds_after_one_retry(mocker):
|
||||
"""Dopo 1 fallimento transient, il retry riesce al 2 tentativo."""
|
||||
import openai
|
||||
def test_topic_conflict_409_recovers():
|
||||
transport = _mock_transport([
|
||||
httpx.Response(409, json={"success": False, "error": {"code": "CONFLICT"}}),
|
||||
httpx.Response(200, json={"success": True, "data": [
|
||||
{"id": "topic-existing", "name": "swarm-wrong"},
|
||||
]}),
|
||||
])
|
||||
client = _make_client(transport)
|
||||
|
||||
fake_response = mocker.MagicMock()
|
||||
fake_response.choices = [
|
||||
mocker.MagicMock(message=mocker.MagicMock(content="(strategy ...)"))
|
||||
]
|
||||
fake_response.usage = mocker.MagicMock(prompt_tokens=100, completion_tokens=200)
|
||||
with pytest.raises(OpusAgentError, match="conflict but not found"):
|
||||
client.complete(make_genome(ModelTier.C), system="sys", user="usr")
|
||||
|
||||
fake_openai = mocker.MagicMock()
|
||||
fake_openai.chat.completions.create.side_effect = [
|
||||
openai.APITimeoutError(request=mocker.MagicMock()),
|
||||
fake_response,
|
||||
]
|
||||
mocker.patch("multi_swarm_core.llm.client.OpenAI", return_value=fake_openai)
|
||||
|
||||
client = LLMClient(openrouter_api_key="or-x")
|
||||
g = make_genome(ModelTier.C)
|
||||
out = client.complete(g, system="sys", user="usr")
|
||||
def test_empty_response_raises():
|
||||
transport = _mock_transport([
|
||||
httpx.Response(201, json=TOPIC_RESPONSE),
|
||||
httpx.Response(202, json=REQUEST_ACCEPTED),
|
||||
httpx.Response(200, json={
|
||||
"success": True,
|
||||
"data": {"id": "req-456", "status": "completed", "result": ""},
|
||||
}),
|
||||
])
|
||||
client = _make_client(transport)
|
||||
|
||||
assert isinstance(out, CompletionResult)
|
||||
assert out.text == "(strategy ...)"
|
||||
assert fake_openai.chat.completions.create.call_count == 2
|
||||
with pytest.raises(EmptyCompletionError):
|
||||
client.complete(make_genome(ModelTier.C), system="sys", user="usr")
|
||||
|
||||
|
||||
def test_request_failed_raises_opus_agent_error():
|
||||
transport = _mock_transport([
|
||||
httpx.Response(201, json=TOPIC_RESPONSE),
|
||||
httpx.Response(202, json=REQUEST_ACCEPTED),
|
||||
httpx.Response(200, json={
|
||||
"success": True,
|
||||
"data": {"id": "req-456", "status": "failed", "error": "model overloaded"},
|
||||
}),
|
||||
])
|
||||
client = _make_client(transport)
|
||||
|
||||
with pytest.raises(OpusAgentError, match="failed"):
|
||||
client.complete(make_genome(ModelTier.C), system="sys", user="usr")
|
||||
|
||||
|
||||
def test_rate_limit_429_is_retryable():
|
||||
transport = _mock_transport([
|
||||
httpx.Response(201, json=TOPIC_RESPONSE),
|
||||
httpx.Response(429, json={"success": False}),
|
||||
httpx.Response(202, json=REQUEST_ACCEPTED),
|
||||
httpx.Response(200, json=_completed_response("after retry")),
|
||||
])
|
||||
client = _make_client(transport)
|
||||
out = client.complete(make_genome(ModelTier.C), system="sys", user="usr")
|
||||
|
||||
assert out.text == "after retry"
|
||||
|
||||
|
||||
def test_polling_waits_for_completion():
|
||||
transport = _mock_transport([
|
||||
httpx.Response(201, json=TOPIC_RESPONSE),
|
||||
httpx.Response(202, json=REQUEST_ACCEPTED),
|
||||
httpx.Response(200, json={
|
||||
"success": True,
|
||||
"data": {"id": "req-456", "status": "processing"},
|
||||
}),
|
||||
httpx.Response(200, json={
|
||||
"success": True,
|
||||
"data": {"id": "req-456", "status": "processing"},
|
||||
}),
|
||||
httpx.Response(200, json=_completed_response("done")),
|
||||
])
|
||||
client = _make_client(transport)
|
||||
out = client.complete(make_genome(ModelTier.C), system="sys", user="usr")
|
||||
|
||||
assert out.text == "done"
|
||||
|
||||
|
||||
def test_tokens_are_zero():
|
||||
transport = _mock_transport([
|
||||
httpx.Response(201, json=TOPIC_RESPONSE),
|
||||
httpx.Response(202, json=REQUEST_ACCEPTED),
|
||||
httpx.Response(200, json=_completed_response()),
|
||||
])
|
||||
client = _make_client(transport)
|
||||
out = client.complete(make_genome(ModelTier.C), system="sys", user="usr")
|
||||
|
||||
assert out.input_tokens == 0
|
||||
assert out.output_tokens == 0
|
||||
|
||||
Reference in New Issue
Block a user