61f2124b1c
Replace the char-count estimate (EST_OUTPUT_CHARS) with ContractProgress:
points are earned only when real output units complete — first token (5),
each closed contract key / array item (5-90), root closed (95), stored (100).
Reasoning/thinking streams no longer fake progress: the first thinking
delta fires the first-token milestone ('the model is alive') and holds.
Surface the job's tone through ProcessingJobOut/JobInfo so the bar can
label itself 'Summarizing dry wit…'. Test rewritten to pin the new
honest-thinking semantics.
251 lines
9.9 KiB
Python
251 lines
9.9 KiB
Python
"""Adapter wire-protocol tests (M7): JSON shapes in/out, error mapping.
|
|
|
|
HTTP adapters take an injectable httpx client; faster-whisper is an
|
|
optional dependency and is only exercised when installed.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import importlib.util
|
|
|
|
import httpx
|
|
import pytest
|
|
|
|
from shonar.core.config import Settings
|
|
from shonar.services import ai
|
|
from shonar.services.ai import ProviderConfigError, SummaryResult
|
|
from shonar.services.ai.ollama import OllamaProvider
|
|
from shonar.services.ai.openai_compat import OpenAICompatProvider
|
|
from shonar.services.ai.whisper_http import WhisperHttpProvider
|
|
|
|
|
|
def mock_client(handler) -> httpx.AsyncClient:
|
|
return httpx.AsyncClient(transport=httpx.MockTransport(handler))
|
|
|
|
|
|
# --- factories -----------------------------------------------------------------
|
|
|
|
|
|
def test_factories_none_means_skip():
|
|
s = Settings(transcription_provider="none", llm_provider="none")
|
|
assert ai.get_transcription_provider(s) is None
|
|
assert ai.get_llm_provider(s) is None
|
|
|
|
|
|
def test_factories_unknown_names_raise_config_error():
|
|
s = Settings(transcription_provider="whisper-9k")
|
|
with pytest.raises(ProviderConfigError):
|
|
ai.get_transcription_provider(s)
|
|
s = Settings(llm_provider="clippy")
|
|
with pytest.raises(ProviderConfigError):
|
|
ai.get_llm_provider(s)
|
|
|
|
|
|
def test_factories_missing_urls_raise_config_error():
|
|
s = Settings(transcription_provider="whisper_http", transcription_base_url="")
|
|
with pytest.raises(ProviderConfigError):
|
|
ai.get_transcription_provider(s)
|
|
s = Settings(llm_provider="openai_compat", llm_base_url="http://x", llm_model="")
|
|
with pytest.raises(ProviderConfigError):
|
|
ai.get_llm_provider(s)
|
|
|
|
|
|
def test_faster_whisper_missing_dep_is_config_error():
|
|
if importlib.util.find_spec("faster_whisper") is not None:
|
|
pytest.skip("faster-whisper installed; missing-dep path N/A")
|
|
# Constructor validates eagerly so misconfiguration fails at startup,
|
|
# not on the first recording.
|
|
with pytest.raises(ProviderConfigError):
|
|
ai.get_transcription_provider(Settings(transcription_provider="faster_whisper"))
|
|
|
|
|
|
# --- whisper_http -----------------------------------------------------------------
|
|
|
|
|
|
def whisper_ok(request: httpx.Request) -> httpx.Response:
|
|
assert request.url.path == "/v1/audio/transcriptions"
|
|
assert request.method == "POST"
|
|
return httpx.Response(200, json={
|
|
"text": "hello world",
|
|
"language": "en",
|
|
"segments": [{"start": 0.0, "end": 1.2, "text": "hello world"}],
|
|
})
|
|
|
|
|
|
async def test_whisper_http_happy_path():
|
|
p = WhisperHttpProvider("http://stt:8000", model="small",
|
|
http_client=mock_client(whisper_ok))
|
|
res = await p.transcribe(b"\x00" * 16, "audio/wav")
|
|
assert res.text == "hello world"
|
|
assert res.language == "en"
|
|
assert [(s.start, s.end, s.text) for s in res.segments] == [(0.0, 1.2, "hello world")]
|
|
assert res.model == "small"
|
|
|
|
|
|
async def test_whisper_http_401_is_config_error():
|
|
async def denied(request: httpx.Request) -> httpx.Response:
|
|
return httpx.Response(401, json={"detail": "nope"})
|
|
p = WhisperHttpProvider("http://stt:8000", http_client=mock_client(denied))
|
|
with pytest.raises(ProviderConfigError):
|
|
await p.transcribe(b"\x00" * 16, "audio/wav")
|
|
|
|
|
|
async def test_whisper_http_503_is_transient():
|
|
async def busy(request: httpx.Request) -> httpx.Response:
|
|
return httpx.Response(503, text="overloaded")
|
|
from shonar.services.ai import ProviderTransientError
|
|
|
|
p = WhisperHttpProvider("http://stt:8000", http_client=mock_client(busy))
|
|
with pytest.raises(ProviderTransientError):
|
|
await p.transcribe(b"\x00" * 16, "audio/wav")
|
|
|
|
|
|
# --- openai_compat ------------------------------------------------------------------
|
|
|
|
|
|
def chat_ok(request: httpx.Request) -> httpx.Response:
|
|
assert request.url.path == "/v1/chat/completions"
|
|
body = {
|
|
"short": "Standup.",
|
|
"detailed": "The team met.",
|
|
"key_points": ["a", "b"],
|
|
"decisions": ["ship"],
|
|
"action_items": [{"not": "a string"}, "call ana"],
|
|
"questions": [],
|
|
"extra_key": "ignored",
|
|
}
|
|
import json as _json
|
|
|
|
return httpx.Response(200, json={"choices": [{"message": {"content": _json.dumps(body)}}]})
|
|
|
|
|
|
async def test_openai_compat_parses_and_sanitizes():
|
|
p = OpenAICompatProvider("http://llm:8000", model="qwen",
|
|
http_client=mock_client(chat_ok))
|
|
res = await p.summarize("a very long meeting transcript", title="Standup")
|
|
assert isinstance(res, SummaryResult)
|
|
assert res.short == "Standup."
|
|
assert res.action_items == ("call ana",) # non-strings dropped
|
|
assert res.model == "qwen"
|
|
|
|
|
|
async def test_openai_compat_streams_with_progress():
|
|
"""stream:true replies (SSE deltas) reassemble and tick progress."""
|
|
import json as _json
|
|
|
|
def sse(request: httpx.Request) -> httpx.Response:
|
|
payload = _json.loads(request.content.decode())
|
|
assert payload["stream"] is True
|
|
pieces = ['{"short": "Stand', 'up.", "detailed": "The team met."',
|
|
', "key_points": ["a"]', "}"]
|
|
lines = [f"data: {_json.dumps({'choices': [{'delta': {'content': p}}]})}"
|
|
for p in pieces]
|
|
lines.append("data: [DONE]")
|
|
return httpx.Response(
|
|
200, content=("\n\n".join(lines) + "\n\n").encode(),
|
|
headers={"content-type": "text/event-stream"})
|
|
|
|
p = OpenAICompatProvider("http://llm:8000", model="qwen",
|
|
http_client=mock_client(sse))
|
|
seen_pcts = []
|
|
res = await p.summarize("t", on_progress=seen_pcts.append)
|
|
assert res.short == "Standup."
|
|
assert res.key_points == ("a",)
|
|
assert seen_pcts and seen_pcts == sorted(seen_pcts)
|
|
assert all(0 <= p <= 99 for p in seen_pcts) # never claims 100 early
|
|
|
|
|
|
async def test_openai_compat_thinking_earns_only_first_token():
|
|
"""qwen3-on-llama.cpp streams reasoning_content for most of the run.
|
|
Thinking is real work but NOT contract output: it fires the 5%
|
|
first-token milestone once ("the model is alive") and then holds.
|
|
Progress only climbs when JSON contract units actually close — a
|
|
continuous percent over the thinking span would be a fake estimate."""
|
|
import json as _json
|
|
|
|
def sse(request: httpx.Request) -> httpx.Response:
|
|
think = [f"reasoning chunk {i} " for i in range(40)] # ~800 chars
|
|
lines = [f"data: {_json.dumps({'choices': [{'delta': {'reasoning_content': p}}]})}"
|
|
for p in think]
|
|
answer = _json.dumps({"short": "Done.", "key_points": []})
|
|
lines.append(f"data: {_json.dumps({'choices': [{'delta': {'content': answer}}]})}")
|
|
lines.append("data: [DONE]")
|
|
return httpx.Response(
|
|
200, content=("\n\n".join(lines) + "\n\n").encode(),
|
|
headers={"content-type": "text/event-stream"})
|
|
|
|
p = OpenAICompatProvider("http://llm:8000", model="qwen",
|
|
http_client=mock_client(sse))
|
|
seen_pcts = []
|
|
res = await p.summarize("t", on_progress=seen_pcts.append)
|
|
assert res.short == "Done." # reasoning never enters the reply
|
|
# Thinking earned exactly one tick: the first-token milestone. No
|
|
# fake crawl across the reasoning span.
|
|
assert seen_pcts[0] == 5
|
|
assert seen_pcts.count(5) == 1
|
|
assert max(seen_pcts) <= 99 # never claims 100 before storage
|
|
# Contract units closing does move the bar to the root milestone.
|
|
assert seen_pcts[-1] == 95
|
|
assert seen_pcts == sorted(seen_pcts)
|
|
|
|
|
|
async def test_openai_compat_partial_json_gets_defaults():
|
|
import json as _json
|
|
|
|
async def partial(request: httpx.Request) -> httpx.Response:
|
|
return httpx.Response(200, json={
|
|
"choices": [{"message": {"content": _json.dumps({"short": "Hi."})}}]
|
|
})
|
|
p = OpenAICompatProvider("http://llm:8000", model="qwen",
|
|
http_client=mock_client(partial))
|
|
res = await p.summarize("hi")
|
|
assert res.short == "Hi."
|
|
assert res.detailed == "" and res.key_points == ()
|
|
|
|
|
|
async def test_openai_compat_non_json_is_transient():
|
|
async def garbage(request: httpx.Request) -> httpx.Response:
|
|
return httpx.Response(200, json={
|
|
"choices": [{"message": {"content": "Sure! Here it is..."}}]
|
|
})
|
|
from shonar.services.ai import ProviderTransientError
|
|
|
|
p = OpenAICompatProvider("http://llm:8000", model="qwen",
|
|
http_client=mock_client(garbage))
|
|
with pytest.raises(ProviderTransientError):
|
|
await p.summarize("hi")
|
|
|
|
|
|
# --- ollama ---------------------------------------------------------------------------
|
|
|
|
|
|
async def test_ollama_happy_path():
|
|
import json as _json
|
|
|
|
async def ok(request: httpx.Request) -> httpx.Response:
|
|
assert request.url.path == "/api/chat"
|
|
payload = _json.loads(request.content.decode())
|
|
assert payload["format"] == "json" and payload["stream"] is True
|
|
body = "\n".join([
|
|
_json.dumps({"message": {"content": "{\"short\": \"S."}}),
|
|
_json.dumps({"message": {"content": "\", \"detailed\": \"D.\"}"}}),
|
|
_json.dumps({"done": True}),
|
|
])
|
|
return httpx.Response(
|
|
200, json=None, content=body.encode(),
|
|
headers={"content-type": "application/x-ndjson"})
|
|
p = OllamaProvider("http://ollama:11434", model="llama3",
|
|
http_client=mock_client(ok))
|
|
seen_pcts = []
|
|
res = await p.summarize("meeting notes", on_progress=seen_pcts.append)
|
|
assert res.short == "S." and res.model == "llama3"
|
|
assert seen_pcts and seen_pcts == sorted(seen_pcts) # monotonic progress
|
|
|
|
|
|
async def test_ollama_404_is_config_error():
|
|
async def missing(request: httpx.Request) -> httpx.Response:
|
|
return httpx.Response(404, text="model not found")
|
|
p = OllamaProvider("http://ollama:11434", model="nope",
|
|
http_client=mock_client(missing))
|
|
with pytest.raises(ProviderConfigError):
|
|
await p.summarize("hi")
|