S.H.O.N.A.R._Desktop_Companion/backend/shonar/services/ai/openai_compat.py

"""Summaries via any OpenAI-compatible chat endpoint (self-hosted
vLLM/llama.cpp server, commercial API, …) with JSON mode.
"""

from __future__ import annotations

from collections.abc import AsyncIterator
from contextlib import asynccontextmanager

import httpx

from shonar.services.ai import ProviderConfigError, ProviderTransientError, SummaryResult
from shonar.services.ai._llm import (
    SYSTEM_PROMPT,
    build_user_message,
    make_progress_ticker,
    parse_summary,
)


async def _collect_reply(resp: httpx.Response, ticker) -> str:
    """Reassemble the assistant reply from a (possibly streamed) response.

    Feeds the accumulated content *prefix* to [ticker] as deltas arrive;
    the ticker derives progress from completed JSON contract units only
    (see ContractProgress). Reasoning deltas are real work but not
    contract output — they stream before the JSON and intentionally do
    not move the bar. Servers that ignored "stream": true answer with a
    plain JSON body — that path is handled too."""
    import json as _json

    ctype = resp.headers.get("content-type", "")
    if "text/event-stream" not in ctype:
        body = await resp.aread()
        try:
            content = _json.loads(body)["choices"][0]["message"]["content"]
        except (ValueError, KeyError, IndexError, TypeError) as e:
            raise ProviderTransientError("LLM sent an unreadable reply.") from e
        ticker(content or "")
        return content or ""

    parts: list[str] = []
    async for line in resp.aiter_lines():
        if not line.startswith("data:"):
            continue
        data = line[5:].strip()
        if data == "[DONE]":
            break
        try:
            chunk = _json.loads(data)
            delta = chunk["choices"][0].get("delta") or {}
            piece = delta.get("content") or ""
            # Thinking models (qwen3 on llama.cpp) stream a long
            # reasoning_content channel BEFORE any content. It is real
            # work but not contract output: counting it made the bar
            # rocket to 99 while the JSON had not even started. The
            # honest signal is the JSON itself completing.
            thinking = delta.get("reasoning_content") or ""
        except (ValueError, KeyError, IndexError, TypeError):
            continue  # keep-alives / usage chunks / odd frames: not content
        if piece:
            parts.append(piece)
            ticker("".join(parts))
        elif thinking:
            # First thinking delta proves the model started working:
            # the ticker's 5% "first token" milestone fires once (an
            # empty prefix is a no-op after that). The thinking phase
            # itself earns no further points — it is not contract
            # output — but "the model is alive" is real information.
            ticker("")
    content = "".join(parts)
    if not content:
        raise ProviderTransientError("LLM sent an unreadable reply.")
    return content


class OpenAICompatProvider:
    name = "openai_compat"

    def __init__(
        self,
        base_url: str,
        model: str = "",
        api_key: str = "",
        timeout_s: float = 180.0,
        http_client: httpx.AsyncClient | None = None,
    ) -> None:
        if not base_url.strip():
            raise ProviderConfigError("openai_compat needs SHONAR_LLM_BASE_URL.")
        if not model.strip():
            raise ProviderConfigError("openai_compat needs SHONAR_LLM_MODEL.")
        self.base_url = base_url.rstrip("/")
        self.model = model
        self.api_key = api_key
        self.timeout_s = timeout_s
        self.http_client = http_client

    @asynccontextmanager
    async def _client(self) -> AsyncIterator[httpx.AsyncClient]:
        if self.http_client is not None:
            yield self.http_client
        else:
            async with httpx.AsyncClient(timeout=self.timeout_s) as client:
                yield client

    async def summarize(self, transcript: str, *, title: str | None = None,
                        tone: str | None = None, on_progress=None) -> SummaryResult:
        ticker = make_progress_ticker(on_progress)
        headers = (
            {"Authorization": f"Bearer {self.api_key}"} if self.api_key else {}
        )
        payload = {
            "model": self.model,
            "temperature": 0.2,
            "response_format": {"type": "json_object"},
            # Stream so the UI gets real progress while the model works;
            # the full reply is reassembled from the deltas. Servers that
            # ignore "stream" still work — the non-SSE body is handled too.
            "stream": True,
            "messages": [
                {"role": "system", "content": SYSTEM_PROMPT},
                {"role": "user",
                 "content": build_user_message(transcript, title, tone)},
            ],
        }
        try:
            async with self._client() as client, client.stream(
                "POST", f"{self.base_url}/v1/chat/completions",
                headers=headers, json=payload,
            ) as resp:
                if resp.status_code in (401, 403, 404):
                    raise ProviderConfigError(
                        f"LLM refused the request (HTTP {resp.status_code}).")
                if resp.status_code == 429 or resp.status_code >= 500:
                    raise ProviderTransientError(
                        f"LLM busy (HTTP {resp.status_code}).")
                if resp.status_code != 200:
                    raise ProviderTransientError(
                        f"Summarization failed (HTTP {resp.status_code}).")
                content = await _collect_reply(resp, ticker)
        except (httpx.TimeoutException, httpx.TransportError) as e:
            raise ProviderTransientError(f"LLM unreachable: {type(e).__name__}") from e
        import json as _json

        try:
            data = _json.loads(content)
        except ValueError as e:
            raise ProviderTransientError("LLM reply was not JSON.") from e
        return parse_summary(data, self.model)