avi/S.H.O.N.A.R._Desktop_Companion
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Standalone Shonar Desktop: vendor portable sources + local engine; decouple from ~/Projects/Shonar

Browse source 
- shared/ = portable Android-origin sources vendored from deferred/desktop-server
  (app/build.gradle.kts srcDir repointed; PlaybackController.kt excluded as Android-only)
- backend/ = bundled-lite engine (SQLite + inline queue); .venv symlinked from the
  old checkout, PYTHONPATH pins THIS backend's code over any editable install
- repoRoot() resolves this project dir (env SHONAR_REPO still wins); desktop-dev.sh
  watches shared/ + backend/
- Verified: :app:compileKotlin + :app:test green (23 tests); engine boots on :8010,
  self-migrates, /healthz ok
This commit is contained in:
avi 2026-09-14 17:14:54 -05:00
commit 76c867fca4
136 changed files with 21099 additions and 0 deletions
Download patch file Download diff file Expand all files Collapse all files

3
backend/shonar/db/__init__.py Normal file
View file

@ -0,0 +1,3 @@
"""DB package. Importing this registers every model on Base.metadata."""
from shonar.db import models # noqa: F401

50
backend/shonar/db/base.py Normal file
View file

@ -0,0 +1,50 @@
"""Public SQLAlchemy model base."""
from __future__ import annotations
import uuid
from datetime import UTC, datetime
from sqlalchemy import DateTime, TypeDecorator, Uuid
from sqlalchemy.orm import DeclarativeBase, Mapped, mapped_column
class UTCDT(TypeDecorator):
"""Timezone-aware datetime that survives SQLite round-trips.
Postgres returns aware values (impl is a no-op there); SQLite has no
tz info and hands back naive datetimes, which crash comparisons with
``utcnow()``. Attach UTC on read whenever the driver lost it.
"""
impl = DateTime(timezone=True)
cache_ok = True
def process_result_value(self, value, dialect): # noqa: ANN001, ANN201
if value is not None and value.tzinfo is None:
return value.replace(tzinfo=UTC)
return value
def utcnow() -> datetime:
return datetime.now(UTC)
def new_uuid() -> uuid.UUID:
return uuid.uuid4()
class Base(DeclarativeBase):
pass
class PublicIdMixin:
"""Gives every table a UUID public id; integer PKs stay internal."""
id: Mapped[uuid.UUID] = mapped_column(Uuid, primary_key=True, default=new_uuid)
created_at: Mapped[datetime] = mapped_column(UTCDT(), default=utcnow, nullable=False)
# default= as well as onupdate=: SQLite enforces NOT NULL on insert
# (Postgres silently stored NULL for rows that never got an UPDATE).
updated_at: Mapped[datetime] = mapped_column(
UTCDT(), default=utcnow, onupdate=utcnow, nullable=False
)

33
backend/shonar/db/migrate.py Normal file
View file

@ -0,0 +1,33 @@
"""Programmatic schema migrations for the bundled desktop engine.
Server deployments run ``alembic upgrade head`` in their deploy flow;
the desktop app owns its SQLite file end-to-end, so the API applies
migrations itself at startup (``SHONAR_AUTO_MIGRATE=1``).
"""
from __future__ import annotations
import asyncio
import logging
from pathlib import Path
from alembic import command
from alembic.config import Config
logger = logging.getLogger("shonar.migrate")
# backend/ dir: migrations/ + alembic.ini live here relative to this file.
_BACKEND_DIR = Path(__file__).resolve().parents[2]
async def upgrade_head(database_url: str) -> None:
"""Run 'alembic upgrade head' for `database_url` without blocking the loop."""
def _run() -> None:
cfg = Config(str(_BACKEND_DIR / "alembic.ini"))
cfg.set_main_option("script_location", str(_BACKEND_DIR / "migrations"))
cfg.set_main_option("sqlalchemy.url", database_url)
command.upgrade(cfg, "head")
await asyncio.to_thread(_run)
logger.info("database schema migrated to head")

453
backend/shonar/db/models.py Normal file
View file

@ -0,0 +1,453 @@
"""All persistent models.
Design rules:
- Every client-visible identifier is a UUID (``PublicIdMixin.id``).
- Filesystem/storage paths are NEVER exposed to clients.
- Original uploads are immutable; processed audio lives in separate
``Asset`` rows and never replaces an original.
"""
from __future__ import annotations
import enum
import uuid
from datetime import datetime
# JSON that renders JSONB on PostgreSQL and plain JSON on SQLite (the
# desktop bundled-lite engine runs on SQLite; JSONB does not compile there).
from sqlalchemy import JSON as _JSON
from sqlalchemy import (
BigInteger,
Boolean,
Enum,
Float,
ForeignKey,
Index,
String,
Text,
UniqueConstraint,
)
from sqlalchemy.dialects.postgresql import JSONB
from sqlalchemy.orm import Mapped, mapped_column, relationship
from shonar.db.base import UTCDT, Base, PublicIdMixin, utcnow
JSONType = _JSON().with_variant(JSONB(), "postgresql")
# ---------------------------------------------------------------------------
# Users & auth
# ---------------------------------------------------------------------------
class User(Base, PublicIdMixin):
__tablename__ = "users"
email: Mapped[str] = mapped_column(String(320), unique=True, index=True, nullable=False)
password_hash: Mapped[str] = mapped_column(String(255), nullable=False)
display_name: Mapped[str | None] = mapped_column(String(120))
is_active: Mapped[bool] = mapped_column(Boolean, default=True, nullable=False)
# Feature switches the user controls (mirror of app settings, server truth
# for e.g. whether location metadata may be stored at all).
location_storage_enabled: Mapped[bool] = mapped_column(Boolean, default=False, nullable=False)
deleted_at: Mapped[datetime | None] = mapped_column(UTCDT())
devices: Mapped[list[Device]] = relationship(back_populates="user")
recordings: Mapped[list[Recording]] = relationship(back_populates="user")
class Device(Base, PublicIdMixin):
__tablename__ = "devices"
user_id: Mapped[uuid.UUID] = mapped_column(
ForeignKey("users.id", ondelete="CASCADE"), index=True, nullable=False
)
name: Mapped[str] = mapped_column(String(120), nullable=False)
platform: Mapped[str] = mapped_column(String(40), default="android", nullable=False)
last_seen_at: Mapped[datetime] = mapped_column(UTCDT(), default=utcnow)
revoked_at: Mapped[datetime | None] = mapped_column(UTCDT())
user: Mapped[User] = relationship(back_populates="devices")
class RefreshToken(Base, PublicIdMixin):
"""Rotating refresh tokens, stored hashed, grouped into families for
reuse detection. A refresh consumes one row and issues its replacement
with the same ``family``."""
__tablename__ = "refresh_tokens"
user_id: Mapped[uuid.UUID] = mapped_column(
ForeignKey("users.id", ondelete="CASCADE"), index=True, nullable=False
)
token_hash: Mapped[str] = mapped_column(String(128), unique=True, nullable=False)
family: Mapped[uuid.UUID] = mapped_column(index=True, nullable=False)
device_id: Mapped[uuid.UUID | None] = mapped_column(
ForeignKey("devices.id", ondelete="SET NULL")
)
expires_at: Mapped[datetime] = mapped_column(UTCDT(), nullable=False)
revoked_at: Mapped[datetime | None] = mapped_column(UTCDT())
replaced_by: Mapped[uuid.UUID | None] = mapped_column(
ForeignKey("refresh_tokens.id", ondelete="SET NULL")
)
# ---------------------------------------------------------------------------
# Recordings, assets, uploads
# ---------------------------------------------------------------------------
class ProcessingStatus(enum.StrEnum):
pending_upload = "pending_upload"
uploaded = "uploaded"
queued = "queued"
processing = "processing"
completed = "completed"
failed = "failed"
# No AI configured / requested — audio-only recording, fully usable.
ai_disabled = "ai_disabled"
class Recording(Base, PublicIdMixin):
__tablename__ = "recordings"
user_id: Mapped[uuid.UUID] = mapped_column(
ForeignKey("users.id", ondelete="CASCADE"), index=True, nullable=False
)
device_id: Mapped[uuid.UUID | None] = mapped_column(
ForeignKey("devices.id", ondelete="SET NULL")
)
# Client-generated idempotency id so retried uploads update, not duplicate.
client_recording_id: Mapped[str | None] = mapped_column(String(64), index=True)
title: Mapped[str] = mapped_column(String(300), nullable=False, default="Untitled recording")
recorded_at: Mapped[datetime] = mapped_column(UTCDT(), nullable=False)
duration_seconds: Mapped[float] = mapped_column(Float, default=0.0, nullable=False)
notes: Mapped[str | None] = mapped_column(Text)
# Location is stored ONLY when the user has explicitly enabled it.
latitude: Mapped[float | None] = mapped_column(Float)
longitude: Mapped[float | None] = mapped_column(Float)
location_accuracy_m: Mapped[float | None] = mapped_column(Float)
processing_status: Mapped[ProcessingStatus] = mapped_column(
Enum(
ProcessingStatus,
name="processing_status",
values_callable=lambda e: [m.value for m in e],
),
default=ProcessingStatus.pending_upload,
nullable=False,
index=True,
)
processing_error: Mapped[str | None] = mapped_column(Text)
# Effective faster-whisper model for this recording (e.g. "base").
# Set at finalize time from the per-recording override or the global
# default; NULL means "server default at processing time" (pre-model
# rows). Never rewritten: changing the default affects future rows only.
transcription_model: Mapped[str | None] = mapped_column(String(32))
# The original audio is the Asset row with kind=original for this
# recording (at most one, enforced by a partial unique index). Keeping
# the pointer one-directional avoids a recordings<->assets FK cycle.
deleted_at: Mapped[datetime | None] = mapped_column(UTCDT())
user: Mapped[User] = relationship(back_populates="recordings")
assets: Mapped[list[Asset]] = relationship(
back_populates="recording", cascade="all, delete-orphan"
)
transcripts: Mapped[list[Transcript]] = relationship(
back_populates="recording", cascade="all, delete-orphan"
)
summaries: Mapped[list[Summary]] = relationship(
back_populates="recording", cascade="all, delete-orphan"
)
tags: Mapped[list[Tag]] = relationship(secondary="recording_tags", back_populates="recordings")
__table_args__ = (
Index("ix_recordings_user_recorded", "user_id", "recorded_at"),
Index(
"ix_recordings_user_client_id",
"user_id",
"client_recording_id",
unique=True,
postgresql_where="client_recording_id IS NOT NULL",
),
)
class AssetKind(enum.StrEnum):
original = "original" # user's uploaded file — NEVER modified
normalized = "normalized" # derivative for processing (ffmpeg)
export = "export" # generated export bundle
class Asset(Base, PublicIdMixin):
"""A stored binary object. ``storage_key`` is server-internal only."""
__tablename__ = "assets"
recording_id: Mapped[uuid.UUID | None] = mapped_column(
ForeignKey("recordings.id", ondelete="CASCADE"), index=True
)
user_id: Mapped[uuid.UUID] = mapped_column(
ForeignKey("users.id", ondelete="CASCADE"), index=True, nullable=False
)
kind: Mapped[AssetKind] = mapped_column(
Enum(AssetKind, name="asset_kind", values_callable=lambda e: [m.value for m in e]),
nullable=False,
)
storage_key: Mapped[str] = mapped_column(String(500), nullable=False)
mime_type: Mapped[str] = mapped_column(String(100), nullable=False)
size_bytes: Mapped[int] = mapped_column(BigInteger, nullable=False, default=0)
checksum_sha256: Mapped[str] = mapped_column(String(64), nullable=False)
recording: Mapped[Recording | None] = relationship(back_populates="assets")
__table_args__ = (
# At most one immutable "original" per recording.
Index(
"uq_assets_one_original",
"recording_id",
unique=True,
postgresql_where="kind = 'original'",
),
)
class UploadSessionStatus(enum.StrEnum):
open = "open"
finalizing = "finalizing"
completed = "completed"
aborted = "aborted"
expired = "expired"
class UploadSession(Base, PublicIdMixin):
"""Chunked, resumable upload session."""
__tablename__ = "upload_sessions"
user_id: Mapped[uuid.UUID] = mapped_column(
ForeignKey("users.id", ondelete="CASCADE"), index=True, nullable=False
)
recording_id: Mapped[uuid.UUID | None] = mapped_column(
ForeignKey("recordings.id", ondelete="CASCADE")
)
client_recording_id: Mapped[str | None] = mapped_column(String(64))
title: Mapped[str | None] = mapped_column(String(300))
# Optional per-recording transcription model override chosen on the
# upload screen. Validated at finalize time; the finalize body wins when
# both specify one.
transcription_model: Mapped[str | None] = mapped_column(String(32))
declared_mime_type: Mapped[str] = mapped_column(String(100), nullable=False)
declared_size_bytes: Mapped[int] = mapped_column(BigInteger, nullable=False)
chunk_size_bytes: Mapped[int] = mapped_column(
BigInteger, nullable=False, default=8 * 1024 * 1024
)
status: Mapped[UploadSessionStatus] = mapped_column(
Enum(
UploadSessionStatus,
name="upload_session_status",
values_callable=lambda e: [m.value for m in e],
),
default=UploadSessionStatus.open,
nullable=False,
)
expires_at: Mapped[datetime] = mapped_column(UTCDT(), nullable=False)
completed_asset_id: Mapped[uuid.UUID | None] = mapped_column(
ForeignKey("assets.id", ondelete="SET NULL")
)
chunks: Mapped[list[UploadChunk]] = relationship(
back_populates="session", cascade="all, delete-orphan"
)
class UploadChunk(Base):
__tablename__ = "upload_chunks"
id: Mapped[int] = mapped_column(primary_key=True)
session_id: Mapped[uuid.UUID] = mapped_column(
ForeignKey("upload_sessions.id", ondelete="CASCADE"), index=True, nullable=False
)
chunk_index: Mapped[int] = mapped_column(nullable=False)
size_bytes: Mapped[int] = mapped_column(BigInteger, nullable=False)
checksum_sha256: Mapped[str] = mapped_column(String(64), nullable=False)
storage_key: Mapped[str] = mapped_column(String(500), nullable=False)
created_at: Mapped[datetime] = mapped_column(UTCDT(), default=utcnow)
session: Mapped[UploadSession] = relationship(back_populates="chunks")
__table_args__ = (UniqueConstraint("session_id", "chunk_index"),)
# ---------------------------------------------------------------------------
# Transcript / summary / tags
# ---------------------------------------------------------------------------
class Transcript(Base, PublicIdMixin):
__tablename__ = "transcripts"
recording_id: Mapped[uuid.UUID] = mapped_column(
ForeignKey("recordings.id", ondelete="CASCADE"), index=True, nullable=False
)
# Versioned: regenerated transcripts supersede older rows; the newest
# non-superseded row is authoritative. ``edited_by_user`` rows win.
version: Mapped[int] = mapped_column(default=1, nullable=False)
superseded_at: Mapped[datetime | None] = mapped_column(UTCDT())
language: Mapped[str | None] = mapped_column(String(16))
provider: Mapped[str] = mapped_column(String(80), nullable=False, default="manual")
model: Mapped[str | None] = mapped_column(String(120))
# Full text, plus segments: [{"start": 0.0, "end": 2.5, "text": "...",
# "speaker": "S1"|null}, ...]
text: Mapped[str] = mapped_column(Text, nullable=False, default="")
segments: Mapped[list | None] = mapped_column(JSONType)
edited_by_user: Mapped[bool] = mapped_column(Boolean, default=False, nullable=False)
recording: Mapped[Recording] = relationship(back_populates="transcripts")
__table_args__ = (Index("ix_transcripts_recording_version", "recording_id", "version"),)
class Summary(Base, PublicIdMixin):
"""Structured AI summary — user-editable.
JSON shape of ``content``:
{
"short": str,
"detailed": str,
"key_points": [str],
"decisions": [str],
"action_items": [str],
"questions": [str]
}
"""
__tablename__ = "summaries"
recording_id: Mapped[uuid.UUID] = mapped_column(
ForeignKey("recordings.id", ondelete="CASCADE"), index=True, nullable=False
)
version: Mapped[int] = mapped_column(default=1, nullable=False)
superseded_at: Mapped[datetime | None] = mapped_column(UTCDT())
provider: Mapped[str] = mapped_column(String(80), nullable=False, default="manual")
model: Mapped[str | None] = mapped_column(String(120))
content: Mapped[dict] = mapped_column(JSONType, nullable=False, default=dict)
edited_by_user: Mapped[bool] = mapped_column(Boolean, default=False, nullable=False)
recording: Mapped[Recording] = relationship(back_populates="summaries")
class Tag(Base, PublicIdMixin):
__tablename__ = "tags"
user_id: Mapped[uuid.UUID] = mapped_column(
ForeignKey("users.id", ondelete="CASCADE"), index=True, nullable=False
)
name: Mapped[str] = mapped_column(String(80), nullable=False)
recordings: Mapped[list[Recording]] = relationship(
secondary="recording_tags", back_populates="tags"
)
__table_args__ = (UniqueConstraint("user_id", "name"),)
class RecordingTag(Base):
__tablename__ = "recording_tags"
recording_id: Mapped[uuid.UUID] = mapped_column(
ForeignKey("recordings.id", ondelete="CASCADE"), primary_key=True
)
tag_id: Mapped[uuid.UUID] = mapped_column(
ForeignKey("tags.id", ondelete="CASCADE"), primary_key=True
)
# ---------------------------------------------------------------------------
# Processing jobs
# ---------------------------------------------------------------------------
class JobStatus(enum.StrEnum):
queued = "queued"
running = "running"
succeeded = "succeeded"
failed = "failed"
skipped = "skipped" # e.g. AI disabled, or user-edited output protected
class JobType(enum.StrEnum):
normalize_audio = "normalize_audio"
transcribe = "transcribe"
summarize = "summarize"
class ProcessingJob(Base, PublicIdMixin):
"""One pipeline step for one recording. Idempotent: reruns overwrite
derived outputs (unless user-edited) and never touch originals."""
__tablename__ = "processing_jobs"
recording_id: Mapped[uuid.UUID] = mapped_column(
ForeignKey("recordings.id", ondelete="CASCADE"), index=True, nullable=False
)
job_type: Mapped[JobType] = mapped_column(
Enum(JobType, name="job_type", values_callable=lambda e: [m.value for m in e]),
nullable=False,
)
status: Mapped[JobStatus] = mapped_column(
Enum(JobStatus, name="job_status", values_callable=lambda e: [m.value for m in e]),
default=JobStatus.queued,
nullable=False,
index=True,
)
attempt: Mapped[int] = mapped_column(default=0, nullable=False)
max_attempts: Mapped[int] = mapped_column(default=3, nullable=False)
error: Mapped[str | None] = mapped_column(Text)
# Fine-grained phase for progress display (e.g. transcribe jobs report
# "loading-model" then "transcribing"). Nullable: older rows predate it.
# Never parsed by pipeline logic — display only.
stage: Mapped[str | None] = mapped_column(String(32))
# 0-100 work estimate within the current stage, when known.
progress: Mapped[int | None] = mapped_column()
started_at: Mapped[datetime | None] = mapped_column(UTCDT())
finished_at: Mapped[datetime | None] = mapped_column(UTCDT())
# Opaque arq task handle for observability.
task_handle: Mapped[str | None] = mapped_column(String(120))
__table_args__ = (Index("ix_processing_jobs_recording_type", "recording_id", "job_type"),)
class ExportJob(Base, PublicIdMixin):
__tablename__ = "export_jobs"
user_id: Mapped[uuid.UUID] = mapped_column(
ForeignKey("users.id", ondelete="CASCADE"), index=True, nullable=False
)
recording_id: Mapped[uuid.UUID] = mapped_column(
ForeignKey("recordings.id", ondelete="CASCADE"), index=True, nullable=False
)
# "audio", "transcript_txt", "notes_md", "bundle_zip"
export_type: Mapped[str] = mapped_column(String(40), nullable=False)
status: Mapped[JobStatus] = mapped_column(
Enum(JobStatus, name="export_job_status", values_callable=lambda e: [m.value for m in e]),
default=JobStatus.queued,
nullable=False,
)
asset_id: Mapped[uuid.UUID | None] = mapped_column(ForeignKey("assets.id", ondelete="SET NULL"))
error: Mapped[str | None] = mapped_column(Text)
# Ensure full-text search columns exist on Postgres (added via migration as
# tsvector generated columns; see migrations/versions/*_fts.py).
class AppSetting(Base, PublicIdMixin):
"""Server-wide settings editable at runtime (global transcription model
default, …). Single row per key; the desktop Settings page writes here."""
__tablename__ = "app_settings"
key: Mapped[str] = mapped_column(String(120), nullable=False, unique=True)
value: Mapped[dict] = mapped_column(JSONType, nullable=False, default=dict)

76
backend/shonar/db/session.py Normal file
View file

@ -0,0 +1,76 @@
"""Async SQLAlchemy engine/session management."""
from __future__ import annotations
from collections.abc import AsyncIterator
from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker, create_async_engine
from shonar.core.config import get_settings
_engine = None
_session_factory: async_sessionmaker[AsyncSession] | None = None
def get_engine():
global _engine, _session_factory
if _engine is None:
settings = get_settings()
kwargs: dict = {"pool_pre_ping": True}
url = settings.database_url
# SQLite (tests, desktop bundled engine) does not support the pg
# pool sizing kwargs.
if url.startswith("sqlite"):
kwargs = {}
else:
kwargs.update(pool_size=settings.db_pool_size, max_overflow=settings.db_max_overflow)
_engine = create_async_engine(url, **kwargs)
if url.startswith("sqlite"):
_configure_sqlite(_engine)
_session_factory = async_sessionmaker(_engine, expire_on_commit=False)
return _engine
def _configure_sqlite(engine) -> None: # noqa: ANN001
"""Per-connection SQLite pragmas.
foreign_keys is OFF by default in SQLite; the schema leans on
ON DELETE CASCADE, so every connection must enable it. WAL lets the
inline-queue writer and API readers coexist without SQLITE_BUSY
storms on the desktop box.
"""
from sqlalchemy import event
@event.listens_for(engine.sync_engine, "connect")
def _pragmas(dbapi_conn, _record): # noqa: ANN001
cur = dbapi_conn.cursor()
cur.execute("PRAGMA foreign_keys=ON")
cur.execute("PRAGMA journal_mode=WAL")
cur.execute("PRAGMA busy_timeout=5000")
cur.close()
async def dispose_engine() -> None:
global _engine, _session_factory
if _engine is not None:
await _engine.dispose()
_engine = None
_session_factory = None
async def get_session() -> AsyncIterator[AsyncSession]:
"""FastAPI dependency yielding a database session."""
assert _session_factory is not None, "engine not initialised"
async with _session_factory() as session:
try:
yield session
await session.commit()
except Exception:
await session.rollback()
raise
def session_factory() -> async_sessionmaker[AsyncSession]:
"""Shareable session factory for the worker (outside requests)."""
assert _session_factory is not None, "engine not initialised"
return _session_factory