Padreug
09a5d6ed55
Three small fixes shaken out by live testing on aio-demo: 1. fava_client.add_account: when the target file has no Open directives yet (e.g. the empty accounts/users.beancount seed), append at end of file instead of inserting at index 0. Keeps the seed header comments at the top where they belong. 2. account_sync.sync_single_account_from_beancount: read the full user_id from Beancount metadata when present, fall back to the name-derived 8-char prefix otherwise. crud.get_or_create_user_account writes the full 32-char user_id into Beancount metadata when creating per-user accounts; the sync function was only looking at the account name and returning the prefix, so the post-sync `WHERE user_id=:user_id` query in crud.py missed the row and fell through the UNIQUE-constraint recovery path. Three lines of warning noise per user-account creation. 3. tasks.wait_for_account_sync: await `wait_for_fava_client()` (new helper backed by an asyncio.Event in fava_client.py) before the first sync iteration. Previously the sync task started in libra_start() raced the fire-and-forget `_init_fava()` coroutine and reliably crashed the first run with "Fava client not initialized". Refs: aiolabs/libra#28
414 lines
14 KiB
Python
414 lines
14 KiB
Python
"""
|
|
Account Synchronization Module
|
|
|
|
Syncs accounts from Beancount (source of truth) to Libra DB (metadata store).
|
|
|
|
This implements the hybrid approach:
|
|
- Beancount owns account existence (Open directives)
|
|
- Libra DB stores permissions and user associations
|
|
- Background sync keeps them in sync
|
|
|
|
Related: ACCOUNTS-TABLE-REMOVAL-FEASIBILITY.md - Phase 2 implementation
|
|
"""
|
|
|
|
from datetime import datetime
|
|
from typing import Optional
|
|
from loguru import logger
|
|
|
|
from .crud import (
|
|
create_account,
|
|
get_account_by_name,
|
|
get_all_accounts,
|
|
update_account_is_active,
|
|
)
|
|
from .fava_client import get_fava_client
|
|
from .models import AccountType, CreateAccount
|
|
|
|
|
|
def infer_account_type_from_name(account_name: str) -> AccountType:
|
|
"""
|
|
Infer Beancount account type from hierarchical name.
|
|
|
|
Args:
|
|
account_name: Hierarchical account name (e.g., "Expenses:Food:Groceries")
|
|
|
|
Returns:
|
|
AccountType enum value
|
|
|
|
Examples:
|
|
"Assets:Cash" → AccountType.ASSET
|
|
"Liabilities:PayPal" → AccountType.LIABILITY
|
|
"Expenses:Food" → AccountType.EXPENSE
|
|
"Income:Services" → AccountType.REVENUE
|
|
"Equity:Opening-Balances" → AccountType.EQUITY
|
|
"""
|
|
root = account_name.split(":")[0]
|
|
|
|
type_map = {
|
|
"Assets": AccountType.ASSET,
|
|
"Liabilities": AccountType.LIABILITY,
|
|
"Expenses": AccountType.EXPENSE,
|
|
"Income": AccountType.REVENUE,
|
|
"Equity": AccountType.EQUITY,
|
|
}
|
|
|
|
# Default to ASSET if unknown (shouldn't happen with valid Beancount)
|
|
return type_map.get(root, AccountType.ASSET)
|
|
|
|
|
|
def extract_user_id_from_account_name(account_name: str) -> Optional[str]:
|
|
"""
|
|
Extract user ID from account name if it's a user-specific account.
|
|
|
|
Args:
|
|
account_name: Hierarchical account name
|
|
|
|
Returns:
|
|
User ID if found, None otherwise
|
|
|
|
Examples:
|
|
"Assets:Receivable:User-abc123def" → "abc123def456ghi789"
|
|
"Liabilities:Payable:User-abc123" → "abc123def456ghi789"
|
|
"Expenses:Food" → None
|
|
"""
|
|
if ":User-" not in account_name:
|
|
return None
|
|
|
|
# Extract the part after "User-"
|
|
parts = account_name.split(":User-")
|
|
if len(parts) < 2:
|
|
return None
|
|
|
|
# First 8 characters are the user ID prefix
|
|
user_id_prefix = parts[1]
|
|
|
|
# For now, return the prefix (could look up full user ID from DB if needed)
|
|
# Note: get_or_create_user_account() uses 8-char prefix in account names
|
|
return user_id_prefix
|
|
|
|
|
|
async def sync_accounts_from_beancount(force_full_sync: bool = False) -> dict:
|
|
"""
|
|
Sync accounts from Beancount to Libra DB.
|
|
|
|
This ensures Libra DB has metadata entries for all accounts that exist
|
|
in Beancount, enabling permissions and user associations to work properly.
|
|
|
|
New behavior (soft delete + virtual parents):
|
|
- Accounts in Beancount but not in Libra DB: Added as active
|
|
- Accounts in Libra DB but not in Beancount: Marked as inactive (soft delete)
|
|
- Inactive accounts that return to Beancount: Reactivated
|
|
- Missing intermediate parents: Auto-created as virtual accounts
|
|
|
|
Virtual parent auto-generation example:
|
|
Beancount has: "Expenses:Supplies:Food"
|
|
Missing parent: "Expenses:Supplies" (doesn't exist in Beancount)
|
|
→ Auto-create "Expenses:Supplies" as virtual account
|
|
→ Enables granting permission on "Expenses:Supplies" to cover all Supplies:* children
|
|
|
|
Args:
|
|
force_full_sync: If True, re-check all accounts. If False, only add new ones.
|
|
|
|
Returns:
|
|
dict with sync statistics:
|
|
{
|
|
"total_beancount_accounts": 150,
|
|
"total_libra_accounts": 148,
|
|
"accounts_added": 2,
|
|
"accounts_updated": 0,
|
|
"accounts_skipped": 148,
|
|
"accounts_deactivated": 5,
|
|
"accounts_reactivated": 1,
|
|
"virtual_parents_created": 3,
|
|
"errors": []
|
|
}
|
|
"""
|
|
logger.info("Starting account sync from Beancount to Libra DB")
|
|
|
|
fava = get_fava_client()
|
|
|
|
# Get all accounts from Beancount
|
|
try:
|
|
beancount_accounts = await fava.get_all_accounts()
|
|
except Exception as e:
|
|
logger.error(f"Failed to fetch accounts from Beancount: {e}")
|
|
return {
|
|
"total_beancount_accounts": 0,
|
|
"total_libra_accounts": 0,
|
|
"accounts_added": 0,
|
|
"accounts_updated": 0,
|
|
"accounts_skipped": 0,
|
|
"accounts_deactivated": 0,
|
|
"accounts_reactivated": 0,
|
|
"errors": [str(e)],
|
|
}
|
|
|
|
# Get all accounts from Libra DB (including inactive ones for sync)
|
|
libra_accounts = await get_all_accounts(include_inactive=True)
|
|
|
|
# Build lookup maps
|
|
beancount_account_names = {acc["account"] for acc in beancount_accounts}
|
|
libra_accounts_by_name = {acc.name: acc for acc in libra_accounts}
|
|
|
|
stats = {
|
|
"total_beancount_accounts": len(beancount_accounts),
|
|
"total_libra_accounts": len(libra_accounts),
|
|
"accounts_added": 0,
|
|
"accounts_updated": 0,
|
|
"accounts_skipped": 0,
|
|
"accounts_deactivated": 0,
|
|
"accounts_reactivated": 0,
|
|
"virtual_parents_created": 0,
|
|
"errors": [],
|
|
}
|
|
|
|
# Step 1: Sync accounts from Beancount to Libra DB
|
|
for bc_account in beancount_accounts:
|
|
account_name = bc_account["account"]
|
|
|
|
try:
|
|
existing = libra_accounts_by_name.get(account_name)
|
|
|
|
if existing:
|
|
# Account exists in Libra DB
|
|
# Check if it needs to be reactivated
|
|
if not existing.is_active:
|
|
await update_account_is_active(existing.id, True)
|
|
stats["accounts_reactivated"] += 1
|
|
logger.info(f"Reactivated account: {account_name}")
|
|
else:
|
|
stats["accounts_skipped"] += 1
|
|
logger.debug(f"Account already active: {account_name}")
|
|
continue
|
|
|
|
# Create new account in Libra DB
|
|
account_type = infer_account_type_from_name(account_name)
|
|
user_id = extract_user_id_from_account_name(account_name)
|
|
|
|
# Get description from Beancount metadata if available
|
|
description = None
|
|
if "meta" in bc_account and isinstance(bc_account["meta"], dict):
|
|
description = bc_account["meta"].get("description")
|
|
|
|
await create_account(
|
|
CreateAccount(
|
|
name=account_name,
|
|
account_type=account_type,
|
|
description=description,
|
|
user_id=user_id,
|
|
)
|
|
)
|
|
|
|
stats["accounts_added"] += 1
|
|
logger.info(f"Added account from Beancount: {account_name}")
|
|
|
|
except Exception as e:
|
|
error_msg = f"Failed to sync account {account_name}: {e}"
|
|
logger.error(error_msg)
|
|
stats["errors"].append(error_msg)
|
|
|
|
# Step 2: Mark orphaned accounts (in Libra DB but not in Beancount) as inactive
|
|
# SKIP virtual accounts (they're intentionally metadata-only)
|
|
for libra_account in libra_accounts:
|
|
if libra_account.is_virtual:
|
|
# Virtual accounts are metadata-only, never deactivate them
|
|
continue
|
|
|
|
if libra_account.name not in beancount_account_names:
|
|
# Account no longer exists in Beancount
|
|
if libra_account.is_active:
|
|
try:
|
|
await update_account_is_active(libra_account.id, False)
|
|
stats["accounts_deactivated"] += 1
|
|
logger.info(
|
|
f"Deactivated orphaned account: {libra_account.name}"
|
|
)
|
|
except Exception as e:
|
|
error_msg = (
|
|
f"Failed to deactivate account {libra_account.name}: {e}"
|
|
)
|
|
logger.error(error_msg)
|
|
stats["errors"].append(error_msg)
|
|
|
|
# Step 3: Auto-generate virtual intermediate parent accounts
|
|
# For each account in Beancount, check if all parent levels exist
|
|
# If not, create them as virtual accounts
|
|
|
|
# IMPORTANT: Re-fetch accounts from DB after Step 1 added new accounts
|
|
# Otherwise we'll be checking against stale data and miss newly synced children
|
|
current_libra_accounts = await get_all_accounts(include_inactive=True)
|
|
all_account_names = {acc.name for acc in current_libra_accounts}
|
|
|
|
for bc_account in beancount_accounts:
|
|
account_name = bc_account["account"]
|
|
parts = account_name.split(":")
|
|
|
|
# Check each parent level (e.g., for "Expenses:Supplies:Food", check "Expenses:Supplies")
|
|
for i in range(1, len(parts)):
|
|
parent_name = ":".join(parts[:i])
|
|
|
|
# Skip if parent already exists
|
|
if parent_name in all_account_names:
|
|
continue
|
|
|
|
# Create virtual parent account
|
|
try:
|
|
parent_type = infer_account_type_from_name(parent_name)
|
|
await create_account(
|
|
CreateAccount(
|
|
name=parent_name,
|
|
account_type=parent_type,
|
|
description=f"Auto-generated virtual parent for {parent_name}:* accounts",
|
|
is_virtual=True,
|
|
)
|
|
)
|
|
|
|
stats["virtual_parents_created"] += 1
|
|
all_account_names.add(parent_name) # Track so we don't create duplicates
|
|
logger.info(f"Created virtual parent account: {parent_name}")
|
|
|
|
except Exception as e:
|
|
error_msg = f"Failed to create virtual parent {parent_name}: {e}"
|
|
logger.error(error_msg)
|
|
stats["errors"].append(error_msg)
|
|
|
|
logger.info(
|
|
f"Account sync complete: "
|
|
f"{stats['accounts_added']} added, "
|
|
f"{stats['accounts_reactivated']} reactivated, "
|
|
f"{stats['accounts_deactivated']} deactivated, "
|
|
f"{stats['virtual_parents_created']} virtual parents created, "
|
|
f"{stats['accounts_skipped']} skipped, "
|
|
f"{len(stats['errors'])} errors"
|
|
)
|
|
|
|
return stats
|
|
|
|
|
|
async def sync_single_account_from_beancount(account_name: str) -> bool:
|
|
"""
|
|
Sync a single account from Beancount to Libra DB.
|
|
|
|
Useful for ensuring a specific account exists in Libra DB before
|
|
granting permissions on it.
|
|
|
|
Args:
|
|
account_name: Hierarchical account name (e.g., "Expenses:Food")
|
|
|
|
Returns:
|
|
True if account was created/updated, False if it already existed or failed
|
|
"""
|
|
logger.debug(f"Syncing single account: {account_name}")
|
|
|
|
# Check if already exists
|
|
existing = await get_account_by_name(account_name)
|
|
if existing:
|
|
logger.debug(f"Account already exists: {account_name}")
|
|
return False
|
|
|
|
# Get from Beancount
|
|
fava = get_fava_client()
|
|
try:
|
|
all_accounts = await fava.get_all_accounts()
|
|
bc_account = next(
|
|
(acc for acc in all_accounts if acc["account"] == account_name), None
|
|
)
|
|
|
|
if not bc_account:
|
|
logger.error(f"Account not found in Beancount: {account_name}")
|
|
return False
|
|
|
|
# Create in Libra DB
|
|
account_type = infer_account_type_from_name(account_name)
|
|
|
|
# Prefer the full user_id stored in Beancount metadata (libra writes it
|
|
# when crud.get_or_create_user_account calls fava.add_account). Fall
|
|
# back to the name-derived 8-char prefix for accounts imported without
|
|
# metadata. This keeps user_id consistent with what the caller will
|
|
# query for, avoiding a churn cycle through the UNIQUE-constraint
|
|
# recovery path in crud.py.
|
|
description = None
|
|
meta_user_id = None
|
|
if "meta" in bc_account and isinstance(bc_account["meta"], dict):
|
|
description = bc_account["meta"].get("description")
|
|
meta_user_id = bc_account["meta"].get("user_id")
|
|
|
|
user_id = meta_user_id or extract_user_id_from_account_name(account_name)
|
|
|
|
await create_account(
|
|
CreateAccount(
|
|
name=account_name,
|
|
account_type=account_type,
|
|
description=description,
|
|
user_id=user_id,
|
|
)
|
|
)
|
|
|
|
logger.info(f"Created account from Beancount: {account_name}")
|
|
return True
|
|
|
|
except Exception as e:
|
|
logger.error(f"Failed to sync account {account_name}: {e}")
|
|
return False
|
|
|
|
|
|
async def ensure_account_exists_in_libra(account_name: str) -> bool:
|
|
"""
|
|
Ensure account exists in Libra DB, creating from Beancount if needed.
|
|
|
|
This is the recommended function to call before granting permissions.
|
|
|
|
Args:
|
|
account_name: Hierarchical account name
|
|
|
|
Returns:
|
|
True if account exists (or was created), False if failed
|
|
"""
|
|
# Check Libra DB first
|
|
existing = await get_account_by_name(account_name)
|
|
if existing:
|
|
return True
|
|
|
|
# Try to sync from Beancount
|
|
return await sync_single_account_from_beancount(account_name)
|
|
|
|
|
|
# Background sync task (can be scheduled with cron or async scheduler)
|
|
async def scheduled_account_sync():
|
|
"""
|
|
Scheduled task to sync accounts from Beancount to Libra DB.
|
|
|
|
Run this periodically (e.g., every hour) to keep Libra DB in sync with Beancount.
|
|
|
|
Example with APScheduler:
|
|
from apscheduler.schedulers.asyncio import AsyncIOScheduler
|
|
|
|
scheduler = AsyncIOScheduler()
|
|
scheduler.add_job(
|
|
scheduled_account_sync,
|
|
'interval',
|
|
hours=1, # Run every hour
|
|
id='account_sync'
|
|
)
|
|
scheduler.start()
|
|
"""
|
|
logger.info("Running scheduled account sync")
|
|
|
|
try:
|
|
stats = await sync_accounts_from_beancount(force_full_sync=False)
|
|
|
|
if stats["accounts_added"] > 0:
|
|
logger.info(
|
|
f"Scheduled sync: Added {stats['accounts_added']} new accounts"
|
|
)
|
|
|
|
if stats["errors"]:
|
|
logger.warning(
|
|
f"Scheduled sync: {len(stats['errors'])} errors encountered"
|
|
)
|
|
|
|
return stats
|
|
|
|
except Exception as e:
|
|
logger.error(f"Scheduled account sync failed: {e}")
|
|
raise
|