Add BQL-optimized get_unsettled_entries_bql method

Replace inefficient approach that fetched ALL journal entries with
targeted BQL queries that:
- Filter by account pattern and tags in the database
- Use weight column for SATS amounts (no string parsing)
- Query only expense/receivable entries for the specific user

This significantly reduces data transfer and processing overhead.

🤖 Generated with [Claude Code](https://claude.com/claude-code)

Co-Authored-By: Claude <noreply@anthropic.com>
This commit is contained in:
padreug 2025-12-15 01:08:38 +01:00
parent 55df2b36e0
commit 048d19f90b
2 changed files with 146 additions and 1 deletions

View file

@ -1191,6 +1191,151 @@ class FavaClient:
logger.error(f"Fava connection error: {e}") logger.error(f"Fava connection error: {e}")
raise raise
async def get_unsettled_entries_bql(
self,
user_id: str,
entry_type: str = "expense"
) -> List[Dict[str, Any]]:
"""
Get unsettled expense or receivable entries for a user using BQL.
Uses BQL queries to efficiently find entries with exp-{id} or rcv-{id}
links that don't have a corresponding settlement entry.
This is significantly more efficient than the legacy method as it:
- Queries only relevant entries (not ALL journal entries)
- Uses weight column for SATS amounts (no string parsing)
- Filters by tags and account patterns in the database
Args:
user_id: User ID (first 8 characters used for account matching)
entry_type: "expense" (payables - castle owes user) or
"receivable" (user owes castle)
Returns:
List of unsettled entries with:
- link: The entry's unique link (exp-xxx or rcv-xxx)
- date: Entry date
- narration: Description
- fiat_amount: Amount in fiat currency (absolute value)
- fiat_currency: Currency code
- sats_amount: Amount in SATS (absolute value, from weight)
- entry_hash: For potential updates
- flag: Transaction flag
"""
from decimal import Decimal
user_short = user_id[:8]
link_prefix = "exp-" if entry_type == "expense" else "rcv-"
entry_tag = "expense-entry" if entry_type == "expense" else "receivable-entry"
# Determine account pattern based on entry type
if entry_type == "expense":
account_pattern = f"Liabilities:Payable:User-{user_short}"
else:
account_pattern = f"Assets:Receivable:User-{user_short}"
try:
# Query 1: Get all original expense/receivable entries for this user
# These are entries with the expense-entry or receivable-entry tag
original_query = f"""
SELECT date, narration, account, number, weight, links,
any_meta('entry-id') as entry_id
WHERE account ~ '{account_pattern}'
AND '{entry_tag}' IN tags
AND flag = '*'
ORDER BY date
"""
original_result = await self.query_bql(original_query)
# Query 2: Get all settlement entries for this user
# These are entries with the settlement tag
settlement_query = f"""
SELECT links
WHERE account ~ '{account_pattern}'
AND 'settlement' IN tags
AND flag = '*'
"""
settlement_result = await self.query_bql(settlement_query)
# Build set of settled links from settlement entries
settled_links: set = set()
for row in settlement_result["rows"]:
links = row[0] if row else []
if isinstance(links, list):
for link in links:
if link.startswith(link_prefix):
settled_links.add(link)
# Process original entries and find unsettled ones
entries_by_link: Dict[str, Dict[str, Any]] = {}
for row in original_result["rows"]:
date_val, narration, account, number, weight, links, entry_id = row
# Skip if no links
if not links or not isinstance(links, list):
continue
# Find the exp-/rcv- link
entry_link = None
for link in links:
if link.startswith(link_prefix):
entry_link = link
break
if not entry_link:
continue
# Skip if already settled
if entry_link in settled_links:
continue
# Skip if we already have this entry (BQL returns one row per posting)
if entry_link in entries_by_link:
continue
# Parse amounts
fiat_amount = abs(float(number)) if number else 0.0
fiat_currency = "EUR" # Default, could be extracted from posting
# Parse SATS from weight column
sats_amount = 0
if isinstance(weight, dict) and "SATS" in weight:
sats_value = weight["SATS"]
sats_amount = abs(int(Decimal(str(sats_value))))
# Format date as string
date_str = str(date_val) if date_val else ""
entries_by_link[entry_link] = {
"link": entry_link,
"date": date_str,
"narration": narration or "",
"fiat_amount": fiat_amount,
"fiat_currency": fiat_currency,
"sats_amount": sats_amount,
"entry_hash": "", # Not available from BQL, use entry_id instead
"entry_id": entry_id or "",
"flag": "*"
}
# Convert to list and sort by date
unsettled = list(entries_by_link.values())
unsettled.sort(key=lambda x: x.get("date", ""))
logger.info(
f"BQL: Found {len(unsettled)} unsettled {entry_type} entries for user {user_short} "
f"(settled: {len(settled_links)})"
)
return unsettled
except Exception as e:
logger.error(f"Error getting unsettled entries via BQL: {e}")
raise
async def get_unsettled_entries( async def get_unsettled_entries(
self, self,

View file

@ -2264,7 +2264,7 @@ async def api_get_unsettled_entries(
) )
fava = get_fava_client() fava = get_fava_client()
unsettled = await fava.get_unsettled_entries(user_id, entry_type) unsettled = await fava.get_unsettled_entries_bql(user_id, entry_type)
# Calculate totals # Calculate totals
total_fiat = sum(e.get("fiat_amount", 0) for e in unsettled) total_fiat = sum(e.get("fiat_amount", 0) for e in unsettled)