Add BQL-optimized get_unsettled_entries_bql method
Replace inefficient approach that fetched ALL journal entries with targeted BQL queries that: - Filter by account pattern and tags in the database - Use weight column for SATS amounts (no string parsing) - Query only expense/receivable entries for the specific user This significantly reduces data transfer and processing overhead. 🤖 Generated with [Claude Code](https://claude.com/claude-code) Co-Authored-By: Claude <noreply@anthropic.com>
This commit is contained in:
parent
55df2b36e0
commit
048d19f90b
2 changed files with 146 additions and 1 deletions
145
fava_client.py
145
fava_client.py
|
|
@ -1191,6 +1191,151 @@ class FavaClient:
|
||||||
logger.error(f"Fava connection error: {e}")
|
logger.error(f"Fava connection error: {e}")
|
||||||
raise
|
raise
|
||||||
|
|
||||||
|
async def get_unsettled_entries_bql(
|
||||||
|
self,
|
||||||
|
user_id: str,
|
||||||
|
entry_type: str = "expense"
|
||||||
|
) -> List[Dict[str, Any]]:
|
||||||
|
"""
|
||||||
|
Get unsettled expense or receivable entries for a user using BQL.
|
||||||
|
|
||||||
|
Uses BQL queries to efficiently find entries with exp-{id} or rcv-{id}
|
||||||
|
links that don't have a corresponding settlement entry.
|
||||||
|
|
||||||
|
This is significantly more efficient than the legacy method as it:
|
||||||
|
- Queries only relevant entries (not ALL journal entries)
|
||||||
|
- Uses weight column for SATS amounts (no string parsing)
|
||||||
|
- Filters by tags and account patterns in the database
|
||||||
|
|
||||||
|
Args:
|
||||||
|
user_id: User ID (first 8 characters used for account matching)
|
||||||
|
entry_type: "expense" (payables - castle owes user) or
|
||||||
|
"receivable" (user owes castle)
|
||||||
|
|
||||||
|
Returns:
|
||||||
|
List of unsettled entries with:
|
||||||
|
- link: The entry's unique link (exp-xxx or rcv-xxx)
|
||||||
|
- date: Entry date
|
||||||
|
- narration: Description
|
||||||
|
- fiat_amount: Amount in fiat currency (absolute value)
|
||||||
|
- fiat_currency: Currency code
|
||||||
|
- sats_amount: Amount in SATS (absolute value, from weight)
|
||||||
|
- entry_hash: For potential updates
|
||||||
|
- flag: Transaction flag
|
||||||
|
"""
|
||||||
|
from decimal import Decimal
|
||||||
|
|
||||||
|
user_short = user_id[:8]
|
||||||
|
link_prefix = "exp-" if entry_type == "expense" else "rcv-"
|
||||||
|
entry_tag = "expense-entry" if entry_type == "expense" else "receivable-entry"
|
||||||
|
|
||||||
|
# Determine account pattern based on entry type
|
||||||
|
if entry_type == "expense":
|
||||||
|
account_pattern = f"Liabilities:Payable:User-{user_short}"
|
||||||
|
else:
|
||||||
|
account_pattern = f"Assets:Receivable:User-{user_short}"
|
||||||
|
|
||||||
|
try:
|
||||||
|
# Query 1: Get all original expense/receivable entries for this user
|
||||||
|
# These are entries with the expense-entry or receivable-entry tag
|
||||||
|
original_query = f"""
|
||||||
|
SELECT date, narration, account, number, weight, links,
|
||||||
|
any_meta('entry-id') as entry_id
|
||||||
|
WHERE account ~ '{account_pattern}'
|
||||||
|
AND '{entry_tag}' IN tags
|
||||||
|
AND flag = '*'
|
||||||
|
ORDER BY date
|
||||||
|
"""
|
||||||
|
|
||||||
|
original_result = await self.query_bql(original_query)
|
||||||
|
|
||||||
|
# Query 2: Get all settlement entries for this user
|
||||||
|
# These are entries with the settlement tag
|
||||||
|
settlement_query = f"""
|
||||||
|
SELECT links
|
||||||
|
WHERE account ~ '{account_pattern}'
|
||||||
|
AND 'settlement' IN tags
|
||||||
|
AND flag = '*'
|
||||||
|
"""
|
||||||
|
|
||||||
|
settlement_result = await self.query_bql(settlement_query)
|
||||||
|
|
||||||
|
# Build set of settled links from settlement entries
|
||||||
|
settled_links: set = set()
|
||||||
|
for row in settlement_result["rows"]:
|
||||||
|
links = row[0] if row else []
|
||||||
|
if isinstance(links, list):
|
||||||
|
for link in links:
|
||||||
|
if link.startswith(link_prefix):
|
||||||
|
settled_links.add(link)
|
||||||
|
|
||||||
|
# Process original entries and find unsettled ones
|
||||||
|
entries_by_link: Dict[str, Dict[str, Any]] = {}
|
||||||
|
|
||||||
|
for row in original_result["rows"]:
|
||||||
|
date_val, narration, account, number, weight, links, entry_id = row
|
||||||
|
|
||||||
|
# Skip if no links
|
||||||
|
if not links or not isinstance(links, list):
|
||||||
|
continue
|
||||||
|
|
||||||
|
# Find the exp-/rcv- link
|
||||||
|
entry_link = None
|
||||||
|
for link in links:
|
||||||
|
if link.startswith(link_prefix):
|
||||||
|
entry_link = link
|
||||||
|
break
|
||||||
|
|
||||||
|
if not entry_link:
|
||||||
|
continue
|
||||||
|
|
||||||
|
# Skip if already settled
|
||||||
|
if entry_link in settled_links:
|
||||||
|
continue
|
||||||
|
|
||||||
|
# Skip if we already have this entry (BQL returns one row per posting)
|
||||||
|
if entry_link in entries_by_link:
|
||||||
|
continue
|
||||||
|
|
||||||
|
# Parse amounts
|
||||||
|
fiat_amount = abs(float(number)) if number else 0.0
|
||||||
|
fiat_currency = "EUR" # Default, could be extracted from posting
|
||||||
|
|
||||||
|
# Parse SATS from weight column
|
||||||
|
sats_amount = 0
|
||||||
|
if isinstance(weight, dict) and "SATS" in weight:
|
||||||
|
sats_value = weight["SATS"]
|
||||||
|
sats_amount = abs(int(Decimal(str(sats_value))))
|
||||||
|
|
||||||
|
# Format date as string
|
||||||
|
date_str = str(date_val) if date_val else ""
|
||||||
|
|
||||||
|
entries_by_link[entry_link] = {
|
||||||
|
"link": entry_link,
|
||||||
|
"date": date_str,
|
||||||
|
"narration": narration or "",
|
||||||
|
"fiat_amount": fiat_amount,
|
||||||
|
"fiat_currency": fiat_currency,
|
||||||
|
"sats_amount": sats_amount,
|
||||||
|
"entry_hash": "", # Not available from BQL, use entry_id instead
|
||||||
|
"entry_id": entry_id or "",
|
||||||
|
"flag": "*"
|
||||||
|
}
|
||||||
|
|
||||||
|
# Convert to list and sort by date
|
||||||
|
unsettled = list(entries_by_link.values())
|
||||||
|
unsettled.sort(key=lambda x: x.get("date", ""))
|
||||||
|
|
||||||
|
logger.info(
|
||||||
|
f"BQL: Found {len(unsettled)} unsettled {entry_type} entries for user {user_short} "
|
||||||
|
f"(settled: {len(settled_links)})"
|
||||||
|
)
|
||||||
|
|
||||||
|
return unsettled
|
||||||
|
|
||||||
|
except Exception as e:
|
||||||
|
logger.error(f"Error getting unsettled entries via BQL: {e}")
|
||||||
|
raise
|
||||||
|
|
||||||
async def get_unsettled_entries(
|
async def get_unsettled_entries(
|
||||||
self,
|
self,
|
||||||
|
|
|
||||||
|
|
@ -2264,7 +2264,7 @@ async def api_get_unsettled_entries(
|
||||||
)
|
)
|
||||||
|
|
||||||
fava = get_fava_client()
|
fava = get_fava_client()
|
||||||
unsettled = await fava.get_unsettled_entries(user_id, entry_type)
|
unsettled = await fava.get_unsettled_entries_bql(user_id, entry_type)
|
||||||
|
|
||||||
# Calculate totals
|
# Calculate totals
|
||||||
total_fiat = sum(e.get("fiat_amount", 0) for e in unsettled)
|
total_fiat = sum(e.get("fiat_amount", 0) for e in unsettled)
|
||||||
|
|
|
||||||
Loading…
Add table
Add a link
Reference in a new issue