From 7173e051feed3e9aaa190b0e4942d8c280679231 Mon Sep 17 00:00:00 2001 From: padreug Date: Sun, 14 Dec 2025 23:58:56 +0100 Subject: [PATCH] Use BQL sum(weight) for efficient SATS balance queries MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Now that the ledger uses @@ SATS price notation, BQL can efficiently aggregate SATS balances using the weight column instead of manual entry-by-entry aggregation. Changes: - fava_client.py: Update get_user_balance_bql() and get_all_user_balances_bql() to use sum(weight) for SATS aggregation (5-10x performance improvement) - views_api.py: Switch all balance endpoints to use BQL methods The weight column returns the @@ price value, enabling server-side filtering and aggregation instead of fetching all entries. 🤖 Generated with [Claude Code](https://claude.com/claude-code) Co-Authored-By: Claude --- fava_client.py | 184 ++++++++++++++++--------------------------------- views_api.py | 22 +++--- 2 files changed, 72 insertions(+), 134 deletions(-) diff --git a/fava_client.py b/fava_client.py index ac99bd2..601907b 100644 --- a/fava_client.py +++ b/fava_client.py @@ -557,103 +557,71 @@ class FavaClient: async def get_user_balance_bql(self, user_id: str) -> Dict[str, Any]: """ - Get user balance using BQL (efficient, replaces 115-line manual aggregation). + Get user balance using BQL with price notation (efficient server-side aggregation). - ⚠️ NOT CURRENTLY USED: This method cannot access SATS balances stored in posting - metadata. It only queries position amounts (EUR/USD). For Castle's current ledger - format, use get_user_balance() instead (manual aggregation with caching). - - This method uses Beancount Query Language for server-side filtering and aggregation, - which would provide 5-10x performance improvement IF SATS were stored as position - amounts instead of metadata. - - FUTURE CONSIDERATION: If Castle's ledger format changes to store SATS as position - amounts (e.g., "100000 SATS {100.00 EUR}"), this method would become feasible and - provide significant performance benefits. - - See: docs/BQL-BALANCE-QUERIES.md for detailed test results and analysis. + Uses sum(weight) to aggregate SATS from @@ price notation. + This provides 5-10x performance improvement over manual aggregation. Args: user_id: User ID Returns: { - "balance": int (sats), # Currently returns 0 (cannot access metadata) - "fiat_balances": {"EUR": Decimal("100.50"), ...}, # Works correctly - "accounts": [{"account": "...", "sats": 150000}, ...] + "balance": int (sats from weight column), + "fiat_balances": {"EUR": Decimal("100.50"), ...}, + "accounts": [{"account": "...", "sats": 150000, "eur": Decimal("100.50")}, ...] } Example: balance = await fava.get_user_balance_bql("af983632") - print(f"Balance: {balance['balance']} sats") # Will be 0 with current ledger format + print(f"Balance: {balance['balance']} sats") """ from decimal import Decimal - import re - # Build BQL query for this user's Payable/Receivable accounts user_id_prefix = user_id[:8] + + # BQL query using sum(weight) for SATS aggregation + # weight column returns the @@ price value (SATS) from price notation query = f""" - SELECT account, sum(position) as balance + SELECT account, sum(number), sum(weight) WHERE account ~ ':User-{user_id_prefix}' AND (account ~ 'Payable' OR account ~ 'Receivable') - AND flag != '!' + AND flag = '*' GROUP BY account """ result = await self.query_bql(query) - # Process results total_sats = 0 fiat_balances = {} accounts = [] for row in result["rows"]: - account_name, position = row + account_name, fiat_sum, weight_sum = row - # Position can be: - # - Dict: {"SATS": "150000", "EUR": "145.50"} - # - String: "150000 SATS" or "145.50 EUR" + # Parse fiat amount (sum of EUR/USD amounts) + fiat_amount = Decimal(str(fiat_sum)) if fiat_sum else Decimal(0) - if isinstance(position, dict): - # Extract SATS - sats_str = position.get("SATS", "0") - sats_amount = int(sats_str) if sats_str else 0 - total_sats += sats_amount + # Parse SATS from weight column + # weight_sum is an Inventory dict like {"SATS": -10442635.00} + sats_amount = 0 + if isinstance(weight_sum, dict) and "SATS" in weight_sum: + sats_value = weight_sum["SATS"] + sats_amount = int(Decimal(str(sats_value))) - accounts.append({ - "account": account_name, - "sats": sats_amount - }) + total_sats += sats_amount - # Extract fiat currencies - for currency in ["EUR", "USD", "GBP"]: - if currency in position: - fiat_str = position[currency] - fiat_amount = Decimal(fiat_str) if fiat_str else Decimal(0) + # Aggregate fiat (assume EUR for now, could be extended) + if fiat_amount != 0: + if "EUR" not in fiat_balances: + fiat_balances["EUR"] = Decimal(0) + fiat_balances["EUR"] += fiat_amount - if currency not in fiat_balances: - fiat_balances[currency] = Decimal(0) - fiat_balances[currency] += fiat_amount - - elif isinstance(position, str): - # Single currency (parse "150000 SATS" or "145.50 EUR") - sats_match = re.match(r'^(-?\d+)\s+SATS$', position) - if sats_match: - sats_amount = int(sats_match.group(1)) - total_sats += sats_amount - accounts.append({ - "account": account_name, - "sats": sats_amount - }) - else: - fiat_match = re.match(r'^(-?[\d.]+)\s+([A-Z]{3})$', position) - if fiat_match and fiat_match.group(2) in ('EUR', 'USD', 'GBP'): - fiat_amount = Decimal(fiat_match.group(1)) - currency = fiat_match.group(2) - - if currency not in fiat_balances: - fiat_balances[currency] = Decimal(0) - fiat_balances[currency] += fiat_amount + accounts.append({ + "account": account_name, + "sats": sats_amount, + "eur": fiat_amount + }) logger.info(f"User {user_id[:8]} balance (BQL): {total_sats} sats, fiat: {dict(fiat_balances)}") @@ -665,28 +633,18 @@ class FavaClient: async def get_all_user_balances_bql(self) -> List[Dict[str, Any]]: """ - Get balances for all users using BQL (efficient admin view). + Get balances for all users using BQL with price notation (efficient admin view). - ⚠️ NOT CURRENTLY USED: This method cannot access SATS balances stored in posting - metadata. It only queries position amounts (EUR/USD). For Castle's current ledger - format, use get_all_user_balances() instead (manual aggregation with caching). - - This method uses Beancount Query Language to query all user balances - in a single efficient query, which would be faster than fetching all entries IF - SATS were stored as position amounts instead of metadata. - - FUTURE CONSIDERATION: If Castle's ledger format changes to store SATS as position - amounts, this method would provide significant performance benefits for admin views. - - See: docs/BQL-BALANCE-QUERIES.md for detailed test results and analysis. + Uses sum(weight) to aggregate SATS from @@ price notation in a single query. + This provides significant performance benefits for admin views. Returns: [ { "user_id": "abc123", - "balance": 100000, # Currently 0 (cannot access metadata) - "fiat_balances": {"EUR": Decimal("100.50")}, # Works correctly - "accounts": [{"account": "...", "sats": 150000}, ...] + "balance": 100000, + "fiat_balances": {"EUR": Decimal("100.50")}, + "accounts": [{"account": "...", "sats": 150000, "eur": Decimal("100.50")}, ...] }, ... ] @@ -694,16 +652,15 @@ class FavaClient: Example: all_balances = await fava.get_all_user_balances_bql() for user in all_balances: - print(f"{user['user_id']}: {user['balance']} sats") # Will be 0 with current format + print(f"{user['user_id']}: {user['balance']} sats") """ from decimal import Decimal - import re - # BQL query for ALL user accounts + # BQL query using sum(weight) for SATS aggregation query = """ - SELECT account, sum(position) as balance + SELECT account, sum(number), sum(weight) WHERE (account ~ 'Payable:User-' OR account ~ 'Receivable:User-') - AND flag != '!' + AND flag = '*' GROUP BY account """ @@ -713,15 +670,13 @@ class FavaClient: user_data = {} for row in result["rows"]: - account_name, position = row + account_name, fiat_sum, weight_sum = row # Extract user_id from account name - # Format: "Liabilities:Payable:User-abc12345" or "Assets:Receivable:User-abc12345" if ":User-" not in account_name: continue user_id_with_prefix = account_name.split(":User-")[1] - # User ID is the first 8 chars (our standard) user_id = user_id_with_prefix[:8] if user_id not in user_data: @@ -732,45 +687,28 @@ class FavaClient: "accounts": [] } - # Process position (same logic as single-user query) - if isinstance(position, dict): - sats_str = position.get("SATS", "0") - sats_amount = int(sats_str) if sats_str else 0 - user_data[user_id]["balance"] += sats_amount + # Parse fiat amount + fiat_amount = Decimal(str(fiat_sum)) if fiat_sum else Decimal(0) - user_data[user_id]["accounts"].append({ - "account": account_name, - "sats": sats_amount - }) + # Parse SATS from weight column + sats_amount = 0 + if isinstance(weight_sum, dict) and "SATS" in weight_sum: + sats_value = weight_sum["SATS"] + sats_amount = int(Decimal(str(sats_value))) - for currency in ["EUR", "USD", "GBP"]: - if currency in position: - fiat_str = position[currency] - fiat_amount = Decimal(fiat_str) if fiat_str else Decimal(0) + user_data[user_id]["balance"] += sats_amount - if currency not in user_data[user_id]["fiat_balances"]: - user_data[user_id]["fiat_balances"][currency] = Decimal(0) - user_data[user_id]["fiat_balances"][currency] += fiat_amount + # Aggregate fiat + if fiat_amount != 0: + if "EUR" not in user_data[user_id]["fiat_balances"]: + user_data[user_id]["fiat_balances"]["EUR"] = Decimal(0) + user_data[user_id]["fiat_balances"]["EUR"] += fiat_amount - elif isinstance(position, str): - # Single currency (parse "150000 SATS" or "145.50 EUR") - sats_match = re.match(r'^(-?\d+)\s+SATS$', position) - if sats_match: - sats_amount = int(sats_match.group(1)) - user_data[user_id]["balance"] += sats_amount - user_data[user_id]["accounts"].append({ - "account": account_name, - "sats": sats_amount - }) - else: - fiat_match = re.match(r'^(-?[\d.]+)\s+([A-Z]{3})$', position) - if fiat_match and fiat_match.group(2) in ('EUR', 'USD', 'GBP'): - fiat_amount = Decimal(fiat_match.group(1)) - currency = fiat_match.group(2) - - if currency not in user_data[user_id]["fiat_balances"]: - user_data[user_id]["fiat_balances"][currency] = Decimal(0) - user_data[user_id]["fiat_balances"][currency] += fiat_amount + user_data[user_id]["accounts"].append({ + "account": account_name, + "sats": sats_amount, + "eur": fiat_amount + }) logger.info(f"Fetched balances for {len(user_data)} users (BQL)") diff --git a/views_api.py b/views_api.py index b8c56f6..814d83f 100644 --- a/views_api.py +++ b/views_api.py @@ -1367,10 +1367,10 @@ async def api_get_my_balance( # If super user, show total castle position if wallet.wallet.user == lnbits_settings.super_user: - all_balances = await fava.get_all_user_balances() + all_balances = await fava.get_all_user_balances_bql() # Calculate total: - # From get_user_balance(): positive = user owes castle, negative = castle owes user + # From get_user_balance_bql(): positive = user owes castle, negative = castle owes user # Positive balances = Users owe Castle (receivables for Castle) # Negative balances = Castle owes users (liabilities for Castle) # Net: positive means castle is owed money, negative means castle owes money @@ -1396,7 +1396,7 @@ async def api_get_my_balance( ) # For regular users, show their individual balance from Fava - balance_data = await fava.get_user_balance(wallet.wallet.user) + balance_data = await fava.get_user_balance_bql(wallet.wallet.user) return UserBalance( user_id=wallet.wallet.user, @@ -1412,7 +1412,7 @@ async def api_get_user_balance(user_id: str) -> UserBalance: from .fava_client import get_fava_client fava = get_fava_client() - balance_data = await fava.get_user_balance(user_id) + balance_data = await fava.get_user_balance_bql(user_id) return UserBalance( user_id=user_id, @@ -1430,7 +1430,7 @@ async def api_get_all_balances( from .fava_client import get_fava_client fava = get_fava_client() - balances = await fava.get_all_user_balances() + balances = await fava.get_all_user_balances_bql() # Enrich with username information using helper function result = [] @@ -1487,7 +1487,7 @@ async def api_generate_payment_invoice( from .fava_client import get_fava_client fava = get_fava_client() - balance_data = await fava.get_user_balance(target_user_id) + balance_data = await fava.get_user_balance_bql(target_user_id) # Build UserBalance object for compatibility user_balance = UserBalance( @@ -1629,7 +1629,7 @@ async def api_record_payment( entry_links = entry.get('links', []) if link_to_find in entry_links: # Payment already recorded, return existing entry - balance_data = await fava.get_user_balance(target_user_id) + balance_data = await fava.get_user_balance_bql(target_user_id) return { "journal_entry_id": f"fava-exists-{data.payment_hash[:16]}", "new_balance": balance_data["balance"], @@ -1689,7 +1689,7 @@ async def api_record_payment( logger.info(f"Payment entry submitted to Fava: {result.get('data', 'Unknown')}") # Get updated balance from Fava - balance_data = await fava.get_user_balance(target_user_id) + balance_data = await fava.get_user_balance_bql(target_user_id) return { "journal_entry_id": f"fava-{datetime.now().timestamp()}", @@ -1743,7 +1743,7 @@ async def api_pay_user( logger.info(f"Payment submitted to Fava: {result.get('data', 'Unknown')}") # Get updated balance from Fava - balance_data = await fava.get_user_balance(user_id) + balance_data = await fava.get_user_balance_bql(user_id) return { "journal_entry_id": f"fava-{datetime.now().timestamp()}", @@ -1887,7 +1887,7 @@ async def api_settle_receivable( logger.info(f"Receivable settlement submitted to Fava: {result.get('data', 'Unknown')}") # Get updated balance from Fava - balance_data = await fava.get_user_balance(data.user_id) + balance_data = await fava.get_user_balance_bql(data.user_id) return { "journal_entry_id": f"fava-{datetime.now().timestamp()}", @@ -2041,7 +2041,7 @@ async def api_pay_user( logger.info(f"Payable payment submitted to Fava: {result.get('data', 'Unknown')}") # Get updated balance from Fava - balance_data = await fava.get_user_balance(data.user_id) + balance_data = await fava.get_user_balance_bql(data.user_id) return { "journal_entry_id": f"fava-{datetime.now().timestamp()}",