aiolabs/castle
1
0
Fork 0
Code Issues 5 Pull requests 2 Projects Releases 5 Packages Wiki Activity Actions

Use BQL sum(weight) for efficient SATS balance queries

Browse source 
Now that the ledger uses @@ SATS price notation, BQL can efficiently
aggregate SATS balances using the weight column instead of manual
entry-by-entry aggregation.

Changes:
- fava_client.py: Update get_user_balance_bql() and get_all_user_balances_bql()
  to use sum(weight) for SATS aggregation (5-10x performance improvement)
- views_api.py: Switch all balance endpoints to use BQL methods

The weight column returns the @@ price value, enabling server-side
filtering and aggregation instead of fetching all entries.

🤖 Generated with [Claude Code](https://claude.com/claude-code)

Co-Authored-By: Claude <noreply@anthropic.com>
This commit is contained in:
padreug 2025-12-14 23:58:56 +01:00
parent dfdcc441a1
commit 7173e051fe
2 changed files with 72 additions and 134 deletions
Download patch file Download diff file Expand all files Collapse all files

184
fava_client.py
View file

@ -557,103 +557,71 @@ class FavaClient:
async def get_user_balance_bql(self, user_id: str) -> Dict[str, Any]: async def get_user_balance_bql(self, user_id: str) -> Dict[str, Any]:
""" """
Get user balance using BQL (efficient, replaces 115-line manual aggregation). Get user balance using BQL with price notation (efficient server-side aggregation).
⚠️ NOT CURRENTLY USED: This method cannot access SATS balances stored in posting Uses sum(weight) to aggregate SATS from @@ price notation.
metadata. It only queries position amounts (EUR/USD). For Castle's current ledger This provides 5-10x performance improvement over manual aggregation.
format, use get_user_balance() instead (manual aggregation with caching).
This method uses Beancount Query Language for server-side filtering and aggregation,
which would provide 5-10x performance improvement IF SATS were stored as position
amounts instead of metadata.
FUTURE CONSIDERATION: If Castle's ledger format changes to store SATS as position
amounts (e.g., "100000 SATS {100.00 EUR}"), this method would become feasible and
provide significant performance benefits.
See: docs/BQL-BALANCE-QUERIES.md for detailed test results and analysis.
Args: Args:
user_id: User ID user_id: User ID
Returns: Returns:
{ {
"balance": int (sats), # Currently returns 0 (cannot access metadata) "balance": int (sats from weight column),
"fiat_balances": {"EUR": Decimal("100.50"), ...}, # Works correctly "fiat_balances": {"EUR": Decimal("100.50"), ...},
"accounts": [{"account": "...", "sats": 150000}, ...] "accounts": [{"account": "...", "sats": 150000, "eur": Decimal("100.50")}, ...]
} }
Example: Example:
balance = await fava.get_user_balance_bql("af983632") balance = await fava.get_user_balance_bql("af983632")
print(f"Balance: {balance['balance']} sats") # Will be 0 with current ledger format print(f"Balance: {balance['balance']} sats")
""" """
from decimal import Decimal from decimal import Decimal
import re
# Build BQL query for this user's Payable/Receivable accounts
user_id_prefix = user_id[:8] user_id_prefix = user_id[:8]
# BQL query using sum(weight) for SATS aggregation
# weight column returns the @@ price value (SATS) from price notation
query = f""" query = f"""
SELECT account, sum(position) as balance SELECT account, sum(number), sum(weight)
WHERE account ~ ':User-{user_id_prefix}' WHERE account ~ ':User-{user_id_prefix}'
AND (account ~ 'Payable' OR account ~ 'Receivable') AND (account ~ 'Payable' OR account ~ 'Receivable')
AND flag != '!' AND flag = '*'
GROUP BY account GROUP BY account
""" """
result = await self.query_bql(query) result = await self.query_bql(query)
# Process results
total_sats = 0 total_sats = 0
fiat_balances = {} fiat_balances = {}
accounts = [] accounts = []
for row in result["rows"]: for row in result["rows"]:
account_name, position = row account_name, fiat_sum, weight_sum = row
# Position can be: # Parse fiat amount (sum of EUR/USD amounts)
# - Dict: {"SATS": "150000", "EUR": "145.50"} fiat_amount = Decimal(str(fiat_sum)) if fiat_sum else Decimal(0)
# - String: "150000 SATS" or "145.50 EUR"
if isinstance(position, dict): # Parse SATS from weight column
# Extract SATS # weight_sum is an Inventory dict like {"SATS": -10442635.00}
sats_str = position.get("SATS", "0") sats_amount = 0
sats_amount = int(sats_str) if sats_str else 0 if isinstance(weight_sum, dict) and "SATS" in weight_sum:
total_sats += sats_amount sats_value = weight_sum["SATS"]
sats_amount = int(Decimal(str(sats_value)))
accounts.append({ total_sats += sats_amount
"account": account_name,
"sats": sats_amount
})
# Extract fiat currencies # Aggregate fiat (assume EUR for now, could be extended)
for currency in ["EUR", "USD", "GBP"]: if fiat_amount != 0:
if currency in position: if "EUR" not in fiat_balances:
fiat_str = position[currency] fiat_balances["EUR"] = Decimal(0)
fiat_amount = Decimal(fiat_str) if fiat_str else Decimal(0) fiat_balances["EUR"] += fiat_amount
if currency not in fiat_balances: accounts.append({
fiat_balances[currency] = Decimal(0) "account": account_name,
fiat_balances[currency] += fiat_amount "sats": sats_amount,
"eur": fiat_amount
elif isinstance(position, str): })
# Single currency (parse "150000 SATS" or "145.50 EUR")
sats_match = re.match(r'^(-?\d+)\s+SATS$', position)
if sats_match:
sats_amount = int(sats_match.group(1))
total_sats += sats_amount
accounts.append({
"account": account_name,
"sats": sats_amount
})
else:
fiat_match = re.match(r'^(-?[\d.]+)\s+([A-Z]{3})$', position)
if fiat_match and fiat_match.group(2) in ('EUR', 'USD', 'GBP'):
fiat_amount = Decimal(fiat_match.group(1))
currency = fiat_match.group(2)
if currency not in fiat_balances:
fiat_balances[currency] = Decimal(0)
fiat_balances[currency] += fiat_amount
logger.info(f"User {user_id[:8]} balance (BQL): {total_sats} sats, fiat: {dict(fiat_balances)}") logger.info(f"User {user_id[:8]} balance (BQL): {total_sats} sats, fiat: {dict(fiat_balances)}")
@ -665,28 +633,18 @@ class FavaClient:
async def get_all_user_balances_bql(self) -> List[Dict[str, Any]]: async def get_all_user_balances_bql(self) -> List[Dict[str, Any]]:
""" """
Get balances for all users using BQL (efficient admin view). Get balances for all users using BQL with price notation (efficient admin view).
⚠️ NOT CURRENTLY USED: This method cannot access SATS balances stored in posting Uses sum(weight) to aggregate SATS from @@ price notation in a single query.
metadata. It only queries position amounts (EUR/USD). For Castle's current ledger This provides significant performance benefits for admin views.
format, use get_all_user_balances() instead (manual aggregation with caching).
This method uses Beancount Query Language to query all user balances
in a single efficient query, which would be faster than fetching all entries IF
SATS were stored as position amounts instead of metadata.
FUTURE CONSIDERATION: If Castle's ledger format changes to store SATS as position
amounts, this method would provide significant performance benefits for admin views.
See: docs/BQL-BALANCE-QUERIES.md for detailed test results and analysis.
Returns: Returns:
[ [
{ {
"user_id": "abc123", "user_id": "abc123",
"balance": 100000, # Currently 0 (cannot access metadata) "balance": 100000,
"fiat_balances": {"EUR": Decimal("100.50")}, # Works correctly "fiat_balances": {"EUR": Decimal("100.50")},
"accounts": [{"account": "...", "sats": 150000}, ...] "accounts": [{"account": "...", "sats": 150000, "eur": Decimal("100.50")}, ...]
}, },
... ...
] ]
@ -694,16 +652,15 @@ class FavaClient:
Example: Example:
all_balances = await fava.get_all_user_balances_bql() all_balances = await fava.get_all_user_balances_bql()
for user in all_balances: for user in all_balances:
print(f"{user['user_id']}: {user['balance']} sats") # Will be 0 with current format print(f"{user['user_id']}: {user['balance']} sats")
""" """
from decimal import Decimal from decimal import Decimal
import re
# BQL query for ALL user accounts # BQL query using sum(weight) for SATS aggregation
query = """ query = """
SELECT account, sum(position) as balance SELECT account, sum(number), sum(weight)
WHERE (account ~ 'Payable:User-' OR account ~ 'Receivable:User-') WHERE (account ~ 'Payable:User-' OR account ~ 'Receivable:User-')
AND flag != '!' AND flag = '*'
GROUP BY account GROUP BY account
""" """
@ -713,15 +670,13 @@ class FavaClient:
user_data = {} user_data = {}
for row in result["rows"]: for row in result["rows"]:
account_name, position = row account_name, fiat_sum, weight_sum = row
# Extract user_id from account name # Extract user_id from account name
# Format: "Liabilities:Payable:User-abc12345" or "Assets:Receivable:User-abc12345"
if ":User-" not in account_name: if ":User-" not in account_name:
continue continue
user_id_with_prefix = account_name.split(":User-")[1] user_id_with_prefix = account_name.split(":User-")[1]
# User ID is the first 8 chars (our standard)
user_id = user_id_with_prefix[:8] user_id = user_id_with_prefix[:8]
if user_id not in user_data: if user_id not in user_data:
@ -732,45 +687,28 @@ class FavaClient:
"accounts": [] "accounts": []
} }
# Process position (same logic as single-user query) # Parse fiat amount
if isinstance(position, dict): fiat_amount = Decimal(str(fiat_sum)) if fiat_sum else Decimal(0)
sats_str = position.get("SATS", "0")
sats_amount = int(sats_str) if sats_str else 0
user_data[user_id]["balance"] += sats_amount
user_data[user_id]["accounts"].append({ # Parse SATS from weight column
"account": account_name, sats_amount = 0
"sats": sats_amount if isinstance(weight_sum, dict) and "SATS" in weight_sum:
}) sats_value = weight_sum["SATS"]
sats_amount = int(Decimal(str(sats_value)))
for currency in ["EUR", "USD", "GBP"]: user_data[user_id]["balance"] += sats_amount
if currency in position:
fiat_str = position[currency]
fiat_amount = Decimal(fiat_str) if fiat_str else Decimal(0)
if currency not in user_data[user_id]["fiat_balances"]: # Aggregate fiat
user_data[user_id]["fiat_balances"][currency] = Decimal(0) if fiat_amount != 0:
user_data[user_id]["fiat_balances"][currency] += fiat_amount if "EUR" not in user_data[user_id]["fiat_balances"]:
user_data[user_id]["fiat_balances"]["EUR"] = Decimal(0)
user_data[user_id]["fiat_balances"]["EUR"] += fiat_amount
elif isinstance(position, str): user_data[user_id]["accounts"].append({
# Single currency (parse "150000 SATS" or "145.50 EUR") "account": account_name,
sats_match = re.match(r'^(-?\d+)\s+SATS$', position) "sats": sats_amount,
if sats_match: "eur": fiat_amount
sats_amount = int(sats_match.group(1)) })
user_data[user_id]["balance"] += sats_amount
user_data[user_id]["accounts"].append({
"account": account_name,
"sats": sats_amount
})
else:
fiat_match = re.match(r'^(-?[\d.]+)\s+([A-Z]{3})$', position)
if fiat_match and fiat_match.group(2) in ('EUR', 'USD', 'GBP'):
fiat_amount = Decimal(fiat_match.group(1))
currency = fiat_match.group(2)
if currency not in user_data[user_id]["fiat_balances"]:
user_data[user_id]["fiat_balances"][currency] = Decimal(0)
user_data[user_id]["fiat_balances"][currency] += fiat_amount
logger.info(f"Fetched balances for {len(user_data)} users (BQL)") logger.info(f"Fetched balances for {len(user_data)} users (BQL)")

22
views_api.py
View file

@ -1367,10 +1367,10 @@ async def api_get_my_balance(
# If super user, show total castle position # If super user, show total castle position
if wallet.wallet.user == lnbits_settings.super_user: if wallet.wallet.user == lnbits_settings.super_user:
all_balances = await fava.get_all_user_balances() all_balances = await fava.get_all_user_balances_bql()
# Calculate total: # Calculate total:
# From get_user_balance(): positive = user owes castle, negative = castle owes user # From get_user_balance_bql(): positive = user owes castle, negative = castle owes user
# Positive balances = Users owe Castle (receivables for Castle) # Positive balances = Users owe Castle (receivables for Castle)
# Negative balances = Castle owes users (liabilities for Castle) # Negative balances = Castle owes users (liabilities for Castle)
# Net: positive means castle is owed money, negative means castle owes money # Net: positive means castle is owed money, negative means castle owes money
@ -1396,7 +1396,7 @@ async def api_get_my_balance(
) )
# For regular users, show their individual balance from Fava # For regular users, show their individual balance from Fava
balance_data = await fava.get_user_balance(wallet.wallet.user) balance_data = await fava.get_user_balance_bql(wallet.wallet.user)
return UserBalance( return UserBalance(
user_id=wallet.wallet.user, user_id=wallet.wallet.user,
@ -1412,7 +1412,7 @@ async def api_get_user_balance(user_id: str) -> UserBalance:
from .fava_client import get_fava_client from .fava_client import get_fava_client
fava = get_fava_client() fava = get_fava_client()
balance_data = await fava.get_user_balance(user_id) balance_data = await fava.get_user_balance_bql(user_id)
return UserBalance( return UserBalance(
user_id=user_id, user_id=user_id,
@ -1430,7 +1430,7 @@ async def api_get_all_balances(
from .fava_client import get_fava_client from .fava_client import get_fava_client
fava = get_fava_client() fava = get_fava_client()
balances = await fava.get_all_user_balances() balances = await fava.get_all_user_balances_bql()
# Enrich with username information using helper function # Enrich with username information using helper function
result = [] result = []
@ -1487,7 +1487,7 @@ async def api_generate_payment_invoice(
from .fava_client import get_fava_client from .fava_client import get_fava_client
fava = get_fava_client() fava = get_fava_client()
balance_data = await fava.get_user_balance(target_user_id) balance_data = await fava.get_user_balance_bql(target_user_id)
# Build UserBalance object for compatibility # Build UserBalance object for compatibility
user_balance = UserBalance( user_balance = UserBalance(
@ -1629,7 +1629,7 @@ async def api_record_payment(
entry_links = entry.get('links', []) entry_links = entry.get('links', [])
if link_to_find in entry_links: if link_to_find in entry_links:
# Payment already recorded, return existing entry # Payment already recorded, return existing entry
balance_data = await fava.get_user_balance(target_user_id) balance_data = await fava.get_user_balance_bql(target_user_id)
return { return {
"journal_entry_id": f"fava-exists-{data.payment_hash[:16]}", "journal_entry_id": f"fava-exists-{data.payment_hash[:16]}",
"new_balance": balance_data["balance"], "new_balance": balance_data["balance"],
@ -1689,7 +1689,7 @@ async def api_record_payment(
logger.info(f"Payment entry submitted to Fava: {result.get('data', 'Unknown')}") logger.info(f"Payment entry submitted to Fava: {result.get('data', 'Unknown')}")
# Get updated balance from Fava # Get updated balance from Fava
balance_data = await fava.get_user_balance(target_user_id) balance_data = await fava.get_user_balance_bql(target_user_id)
return { return {
"journal_entry_id": f"fava-{datetime.now().timestamp()}", "journal_entry_id": f"fava-{datetime.now().timestamp()}",
@ -1743,7 +1743,7 @@ async def api_pay_user(
logger.info(f"Payment submitted to Fava: {result.get('data', 'Unknown')}") logger.info(f"Payment submitted to Fava: {result.get('data', 'Unknown')}")
# Get updated balance from Fava # Get updated balance from Fava
balance_data = await fava.get_user_balance(user_id) balance_data = await fava.get_user_balance_bql(user_id)
return { return {
"journal_entry_id": f"fava-{datetime.now().timestamp()}", "journal_entry_id": f"fava-{datetime.now().timestamp()}",
@ -1887,7 +1887,7 @@ async def api_settle_receivable(
logger.info(f"Receivable settlement submitted to Fava: {result.get('data', 'Unknown')}") logger.info(f"Receivable settlement submitted to Fava: {result.get('data', 'Unknown')}")
# Get updated balance from Fava # Get updated balance from Fava
balance_data = await fava.get_user_balance(data.user_id) balance_data = await fava.get_user_balance_bql(data.user_id)
return { return {
"journal_entry_id": f"fava-{datetime.now().timestamp()}", "journal_entry_id": f"fava-{datetime.now().timestamp()}",
@ -2041,7 +2041,7 @@ async def api_pay_user(
logger.info(f"Payable payment submitted to Fava: {result.get('data', 'Unknown')}") logger.info(f"Payable payment submitted to Fava: {result.get('data', 'Unknown')}")
# Get updated balance from Fava # Get updated balance from Fava
balance_data = await fava.get_user_balance(data.user_id) balance_data = await fava.get_user_balance_bql(data.user_id)
return { return {
"journal_entry_id": f"fava-{datetime.now().timestamp()}", "journal_entry_id": f"fava-{datetime.now().timestamp()}",