From 6e86f53962b6830651d9e16fcbc2b21a02a78051 Mon Sep 17 00:00:00 2001 From: padreug Date: Sun, 11 Jan 2026 14:47:56 +0100 Subject: [PATCH 1/8] refactor: use Decimal instead of float for monetary calculations MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - calculations.py: Use Decimal for commission percentages, exchange rates, and client balances. Added to_decimal() helper for safe float conversion. Changed from banker's rounding to ROUND_HALF_UP. - models.py: Changed all fiat amounts, percentages, and exchange rates to Decimal. Added json_encoders for API serialization. - transaction_processor.py: Convert to Decimal at data ingestion boundary (CSV parsing). Updated all defaults and calculations to use Decimal. - tests: Updated to work with Decimal return types. This prevents floating-point precision issues in financial calculations. All 23 tests pass. 🤖 Generated with [Claude Code](https://claude.ai/code) Co-Authored-By: Claude --- calculations.py | 89 ++++++++++++++++++++++++++----------- models.py | 91 ++++++++++++++++++++++++-------------- tests/test_calculations.py | 18 ++++---- transaction_processor.py | 83 ++++++++++++++++++++-------------- 4 files changed, 180 insertions(+), 101 deletions(-) diff --git a/calculations.py b/calculations.py index a7b3aa9..c16db1e 100644 --- a/calculations.py +++ b/calculations.py @@ -3,16 +3,34 @@ Pure calculation functions for DCA transaction processing. These functions have no external dependencies (no lnbits, no database) and can be easily tested in isolation. + +All monetary calculations use Decimal for precision. Satoshi amounts +remain as int since they are the smallest indivisible unit. """ -from typing import Dict, Tuple +from decimal import Decimal, ROUND_HALF_UP +from typing import Dict, Tuple, Union + +# Type alias for values that can be Decimal or numeric types that will be converted +DecimalLike = Union[Decimal, float, int, str] + + +def to_decimal(value: DecimalLike) -> Decimal: + """Convert a value to Decimal, handling floats carefully.""" + if isinstance(value, Decimal): + return value + # Convert floats via string to avoid binary float precision issues + # e.g., Decimal(0.055) gives 0.054999999... but Decimal("0.055") is exact + if isinstance(value, float): + return Decimal(str(value)) + return Decimal(value) def calculate_commission( crypto_atoms: int, - commission_percentage: float, - discount: float = 0.0 -) -> Tuple[int, int, float]: + commission_percentage: DecimalLike, + discount: DecimalLike = Decimal("0") +) -> Tuple[int, int, Decimal]: """ Calculate commission split from a Lamassu transaction. @@ -34,15 +52,25 @@ def calculate_commission( Tuple of (base_amount_sats, commission_amount_sats, effective_commission_rate) Example: - >>> calculate_commission(266800, 0.03, 0.0) - (259029, 7771, 0.03) + >>> calculate_commission(266800, Decimal("0.03"), Decimal("0")) + (259029, 7771, Decimal('0.03')) """ - if commission_percentage > 0: - effective_commission = commission_percentage * (100 - discount) / 100 - base_crypto_atoms = round(crypto_atoms / (1 + effective_commission)) + # Convert inputs to Decimal for precise calculations + comm_pct = to_decimal(commission_percentage) + disc = to_decimal(discount) + + if comm_pct > 0: + # effective = commission_percentage * (100 - discount) / 100 + effective_commission = comm_pct * (Decimal("100") - disc) / Decimal("100") + + # base = crypto_atoms / (1 + effective_commission), rounded to nearest int + divisor = Decimal("1") + effective_commission + exact_base = Decimal(crypto_atoms) / divisor + # Use ROUND_HALF_UP (standard rounding: 0.5 rounds up) + base_crypto_atoms = int(exact_base.quantize(Decimal("1"), rounding=ROUND_HALF_UP)) commission_amount_sats = crypto_atoms - base_crypto_atoms else: - effective_commission = 0.0 + effective_commission = Decimal("0") base_crypto_atoms = crypto_atoms commission_amount_sats = 0 @@ -51,8 +79,8 @@ def calculate_commission( def calculate_distribution( base_amount_sats: int, - client_balances: Dict[str, float], - min_balance_threshold: float = 0.01 + client_balances: Dict[str, DecimalLike], + min_balance_threshold: DecimalLike = Decimal("0.01") ) -> Dict[str, int]: """ Calculate proportional distribution of sats to clients based on their fiat balances. @@ -69,15 +97,18 @@ def calculate_distribution( Dict of {client_id: allocated_sats} Example: - >>> calculate_distribution(100000, {"a": 500.0, "b": 500.0}) + >>> calculate_distribution(100000, {"a": Decimal("500"), "b": Decimal("500")}) {"a": 50000, "b": 50000} """ - # Filter out clients with balance below threshold - active_balances = { - client_id: balance - for client_id, balance in client_balances.items() - if balance >= min_balance_threshold - } + # Convert threshold to Decimal + threshold = to_decimal(min_balance_threshold) + + # Filter out clients with balance below threshold, converting to Decimal + active_balances: Dict[str, Decimal] = {} + for client_id, balance in client_balances.items(): + bal = to_decimal(balance) + if bal >= threshold: + active_balances[client_id] = bal if not active_balances: return {} @@ -90,11 +121,13 @@ def calculate_distribution( # First pass: calculate base allocations and track for remainder distribution client_calculations = [] distributed_sats = 0 + base_sats_decimal = Decimal(base_amount_sats) for client_id, balance in active_balances.items(): proportion = balance / total_balance - exact_share = base_amount_sats * proportion - allocated_sats = round(exact_share) + exact_share = base_sats_decimal * proportion + # Round to nearest integer using ROUND_HALF_UP + allocated_sats = int(exact_share.quantize(Decimal("1"), rounding=ROUND_HALF_UP)) client_calculations.append({ 'client_id': client_id, @@ -109,8 +142,9 @@ def calculate_distribution( if remainder != 0: # Sort by largest fractional remainder to distribute fairly + # The fractional part is exact_share - allocated_sats client_calculations.sort( - key=lambda x: x['exact_share'] - x['allocated_sats'], + key=lambda x: x['exact_share'] - Decimal(x['allocated_sats']), reverse=True ) @@ -131,7 +165,7 @@ def calculate_distribution( return distributions -def calculate_exchange_rate(base_crypto_atoms: int, fiat_amount: float) -> float: +def calculate_exchange_rate(base_crypto_atoms: int, fiat_amount: DecimalLike) -> Decimal: """ Calculate exchange rate in sats per fiat unit. @@ -140,8 +174,9 @@ def calculate_exchange_rate(base_crypto_atoms: int, fiat_amount: float) -> float fiat_amount: Fiat amount dispensed Returns: - Exchange rate as sats per fiat unit + Exchange rate as sats per fiat unit (Decimal for precision) """ - if fiat_amount <= 0: - return 0.0 - return base_crypto_atoms / fiat_amount + fiat = to_decimal(fiat_amount) + if fiat <= 0: + return Decimal("0") + return Decimal(base_crypto_atoms) / fiat diff --git a/models.py b/models.py index e4bf64d..e7274c1 100644 --- a/models.py +++ b/models.py @@ -1,6 +1,7 @@ # Description: Pydantic data models dictate what is passed between frontend and backend. from datetime import datetime +from decimal import Decimal from typing import Optional from pydantic import BaseModel, validator @@ -12,7 +13,7 @@ class CreateDcaClientData(BaseModel): wallet_id: str username: str dca_mode: str = "flow" # 'flow' or 'fixed' - fixed_mode_daily_limit: Optional[float] = None + fixed_mode_daily_limit: Optional[Decimal] = None class DcaClient(BaseModel): @@ -21,44 +22,52 @@ class DcaClient(BaseModel): wallet_id: str username: Optional[str] dca_mode: str - fixed_mode_daily_limit: Optional[int] + fixed_mode_daily_limit: Optional[Decimal] status: str created_at: datetime updated_at: datetime + class Config: + json_encoders = {Decimal: lambda v: float(v)} + class UpdateDcaClientData(BaseModel): username: Optional[str] = None dca_mode: Optional[str] = None - fixed_mode_daily_limit: Optional[float] = None + fixed_mode_daily_limit: Optional[Decimal] = None status: Optional[str] = None # Deposit Models (Now storing GTQ directly) class CreateDepositData(BaseModel): client_id: str - amount: float # Amount in GTQ (e.g., 150.75) + amount: Decimal # Amount in GTQ (e.g., 150.75) currency: str = "GTQ" notes: Optional[str] = None - - @validator('amount') + + @validator('amount', pre=True) def round_amount_to_cents(cls, v): """Ensure amount is rounded to 2 decimal places for DECIMAL(10,2) storage""" if v is not None: - return round(float(v), 2) + # Convert to Decimal via string to avoid float precision issues + d = Decimal(str(v)) if not isinstance(v, Decimal) else v + return d.quantize(Decimal("0.01")) return v class DcaDeposit(BaseModel): id: str client_id: str - amount: float # Amount in GTQ (e.g., 150.75) + amount: Decimal # Amount in GTQ (e.g., 150.75) currency: str status: str # 'pending' or 'confirmed' notes: Optional[str] created_at: datetime confirmed_at: Optional[datetime] + class Config: + json_encoders = {Decimal: lambda v: float(v)} + class UpdateDepositStatusData(BaseModel): status: str @@ -69,8 +78,8 @@ class UpdateDepositStatusData(BaseModel): class CreateDcaPaymentData(BaseModel): client_id: str amount_sats: int - amount_fiat: float # Amount in GTQ (e.g., 150.75) - exchange_rate: float + amount_fiat: Decimal # Amount in GTQ (e.g., 150.75) + exchange_rate: Decimal transaction_type: str # 'flow', 'fixed', 'manual', 'commission' lamassu_transaction_id: Optional[str] = None payment_hash: Optional[str] = None @@ -81,8 +90,8 @@ class DcaPayment(BaseModel): id: str client_id: str amount_sats: int - amount_fiat: float # Amount in GTQ (e.g., 150.75) - exchange_rate: float + amount_fiat: Decimal # Amount in GTQ (e.g., 150.75) + exchange_rate: Decimal transaction_type: str lamassu_transaction_id: Optional[str] payment_hash: Optional[str] @@ -90,38 +99,47 @@ class DcaPayment(BaseModel): created_at: datetime transaction_time: Optional[datetime] = None # Original ATM transaction time + class Config: + json_encoders = {Decimal: lambda v: float(v)} + # Client Balance Summary (Now storing GTQ directly) class ClientBalanceSummary(BaseModel): client_id: str - total_deposits: float # Total confirmed deposits in GTQ - total_payments: float # Total payments made in GTQ - remaining_balance: float # Available balance for DCA in GTQ + total_deposits: Decimal # Total confirmed deposits in GTQ + total_payments: Decimal # Total payments made in GTQ + remaining_balance: Decimal # Available balance for DCA in GTQ currency: str + class Config: + json_encoders = {Decimal: lambda v: float(v)} + # Transaction Processing Models class LamassuTransaction(BaseModel): transaction_id: str - amount_fiat: float # Amount in GTQ (e.g., 150.75) + amount_fiat: Decimal # Amount in GTQ (e.g., 150.75) amount_crypto: int - exchange_rate: float + exchange_rate: Decimal transaction_type: str # 'cash_in' or 'cash_out' status: str timestamp: datetime + class Config: + json_encoders = {Decimal: lambda v: float(v)} + # Lamassu Transaction Storage Models class CreateLamassuTransactionData(BaseModel): lamassu_transaction_id: str - fiat_amount: float # Amount in GTQ (e.g., 150.75) + fiat_amount: Decimal # Amount in GTQ (e.g., 150.75) crypto_amount: int - commission_percentage: float - discount: float = 0.0 - effective_commission: float + commission_percentage: Decimal + discount: Decimal = Decimal("0") + effective_commission: Decimal commission_amount_sats: int base_amount_sats: int - exchange_rate: float + exchange_rate: Decimal crypto_code: str = "BTC" fiat_code: str = "GTQ" device_id: Optional[str] = None @@ -131,14 +149,14 @@ class CreateLamassuTransactionData(BaseModel): class StoredLamassuTransaction(BaseModel): id: str lamassu_transaction_id: str - fiat_amount: float # Amount in GTQ (e.g., 150.75) + fiat_amount: Decimal # Amount in GTQ (e.g., 150.75) crypto_amount: int - commission_percentage: float - discount: float - effective_commission: float + commission_percentage: Decimal + discount: Decimal + effective_commission: Decimal commission_amount_sats: int base_amount_sats: int - exchange_rate: float + exchange_rate: Decimal crypto_code: str fiat_code: str device_id: Optional[str] @@ -147,6 +165,9 @@ class StoredLamassuTransaction(BaseModel): clients_count: int # Number of clients who received distributions distributions_total_sats: int # Total sats distributed to clients + class Config: + json_encoders = {Decimal: lambda v: float(v)} + # Lamassu Configuration Models class CreateLamassuConfigData(BaseModel): @@ -167,13 +188,14 @@ class CreateLamassuConfigData(BaseModel): ssh_password: Optional[str] = None ssh_private_key: Optional[str] = None # Path to private key file or key content # DCA Client Limits - max_daily_limit_gtq: float = 2000.0 # Maximum daily limit for Fixed mode clients - - @validator('max_daily_limit_gtq') + max_daily_limit_gtq: Decimal = Decimal("2000") # Maximum daily limit for Fixed mode clients + + @validator('max_daily_limit_gtq', pre=True) def round_max_daily_limit(cls, v): """Ensure max daily limit is rounded to 2 decimal places""" if v is not None: - return round(float(v), 2) + d = Decimal(str(v)) if not isinstance(v, Decimal) else v + return d.quantize(Decimal("0.01")) return v @@ -204,7 +226,10 @@ class LamassuConfig(BaseModel): last_poll_time: Optional[datetime] = None last_successful_poll: Optional[datetime] = None # DCA Client Limits - max_daily_limit_gtq: float = 2000.0 # Maximum daily limit for Fixed mode clients + max_daily_limit_gtq: Decimal = Decimal("2000") # Maximum daily limit for Fixed mode clients + + class Config: + json_encoders = {Decimal: lambda v: float(v)} class UpdateLamassuConfigData(BaseModel): @@ -226,6 +251,6 @@ class UpdateLamassuConfigData(BaseModel): ssh_password: Optional[str] = None ssh_private_key: Optional[str] = None # DCA Client Limits - max_daily_limit_gtq: Optional[int] = None + max_daily_limit_gtq: Optional[Decimal] = None diff --git a/tests/test_calculations.py b/tests/test_calculations.py index 04262a0..b175421 100644 --- a/tests/test_calculations.py +++ b/tests/test_calculations.py @@ -10,7 +10,7 @@ from decimal import Decimal from typing import Dict, List, Tuple # Import from the parent package (following lnurlp pattern) -from ..calculations import calculate_commission, calculate_distribution, calculate_exchange_rate +from ..calculations import calculate_commission, calculate_distribution, calculate_exchange_rate, to_decimal # ============================================================================= @@ -245,11 +245,12 @@ class TestDistributionCalculation: # Convert each client's sats back to fiat total_fiat_distributed = sum( - sats / exchange_rate for sats in distributions.values() + Decimal(sats) / exchange_rate for sats in distributions.values() ) # Should equal original fiat amount (within small rounding tolerance) - assert abs(total_fiat_distributed - fiat_amount) < 0.01, \ + fiat_decimal = to_decimal(fiat_amount) + assert abs(total_fiat_distributed - fiat_decimal) < Decimal("0.01"), \ f"Fiat round-trip failed: {total_fiat_distributed:.2f} != {fiat_amount:.2f} " \ f"(crypto={crypto_atoms}, comm={comm_pct}, discount={discount})" @@ -287,11 +288,12 @@ class TestEmpiricalTransactions: "expected_base_sats": 259029, "expected_commission_sats": 7771, "expected_distributions": { - # 259029 / 2 = 129514.5 → both get 129514 or 129515 - # With banker's rounding: 129514.5 → 129514 (even) - # Remainder of 1 sat goes to first client by fractional sort - "client_a": 129515, - "client_b": 129514, + # 259029 / 2 = 129514.5 → both round to 129515 (ROUND_HALF_UP) + # Total = 259030, remainder = -1 + # Both have same fractional (-0.5), client_a is first alphabetically + # So client_a gets -1 adjustment + "client_a": 129514, + "client_b": 129515, }, }, # Add more scenarios from your real data! diff --git a/transaction_processor.py b/transaction_processor.py index 45e0880..1f511df 100644 --- a/transaction_processor.py +++ b/transaction_processor.py @@ -3,6 +3,7 @@ import asyncio import asyncpg from datetime import datetime, timedelta, timezone +from decimal import Decimal from typing import List, Optional, Dict, Any from loguru import logger import socket @@ -492,28 +493,38 @@ class LamassuTransactionProcessor: results = [] for row in reader: # Convert string values to appropriate types + # Use Decimal for monetary and percentage values processed_row = {} for key, value in row.items(): # Handle None/empty values consistently at data ingestion boundary if value == '' or value is None: - if key in ['fiat_amount', 'crypto_amount']: - processed_row[key] = 0 # Default numeric fields to 0 + if key == 'crypto_amount': + processed_row[key] = 0 # Sats are always int + elif key == 'fiat_amount': + processed_row[key] = Decimal("0") # Fiat as Decimal elif key in ['commission_percentage', 'discount']: - processed_row[key] = 0.0 # Default percentage fields to 0.0 + processed_row[key] = Decimal("0") # Percentages as Decimal else: processed_row[key] = None # Keep None for non-numeric fields elif key in ['transaction_id', 'device_id', 'crypto_code', 'fiat_code']: processed_row[key] = str(value) - elif key in ['fiat_amount', 'crypto_amount']: + elif key == 'crypto_amount': try: - processed_row[key] = int(float(value)) + processed_row[key] = int(float(value)) # Sats are always int except (ValueError, TypeError): - processed_row[key] = 0 # Fallback to 0 for invalid values + processed_row[key] = 0 + elif key == 'fiat_amount': + try: + # Convert via string to avoid float precision issues + processed_row[key] = Decimal(str(value)) + except (ValueError, TypeError): + processed_row[key] = Decimal("0") elif key in ['commission_percentage', 'discount']: try: - processed_row[key] = float(value) + # Convert via string to avoid float precision issues + processed_row[key] = Decimal(str(value)) except (ValueError, TypeError): - processed_row[key] = 0.0 # Fallback to 0.0 for invalid values + processed_row[key] = Decimal("0") elif key == 'transaction_time': from datetime import datetime # Parse PostgreSQL timestamp format and ensure it's in UTC for consistency @@ -679,13 +690,13 @@ class LamassuTransactionProcessor: logger.info("No Flow Mode clients found - skipping distribution") return {} - # Extract transaction details - guaranteed clean from data ingestion + # Extract transaction details - guaranteed clean from data ingestion (Decimal types) crypto_atoms = transaction.get("crypto_amount", 0) # Total sats with commission baked in - fiat_amount = transaction.get("fiat_amount", 0) # Actual fiat dispensed (principal only) - commission_percentage = transaction.get("commission_percentage", 0.0) # Already stored as decimal (e.g., 0.045) - discount = transaction.get("discount", 0.0) # Discount percentage + fiat_amount = transaction.get("fiat_amount", Decimal("0")) # Actual fiat dispensed (principal only) + commission_percentage = transaction.get("commission_percentage", Decimal("0")) # Already stored as Decimal (e.g., 0.045) + discount = transaction.get("discount", Decimal("0")) # Discount percentage transaction_time = transaction.get("transaction_time") # ATM transaction timestamp for temporal accuracy - + # Normalize transaction_time to UTC if present if transaction_time is not None: if transaction_time.tzinfo is None: @@ -697,7 +708,7 @@ class LamassuTransactionProcessor: original_tz = transaction_time.tzinfo transaction_time = transaction_time.astimezone(timezone.utc) logger.info(f"Converted transaction time from {original_tz} to UTC") - + # Validate required fields if crypto_atoms is None: logger.error(f"Missing crypto_amount in transaction: {transaction}") @@ -707,10 +718,10 @@ class LamassuTransactionProcessor: return {} if commission_percentage is None: logger.warning(f"Missing commission_percentage in transaction: {transaction}, defaulting to 0") - commission_percentage = 0.0 + commission_percentage = Decimal("0") if discount is None: logger.info(f"Missing discount in transaction: {transaction}, defaulting to 0") - discount = 0.0 + discount = Decimal("0") if transaction_time is None: logger.warning(f"Missing transaction_time in transaction: {transaction}") # Could use current time as fallback, but this indicates a data issue @@ -733,15 +744,16 @@ class LamassuTransactionProcessor: logger.warning("No transaction time available - using current balances (may be inaccurate)") # Get balance summaries for all clients to calculate proportions - client_balances = {} - total_confirmed_deposits = 0 - + client_balances: Dict[str, Decimal] = {} + total_confirmed_deposits = Decimal("0") + min_balance = Decimal("0.01") + for client in flow_clients: # Get balance as of the transaction time for temporal accuracy balance = await get_client_balance_summary(client.id, as_of_time=transaction_time) # Only include clients with positive remaining balance # NOTE: This works for fiat amounts that use cents - if balance.remaining_balance >= 0.01: + if balance.remaining_balance >= min_balance: client_balances[client.id] = balance.remaining_balance total_confirmed_deposits += balance.remaining_balance logger.debug(f"Client {client.id[:8]}... included with balance: {balance.remaining_balance:.2f} GTQ") @@ -766,7 +778,10 @@ class LamassuTransactionProcessor: proportion = client_balances[client_id] / total_confirmed_deposits # Calculate equivalent fiat value in GTQ for tracking purposes - client_fiat_amount = round(client_sats_amount / exchange_rate, 2) if exchange_rate > 0 else 0.0 + if exchange_rate > 0: + client_fiat_amount = (Decimal(client_sats_amount) / exchange_rate).quantize(Decimal("0.01")) + else: + client_fiat_amount = Decimal("0") distributions[client_id] = { "fiat_amount": client_fiat_amount, @@ -1003,20 +1018,20 @@ class LamassuTransactionProcessor: async def store_lamassu_transaction(self, transaction: Dict[str, Any]) -> Optional[str]: """Store the Lamassu transaction in our database for audit and UI""" try: - # Extract transaction data - guaranteed clean from data ingestion boundary + # Extract transaction data - guaranteed clean from data ingestion boundary (Decimal types) crypto_atoms = transaction.get("crypto_amount", 0) - fiat_amount = transaction.get("fiat_amount", 0) - commission_percentage = transaction.get("commission_percentage", 0.0) - discount = transaction.get("discount", 0.0) + fiat_amount = transaction.get("fiat_amount", Decimal("0")) + commission_percentage = transaction.get("commission_percentage", Decimal("0")) + discount = transaction.get("discount", Decimal("0")) transaction_time = transaction.get("transaction_time") - + # Normalize transaction_time to UTC if present if transaction_time is not None: if transaction_time.tzinfo is None: transaction_time = transaction_time.replace(tzinfo=timezone.utc) elif transaction_time.tzinfo != timezone.utc: transaction_time = transaction_time.astimezone(timezone.utc) - + # Calculate commission metrics using the extracted pure function base_crypto_atoms, commission_amount_sats, effective_commission = calculate_commission( crypto_atoms, commission_percentage, discount @@ -1024,11 +1039,13 @@ class LamassuTransactionProcessor: # Calculate exchange rate exchange_rate = calculate_exchange_rate(base_crypto_atoms, fiat_amount) - - # Create transaction data with GTQ amounts + + # Create transaction data with GTQ amounts (Decimal already, just ensure 2 decimal places) + fiat_amount_rounded = fiat_amount.quantize(Decimal("0.01")) if isinstance(fiat_amount, Decimal) else Decimal(str(fiat_amount)).quantize(Decimal("0.01")) + transaction_data = CreateLamassuTransactionData( lamassu_transaction_id=transaction["transaction_id"], - fiat_amount=round(fiat_amount, 2), # Store GTQ with 2 decimal places + fiat_amount=fiat_amount_rounded, crypto_amount=crypto_atoms, commission_percentage=commission_percentage, discount=discount, @@ -1148,9 +1165,9 @@ class LamassuTransactionProcessor: # Calculate commission amount for sending to commission wallet crypto_atoms = transaction.get("crypto_amount", 0) - commission_percentage = transaction.get("commission_percentage", 0.0) - discount = transaction.get("discount", 0.0) - + commission_percentage = transaction.get("commission_percentage", Decimal("0")) + discount = transaction.get("discount", Decimal("0")) + # Calculate commission amount using the extracted pure function _, commission_amount_sats, _ = calculate_commission( crypto_atoms, commission_percentage, discount From 49dd4d1844a926741231ed888dd2ee3d225ad376 Mon Sep 17 00:00:00 2001 From: padreug Date: Sun, 11 Jan 2026 14:51:23 +0100 Subject: [PATCH 2/8] test: add integration tests for CSV parsing and full distribution flow MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Tests verify the complete data flow: - CSV parsing → Decimal conversion (simulates execute_ssh_query) - Commission calculations with parsed Decimal values - Distribution calculations with 2 and 4 client scenarios - Pydantic model creation with Decimal types - Exchange rate precision and round-trip accuracy Uses real Lamassu transaction data (8.75%, 5.5% commission rates, discounts, multiple client configurations). 37 tests now pass (23 unit + 14 integration). 🤖 Generated with [Claude Code](https://claude.ai/code) Co-Authored-By: Claude --- tests/test_integration.py | 379 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 379 insertions(+) create mode 100644 tests/test_integration.py diff --git a/tests/test_integration.py b/tests/test_integration.py new file mode 100644 index 0000000..83cd1f2 --- /dev/null +++ b/tests/test_integration.py @@ -0,0 +1,379 @@ +""" +Integration tests for the full transaction processing flow. + +These tests verify that data flows correctly from: + CSV parsing → Decimal conversion → calculations → model creation + +This gives us confidence that real Lamassu transactions will be +processed correctly end-to-end. +""" + +import pytest +from decimal import Decimal +from datetime import datetime, timezone +from unittest.mock import AsyncMock, MagicMock, patch +from typing import Dict, Any +import csv +import io + +from ..calculations import calculate_commission, calculate_distribution, calculate_exchange_rate +from ..models import ( + CreateDcaPaymentData, + CreateLamassuTransactionData, + ClientBalanceSummary, +) + + +# ============================================================================= +# TEST DATA: Real Lamassu CSV output format +# ============================================================================= + +# This simulates what execute_ssh_query receives from the database +LAMASSU_CSV_DATA = { + "8.75pct_large": """transaction_id,fiat_amount,crypto_amount,transaction_time,device_id,status,commission_percentage,discount,crypto_code,fiat_code +abc123,2000,309200,2025-01-10 14:30:00+00,device1,confirmed,0.0875,0,BTC,GTQ""", + + "5.5pct_no_discount": """transaction_id,fiat_amount,crypto_amount,transaction_time,device_id,status,commission_percentage,discount,crypto_code,fiat_code +def456,2000,309500,2025-01-10 15:00:00+00,device1,confirmed,0.055,0,BTC,GTQ""", + + "5.5pct_90pct_discount": """transaction_id,fiat_amount,crypto_amount,transaction_time,device_id,status,commission_percentage,discount,crypto_code,fiat_code +ghi789,800,115000,2025-01-10 16:00:00+00,device1,confirmed,0.055,90,BTC,GTQ""", + + "5.5pct_1300gtq_4clients": """transaction_id,fiat_amount,crypto_amount,transaction_time,device_id,status,commission_percentage,discount,crypto_code,fiat_code +jkl012,1300,205600,2025-01-10 17:00:00+00,device1,confirmed,0.055,0,BTC,GTQ""", +} + + +def parse_csv_like_transaction_processor(csv_data: str) -> Dict[str, Any]: + """ + Parse CSV data exactly like transaction_processor.execute_ssh_query does. + + This is a copy of the parsing logic to test it in isolation. + """ + reader = csv.DictReader(io.StringIO(csv_data)) + results = [] + for row in reader: + processed_row = {} + for key, value in row.items(): + if value == '' or value is None: + if key == 'crypto_amount': + processed_row[key] = 0 + elif key == 'fiat_amount': + processed_row[key] = Decimal("0") + elif key in ['commission_percentage', 'discount']: + processed_row[key] = Decimal("0") + else: + processed_row[key] = None + elif key in ['transaction_id', 'device_id', 'crypto_code', 'fiat_code', 'status']: + processed_row[key] = str(value) + elif key == 'crypto_amount': + try: + processed_row[key] = int(float(value)) + except (ValueError, TypeError): + processed_row[key] = 0 + elif key == 'fiat_amount': + try: + processed_row[key] = Decimal(str(value)) + except (ValueError, TypeError): + processed_row[key] = Decimal("0") + elif key in ['commission_percentage', 'discount']: + try: + processed_row[key] = Decimal(str(value)) + except (ValueError, TypeError): + processed_row[key] = Decimal("0") + elif key == 'transaction_time': + timestamp_str = value + if timestamp_str.endswith('+00'): + timestamp_str = timestamp_str + ':00' + elif timestamp_str.endswith('Z'): + timestamp_str = timestamp_str.replace('Z', '+00:00') + try: + dt = datetime.fromisoformat(timestamp_str) + except ValueError: + dt = datetime.now(timezone.utc) + if dt.tzinfo is None: + dt = dt.replace(tzinfo=timezone.utc) + elif dt.tzinfo != timezone.utc: + dt = dt.astimezone(timezone.utc) + processed_row[key] = dt + else: + processed_row[key] = value + results.append(processed_row) + return results[0] if results else {} + + +# ============================================================================= +# CSV PARSING TESTS +# ============================================================================= + +class TestCsvParsing: + """Test that CSV parsing produces correct Decimal types.""" + + def test_parse_8_75pct_transaction(self): + """Parse 8.75% commission transaction.""" + tx = parse_csv_like_transaction_processor(LAMASSU_CSV_DATA["8.75pct_large"]) + + assert tx["transaction_id"] == "abc123" + assert tx["crypto_amount"] == 309200 + assert tx["fiat_amount"] == Decimal("2000") + assert tx["commission_percentage"] == Decimal("0.0875") + assert tx["discount"] == Decimal("0") + assert isinstance(tx["fiat_amount"], Decimal) + assert isinstance(tx["commission_percentage"], Decimal) + assert isinstance(tx["discount"], Decimal) + + def test_parse_5_5pct_with_discount(self): + """Parse 5.5% commission with 90% discount.""" + tx = parse_csv_like_transaction_processor(LAMASSU_CSV_DATA["5.5pct_90pct_discount"]) + + assert tx["crypto_amount"] == 115000 + assert tx["fiat_amount"] == Decimal("800") + assert tx["commission_percentage"] == Decimal("0.055") + assert tx["discount"] == Decimal("90") + + def test_timestamp_parsing(self): + """Verify timestamp is parsed to UTC datetime.""" + tx = parse_csv_like_transaction_processor(LAMASSU_CSV_DATA["8.75pct_large"]) + + assert isinstance(tx["transaction_time"], datetime) + assert tx["transaction_time"].tzinfo == timezone.utc + + +# ============================================================================= +# END-TO-END CALCULATION TESTS +# ============================================================================= + +class TestEndToEndCalculations: + """ + Test the full flow: CSV → Decimal → calculations → expected results. + + These use the same empirical data as test_calculations.py but verify + the data flows correctly through parsing. + """ + + @pytest.mark.parametrize("csv_key,expected_base,expected_commission", [ + ("8.75pct_large", 284322, 24878), + ("5.5pct_no_discount", 293365, 16135), + ("5.5pct_90pct_discount", 114371, 629), + ("5.5pct_1300gtq_4clients", 194882, 10718), + ]) + def test_csv_to_commission_calculation(self, csv_key, expected_base, expected_commission): + """Verify CSV parsing → commission calculation produces expected results.""" + # Parse CSV like transaction_processor does + tx = parse_csv_like_transaction_processor(LAMASSU_CSV_DATA[csv_key]) + + # Calculate commission using parsed Decimal values + base, commission, effective = calculate_commission( + tx["crypto_amount"], + tx["commission_percentage"], + tx["discount"] + ) + + assert base == expected_base, f"Base mismatch for {csv_key}" + assert commission == expected_commission, f"Commission mismatch for {csv_key}" + assert base + commission == tx["crypto_amount"], "Invariant: base + commission = total" + + def test_full_distribution_flow_two_equal_clients(self): + """Test full flow with two equal-balance clients.""" + tx = parse_csv_like_transaction_processor(LAMASSU_CSV_DATA["8.75pct_large"]) + + # Calculate commission + base_sats, commission_sats, effective = calculate_commission( + tx["crypto_amount"], + tx["commission_percentage"], + tx["discount"] + ) + + # Simulate client balances (as would come from database) + client_balances = { + "client_a": Decimal("1000.00"), + "client_b": Decimal("1000.00"), + } + + # Calculate distribution + distributions = calculate_distribution(base_sats, client_balances) + + # Verify results + assert sum(distributions.values()) == base_sats + assert len(distributions) == 2 + # With equal balances, should be roughly equal (±1 sat for rounding) + assert abs(distributions["client_a"] - distributions["client_b"]) <= 1 + + def test_full_distribution_flow_four_clients(self): + """Test the 1300 GTQ transaction with 4 clients of varying balances.""" + tx = parse_csv_like_transaction_processor(LAMASSU_CSV_DATA["5.5pct_1300gtq_4clients"]) + + # Calculate commission + base_sats, commission_sats, effective = calculate_commission( + tx["crypto_amount"], + tx["commission_percentage"], + tx["discount"] + ) + + assert base_sats == 194882 + assert commission_sats == 10718 + + # Use the actual balance proportions from the real scenario + client_balances = { + "client_a": Decimal("1"), + "client_b": Decimal("986"), + "client_c": Decimal("14"), + "client_d": Decimal("4"), + } + + distributions = calculate_distribution(base_sats, client_balances) + + # Verify invariant + assert sum(distributions.values()) == base_sats + + # Verify proportions are reasonable + total_balance = sum(client_balances.values()) + for client_id, sats in distributions.items(): + expected_proportion = client_balances[client_id] / total_balance + actual_proportion = Decimal(sats) / Decimal(base_sats) + # Allow 1% tolerance for rounding + assert abs(actual_proportion - expected_proportion) < Decimal("0.01"), \ + f"Client {client_id} proportion off: {actual_proportion} vs {expected_proportion}" + + +# ============================================================================= +# MODEL CREATION TESTS +# ============================================================================= + +class TestModelCreation: + """Test that Pydantic models accept Decimal values correctly.""" + + def test_create_lamassu_transaction_data_with_decimals(self): + """Verify CreateLamassuTransactionData accepts Decimal values.""" + tx = parse_csv_like_transaction_processor(LAMASSU_CSV_DATA["8.75pct_large"]) + + base_sats, commission_sats, effective = calculate_commission( + tx["crypto_amount"], + tx["commission_percentage"], + tx["discount"] + ) + exchange_rate = calculate_exchange_rate(base_sats, tx["fiat_amount"]) + + # This should not raise any validation errors + data = CreateLamassuTransactionData( + lamassu_transaction_id=tx["transaction_id"], + fiat_amount=tx["fiat_amount"], + crypto_amount=tx["crypto_amount"], + commission_percentage=tx["commission_percentage"], + discount=tx["discount"], + effective_commission=effective, + commission_amount_sats=commission_sats, + base_amount_sats=base_sats, + exchange_rate=exchange_rate, + crypto_code=tx["crypto_code"], + fiat_code=tx["fiat_code"], + device_id=tx["device_id"], + transaction_time=tx["transaction_time"], + ) + + assert data.fiat_amount == Decimal("2000") + assert data.commission_percentage == Decimal("0.0875") + assert data.base_amount_sats == 284322 + assert data.commission_amount_sats == 24878 + + def test_create_dca_payment_data_with_decimals(self): + """Verify CreateDcaPaymentData accepts Decimal values.""" + tx = parse_csv_like_transaction_processor(LAMASSU_CSV_DATA["5.5pct_no_discount"]) + + base_sats, _, _ = calculate_commission( + tx["crypto_amount"], + tx["commission_percentage"], + tx["discount"] + ) + exchange_rate = calculate_exchange_rate(base_sats, tx["fiat_amount"]) + + # Simulate a client getting half the distribution + client_sats = base_sats // 2 + client_fiat = (Decimal(client_sats) / exchange_rate).quantize(Decimal("0.01")) + + # This should not raise any validation errors + data = CreateDcaPaymentData( + client_id="test_client_123", + amount_sats=client_sats, + amount_fiat=client_fiat, + exchange_rate=exchange_rate, + transaction_type="flow", + lamassu_transaction_id=tx["transaction_id"], + transaction_time=tx["transaction_time"], + ) + + assert isinstance(data.amount_fiat, Decimal) + assert isinstance(data.exchange_rate, Decimal) + assert data.amount_sats == client_sats + + def test_client_balance_summary_with_decimals(self): + """Verify ClientBalanceSummary accepts Decimal values.""" + summary = ClientBalanceSummary( + client_id="test_client", + total_deposits=Decimal("5000.00"), + total_payments=Decimal("1234.56"), + remaining_balance=Decimal("3765.44"), + currency="GTQ", + ) + + assert summary.remaining_balance == Decimal("3765.44") + assert isinstance(summary.total_deposits, Decimal) + + +# ============================================================================= +# EXCHANGE RATE PRECISION TESTS +# ============================================================================= + +class TestExchangeRatePrecision: + """Test that exchange rate calculations maintain precision.""" + + def test_exchange_rate_round_trip(self): + """Verify sats → fiat → sats round-trip maintains precision.""" + tx = parse_csv_like_transaction_processor(LAMASSU_CSV_DATA["5.5pct_1300gtq_4clients"]) + + base_sats, _, _ = calculate_commission( + tx["crypto_amount"], + tx["commission_percentage"], + tx["discount"] + ) + + exchange_rate = calculate_exchange_rate(base_sats, tx["fiat_amount"]) + + # Convert sats to fiat and back + fiat_equivalent = Decimal(base_sats) / exchange_rate + sats_back = int((fiat_equivalent * exchange_rate).quantize(Decimal("1"))) + + # Should be within 1 sat of original + assert abs(sats_back - base_sats) <= 1 + + def test_per_client_fiat_sums_to_total(self): + """Verify per-client fiat amounts sum to total fiat (within tolerance).""" + tx = parse_csv_like_transaction_processor(LAMASSU_CSV_DATA["5.5pct_1300gtq_4clients"]) + + base_sats, _, _ = calculate_commission( + tx["crypto_amount"], + tx["commission_percentage"], + tx["discount"] + ) + + exchange_rate = calculate_exchange_rate(base_sats, tx["fiat_amount"]) + + client_balances = { + "client_a": Decimal("1"), + "client_b": Decimal("986"), + "client_c": Decimal("14"), + "client_d": Decimal("4"), + } + + distributions = calculate_distribution(base_sats, client_balances) + + # Calculate per-client fiat and sum + total_fiat_distributed = Decimal("0") + for client_id, sats in distributions.items(): + client_fiat = (Decimal(sats) / exchange_rate).quantize(Decimal("0.01")) + total_fiat_distributed += client_fiat + + # Should be within 0.05 GTQ of original (accounting for per-client rounding) + # This is the 0.01 discrepancy we discussed, multiplied by number of clients + assert abs(total_fiat_distributed - tx["fiat_amount"]) < Decimal("0.05"), \ + f"Total distributed {total_fiat_distributed} vs original {tx['fiat_amount']}" From d2450474875c24a6cc2502e09c0d10208a4ca47f Mon Sep 17 00:00:00 2001 From: padreug Date: Sun, 11 Jan 2026 14:56:29 +0100 Subject: [PATCH 3/8] test: add CRUD layer tests with mocked database MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Tests verify Decimal values flow correctly through CRUD operations: - create_deposit passes Decimal amount to db.execute() - create_dca_payment passes Decimal fiat and exchange_rate - create_lamassu_transaction passes all 5 Decimal fields - get_client_balance_summary returns Decimal types 41 tests now pass (23 unit + 18 integration). 🤖 Generated with [Claude Code](https://claude.ai/code) Co-Authored-By: Claude --- tests/test_integration.py | 243 +++++++++++++++++++++++++++++++++++++- 1 file changed, 242 insertions(+), 1 deletion(-) diff --git a/tests/test_integration.py b/tests/test_integration.py index 83cd1f2..9fe2623 100644 --- a/tests/test_integration.py +++ b/tests/test_integration.py @@ -2,7 +2,7 @@ Integration tests for the full transaction processing flow. These tests verify that data flows correctly from: - CSV parsing → Decimal conversion → calculations → model creation + CSV parsing → Decimal conversion → calculations → model creation → CRUD This gives us confidence that real Lamassu transactions will be processed correctly end-to-end. @@ -21,6 +21,10 @@ from ..models import ( CreateDcaPaymentData, CreateLamassuTransactionData, ClientBalanceSummary, + CreateDepositData, + DcaDeposit, + DcaPayment, + StoredLamassuTransaction, ) @@ -377,3 +381,240 @@ class TestExchangeRatePrecision: # This is the 0.01 discrepancy we discussed, multiplied by number of clients assert abs(total_fiat_distributed - tx["fiat_amount"]) < Decimal("0.05"), \ f"Total distributed {total_fiat_distributed} vs original {tx['fiat_amount']}" + + +# ============================================================================= +# CRUD LAYER TESTS (with mocked database) +# ============================================================================= + +class TestCrudLayerDecimalHandling: + """ + Test that CRUD operations correctly pass Decimal values to the database. + + These tests mock the database layer to verify: + 1. Decimal values from models are passed correctly to db.execute() + 2. The correct SQL parameters include Decimal types + """ + + @pytest.mark.asyncio + async def test_create_deposit_passes_decimal_amount(self): + """Verify create_deposit passes Decimal amount to database.""" + from unittest.mock import patch, AsyncMock + + # Create deposit data with Decimal + deposit_data = CreateDepositData( + client_id="test_client_123", + amount=Decimal("1500.75"), + currency="GTQ", + notes="Test deposit" + ) + + # Verify the model stored it as Decimal + assert isinstance(deposit_data.amount, Decimal) + assert deposit_data.amount == Decimal("1500.75") + + # Mock the database + mock_db = AsyncMock() + mock_db.execute = AsyncMock() + mock_db.fetchone = AsyncMock(return_value=DcaDeposit( + id="deposit_123", + client_id="test_client_123", + amount=Decimal("1500.75"), + currency="GTQ", + status="pending", + notes="Test deposit", + created_at=datetime.now(timezone.utc), + confirmed_at=None + )) + + with patch('satmachineadmin.crud.db', mock_db): + from ..crud import create_deposit + result = await create_deposit(deposit_data) + + # Verify db.execute was called + mock_db.execute.assert_called_once() + + # Get the parameters passed to execute + call_args = mock_db.execute.call_args + params = call_args[0][1] # Second positional arg is the params dict + + # Verify the amount parameter is Decimal + assert isinstance(params["amount"], Decimal) + assert params["amount"] == Decimal("1500.75") + + @pytest.mark.asyncio + async def test_create_dca_payment_passes_decimal_values(self): + """Verify create_dca_payment passes Decimal fiat and exchange_rate.""" + from unittest.mock import patch, AsyncMock + + # Parse a real transaction + tx = parse_csv_like_transaction_processor(LAMASSU_CSV_DATA["5.5pct_no_discount"]) + base_sats, _, _ = calculate_commission( + tx["crypto_amount"], + tx["commission_percentage"], + tx["discount"] + ) + exchange_rate = calculate_exchange_rate(base_sats, tx["fiat_amount"]) + + # Create payment data + client_sats = 146682 # Half of base + client_fiat = (Decimal(client_sats) / exchange_rate).quantize(Decimal("0.01")) + + payment_data = CreateDcaPaymentData( + client_id="test_client", + amount_sats=client_sats, + amount_fiat=client_fiat, + exchange_rate=exchange_rate, + transaction_type="flow", + lamassu_transaction_id="def456", + transaction_time=tx["transaction_time"], + ) + + # Verify model has Decimal types + assert isinstance(payment_data.amount_fiat, Decimal) + assert isinstance(payment_data.exchange_rate, Decimal) + + # Mock database + mock_db = AsyncMock() + mock_db.execute = AsyncMock() + mock_db.fetchone = AsyncMock(return_value=DcaPayment( + id="payment_123", + client_id="test_client", + amount_sats=client_sats, + amount_fiat=client_fiat, + exchange_rate=exchange_rate, + transaction_type="flow", + lamassu_transaction_id="def456", + payment_hash=None, + status="pending", + created_at=datetime.now(timezone.utc), + transaction_time=tx["transaction_time"], + )) + + with patch('satmachineadmin.crud.db', mock_db): + from ..crud import create_dca_payment + result = await create_dca_payment(payment_data) + + # Verify db.execute was called + mock_db.execute.assert_called_once() + + # Get params + call_args = mock_db.execute.call_args + params = call_args[0][1] + + # Verify Decimal types in params + assert isinstance(params["amount_fiat"], Decimal) + assert isinstance(params["exchange_rate"], Decimal) + + @pytest.mark.asyncio + async def test_create_lamassu_transaction_passes_all_decimals(self): + """Verify create_lamassu_transaction passes all Decimal fields.""" + from unittest.mock import patch, AsyncMock + + tx = parse_csv_like_transaction_processor(LAMASSU_CSV_DATA["8.75pct_large"]) + base_sats, commission_sats, effective = calculate_commission( + tx["crypto_amount"], + tx["commission_percentage"], + tx["discount"] + ) + exchange_rate = calculate_exchange_rate(base_sats, tx["fiat_amount"]) + + transaction_data = CreateLamassuTransactionData( + lamassu_transaction_id=tx["transaction_id"], + fiat_amount=tx["fiat_amount"], + crypto_amount=tx["crypto_amount"], + commission_percentage=tx["commission_percentage"], + discount=tx["discount"], + effective_commission=effective, + commission_amount_sats=commission_sats, + base_amount_sats=base_sats, + exchange_rate=exchange_rate, + crypto_code=tx["crypto_code"], + fiat_code=tx["fiat_code"], + device_id=tx["device_id"], + transaction_time=tx["transaction_time"], + ) + + # Verify all Decimal fields + assert isinstance(transaction_data.fiat_amount, Decimal) + assert isinstance(transaction_data.commission_percentage, Decimal) + assert isinstance(transaction_data.discount, Decimal) + assert isinstance(transaction_data.effective_commission, Decimal) + assert isinstance(transaction_data.exchange_rate, Decimal) + + # Mock database + mock_db = AsyncMock() + mock_db.execute = AsyncMock() + mock_db.fetchone = AsyncMock(return_value=StoredLamassuTransaction( + id="tx_123", + lamassu_transaction_id=tx["transaction_id"], + fiat_amount=tx["fiat_amount"], + crypto_amount=tx["crypto_amount"], + commission_percentage=tx["commission_percentage"], + discount=tx["discount"], + effective_commission=effective, + commission_amount_sats=commission_sats, + base_amount_sats=base_sats, + exchange_rate=exchange_rate, + crypto_code=tx["crypto_code"], + fiat_code=tx["fiat_code"], + device_id=tx["device_id"], + transaction_time=tx["transaction_time"], + processed_at=datetime.now(timezone.utc), + clients_count=0, + distributions_total_sats=0, + )) + + with patch('satmachineadmin.crud.db', mock_db): + from ..crud import create_lamassu_transaction + result = await create_lamassu_transaction(transaction_data) + + # Verify db.execute was called + mock_db.execute.assert_called_once() + + # Get params + call_args = mock_db.execute.call_args + params = call_args[0][1] + + # Verify all Decimal fields in params + assert isinstance(params["fiat_amount"], Decimal), f"fiat_amount is {type(params['fiat_amount'])}" + assert isinstance(params["commission_percentage"], Decimal) + assert isinstance(params["discount"], Decimal) + assert isinstance(params["effective_commission"], Decimal) + assert isinstance(params["exchange_rate"], Decimal) + + # Verify values match + assert params["fiat_amount"] == Decimal("2000") + assert params["commission_percentage"] == Decimal("0.0875") + assert params["base_amount_sats"] == 284322 + assert params["commission_amount_sats"] == 24878 + + @pytest.mark.asyncio + async def test_client_balance_summary_returns_decimals(self): + """Verify get_client_balance_summary returns Decimal types.""" + from unittest.mock import patch, AsyncMock + + # Mock database responses + mock_db = AsyncMock() + + # Mock deposits query result + mock_db.fetchone = AsyncMock(side_effect=[ + # First call: deposits sum + {"total": Decimal("5000.00"), "currency": "GTQ"}, + # Second call: payments sum + {"total": Decimal("1234.56")}, + ]) + + with patch('satmachineadmin.crud.db', mock_db): + from ..crud import get_client_balance_summary + result = await get_client_balance_summary("test_client") + + # Verify result has Decimal types + assert isinstance(result.total_deposits, Decimal) + assert isinstance(result.total_payments, Decimal) + assert isinstance(result.remaining_balance, Decimal) + + # Verify values + assert result.total_deposits == Decimal("5000.00") + assert result.total_payments == Decimal("1234.56") + assert result.remaining_balance == Decimal("3765.44") From 904b3f1d61931dbe09584105e8c0b158e4267326 Mon Sep 17 00:00:00 2001 From: padreug Date: Sun, 11 Jan 2026 15:09:45 +0100 Subject: [PATCH 4/8] fix: add SQLite compatibility for Decimal types MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit SQLite doesn't support Decimal natively - it stores DECIMAL columns as REAL (float). This caused sqlite3.ProgrammingError when writing Decimal values. Changes: - Add prepare_for_db() helper to convert Decimal→float before writes - Add Pydantic validators to convert float→Decimal on model creation - Update CRUD layer tests to verify float params for SQLite - Add SQLite round-trip tests to verify precision is preserved The data flow is now: Decimal (calculations) → float (prepare_for_db) → SQLite → float → Decimal (validators) 🤖 Generated with [Claude Code](https://claude.com/claude-code) Co-Authored-By: Claude Opus 4.5 --- crud.py | 70 ++++++---- models.py | 37 +++++ tests/test_integration.py | 276 +++++++++++++++++++++++++++++++++----- 3 files changed, 323 insertions(+), 60 deletions(-) diff --git a/crud.py b/crud.py index 94c1d20..2516f52 100644 --- a/crud.py +++ b/crud.py @@ -1,5 +1,6 @@ # Description: This file contains the CRUD operations for talking to the database. +from decimal import Decimal from typing import List, Optional, Union from datetime import datetime, timezone @@ -18,16 +19,33 @@ from .models import ( db = Database("ext_satoshimachine") +def prepare_for_db(values: dict) -> dict: + """ + Convert Decimal values to float for SQLite compatibility. + + SQLite doesn't support Decimal natively - it stores DECIMAL as REAL (float). + This function converts Decimal values to float before database writes. + The Pydantic models handle converting float back to Decimal on read. + """ + result = {} + for k, v in values.items(): + if isinstance(v, Decimal): + result[k] = float(v) + else: + result[k] = v + return result + + # DCA Client CRUD Operations async def create_dca_client(data: CreateDcaClientData) -> DcaClient: client_id = urlsafe_short_hash() await db.execute( """ - INSERT INTO satoshimachine.dca_clients + INSERT INTO satoshimachine.dca_clients (id, user_id, wallet_id, username, dca_mode, fixed_mode_daily_limit, status, created_at, updated_at) VALUES (:id, :user_id, :wallet_id, :username, :dca_mode, :fixed_mode_daily_limit, :status, :created_at, :updated_at) """, - { + prepare_for_db({ "id": client_id, "user_id": data.user_id, "wallet_id": data.wallet_id, @@ -37,7 +55,7 @@ async def create_dca_client(data: CreateDcaClientData) -> DcaClient: "status": "active", "created_at": datetime.now(), "updated_at": datetime.now() - } + }) ) return await get_dca_client(client_id) @@ -69,14 +87,14 @@ async def update_dca_client(client_id: str, data: UpdateDcaClientData) -> Option update_data = {k: v for k, v in data.dict().items() if v is not None} if not update_data: return await get_dca_client(client_id) - + update_data["updated_at"] = datetime.now() set_clause = ", ".join([f"{k} = :{k}" for k in update_data.keys()]) update_data["id"] = client_id - + await db.execute( f"UPDATE satoshimachine.dca_clients SET {set_clause} WHERE id = :id", - update_data + prepare_for_db(update_data) ) return await get_dca_client(client_id) @@ -93,11 +111,11 @@ async def create_deposit(data: CreateDepositData) -> DcaDeposit: deposit_id = urlsafe_short_hash() await db.execute( """ - INSERT INTO satoshimachine.dca_deposits + INSERT INTO satoshimachine.dca_deposits (id, client_id, amount, currency, status, notes, created_at) VALUES (:id, :client_id, :amount, :currency, :status, :notes, :created_at) """, - { + prepare_for_db({ "id": deposit_id, "client_id": data.client_id, "amount": data.amount, @@ -105,7 +123,7 @@ async def create_deposit(data: CreateDepositData) -> DcaDeposit: "status": "pending", "notes": data.notes, "created_at": datetime.now() - } + }) ) return await get_deposit(deposit_id) @@ -158,13 +176,13 @@ async def create_dca_payment(data: CreateDcaPaymentData) -> DcaPayment: payment_id = urlsafe_short_hash() await db.execute( """ - INSERT INTO satoshimachine.dca_payments - (id, client_id, amount_sats, amount_fiat, exchange_rate, transaction_type, + INSERT INTO satoshimachine.dca_payments + (id, client_id, amount_sats, amount_fiat, exchange_rate, transaction_type, lamassu_transaction_id, payment_hash, status, created_at, transaction_time) VALUES (:id, :client_id, :amount_sats, :amount_fiat, :exchange_rate, :transaction_type, :lamassu_transaction_id, :payment_hash, :status, :created_at, :transaction_time) """, - { + prepare_for_db({ "id": payment_id, "client_id": data.client_id, "amount_sats": data.amount_sats, @@ -176,7 +194,7 @@ async def create_dca_payment(data: CreateDcaPaymentData) -> DcaPayment: "status": "pending", "created_at": datetime.now(), "transaction_time": data.transaction_time - } + }) ) return await get_dca_payment(payment_id) @@ -295,22 +313,22 @@ async def get_fixed_mode_clients() -> List[DcaClient]: # Lamassu Configuration CRUD Operations async def create_lamassu_config(data: CreateLamassuConfigData) -> LamassuConfig: config_id = urlsafe_short_hash() - + # Deactivate any existing configs first (only one active config allowed) await db.execute( "UPDATE satoshimachine.lamassu_config SET is_active = false, updated_at = :updated_at", {"updated_at": datetime.now()} ) - + await db.execute( """ - INSERT INTO satoshimachine.lamassu_config + INSERT INTO satoshimachine.lamassu_config (id, host, port, database_name, username, password, source_wallet_id, commission_wallet_id, is_active, created_at, updated_at, use_ssh_tunnel, ssh_host, ssh_port, ssh_username, ssh_password, ssh_private_key, max_daily_limit_gtq) VALUES (:id, :host, :port, :database_name, :username, :password, :source_wallet_id, :commission_wallet_id, :is_active, :created_at, :updated_at, :use_ssh_tunnel, :ssh_host, :ssh_port, :ssh_username, :ssh_password, :ssh_private_key, :max_daily_limit_gtq) """, - { + prepare_for_db({ "id": config_id, "host": data.host, "port": data.port, @@ -329,7 +347,7 @@ async def create_lamassu_config(data: CreateLamassuConfigData) -> LamassuConfig: "ssh_password": data.ssh_password, "ssh_private_key": data.ssh_private_key, "max_daily_limit_gtq": data.max_daily_limit_gtq - } + }) ) return await get_lamassu_config(config_id) @@ -360,14 +378,14 @@ async def update_lamassu_config(config_id: str, data: UpdateLamassuConfigData) - update_data = {k: v for k, v in data.dict().items() if v is not None} if not update_data: return await get_lamassu_config(config_id) - + update_data["updated_at"] = datetime.now() set_clause = ", ".join([f"{k} = :{k}" for k in update_data.keys()]) update_data["id"] = config_id - + await db.execute( f"UPDATE satoshimachine.lamassu_config SET {set_clause} WHERE id = :id", - update_data + prepare_for_db(update_data) ) return await get_lamassu_config(config_id) @@ -436,9 +454,9 @@ async def create_lamassu_transaction(data: CreateLamassuTransactionData) -> Stor transaction_id = urlsafe_short_hash() await db.execute( """ - INSERT INTO satoshimachine.lamassu_transactions - (id, lamassu_transaction_id, fiat_amount, crypto_amount, commission_percentage, - discount, effective_commission, commission_amount_sats, base_amount_sats, + INSERT INTO satoshimachine.lamassu_transactions + (id, lamassu_transaction_id, fiat_amount, crypto_amount, commission_percentage, + discount, effective_commission, commission_amount_sats, base_amount_sats, exchange_rate, crypto_code, fiat_code, device_id, transaction_time, processed_at, clients_count, distributions_total_sats) VALUES (:id, :lamassu_transaction_id, :fiat_amount, :crypto_amount, :commission_percentage, @@ -446,7 +464,7 @@ async def create_lamassu_transaction(data: CreateLamassuTransactionData) -> Stor :exchange_rate, :crypto_code, :fiat_code, :device_id, :transaction_time, :processed_at, :clients_count, :distributions_total_sats) """, - { + prepare_for_db({ "id": transaction_id, "lamassu_transaction_id": data.lamassu_transaction_id, "fiat_amount": data.fiat_amount, @@ -464,7 +482,7 @@ async def create_lamassu_transaction(data: CreateLamassuTransactionData) -> Stor "processed_at": datetime.now(), "clients_count": 0, # Will be updated after distributions "distributions_total_sats": 0 # Will be updated after distributions - } + }) ) return await get_lamassu_transaction(transaction_id) diff --git a/models.py b/models.py index e7274c1..3e6aa2a 100644 --- a/models.py +++ b/models.py @@ -7,6 +7,16 @@ from typing import Optional from pydantic import BaseModel, validator +def _to_decimal(v): + """Convert a value to Decimal, handling floats from database.""" + if v is None: + return None + if isinstance(v, Decimal): + return v + # Convert via string to avoid float precision issues + return Decimal(str(v)) + + # DCA Client Models class CreateDcaClientData(BaseModel): user_id: str @@ -27,6 +37,10 @@ class DcaClient(BaseModel): created_at: datetime updated_at: datetime + @validator('fixed_mode_daily_limit', pre=True) + def convert_fixed_mode_daily_limit(cls, v): + return _to_decimal(v) + class Config: json_encoders = {Decimal: lambda v: float(v)} @@ -65,6 +79,10 @@ class DcaDeposit(BaseModel): created_at: datetime confirmed_at: Optional[datetime] + @validator('amount', pre=True) + def convert_amount(cls, v): + return _to_decimal(v) + class Config: json_encoders = {Decimal: lambda v: float(v)} @@ -99,6 +117,10 @@ class DcaPayment(BaseModel): created_at: datetime transaction_time: Optional[datetime] = None # Original ATM transaction time + @validator('amount_fiat', 'exchange_rate', pre=True) + def convert_decimals(cls, v): + return _to_decimal(v) + class Config: json_encoders = {Decimal: lambda v: float(v)} @@ -111,6 +133,10 @@ class ClientBalanceSummary(BaseModel): remaining_balance: Decimal # Available balance for DCA in GTQ currency: str + @validator('total_deposits', 'total_payments', 'remaining_balance', pre=True) + def convert_decimals(cls, v): + return _to_decimal(v) + class Config: json_encoders = {Decimal: lambda v: float(v)} @@ -165,6 +191,13 @@ class StoredLamassuTransaction(BaseModel): clients_count: int # Number of clients who received distributions distributions_total_sats: int # Total sats distributed to clients + @validator( + 'fiat_amount', 'commission_percentage', 'discount', + 'effective_commission', 'exchange_rate', pre=True + ) + def convert_decimals(cls, v): + return _to_decimal(v) + class Config: json_encoders = {Decimal: lambda v: float(v)} @@ -228,6 +261,10 @@ class LamassuConfig(BaseModel): # DCA Client Limits max_daily_limit_gtq: Decimal = Decimal("2000") # Maximum daily limit for Fixed mode clients + @validator('max_daily_limit_gtq', pre=True) + def convert_max_daily_limit(cls, v): + return _to_decimal(v) + class Config: json_encoders = {Decimal: lambda v: float(v)} diff --git a/tests/test_integration.py b/tests/test_integration.py index 9fe2623..fc1ff62 100644 --- a/tests/test_integration.py +++ b/tests/test_integration.py @@ -389,16 +389,16 @@ class TestExchangeRatePrecision: class TestCrudLayerDecimalHandling: """ - Test that CRUD operations correctly pass Decimal values to the database. + Test that CRUD operations handle Decimal→float conversion for SQLite. These tests mock the database layer to verify: - 1. Decimal values from models are passed correctly to db.execute() - 2. The correct SQL parameters include Decimal types + 1. Decimal values from models are converted to float via prepare_for_db() + 2. Float values from SQLite are converted back to Decimal via model validators """ @pytest.mark.asyncio - async def test_create_deposit_passes_decimal_amount(self): - """Verify create_deposit passes Decimal amount to database.""" + async def test_create_deposit_passes_float_amount_for_sqlite(self): + """Verify create_deposit passes float amount to database (SQLite compatibility).""" from unittest.mock import patch, AsyncMock # Create deposit data with Decimal @@ -419,7 +419,7 @@ class TestCrudLayerDecimalHandling: mock_db.fetchone = AsyncMock(return_value=DcaDeposit( id="deposit_123", client_id="test_client_123", - amount=Decimal("1500.75"), + amount=1500.75, # float from SQLite currency="GTQ", status="pending", notes="Test deposit", @@ -438,13 +438,13 @@ class TestCrudLayerDecimalHandling: call_args = mock_db.execute.call_args params = call_args[0][1] # Second positional arg is the params dict - # Verify the amount parameter is Decimal - assert isinstance(params["amount"], Decimal) - assert params["amount"] == Decimal("1500.75") + # Verify the amount parameter is float (for SQLite compatibility) + assert isinstance(params["amount"], float) + assert params["amount"] == 1500.75 @pytest.mark.asyncio - async def test_create_dca_payment_passes_decimal_values(self): - """Verify create_dca_payment passes Decimal fiat and exchange_rate.""" + async def test_create_dca_payment_passes_float_values_for_sqlite(self): + """Verify create_dca_payment passes float fiat and exchange_rate (SQLite compatibility).""" from unittest.mock import patch, AsyncMock # Parse a real transaction @@ -474,15 +474,15 @@ class TestCrudLayerDecimalHandling: assert isinstance(payment_data.amount_fiat, Decimal) assert isinstance(payment_data.exchange_rate, Decimal) - # Mock database + # Mock database - returns float like SQLite would mock_db = AsyncMock() mock_db.execute = AsyncMock() mock_db.fetchone = AsyncMock(return_value=DcaPayment( id="payment_123", client_id="test_client", amount_sats=client_sats, - amount_fiat=client_fiat, - exchange_rate=exchange_rate, + amount_fiat=float(client_fiat), # float from SQLite + exchange_rate=float(exchange_rate), # float from SQLite transaction_type="flow", lamassu_transaction_id="def456", payment_hash=None, @@ -502,13 +502,13 @@ class TestCrudLayerDecimalHandling: call_args = mock_db.execute.call_args params = call_args[0][1] - # Verify Decimal types in params - assert isinstance(params["amount_fiat"], Decimal) - assert isinstance(params["exchange_rate"], Decimal) + # Verify float types in params (SQLite compatibility) + assert isinstance(params["amount_fiat"], float) + assert isinstance(params["exchange_rate"], float) @pytest.mark.asyncio - async def test_create_lamassu_transaction_passes_all_decimals(self): - """Verify create_lamassu_transaction passes all Decimal fields.""" + async def test_create_lamassu_transaction_passes_floats_for_sqlite(self): + """Verify create_lamassu_transaction passes float fields (SQLite compatibility).""" from unittest.mock import patch, AsyncMock tx = parse_csv_like_transaction_processor(LAMASSU_CSV_DATA["8.75pct_large"]) @@ -535,27 +535,27 @@ class TestCrudLayerDecimalHandling: transaction_time=tx["transaction_time"], ) - # Verify all Decimal fields + # Verify all Decimal fields in the model assert isinstance(transaction_data.fiat_amount, Decimal) assert isinstance(transaction_data.commission_percentage, Decimal) assert isinstance(transaction_data.discount, Decimal) assert isinstance(transaction_data.effective_commission, Decimal) assert isinstance(transaction_data.exchange_rate, Decimal) - # Mock database + # Mock database - returns floats like SQLite would mock_db = AsyncMock() mock_db.execute = AsyncMock() mock_db.fetchone = AsyncMock(return_value=StoredLamassuTransaction( id="tx_123", lamassu_transaction_id=tx["transaction_id"], - fiat_amount=tx["fiat_amount"], + fiat_amount=float(tx["fiat_amount"]), # float from SQLite crypto_amount=tx["crypto_amount"], - commission_percentage=tx["commission_percentage"], - discount=tx["discount"], - effective_commission=effective, + commission_percentage=float(tx["commission_percentage"]), # float from SQLite + discount=float(tx["discount"]), # float from SQLite + effective_commission=float(effective), # float from SQLite commission_amount_sats=commission_sats, base_amount_sats=base_sats, - exchange_rate=exchange_rate, + exchange_rate=float(exchange_rate), # float from SQLite crypto_code=tx["crypto_code"], fiat_code=tx["fiat_code"], device_id=tx["device_id"], @@ -576,16 +576,16 @@ class TestCrudLayerDecimalHandling: call_args = mock_db.execute.call_args params = call_args[0][1] - # Verify all Decimal fields in params - assert isinstance(params["fiat_amount"], Decimal), f"fiat_amount is {type(params['fiat_amount'])}" - assert isinstance(params["commission_percentage"], Decimal) - assert isinstance(params["discount"], Decimal) - assert isinstance(params["effective_commission"], Decimal) - assert isinstance(params["exchange_rate"], Decimal) + # Verify all Decimal fields are converted to float for SQLite + assert isinstance(params["fiat_amount"], float), f"fiat_amount is {type(params['fiat_amount'])}" + assert isinstance(params["commission_percentage"], float) + assert isinstance(params["discount"], float) + assert isinstance(params["effective_commission"], float) + assert isinstance(params["exchange_rate"], float) # Verify values match - assert params["fiat_amount"] == Decimal("2000") - assert params["commission_percentage"] == Decimal("0.0875") + assert params["fiat_amount"] == 2000.0 + assert params["commission_percentage"] == 0.0875 assert params["base_amount_sats"] == 284322 assert params["commission_amount_sats"] == 24878 @@ -618,3 +618,211 @@ class TestCrudLayerDecimalHandling: assert result.total_deposits == Decimal("5000.00") assert result.total_payments == Decimal("1234.56") assert result.remaining_balance == Decimal("3765.44") + + +# ============================================================================= +# SQLITE ROUND-TRIP TESTS +# ============================================================================= + +class TestSqliteDecimalRoundTrip: + """ + Test that Decimal values survive the SQLite round-trip: + Decimal → float (prepare_for_db) → SQLite REAL → float → Decimal (validator) + + SQLite doesn't support Decimal natively - it stores DECIMAL as REAL (float). + We use prepare_for_db() to convert Decimal→float before writing, + and Pydantic validators with pre=True to convert float→Decimal on read. + """ + + def test_prepare_for_db_converts_decimals_to_float(self): + """Verify prepare_for_db converts Decimal to float.""" + from ..crud import prepare_for_db + + values = { + "amount": Decimal("1500.75"), + "exchange_rate": Decimal("146.6825"), + "commission": Decimal("0.0875"), + "name": "test", # Non-Decimal should pass through + "count": 42, # Non-Decimal should pass through + } + + result = prepare_for_db(values) + + # Decimals should be converted to float + assert isinstance(result["amount"], float) + assert isinstance(result["exchange_rate"], float) + assert isinstance(result["commission"], float) + + # Values should be preserved + assert result["amount"] == 1500.75 + assert result["exchange_rate"] == 146.6825 + assert result["commission"] == 0.0875 + + # Non-Decimals should pass through unchanged + assert result["name"] == "test" + assert result["count"] == 42 + + def test_model_validator_converts_float_to_decimal(self): + """Verify model validators convert float back to Decimal.""" + # Simulate what comes back from SQLite (floats) + db_row = { + "id": "deposit_123", + "client_id": "client_abc", + "amount": 1500.75, # float from SQLite + "currency": "GTQ", + "status": "pending", + "notes": "Test", + "created_at": datetime.now(timezone.utc), + "confirmed_at": None, + } + + # Create model from "database" data + deposit = DcaDeposit(**db_row) + + # Validator should have converted float → Decimal + assert isinstance(deposit.amount, Decimal) + assert deposit.amount == Decimal("1500.75") + + def test_payment_model_converts_multiple_floats(self): + """Verify DcaPayment converts all float fields to Decimal.""" + db_row = { + "id": "payment_123", + "client_id": "client_abc", + "amount_sats": 146682, + "amount_fiat": 1000.50, # float from SQLite + "exchange_rate": 146.6825, # float from SQLite + "transaction_type": "flow", + "lamassu_transaction_id": "tx_456", + "payment_hash": None, + "status": "confirmed", + "created_at": datetime.now(timezone.utc), + "transaction_time": datetime.now(timezone.utc), + } + + payment = DcaPayment(**db_row) + + assert isinstance(payment.amount_fiat, Decimal) + assert isinstance(payment.exchange_rate, Decimal) + assert payment.amount_fiat == Decimal("1000.50") + assert payment.exchange_rate == Decimal("146.6825") + + def test_stored_transaction_converts_all_decimal_fields(self): + """Verify StoredLamassuTransaction converts all float fields.""" + db_row = { + "id": "tx_123", + "lamassu_transaction_id": "lamassu_abc", + "fiat_amount": 2000.00, # float from SQLite + "crypto_amount": 309200, + "commission_percentage": 0.0875, # float from SQLite + "discount": 0.0, # float from SQLite + "effective_commission": 0.0875, # float from SQLite + "commission_amount_sats": 24878, + "base_amount_sats": 284322, + "exchange_rate": 142.161, # float from SQLite + "crypto_code": "BTC", + "fiat_code": "GTQ", + "device_id": "device1", + "transaction_time": datetime.now(timezone.utc), + "processed_at": datetime.now(timezone.utc), + "clients_count": 2, + "distributions_total_sats": 284322, + } + + tx = StoredLamassuTransaction(**db_row) + + # All Decimal fields should be converted + assert isinstance(tx.fiat_amount, Decimal) + assert isinstance(tx.commission_percentage, Decimal) + assert isinstance(tx.discount, Decimal) + assert isinstance(tx.effective_commission, Decimal) + assert isinstance(tx.exchange_rate, Decimal) + + # Values should be preserved + assert tx.fiat_amount == Decimal("2000.00") + assert tx.commission_percentage == Decimal("0.0875") + assert tx.discount == Decimal("0.0") + assert tx.effective_commission == Decimal("0.0875") + + def test_full_round_trip_preserves_precision(self): + """ + Test complete round-trip: Decimal → prepare_for_db → float → Decimal. + + This simulates what happens in production: + 1. We have Decimal values from calculations + 2. prepare_for_db converts them to float for SQLite + 3. SQLite stores as REAL + 4. We read back as float + 5. Model validators convert back to Decimal + """ + from ..crud import prepare_for_db + + # Original Decimal values (from calculations) + original = { + "fiat_amount": Decimal("1300.00"), + "commission_percentage": Decimal("0.055"), + "discount": Decimal("0"), + "effective_commission": Decimal("0.055"), + "exchange_rate": Decimal("149.90215490109"), # High precision + } + + # Step 1: Convert for database storage + for_db = prepare_for_db(original) + + # Verify all are floats + for key in original: + assert isinstance(for_db[key], float), f"{key} should be float" + + # Step 2: Simulate SQLite storage and retrieval + # (SQLite would store these as REAL and return as float) + from_db = for_db.copy() + + # Step 3: Create a model (simulating what happens when reading) + db_row = { + "id": "tx_test", + "lamassu_transaction_id": "test_123", + "fiat_amount": from_db["fiat_amount"], + "crypto_amount": 205600, + "commission_percentage": from_db["commission_percentage"], + "discount": from_db["discount"], + "effective_commission": from_db["effective_commission"], + "commission_amount_sats": 10718, + "base_amount_sats": 194882, + "exchange_rate": from_db["exchange_rate"], + "crypto_code": "BTC", + "fiat_code": "GTQ", + "device_id": "device1", + "transaction_time": datetime.now(timezone.utc), + "processed_at": datetime.now(timezone.utc), + "clients_count": 1, + "distributions_total_sats": 194882, + } + + tx = StoredLamassuTransaction(**db_row) + + # Step 4: Verify Decimal types restored + assert isinstance(tx.fiat_amount, Decimal) + assert isinstance(tx.commission_percentage, Decimal) + assert isinstance(tx.exchange_rate, Decimal) + + # Step 5: Verify precision for 2 decimal place values + assert tx.fiat_amount == Decimal("1300.0") # SQLite preserves this exactly + assert tx.commission_percentage == Decimal("0.055") + assert tx.discount == Decimal("0.0") + + def test_client_balance_summary_float_conversion(self): + """Verify ClientBalanceSummary converts floats from aggregation queries.""" + # SQLite SUM() returns float + db_row = { + "client_id": "client_123", + "total_deposits": 5000.00, # float from SUM() + "total_payments": 1234.56, # float from SUM() + "remaining_balance": 3765.44, # calculated float + "currency": "GTQ", + } + + summary = ClientBalanceSummary(**db_row) + + assert isinstance(summary.total_deposits, Decimal) + assert isinstance(summary.total_payments, Decimal) + assert isinstance(summary.remaining_balance, Decimal) + assert summary.remaining_balance == Decimal("3765.44") From a703bb32d3065d2f2536f5e966300b743892c963 Mon Sep 17 00:00:00 2001 From: padreug Date: Sun, 11 Jan 2026 15:17:07 +0100 Subject: [PATCH 5/8] fix: cast amount to float for LNbits create_invoice API MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit LNbits create_invoice expects `amount: float` but we were passing int. While Python's duck typing handles this, explicit casting ensures type correctness and matches the API contract. 🤖 Generated with [Claude Code](https://claude.com/claude-code) Co-Authored-By: Claude Opus 4.5 --- transaction_processor.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/transaction_processor.py b/transaction_processor.py index 1f511df..deaac47 100644 --- a/transaction_processor.py +++ b/transaction_processor.py @@ -935,7 +935,7 @@ class LamassuTransactionProcessor: } new_payment = await create_invoice( wallet_id=target_wallet.id, - amount=amount_sats, # LNBits create_invoice expects sats + amount=float(amount_sats), # LNBits expects float for amount internal=True, # Internal transfer within LNBits memo=memo, extra=extra @@ -1102,7 +1102,7 @@ class LamassuTransactionProcessor: commission_payment = await create_invoice( wallet_id=admin_config.commission_wallet_id, - amount=commission_amount_sats, + amount=float(commission_amount_sats), # LNBits expects float internal=True, memo=commission_memo, extra={ From 49f3670bac46d36d480d70fb32fcfe0f392a4256 Mon Sep 17 00:00:00 2001 From: padreug Date: Sun, 11 Jan 2026 15:40:17 +0100 Subject: [PATCH 6/8] fix: cast amount to float for LNbits create_invoice API MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit LNbits create_invoice expects amount as float, not int. Added explicit float() cast to both DCA distribution and commission payment invoice creation calls. 🤖 Generated with [Claude Code](https://claude.com/claude-code) Co-Authored-By: Claude Opus 4.5 --- transaction_processor.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/transaction_processor.py b/transaction_processor.py index 45e0880..a848998 100644 --- a/transaction_processor.py +++ b/transaction_processor.py @@ -920,7 +920,7 @@ class LamassuTransactionProcessor: } new_payment = await create_invoice( wallet_id=target_wallet.id, - amount=amount_sats, # LNBits create_invoice expects sats + amount=float(amount_sats), # LNBits create_invoice expects float internal=True, # Internal transfer within LNBits memo=memo, extra=extra @@ -1085,7 +1085,7 @@ class LamassuTransactionProcessor: commission_payment = await create_invoice( wallet_id=admin_config.commission_wallet_id, - amount=commission_amount_sats, + amount=float(commission_amount_sats), # LNbits create_invoice expects float internal=True, memo=commission_memo, extra={ From 545a0284a7c9a93b8b4d0fc540c9bf029a0cb464 Mon Sep 17 00:00:00 2001 From: padreug Date: Sun, 11 Jan 2026 15:54:48 +0100 Subject: [PATCH 7/8] fix: cap DCA allocations when ATM cash exceeds tracked balances MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit When there's a sync mismatch (more cash in ATM than tracked client balances), cap each client's allocation to their remaining fiat balance equivalent in sats. Orphan sats stay in the source wallet. This prevents over-allocation when deposits haven't been recorded yet or when there's a timing mismatch between ATM transactions and balance tracking. - Detect sync mismatch: total_confirmed_deposits < fiat_amount - In sync mismatch mode: allocate based on client balance, not tx amount - Track orphan_sats that couldn't be distributed - Normal mode unchanged: proportional distribution using calculate_distribution() 🤖 Generated with [Claude Code](https://claude.com/claude-code) Co-Authored-By: Claude Opus 4.5 --- transaction_processor.py | 131 +++++++++++++++++++++++++++------------ 1 file changed, 91 insertions(+), 40 deletions(-) diff --git a/transaction_processor.py b/transaction_processor.py index a848998..661e5ab 100644 --- a/transaction_processor.py +++ b/transaction_processor.py @@ -669,15 +669,21 @@ class LamassuTransactionProcessor: logger.error(f"Error fetching transactions from Lamassu database: {e}") return [] - async def calculate_distribution_amounts(self, transaction: Dict[str, Any]) -> Dict[str, int]: - """Calculate how much each Flow Mode client should receive""" + async def calculate_distribution_amounts(self, transaction: Dict[str, Any]) -> tuple[Dict[str, Any], int]: + """Calculate how much each Flow Mode client should receive. + + Returns: + tuple: (distributions dict, orphan_sats int) + - distributions: {client_id: {fiat_amount, sats_amount, exchange_rate}} + - orphan_sats: sats that couldn't be distributed due to sync mismatch + """ try: # Get all active Flow Mode clients flow_clients = await get_flow_mode_clients() - + if not flow_clients: logger.info("No Flow Mode clients found - skipping distribution") - return {} + return {}, 0 # Extract transaction details - guaranteed clean from data ingestion crypto_atoms = transaction.get("crypto_amount", 0) # Total sats with commission baked in @@ -701,10 +707,10 @@ class LamassuTransactionProcessor: # Validate required fields if crypto_atoms is None: logger.error(f"Missing crypto_amount in transaction: {transaction}") - return {} + return {}, 0 if fiat_amount is None: logger.error(f"Missing fiat_amount in transaction: {transaction}") - return {} + return {}, 0 if commission_percentage is None: logger.warning(f"Missing commission_percentage in transaction: {transaction}, defaulting to 0") commission_percentage = 0.0 @@ -750,37 +756,76 @@ class LamassuTransactionProcessor: if total_confirmed_deposits == 0: logger.info("No clients with remaining DCA balance - skipping distribution") - return {} + return {}, 0 - # Calculate sat allocations using the extracted pure function - sat_allocations = calculate_distribution(base_crypto_atoms, client_balances) + # Detect sync mismatch: more money in ATM than tracked client balances + sync_mismatch = total_confirmed_deposits < fiat_amount + if sync_mismatch: + orphan_fiat = fiat_amount - total_confirmed_deposits + logger.warning( + f"Sync mismatch detected: tracked balances ({total_confirmed_deposits:.2f} GTQ) " + f"< transaction ({fiat_amount} GTQ). Orphan amount: {orphan_fiat:.2f} GTQ" + ) - if not sat_allocations: - logger.info("No allocations calculated - skipping distribution") - return {} - - # Build final distributions dict with additional tracking fields + # Calculate distribution amounts distributions = {} - for client_id, client_sats_amount in sat_allocations.items(): - # Calculate proportion for logging - proportion = client_balances[client_id] / total_confirmed_deposits - # Calculate equivalent fiat value in GTQ for tracking purposes - client_fiat_amount = round(client_sats_amount / exchange_rate, 2) if exchange_rate > 0 else 0.0 + if sync_mismatch: + # SYNC MISMATCH MODE: Cap each client's allocation to their remaining fiat balance + # Each client gets sats equivalent to their full remaining balance + for client_id, client_balance in client_balances.items(): + # Calculate sats equivalent to this client's remaining fiat balance + client_sats_amount = round(client_balance * exchange_rate) + proportion = client_balance / total_confirmed_deposits - distributions[client_id] = { - "fiat_amount": client_fiat_amount, - "sats_amount": client_sats_amount, - "exchange_rate": exchange_rate - } + # Calculate equivalent fiat value in GTQ for tracking purposes + client_fiat_amount = round(client_sats_amount / exchange_rate, 2) if exchange_rate > 0 else 0.0 - logger.info(f"Client {client_id[:8]}... gets {client_sats_amount} sats (≈{client_fiat_amount:.2f} GTQ, {proportion:.2%} share)") - - # Verification: ensure total distribution equals base amount - total_distributed = sum(dist["sats_amount"] for dist in distributions.values()) - if total_distributed != base_crypto_atoms: - logger.error(f"Distribution mismatch! Expected: {base_crypto_atoms} sats, Distributed: {total_distributed} sats") - raise ValueError(f"Satoshi distribution calculation error: {base_crypto_atoms} != {total_distributed}") + distributions[client_id] = { + "fiat_amount": client_fiat_amount, + "sats_amount": client_sats_amount, + "exchange_rate": exchange_rate + } + + logger.info(f"Client {client_id[:8]}... gets {client_sats_amount} sats (≈{client_fiat_amount:.2f} GTQ, {proportion:.2%} share)") + + # Calculate orphan sats (difference between base amount and distributed) + total_distributed = sum(dist["sats_amount"] for dist in distributions.values()) + orphan_sats = base_crypto_atoms - total_distributed + logger.info( + f"Sync mismatch distribution: {total_distributed} sats to clients, " + f"{orphan_sats} sats orphaned (staying in source wallet)" + ) + else: + # NORMAL MODE: Proportional distribution based on transaction amount + sat_allocations = calculate_distribution(base_crypto_atoms, client_balances) + + if not sat_allocations: + logger.info("No allocations calculated - skipping distribution") + return {}, 0 + + # Build final distributions dict with additional tracking fields + for client_id, client_sats_amount in sat_allocations.items(): + # Calculate proportion for logging + proportion = client_balances[client_id] / total_confirmed_deposits + + # Calculate equivalent fiat value in GTQ for tracking purposes + client_fiat_amount = round(client_sats_amount / exchange_rate, 2) if exchange_rate > 0 else 0.0 + + distributions[client_id] = { + "fiat_amount": client_fiat_amount, + "sats_amount": client_sats_amount, + "exchange_rate": exchange_rate + } + + logger.info(f"Client {client_id[:8]}... gets {client_sats_amount} sats (≈{client_fiat_amount:.2f} GTQ, {proportion:.2%} share)") + + # Verification: ensure total distribution equals base amount + total_distributed = sum(dist["sats_amount"] for dist in distributions.values()) + if total_distributed != base_crypto_atoms: + logger.error(f"Distribution mismatch! Expected: {base_crypto_atoms} sats, Distributed: {total_distributed} sats") + raise ValueError(f"Satoshi distribution calculation error: {base_crypto_atoms} != {total_distributed}") + orphan_sats = 0 # Safety check: Re-verify all clients still have positive balances before finalizing distributions # This prevents race conditions where balances changed during calculation @@ -800,18 +845,18 @@ class LamassuTransactionProcessor: # Recalculate proportions if some clients were rejected if len(final_distributions) == 0: logger.info("All clients rejected due to negative balances - no distributions") - return {} - + return {}, orphan_sats + # For simplicity, we'll still return the original distributions but log the warning # In a production system, you might want to recalculate the entire distribution logger.warning("Proceeding with original distribution despite balance warnings - manual review recommended") - + logger.info(f"Distribution verified: {total_distributed} sats distributed across {len(distributions)} clients (clients with positive allocations only)") - return distributions - + return distributions, orphan_sats + except Exception as e: logger.error(f"Error calculating distribution amounts: {e}") - return {} + return {}, 0 async def distribute_to_clients(self, transaction: Dict[str, Any], distributions: Dict[str, Dict[str, int]]) -> None: """Send Bitcoin payments to DCA clients""" @@ -1140,10 +1185,16 @@ class LamassuTransactionProcessor: stored_transaction = await self.store_lamassu_transaction(transaction) # Calculate distribution amounts - distributions = await self.calculate_distribution_amounts(transaction) - + distributions, orphan_sats = await self.calculate_distribution_amounts(transaction) + if not distributions: - logger.info(f"No distributions calculated for transaction {transaction_id}") + if orphan_sats > 0: + logger.warning( + f"No client distributions for transaction {transaction_id}, " + f"but {orphan_sats} orphan sats remain in source wallet" + ) + else: + logger.info(f"No distributions calculated for transaction {transaction_id}") return # Calculate commission amount for sending to commission wallet From 6eb076d5f65977255ba8f3fa2613e30c22935b70 Mon Sep 17 00:00:00 2001 From: padreug Date: Sun, 11 Jan 2026 16:10:48 +0100 Subject: [PATCH 8/8] chore: bump version to 0.0.4 --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index c5c417d..885fe45 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "satmachineadmin" -version = "0.0.0" +version = "0.0.4" description = "Eightball is a simple API that allows you to create a random number generator." authors = ["benarc", "dni "]