diff --git a/calculations.py b/calculations.py index c16db1e..a7b3aa9 100644 --- a/calculations.py +++ b/calculations.py @@ -3,34 +3,16 @@ Pure calculation functions for DCA transaction processing. These functions have no external dependencies (no lnbits, no database) and can be easily tested in isolation. - -All monetary calculations use Decimal for precision. Satoshi amounts -remain as int since they are the smallest indivisible unit. """ -from decimal import Decimal, ROUND_HALF_UP -from typing import Dict, Tuple, Union - -# Type alias for values that can be Decimal or numeric types that will be converted -DecimalLike = Union[Decimal, float, int, str] - - -def to_decimal(value: DecimalLike) -> Decimal: - """Convert a value to Decimal, handling floats carefully.""" - if isinstance(value, Decimal): - return value - # Convert floats via string to avoid binary float precision issues - # e.g., Decimal(0.055) gives 0.054999999... but Decimal("0.055") is exact - if isinstance(value, float): - return Decimal(str(value)) - return Decimal(value) +from typing import Dict, Tuple def calculate_commission( crypto_atoms: int, - commission_percentage: DecimalLike, - discount: DecimalLike = Decimal("0") -) -> Tuple[int, int, Decimal]: + commission_percentage: float, + discount: float = 0.0 +) -> Tuple[int, int, float]: """ Calculate commission split from a Lamassu transaction. @@ -52,25 +34,15 @@ def calculate_commission( Tuple of (base_amount_sats, commission_amount_sats, effective_commission_rate) Example: - >>> calculate_commission(266800, Decimal("0.03"), Decimal("0")) - (259029, 7771, Decimal('0.03')) + >>> calculate_commission(266800, 0.03, 0.0) + (259029, 7771, 0.03) """ - # Convert inputs to Decimal for precise calculations - comm_pct = to_decimal(commission_percentage) - disc = to_decimal(discount) - - if comm_pct > 0: - # effective = commission_percentage * (100 - discount) / 100 - effective_commission = comm_pct * (Decimal("100") - disc) / Decimal("100") - - # base = crypto_atoms / (1 + effective_commission), rounded to nearest int - divisor = Decimal("1") + effective_commission - exact_base = Decimal(crypto_atoms) / divisor - # Use ROUND_HALF_UP (standard rounding: 0.5 rounds up) - base_crypto_atoms = int(exact_base.quantize(Decimal("1"), rounding=ROUND_HALF_UP)) + if commission_percentage > 0: + effective_commission = commission_percentage * (100 - discount) / 100 + base_crypto_atoms = round(crypto_atoms / (1 + effective_commission)) commission_amount_sats = crypto_atoms - base_crypto_atoms else: - effective_commission = Decimal("0") + effective_commission = 0.0 base_crypto_atoms = crypto_atoms commission_amount_sats = 0 @@ -79,8 +51,8 @@ def calculate_commission( def calculate_distribution( base_amount_sats: int, - client_balances: Dict[str, DecimalLike], - min_balance_threshold: DecimalLike = Decimal("0.01") + client_balances: Dict[str, float], + min_balance_threshold: float = 0.01 ) -> Dict[str, int]: """ Calculate proportional distribution of sats to clients based on their fiat balances. @@ -97,18 +69,15 @@ def calculate_distribution( Dict of {client_id: allocated_sats} Example: - >>> calculate_distribution(100000, {"a": Decimal("500"), "b": Decimal("500")}) + >>> calculate_distribution(100000, {"a": 500.0, "b": 500.0}) {"a": 50000, "b": 50000} """ - # Convert threshold to Decimal - threshold = to_decimal(min_balance_threshold) - - # Filter out clients with balance below threshold, converting to Decimal - active_balances: Dict[str, Decimal] = {} - for client_id, balance in client_balances.items(): - bal = to_decimal(balance) - if bal >= threshold: - active_balances[client_id] = bal + # Filter out clients with balance below threshold + active_balances = { + client_id: balance + for client_id, balance in client_balances.items() + if balance >= min_balance_threshold + } if not active_balances: return {} @@ -121,13 +90,11 @@ def calculate_distribution( # First pass: calculate base allocations and track for remainder distribution client_calculations = [] distributed_sats = 0 - base_sats_decimal = Decimal(base_amount_sats) for client_id, balance in active_balances.items(): proportion = balance / total_balance - exact_share = base_sats_decimal * proportion - # Round to nearest integer using ROUND_HALF_UP - allocated_sats = int(exact_share.quantize(Decimal("1"), rounding=ROUND_HALF_UP)) + exact_share = base_amount_sats * proportion + allocated_sats = round(exact_share) client_calculations.append({ 'client_id': client_id, @@ -142,9 +109,8 @@ def calculate_distribution( if remainder != 0: # Sort by largest fractional remainder to distribute fairly - # The fractional part is exact_share - allocated_sats client_calculations.sort( - key=lambda x: x['exact_share'] - Decimal(x['allocated_sats']), + key=lambda x: x['exact_share'] - x['allocated_sats'], reverse=True ) @@ -165,7 +131,7 @@ def calculate_distribution( return distributions -def calculate_exchange_rate(base_crypto_atoms: int, fiat_amount: DecimalLike) -> Decimal: +def calculate_exchange_rate(base_crypto_atoms: int, fiat_amount: float) -> float: """ Calculate exchange rate in sats per fiat unit. @@ -174,9 +140,8 @@ def calculate_exchange_rate(base_crypto_atoms: int, fiat_amount: DecimalLike) -> fiat_amount: Fiat amount dispensed Returns: - Exchange rate as sats per fiat unit (Decimal for precision) + Exchange rate as sats per fiat unit """ - fiat = to_decimal(fiat_amount) - if fiat <= 0: - return Decimal("0") - return Decimal(base_crypto_atoms) / fiat + if fiat_amount <= 0: + return 0.0 + return base_crypto_atoms / fiat_amount diff --git a/crud.py b/crud.py index 2516f52..94c1d20 100644 --- a/crud.py +++ b/crud.py @@ -1,6 +1,5 @@ # Description: This file contains the CRUD operations for talking to the database. -from decimal import Decimal from typing import List, Optional, Union from datetime import datetime, timezone @@ -19,33 +18,16 @@ from .models import ( db = Database("ext_satoshimachine") -def prepare_for_db(values: dict) -> dict: - """ - Convert Decimal values to float for SQLite compatibility. - - SQLite doesn't support Decimal natively - it stores DECIMAL as REAL (float). - This function converts Decimal values to float before database writes. - The Pydantic models handle converting float back to Decimal on read. - """ - result = {} - for k, v in values.items(): - if isinstance(v, Decimal): - result[k] = float(v) - else: - result[k] = v - return result - - # DCA Client CRUD Operations async def create_dca_client(data: CreateDcaClientData) -> DcaClient: client_id = urlsafe_short_hash() await db.execute( """ - INSERT INTO satoshimachine.dca_clients + INSERT INTO satoshimachine.dca_clients (id, user_id, wallet_id, username, dca_mode, fixed_mode_daily_limit, status, created_at, updated_at) VALUES (:id, :user_id, :wallet_id, :username, :dca_mode, :fixed_mode_daily_limit, :status, :created_at, :updated_at) """, - prepare_for_db({ + { "id": client_id, "user_id": data.user_id, "wallet_id": data.wallet_id, @@ -55,7 +37,7 @@ async def create_dca_client(data: CreateDcaClientData) -> DcaClient: "status": "active", "created_at": datetime.now(), "updated_at": datetime.now() - }) + } ) return await get_dca_client(client_id) @@ -87,14 +69,14 @@ async def update_dca_client(client_id: str, data: UpdateDcaClientData) -> Option update_data = {k: v for k, v in data.dict().items() if v is not None} if not update_data: return await get_dca_client(client_id) - + update_data["updated_at"] = datetime.now() set_clause = ", ".join([f"{k} = :{k}" for k in update_data.keys()]) update_data["id"] = client_id - + await db.execute( f"UPDATE satoshimachine.dca_clients SET {set_clause} WHERE id = :id", - prepare_for_db(update_data) + update_data ) return await get_dca_client(client_id) @@ -111,11 +93,11 @@ async def create_deposit(data: CreateDepositData) -> DcaDeposit: deposit_id = urlsafe_short_hash() await db.execute( """ - INSERT INTO satoshimachine.dca_deposits + INSERT INTO satoshimachine.dca_deposits (id, client_id, amount, currency, status, notes, created_at) VALUES (:id, :client_id, :amount, :currency, :status, :notes, :created_at) """, - prepare_for_db({ + { "id": deposit_id, "client_id": data.client_id, "amount": data.amount, @@ -123,7 +105,7 @@ async def create_deposit(data: CreateDepositData) -> DcaDeposit: "status": "pending", "notes": data.notes, "created_at": datetime.now() - }) + } ) return await get_deposit(deposit_id) @@ -176,13 +158,13 @@ async def create_dca_payment(data: CreateDcaPaymentData) -> DcaPayment: payment_id = urlsafe_short_hash() await db.execute( """ - INSERT INTO satoshimachine.dca_payments - (id, client_id, amount_sats, amount_fiat, exchange_rate, transaction_type, + INSERT INTO satoshimachine.dca_payments + (id, client_id, amount_sats, amount_fiat, exchange_rate, transaction_type, lamassu_transaction_id, payment_hash, status, created_at, transaction_time) VALUES (:id, :client_id, :amount_sats, :amount_fiat, :exchange_rate, :transaction_type, :lamassu_transaction_id, :payment_hash, :status, :created_at, :transaction_time) """, - prepare_for_db({ + { "id": payment_id, "client_id": data.client_id, "amount_sats": data.amount_sats, @@ -194,7 +176,7 @@ async def create_dca_payment(data: CreateDcaPaymentData) -> DcaPayment: "status": "pending", "created_at": datetime.now(), "transaction_time": data.transaction_time - }) + } ) return await get_dca_payment(payment_id) @@ -313,22 +295,22 @@ async def get_fixed_mode_clients() -> List[DcaClient]: # Lamassu Configuration CRUD Operations async def create_lamassu_config(data: CreateLamassuConfigData) -> LamassuConfig: config_id = urlsafe_short_hash() - + # Deactivate any existing configs first (only one active config allowed) await db.execute( "UPDATE satoshimachine.lamassu_config SET is_active = false, updated_at = :updated_at", {"updated_at": datetime.now()} ) - + await db.execute( """ - INSERT INTO satoshimachine.lamassu_config + INSERT INTO satoshimachine.lamassu_config (id, host, port, database_name, username, password, source_wallet_id, commission_wallet_id, is_active, created_at, updated_at, use_ssh_tunnel, ssh_host, ssh_port, ssh_username, ssh_password, ssh_private_key, max_daily_limit_gtq) VALUES (:id, :host, :port, :database_name, :username, :password, :source_wallet_id, :commission_wallet_id, :is_active, :created_at, :updated_at, :use_ssh_tunnel, :ssh_host, :ssh_port, :ssh_username, :ssh_password, :ssh_private_key, :max_daily_limit_gtq) """, - prepare_for_db({ + { "id": config_id, "host": data.host, "port": data.port, @@ -347,7 +329,7 @@ async def create_lamassu_config(data: CreateLamassuConfigData) -> LamassuConfig: "ssh_password": data.ssh_password, "ssh_private_key": data.ssh_private_key, "max_daily_limit_gtq": data.max_daily_limit_gtq - }) + } ) return await get_lamassu_config(config_id) @@ -378,14 +360,14 @@ async def update_lamassu_config(config_id: str, data: UpdateLamassuConfigData) - update_data = {k: v for k, v in data.dict().items() if v is not None} if not update_data: return await get_lamassu_config(config_id) - + update_data["updated_at"] = datetime.now() set_clause = ", ".join([f"{k} = :{k}" for k in update_data.keys()]) update_data["id"] = config_id - + await db.execute( f"UPDATE satoshimachine.lamassu_config SET {set_clause} WHERE id = :id", - prepare_for_db(update_data) + update_data ) return await get_lamassu_config(config_id) @@ -454,9 +436,9 @@ async def create_lamassu_transaction(data: CreateLamassuTransactionData) -> Stor transaction_id = urlsafe_short_hash() await db.execute( """ - INSERT INTO satoshimachine.lamassu_transactions - (id, lamassu_transaction_id, fiat_amount, crypto_amount, commission_percentage, - discount, effective_commission, commission_amount_sats, base_amount_sats, + INSERT INTO satoshimachine.lamassu_transactions + (id, lamassu_transaction_id, fiat_amount, crypto_amount, commission_percentage, + discount, effective_commission, commission_amount_sats, base_amount_sats, exchange_rate, crypto_code, fiat_code, device_id, transaction_time, processed_at, clients_count, distributions_total_sats) VALUES (:id, :lamassu_transaction_id, :fiat_amount, :crypto_amount, :commission_percentage, @@ -464,7 +446,7 @@ async def create_lamassu_transaction(data: CreateLamassuTransactionData) -> Stor :exchange_rate, :crypto_code, :fiat_code, :device_id, :transaction_time, :processed_at, :clients_count, :distributions_total_sats) """, - prepare_for_db({ + { "id": transaction_id, "lamassu_transaction_id": data.lamassu_transaction_id, "fiat_amount": data.fiat_amount, @@ -482,7 +464,7 @@ async def create_lamassu_transaction(data: CreateLamassuTransactionData) -> Stor "processed_at": datetime.now(), "clients_count": 0, # Will be updated after distributions "distributions_total_sats": 0 # Will be updated after distributions - }) + } ) return await get_lamassu_transaction(transaction_id) diff --git a/models.py b/models.py index 3e6aa2a..e4bf64d 100644 --- a/models.py +++ b/models.py @@ -1,29 +1,18 @@ # Description: Pydantic data models dictate what is passed between frontend and backend. from datetime import datetime -from decimal import Decimal from typing import Optional from pydantic import BaseModel, validator -def _to_decimal(v): - """Convert a value to Decimal, handling floats from database.""" - if v is None: - return None - if isinstance(v, Decimal): - return v - # Convert via string to avoid float precision issues - return Decimal(str(v)) - - # DCA Client Models class CreateDcaClientData(BaseModel): user_id: str wallet_id: str username: str dca_mode: str = "flow" # 'flow' or 'fixed' - fixed_mode_daily_limit: Optional[Decimal] = None + fixed_mode_daily_limit: Optional[float] = None class DcaClient(BaseModel): @@ -32,60 +21,44 @@ class DcaClient(BaseModel): wallet_id: str username: Optional[str] dca_mode: str - fixed_mode_daily_limit: Optional[Decimal] + fixed_mode_daily_limit: Optional[int] status: str created_at: datetime updated_at: datetime - @validator('fixed_mode_daily_limit', pre=True) - def convert_fixed_mode_daily_limit(cls, v): - return _to_decimal(v) - - class Config: - json_encoders = {Decimal: lambda v: float(v)} - class UpdateDcaClientData(BaseModel): username: Optional[str] = None dca_mode: Optional[str] = None - fixed_mode_daily_limit: Optional[Decimal] = None + fixed_mode_daily_limit: Optional[float] = None status: Optional[str] = None # Deposit Models (Now storing GTQ directly) class CreateDepositData(BaseModel): client_id: str - amount: Decimal # Amount in GTQ (e.g., 150.75) + amount: float # Amount in GTQ (e.g., 150.75) currency: str = "GTQ" notes: Optional[str] = None - - @validator('amount', pre=True) + + @validator('amount') def round_amount_to_cents(cls, v): """Ensure amount is rounded to 2 decimal places for DECIMAL(10,2) storage""" if v is not None: - # Convert to Decimal via string to avoid float precision issues - d = Decimal(str(v)) if not isinstance(v, Decimal) else v - return d.quantize(Decimal("0.01")) + return round(float(v), 2) return v class DcaDeposit(BaseModel): id: str client_id: str - amount: Decimal # Amount in GTQ (e.g., 150.75) + amount: float # Amount in GTQ (e.g., 150.75) currency: str status: str # 'pending' or 'confirmed' notes: Optional[str] created_at: datetime confirmed_at: Optional[datetime] - @validator('amount', pre=True) - def convert_amount(cls, v): - return _to_decimal(v) - - class Config: - json_encoders = {Decimal: lambda v: float(v)} - class UpdateDepositStatusData(BaseModel): status: str @@ -96,8 +69,8 @@ class UpdateDepositStatusData(BaseModel): class CreateDcaPaymentData(BaseModel): client_id: str amount_sats: int - amount_fiat: Decimal # Amount in GTQ (e.g., 150.75) - exchange_rate: Decimal + amount_fiat: float # Amount in GTQ (e.g., 150.75) + exchange_rate: float transaction_type: str # 'flow', 'fixed', 'manual', 'commission' lamassu_transaction_id: Optional[str] = None payment_hash: Optional[str] = None @@ -108,8 +81,8 @@ class DcaPayment(BaseModel): id: str client_id: str amount_sats: int - amount_fiat: Decimal # Amount in GTQ (e.g., 150.75) - exchange_rate: Decimal + amount_fiat: float # Amount in GTQ (e.g., 150.75) + exchange_rate: float transaction_type: str lamassu_transaction_id: Optional[str] payment_hash: Optional[str] @@ -117,55 +90,38 @@ class DcaPayment(BaseModel): created_at: datetime transaction_time: Optional[datetime] = None # Original ATM transaction time - @validator('amount_fiat', 'exchange_rate', pre=True) - def convert_decimals(cls, v): - return _to_decimal(v) - - class Config: - json_encoders = {Decimal: lambda v: float(v)} - # Client Balance Summary (Now storing GTQ directly) class ClientBalanceSummary(BaseModel): client_id: str - total_deposits: Decimal # Total confirmed deposits in GTQ - total_payments: Decimal # Total payments made in GTQ - remaining_balance: Decimal # Available balance for DCA in GTQ + total_deposits: float # Total confirmed deposits in GTQ + total_payments: float # Total payments made in GTQ + remaining_balance: float # Available balance for DCA in GTQ currency: str - @validator('total_deposits', 'total_payments', 'remaining_balance', pre=True) - def convert_decimals(cls, v): - return _to_decimal(v) - - class Config: - json_encoders = {Decimal: lambda v: float(v)} - # Transaction Processing Models class LamassuTransaction(BaseModel): transaction_id: str - amount_fiat: Decimal # Amount in GTQ (e.g., 150.75) + amount_fiat: float # Amount in GTQ (e.g., 150.75) amount_crypto: int - exchange_rate: Decimal + exchange_rate: float transaction_type: str # 'cash_in' or 'cash_out' status: str timestamp: datetime - class Config: - json_encoders = {Decimal: lambda v: float(v)} - # Lamassu Transaction Storage Models class CreateLamassuTransactionData(BaseModel): lamassu_transaction_id: str - fiat_amount: Decimal # Amount in GTQ (e.g., 150.75) + fiat_amount: float # Amount in GTQ (e.g., 150.75) crypto_amount: int - commission_percentage: Decimal - discount: Decimal = Decimal("0") - effective_commission: Decimal + commission_percentage: float + discount: float = 0.0 + effective_commission: float commission_amount_sats: int base_amount_sats: int - exchange_rate: Decimal + exchange_rate: float crypto_code: str = "BTC" fiat_code: str = "GTQ" device_id: Optional[str] = None @@ -175,14 +131,14 @@ class CreateLamassuTransactionData(BaseModel): class StoredLamassuTransaction(BaseModel): id: str lamassu_transaction_id: str - fiat_amount: Decimal # Amount in GTQ (e.g., 150.75) + fiat_amount: float # Amount in GTQ (e.g., 150.75) crypto_amount: int - commission_percentage: Decimal - discount: Decimal - effective_commission: Decimal + commission_percentage: float + discount: float + effective_commission: float commission_amount_sats: int base_amount_sats: int - exchange_rate: Decimal + exchange_rate: float crypto_code: str fiat_code: str device_id: Optional[str] @@ -191,16 +147,6 @@ class StoredLamassuTransaction(BaseModel): clients_count: int # Number of clients who received distributions distributions_total_sats: int # Total sats distributed to clients - @validator( - 'fiat_amount', 'commission_percentage', 'discount', - 'effective_commission', 'exchange_rate', pre=True - ) - def convert_decimals(cls, v): - return _to_decimal(v) - - class Config: - json_encoders = {Decimal: lambda v: float(v)} - # Lamassu Configuration Models class CreateLamassuConfigData(BaseModel): @@ -221,14 +167,13 @@ class CreateLamassuConfigData(BaseModel): ssh_password: Optional[str] = None ssh_private_key: Optional[str] = None # Path to private key file or key content # DCA Client Limits - max_daily_limit_gtq: Decimal = Decimal("2000") # Maximum daily limit for Fixed mode clients - - @validator('max_daily_limit_gtq', pre=True) + max_daily_limit_gtq: float = 2000.0 # Maximum daily limit for Fixed mode clients + + @validator('max_daily_limit_gtq') def round_max_daily_limit(cls, v): """Ensure max daily limit is rounded to 2 decimal places""" if v is not None: - d = Decimal(str(v)) if not isinstance(v, Decimal) else v - return d.quantize(Decimal("0.01")) + return round(float(v), 2) return v @@ -259,14 +204,7 @@ class LamassuConfig(BaseModel): last_poll_time: Optional[datetime] = None last_successful_poll: Optional[datetime] = None # DCA Client Limits - max_daily_limit_gtq: Decimal = Decimal("2000") # Maximum daily limit for Fixed mode clients - - @validator('max_daily_limit_gtq', pre=True) - def convert_max_daily_limit(cls, v): - return _to_decimal(v) - - class Config: - json_encoders = {Decimal: lambda v: float(v)} + max_daily_limit_gtq: float = 2000.0 # Maximum daily limit for Fixed mode clients class UpdateLamassuConfigData(BaseModel): @@ -288,6 +226,6 @@ class UpdateLamassuConfigData(BaseModel): ssh_password: Optional[str] = None ssh_private_key: Optional[str] = None # DCA Client Limits - max_daily_limit_gtq: Optional[Decimal] = None + max_daily_limit_gtq: Optional[int] = None diff --git a/pyproject.toml b/pyproject.toml index c5c417d..885fe45 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "satmachineadmin" -version = "0.0.0" +version = "0.0.4" description = "Eightball is a simple API that allows you to create a random number generator." authors = ["benarc", "dni "] diff --git a/tests/test_calculations.py b/tests/test_calculations.py index b175421..04262a0 100644 --- a/tests/test_calculations.py +++ b/tests/test_calculations.py @@ -10,7 +10,7 @@ from decimal import Decimal from typing import Dict, List, Tuple # Import from the parent package (following lnurlp pattern) -from ..calculations import calculate_commission, calculate_distribution, calculate_exchange_rate, to_decimal +from ..calculations import calculate_commission, calculate_distribution, calculate_exchange_rate # ============================================================================= @@ -245,12 +245,11 @@ class TestDistributionCalculation: # Convert each client's sats back to fiat total_fiat_distributed = sum( - Decimal(sats) / exchange_rate for sats in distributions.values() + sats / exchange_rate for sats in distributions.values() ) # Should equal original fiat amount (within small rounding tolerance) - fiat_decimal = to_decimal(fiat_amount) - assert abs(total_fiat_distributed - fiat_decimal) < Decimal("0.01"), \ + assert abs(total_fiat_distributed - fiat_amount) < 0.01, \ f"Fiat round-trip failed: {total_fiat_distributed:.2f} != {fiat_amount:.2f} " \ f"(crypto={crypto_atoms}, comm={comm_pct}, discount={discount})" @@ -288,12 +287,11 @@ class TestEmpiricalTransactions: "expected_base_sats": 259029, "expected_commission_sats": 7771, "expected_distributions": { - # 259029 / 2 = 129514.5 → both round to 129515 (ROUND_HALF_UP) - # Total = 259030, remainder = -1 - # Both have same fractional (-0.5), client_a is first alphabetically - # So client_a gets -1 adjustment - "client_a": 129514, - "client_b": 129515, + # 259029 / 2 = 129514.5 → both get 129514 or 129515 + # With banker's rounding: 129514.5 → 129514 (even) + # Remainder of 1 sat goes to first client by fractional sort + "client_a": 129515, + "client_b": 129514, }, }, # Add more scenarios from your real data! diff --git a/tests/test_integration.py b/tests/test_integration.py deleted file mode 100644 index fc1ff62..0000000 --- a/tests/test_integration.py +++ /dev/null @@ -1,828 +0,0 @@ -""" -Integration tests for the full transaction processing flow. - -These tests verify that data flows correctly from: - CSV parsing → Decimal conversion → calculations → model creation → CRUD - -This gives us confidence that real Lamassu transactions will be -processed correctly end-to-end. -""" - -import pytest -from decimal import Decimal -from datetime import datetime, timezone -from unittest.mock import AsyncMock, MagicMock, patch -from typing import Dict, Any -import csv -import io - -from ..calculations import calculate_commission, calculate_distribution, calculate_exchange_rate -from ..models import ( - CreateDcaPaymentData, - CreateLamassuTransactionData, - ClientBalanceSummary, - CreateDepositData, - DcaDeposit, - DcaPayment, - StoredLamassuTransaction, -) - - -# ============================================================================= -# TEST DATA: Real Lamassu CSV output format -# ============================================================================= - -# This simulates what execute_ssh_query receives from the database -LAMASSU_CSV_DATA = { - "8.75pct_large": """transaction_id,fiat_amount,crypto_amount,transaction_time,device_id,status,commission_percentage,discount,crypto_code,fiat_code -abc123,2000,309200,2025-01-10 14:30:00+00,device1,confirmed,0.0875,0,BTC,GTQ""", - - "5.5pct_no_discount": """transaction_id,fiat_amount,crypto_amount,transaction_time,device_id,status,commission_percentage,discount,crypto_code,fiat_code -def456,2000,309500,2025-01-10 15:00:00+00,device1,confirmed,0.055,0,BTC,GTQ""", - - "5.5pct_90pct_discount": """transaction_id,fiat_amount,crypto_amount,transaction_time,device_id,status,commission_percentage,discount,crypto_code,fiat_code -ghi789,800,115000,2025-01-10 16:00:00+00,device1,confirmed,0.055,90,BTC,GTQ""", - - "5.5pct_1300gtq_4clients": """transaction_id,fiat_amount,crypto_amount,transaction_time,device_id,status,commission_percentage,discount,crypto_code,fiat_code -jkl012,1300,205600,2025-01-10 17:00:00+00,device1,confirmed,0.055,0,BTC,GTQ""", -} - - -def parse_csv_like_transaction_processor(csv_data: str) -> Dict[str, Any]: - """ - Parse CSV data exactly like transaction_processor.execute_ssh_query does. - - This is a copy of the parsing logic to test it in isolation. - """ - reader = csv.DictReader(io.StringIO(csv_data)) - results = [] - for row in reader: - processed_row = {} - for key, value in row.items(): - if value == '' or value is None: - if key == 'crypto_amount': - processed_row[key] = 0 - elif key == 'fiat_amount': - processed_row[key] = Decimal("0") - elif key in ['commission_percentage', 'discount']: - processed_row[key] = Decimal("0") - else: - processed_row[key] = None - elif key in ['transaction_id', 'device_id', 'crypto_code', 'fiat_code', 'status']: - processed_row[key] = str(value) - elif key == 'crypto_amount': - try: - processed_row[key] = int(float(value)) - except (ValueError, TypeError): - processed_row[key] = 0 - elif key == 'fiat_amount': - try: - processed_row[key] = Decimal(str(value)) - except (ValueError, TypeError): - processed_row[key] = Decimal("0") - elif key in ['commission_percentage', 'discount']: - try: - processed_row[key] = Decimal(str(value)) - except (ValueError, TypeError): - processed_row[key] = Decimal("0") - elif key == 'transaction_time': - timestamp_str = value - if timestamp_str.endswith('+00'): - timestamp_str = timestamp_str + ':00' - elif timestamp_str.endswith('Z'): - timestamp_str = timestamp_str.replace('Z', '+00:00') - try: - dt = datetime.fromisoformat(timestamp_str) - except ValueError: - dt = datetime.now(timezone.utc) - if dt.tzinfo is None: - dt = dt.replace(tzinfo=timezone.utc) - elif dt.tzinfo != timezone.utc: - dt = dt.astimezone(timezone.utc) - processed_row[key] = dt - else: - processed_row[key] = value - results.append(processed_row) - return results[0] if results else {} - - -# ============================================================================= -# CSV PARSING TESTS -# ============================================================================= - -class TestCsvParsing: - """Test that CSV parsing produces correct Decimal types.""" - - def test_parse_8_75pct_transaction(self): - """Parse 8.75% commission transaction.""" - tx = parse_csv_like_transaction_processor(LAMASSU_CSV_DATA["8.75pct_large"]) - - assert tx["transaction_id"] == "abc123" - assert tx["crypto_amount"] == 309200 - assert tx["fiat_amount"] == Decimal("2000") - assert tx["commission_percentage"] == Decimal("0.0875") - assert tx["discount"] == Decimal("0") - assert isinstance(tx["fiat_amount"], Decimal) - assert isinstance(tx["commission_percentage"], Decimal) - assert isinstance(tx["discount"], Decimal) - - def test_parse_5_5pct_with_discount(self): - """Parse 5.5% commission with 90% discount.""" - tx = parse_csv_like_transaction_processor(LAMASSU_CSV_DATA["5.5pct_90pct_discount"]) - - assert tx["crypto_amount"] == 115000 - assert tx["fiat_amount"] == Decimal("800") - assert tx["commission_percentage"] == Decimal("0.055") - assert tx["discount"] == Decimal("90") - - def test_timestamp_parsing(self): - """Verify timestamp is parsed to UTC datetime.""" - tx = parse_csv_like_transaction_processor(LAMASSU_CSV_DATA["8.75pct_large"]) - - assert isinstance(tx["transaction_time"], datetime) - assert tx["transaction_time"].tzinfo == timezone.utc - - -# ============================================================================= -# END-TO-END CALCULATION TESTS -# ============================================================================= - -class TestEndToEndCalculations: - """ - Test the full flow: CSV → Decimal → calculations → expected results. - - These use the same empirical data as test_calculations.py but verify - the data flows correctly through parsing. - """ - - @pytest.mark.parametrize("csv_key,expected_base,expected_commission", [ - ("8.75pct_large", 284322, 24878), - ("5.5pct_no_discount", 293365, 16135), - ("5.5pct_90pct_discount", 114371, 629), - ("5.5pct_1300gtq_4clients", 194882, 10718), - ]) - def test_csv_to_commission_calculation(self, csv_key, expected_base, expected_commission): - """Verify CSV parsing → commission calculation produces expected results.""" - # Parse CSV like transaction_processor does - tx = parse_csv_like_transaction_processor(LAMASSU_CSV_DATA[csv_key]) - - # Calculate commission using parsed Decimal values - base, commission, effective = calculate_commission( - tx["crypto_amount"], - tx["commission_percentage"], - tx["discount"] - ) - - assert base == expected_base, f"Base mismatch for {csv_key}" - assert commission == expected_commission, f"Commission mismatch for {csv_key}" - assert base + commission == tx["crypto_amount"], "Invariant: base + commission = total" - - def test_full_distribution_flow_two_equal_clients(self): - """Test full flow with two equal-balance clients.""" - tx = parse_csv_like_transaction_processor(LAMASSU_CSV_DATA["8.75pct_large"]) - - # Calculate commission - base_sats, commission_sats, effective = calculate_commission( - tx["crypto_amount"], - tx["commission_percentage"], - tx["discount"] - ) - - # Simulate client balances (as would come from database) - client_balances = { - "client_a": Decimal("1000.00"), - "client_b": Decimal("1000.00"), - } - - # Calculate distribution - distributions = calculate_distribution(base_sats, client_balances) - - # Verify results - assert sum(distributions.values()) == base_sats - assert len(distributions) == 2 - # With equal balances, should be roughly equal (±1 sat for rounding) - assert abs(distributions["client_a"] - distributions["client_b"]) <= 1 - - def test_full_distribution_flow_four_clients(self): - """Test the 1300 GTQ transaction with 4 clients of varying balances.""" - tx = parse_csv_like_transaction_processor(LAMASSU_CSV_DATA["5.5pct_1300gtq_4clients"]) - - # Calculate commission - base_sats, commission_sats, effective = calculate_commission( - tx["crypto_amount"], - tx["commission_percentage"], - tx["discount"] - ) - - assert base_sats == 194882 - assert commission_sats == 10718 - - # Use the actual balance proportions from the real scenario - client_balances = { - "client_a": Decimal("1"), - "client_b": Decimal("986"), - "client_c": Decimal("14"), - "client_d": Decimal("4"), - } - - distributions = calculate_distribution(base_sats, client_balances) - - # Verify invariant - assert sum(distributions.values()) == base_sats - - # Verify proportions are reasonable - total_balance = sum(client_balances.values()) - for client_id, sats in distributions.items(): - expected_proportion = client_balances[client_id] / total_balance - actual_proportion = Decimal(sats) / Decimal(base_sats) - # Allow 1% tolerance for rounding - assert abs(actual_proportion - expected_proportion) < Decimal("0.01"), \ - f"Client {client_id} proportion off: {actual_proportion} vs {expected_proportion}" - - -# ============================================================================= -# MODEL CREATION TESTS -# ============================================================================= - -class TestModelCreation: - """Test that Pydantic models accept Decimal values correctly.""" - - def test_create_lamassu_transaction_data_with_decimals(self): - """Verify CreateLamassuTransactionData accepts Decimal values.""" - tx = parse_csv_like_transaction_processor(LAMASSU_CSV_DATA["8.75pct_large"]) - - base_sats, commission_sats, effective = calculate_commission( - tx["crypto_amount"], - tx["commission_percentage"], - tx["discount"] - ) - exchange_rate = calculate_exchange_rate(base_sats, tx["fiat_amount"]) - - # This should not raise any validation errors - data = CreateLamassuTransactionData( - lamassu_transaction_id=tx["transaction_id"], - fiat_amount=tx["fiat_amount"], - crypto_amount=tx["crypto_amount"], - commission_percentage=tx["commission_percentage"], - discount=tx["discount"], - effective_commission=effective, - commission_amount_sats=commission_sats, - base_amount_sats=base_sats, - exchange_rate=exchange_rate, - crypto_code=tx["crypto_code"], - fiat_code=tx["fiat_code"], - device_id=tx["device_id"], - transaction_time=tx["transaction_time"], - ) - - assert data.fiat_amount == Decimal("2000") - assert data.commission_percentage == Decimal("0.0875") - assert data.base_amount_sats == 284322 - assert data.commission_amount_sats == 24878 - - def test_create_dca_payment_data_with_decimals(self): - """Verify CreateDcaPaymentData accepts Decimal values.""" - tx = parse_csv_like_transaction_processor(LAMASSU_CSV_DATA["5.5pct_no_discount"]) - - base_sats, _, _ = calculate_commission( - tx["crypto_amount"], - tx["commission_percentage"], - tx["discount"] - ) - exchange_rate = calculate_exchange_rate(base_sats, tx["fiat_amount"]) - - # Simulate a client getting half the distribution - client_sats = base_sats // 2 - client_fiat = (Decimal(client_sats) / exchange_rate).quantize(Decimal("0.01")) - - # This should not raise any validation errors - data = CreateDcaPaymentData( - client_id="test_client_123", - amount_sats=client_sats, - amount_fiat=client_fiat, - exchange_rate=exchange_rate, - transaction_type="flow", - lamassu_transaction_id=tx["transaction_id"], - transaction_time=tx["transaction_time"], - ) - - assert isinstance(data.amount_fiat, Decimal) - assert isinstance(data.exchange_rate, Decimal) - assert data.amount_sats == client_sats - - def test_client_balance_summary_with_decimals(self): - """Verify ClientBalanceSummary accepts Decimal values.""" - summary = ClientBalanceSummary( - client_id="test_client", - total_deposits=Decimal("5000.00"), - total_payments=Decimal("1234.56"), - remaining_balance=Decimal("3765.44"), - currency="GTQ", - ) - - assert summary.remaining_balance == Decimal("3765.44") - assert isinstance(summary.total_deposits, Decimal) - - -# ============================================================================= -# EXCHANGE RATE PRECISION TESTS -# ============================================================================= - -class TestExchangeRatePrecision: - """Test that exchange rate calculations maintain precision.""" - - def test_exchange_rate_round_trip(self): - """Verify sats → fiat → sats round-trip maintains precision.""" - tx = parse_csv_like_transaction_processor(LAMASSU_CSV_DATA["5.5pct_1300gtq_4clients"]) - - base_sats, _, _ = calculate_commission( - tx["crypto_amount"], - tx["commission_percentage"], - tx["discount"] - ) - - exchange_rate = calculate_exchange_rate(base_sats, tx["fiat_amount"]) - - # Convert sats to fiat and back - fiat_equivalent = Decimal(base_sats) / exchange_rate - sats_back = int((fiat_equivalent * exchange_rate).quantize(Decimal("1"))) - - # Should be within 1 sat of original - assert abs(sats_back - base_sats) <= 1 - - def test_per_client_fiat_sums_to_total(self): - """Verify per-client fiat amounts sum to total fiat (within tolerance).""" - tx = parse_csv_like_transaction_processor(LAMASSU_CSV_DATA["5.5pct_1300gtq_4clients"]) - - base_sats, _, _ = calculate_commission( - tx["crypto_amount"], - tx["commission_percentage"], - tx["discount"] - ) - - exchange_rate = calculate_exchange_rate(base_sats, tx["fiat_amount"]) - - client_balances = { - "client_a": Decimal("1"), - "client_b": Decimal("986"), - "client_c": Decimal("14"), - "client_d": Decimal("4"), - } - - distributions = calculate_distribution(base_sats, client_balances) - - # Calculate per-client fiat and sum - total_fiat_distributed = Decimal("0") - for client_id, sats in distributions.items(): - client_fiat = (Decimal(sats) / exchange_rate).quantize(Decimal("0.01")) - total_fiat_distributed += client_fiat - - # Should be within 0.05 GTQ of original (accounting for per-client rounding) - # This is the 0.01 discrepancy we discussed, multiplied by number of clients - assert abs(total_fiat_distributed - tx["fiat_amount"]) < Decimal("0.05"), \ - f"Total distributed {total_fiat_distributed} vs original {tx['fiat_amount']}" - - -# ============================================================================= -# CRUD LAYER TESTS (with mocked database) -# ============================================================================= - -class TestCrudLayerDecimalHandling: - """ - Test that CRUD operations handle Decimal→float conversion for SQLite. - - These tests mock the database layer to verify: - 1. Decimal values from models are converted to float via prepare_for_db() - 2. Float values from SQLite are converted back to Decimal via model validators - """ - - @pytest.mark.asyncio - async def test_create_deposit_passes_float_amount_for_sqlite(self): - """Verify create_deposit passes float amount to database (SQLite compatibility).""" - from unittest.mock import patch, AsyncMock - - # Create deposit data with Decimal - deposit_data = CreateDepositData( - client_id="test_client_123", - amount=Decimal("1500.75"), - currency="GTQ", - notes="Test deposit" - ) - - # Verify the model stored it as Decimal - assert isinstance(deposit_data.amount, Decimal) - assert deposit_data.amount == Decimal("1500.75") - - # Mock the database - mock_db = AsyncMock() - mock_db.execute = AsyncMock() - mock_db.fetchone = AsyncMock(return_value=DcaDeposit( - id="deposit_123", - client_id="test_client_123", - amount=1500.75, # float from SQLite - currency="GTQ", - status="pending", - notes="Test deposit", - created_at=datetime.now(timezone.utc), - confirmed_at=None - )) - - with patch('satmachineadmin.crud.db', mock_db): - from ..crud import create_deposit - result = await create_deposit(deposit_data) - - # Verify db.execute was called - mock_db.execute.assert_called_once() - - # Get the parameters passed to execute - call_args = mock_db.execute.call_args - params = call_args[0][1] # Second positional arg is the params dict - - # Verify the amount parameter is float (for SQLite compatibility) - assert isinstance(params["amount"], float) - assert params["amount"] == 1500.75 - - @pytest.mark.asyncio - async def test_create_dca_payment_passes_float_values_for_sqlite(self): - """Verify create_dca_payment passes float fiat and exchange_rate (SQLite compatibility).""" - from unittest.mock import patch, AsyncMock - - # Parse a real transaction - tx = parse_csv_like_transaction_processor(LAMASSU_CSV_DATA["5.5pct_no_discount"]) - base_sats, _, _ = calculate_commission( - tx["crypto_amount"], - tx["commission_percentage"], - tx["discount"] - ) - exchange_rate = calculate_exchange_rate(base_sats, tx["fiat_amount"]) - - # Create payment data - client_sats = 146682 # Half of base - client_fiat = (Decimal(client_sats) / exchange_rate).quantize(Decimal("0.01")) - - payment_data = CreateDcaPaymentData( - client_id="test_client", - amount_sats=client_sats, - amount_fiat=client_fiat, - exchange_rate=exchange_rate, - transaction_type="flow", - lamassu_transaction_id="def456", - transaction_time=tx["transaction_time"], - ) - - # Verify model has Decimal types - assert isinstance(payment_data.amount_fiat, Decimal) - assert isinstance(payment_data.exchange_rate, Decimal) - - # Mock database - returns float like SQLite would - mock_db = AsyncMock() - mock_db.execute = AsyncMock() - mock_db.fetchone = AsyncMock(return_value=DcaPayment( - id="payment_123", - client_id="test_client", - amount_sats=client_sats, - amount_fiat=float(client_fiat), # float from SQLite - exchange_rate=float(exchange_rate), # float from SQLite - transaction_type="flow", - lamassu_transaction_id="def456", - payment_hash=None, - status="pending", - created_at=datetime.now(timezone.utc), - transaction_time=tx["transaction_time"], - )) - - with patch('satmachineadmin.crud.db', mock_db): - from ..crud import create_dca_payment - result = await create_dca_payment(payment_data) - - # Verify db.execute was called - mock_db.execute.assert_called_once() - - # Get params - call_args = mock_db.execute.call_args - params = call_args[0][1] - - # Verify float types in params (SQLite compatibility) - assert isinstance(params["amount_fiat"], float) - assert isinstance(params["exchange_rate"], float) - - @pytest.mark.asyncio - async def test_create_lamassu_transaction_passes_floats_for_sqlite(self): - """Verify create_lamassu_transaction passes float fields (SQLite compatibility).""" - from unittest.mock import patch, AsyncMock - - tx = parse_csv_like_transaction_processor(LAMASSU_CSV_DATA["8.75pct_large"]) - base_sats, commission_sats, effective = calculate_commission( - tx["crypto_amount"], - tx["commission_percentage"], - tx["discount"] - ) - exchange_rate = calculate_exchange_rate(base_sats, tx["fiat_amount"]) - - transaction_data = CreateLamassuTransactionData( - lamassu_transaction_id=tx["transaction_id"], - fiat_amount=tx["fiat_amount"], - crypto_amount=tx["crypto_amount"], - commission_percentage=tx["commission_percentage"], - discount=tx["discount"], - effective_commission=effective, - commission_amount_sats=commission_sats, - base_amount_sats=base_sats, - exchange_rate=exchange_rate, - crypto_code=tx["crypto_code"], - fiat_code=tx["fiat_code"], - device_id=tx["device_id"], - transaction_time=tx["transaction_time"], - ) - - # Verify all Decimal fields in the model - assert isinstance(transaction_data.fiat_amount, Decimal) - assert isinstance(transaction_data.commission_percentage, Decimal) - assert isinstance(transaction_data.discount, Decimal) - assert isinstance(transaction_data.effective_commission, Decimal) - assert isinstance(transaction_data.exchange_rate, Decimal) - - # Mock database - returns floats like SQLite would - mock_db = AsyncMock() - mock_db.execute = AsyncMock() - mock_db.fetchone = AsyncMock(return_value=StoredLamassuTransaction( - id="tx_123", - lamassu_transaction_id=tx["transaction_id"], - fiat_amount=float(tx["fiat_amount"]), # float from SQLite - crypto_amount=tx["crypto_amount"], - commission_percentage=float(tx["commission_percentage"]), # float from SQLite - discount=float(tx["discount"]), # float from SQLite - effective_commission=float(effective), # float from SQLite - commission_amount_sats=commission_sats, - base_amount_sats=base_sats, - exchange_rate=float(exchange_rate), # float from SQLite - crypto_code=tx["crypto_code"], - fiat_code=tx["fiat_code"], - device_id=tx["device_id"], - transaction_time=tx["transaction_time"], - processed_at=datetime.now(timezone.utc), - clients_count=0, - distributions_total_sats=0, - )) - - with patch('satmachineadmin.crud.db', mock_db): - from ..crud import create_lamassu_transaction - result = await create_lamassu_transaction(transaction_data) - - # Verify db.execute was called - mock_db.execute.assert_called_once() - - # Get params - call_args = mock_db.execute.call_args - params = call_args[0][1] - - # Verify all Decimal fields are converted to float for SQLite - assert isinstance(params["fiat_amount"], float), f"fiat_amount is {type(params['fiat_amount'])}" - assert isinstance(params["commission_percentage"], float) - assert isinstance(params["discount"], float) - assert isinstance(params["effective_commission"], float) - assert isinstance(params["exchange_rate"], float) - - # Verify values match - assert params["fiat_amount"] == 2000.0 - assert params["commission_percentage"] == 0.0875 - assert params["base_amount_sats"] == 284322 - assert params["commission_amount_sats"] == 24878 - - @pytest.mark.asyncio - async def test_client_balance_summary_returns_decimals(self): - """Verify get_client_balance_summary returns Decimal types.""" - from unittest.mock import patch, AsyncMock - - # Mock database responses - mock_db = AsyncMock() - - # Mock deposits query result - mock_db.fetchone = AsyncMock(side_effect=[ - # First call: deposits sum - {"total": Decimal("5000.00"), "currency": "GTQ"}, - # Second call: payments sum - {"total": Decimal("1234.56")}, - ]) - - with patch('satmachineadmin.crud.db', mock_db): - from ..crud import get_client_balance_summary - result = await get_client_balance_summary("test_client") - - # Verify result has Decimal types - assert isinstance(result.total_deposits, Decimal) - assert isinstance(result.total_payments, Decimal) - assert isinstance(result.remaining_balance, Decimal) - - # Verify values - assert result.total_deposits == Decimal("5000.00") - assert result.total_payments == Decimal("1234.56") - assert result.remaining_balance == Decimal("3765.44") - - -# ============================================================================= -# SQLITE ROUND-TRIP TESTS -# ============================================================================= - -class TestSqliteDecimalRoundTrip: - """ - Test that Decimal values survive the SQLite round-trip: - Decimal → float (prepare_for_db) → SQLite REAL → float → Decimal (validator) - - SQLite doesn't support Decimal natively - it stores DECIMAL as REAL (float). - We use prepare_for_db() to convert Decimal→float before writing, - and Pydantic validators with pre=True to convert float→Decimal on read. - """ - - def test_prepare_for_db_converts_decimals_to_float(self): - """Verify prepare_for_db converts Decimal to float.""" - from ..crud import prepare_for_db - - values = { - "amount": Decimal("1500.75"), - "exchange_rate": Decimal("146.6825"), - "commission": Decimal("0.0875"), - "name": "test", # Non-Decimal should pass through - "count": 42, # Non-Decimal should pass through - } - - result = prepare_for_db(values) - - # Decimals should be converted to float - assert isinstance(result["amount"], float) - assert isinstance(result["exchange_rate"], float) - assert isinstance(result["commission"], float) - - # Values should be preserved - assert result["amount"] == 1500.75 - assert result["exchange_rate"] == 146.6825 - assert result["commission"] == 0.0875 - - # Non-Decimals should pass through unchanged - assert result["name"] == "test" - assert result["count"] == 42 - - def test_model_validator_converts_float_to_decimal(self): - """Verify model validators convert float back to Decimal.""" - # Simulate what comes back from SQLite (floats) - db_row = { - "id": "deposit_123", - "client_id": "client_abc", - "amount": 1500.75, # float from SQLite - "currency": "GTQ", - "status": "pending", - "notes": "Test", - "created_at": datetime.now(timezone.utc), - "confirmed_at": None, - } - - # Create model from "database" data - deposit = DcaDeposit(**db_row) - - # Validator should have converted float → Decimal - assert isinstance(deposit.amount, Decimal) - assert deposit.amount == Decimal("1500.75") - - def test_payment_model_converts_multiple_floats(self): - """Verify DcaPayment converts all float fields to Decimal.""" - db_row = { - "id": "payment_123", - "client_id": "client_abc", - "amount_sats": 146682, - "amount_fiat": 1000.50, # float from SQLite - "exchange_rate": 146.6825, # float from SQLite - "transaction_type": "flow", - "lamassu_transaction_id": "tx_456", - "payment_hash": None, - "status": "confirmed", - "created_at": datetime.now(timezone.utc), - "transaction_time": datetime.now(timezone.utc), - } - - payment = DcaPayment(**db_row) - - assert isinstance(payment.amount_fiat, Decimal) - assert isinstance(payment.exchange_rate, Decimal) - assert payment.amount_fiat == Decimal("1000.50") - assert payment.exchange_rate == Decimal("146.6825") - - def test_stored_transaction_converts_all_decimal_fields(self): - """Verify StoredLamassuTransaction converts all float fields.""" - db_row = { - "id": "tx_123", - "lamassu_transaction_id": "lamassu_abc", - "fiat_amount": 2000.00, # float from SQLite - "crypto_amount": 309200, - "commission_percentage": 0.0875, # float from SQLite - "discount": 0.0, # float from SQLite - "effective_commission": 0.0875, # float from SQLite - "commission_amount_sats": 24878, - "base_amount_sats": 284322, - "exchange_rate": 142.161, # float from SQLite - "crypto_code": "BTC", - "fiat_code": "GTQ", - "device_id": "device1", - "transaction_time": datetime.now(timezone.utc), - "processed_at": datetime.now(timezone.utc), - "clients_count": 2, - "distributions_total_sats": 284322, - } - - tx = StoredLamassuTransaction(**db_row) - - # All Decimal fields should be converted - assert isinstance(tx.fiat_amount, Decimal) - assert isinstance(tx.commission_percentage, Decimal) - assert isinstance(tx.discount, Decimal) - assert isinstance(tx.effective_commission, Decimal) - assert isinstance(tx.exchange_rate, Decimal) - - # Values should be preserved - assert tx.fiat_amount == Decimal("2000.00") - assert tx.commission_percentage == Decimal("0.0875") - assert tx.discount == Decimal("0.0") - assert tx.effective_commission == Decimal("0.0875") - - def test_full_round_trip_preserves_precision(self): - """ - Test complete round-trip: Decimal → prepare_for_db → float → Decimal. - - This simulates what happens in production: - 1. We have Decimal values from calculations - 2. prepare_for_db converts them to float for SQLite - 3. SQLite stores as REAL - 4. We read back as float - 5. Model validators convert back to Decimal - """ - from ..crud import prepare_for_db - - # Original Decimal values (from calculations) - original = { - "fiat_amount": Decimal("1300.00"), - "commission_percentage": Decimal("0.055"), - "discount": Decimal("0"), - "effective_commission": Decimal("0.055"), - "exchange_rate": Decimal("149.90215490109"), # High precision - } - - # Step 1: Convert for database storage - for_db = prepare_for_db(original) - - # Verify all are floats - for key in original: - assert isinstance(for_db[key], float), f"{key} should be float" - - # Step 2: Simulate SQLite storage and retrieval - # (SQLite would store these as REAL and return as float) - from_db = for_db.copy() - - # Step 3: Create a model (simulating what happens when reading) - db_row = { - "id": "tx_test", - "lamassu_transaction_id": "test_123", - "fiat_amount": from_db["fiat_amount"], - "crypto_amount": 205600, - "commission_percentage": from_db["commission_percentage"], - "discount": from_db["discount"], - "effective_commission": from_db["effective_commission"], - "commission_amount_sats": 10718, - "base_amount_sats": 194882, - "exchange_rate": from_db["exchange_rate"], - "crypto_code": "BTC", - "fiat_code": "GTQ", - "device_id": "device1", - "transaction_time": datetime.now(timezone.utc), - "processed_at": datetime.now(timezone.utc), - "clients_count": 1, - "distributions_total_sats": 194882, - } - - tx = StoredLamassuTransaction(**db_row) - - # Step 4: Verify Decimal types restored - assert isinstance(tx.fiat_amount, Decimal) - assert isinstance(tx.commission_percentage, Decimal) - assert isinstance(tx.exchange_rate, Decimal) - - # Step 5: Verify precision for 2 decimal place values - assert tx.fiat_amount == Decimal("1300.0") # SQLite preserves this exactly - assert tx.commission_percentage == Decimal("0.055") - assert tx.discount == Decimal("0.0") - - def test_client_balance_summary_float_conversion(self): - """Verify ClientBalanceSummary converts floats from aggregation queries.""" - # SQLite SUM() returns float - db_row = { - "client_id": "client_123", - "total_deposits": 5000.00, # float from SUM() - "total_payments": 1234.56, # float from SUM() - "remaining_balance": 3765.44, # calculated float - "currency": "GTQ", - } - - summary = ClientBalanceSummary(**db_row) - - assert isinstance(summary.total_deposits, Decimal) - assert isinstance(summary.total_payments, Decimal) - assert isinstance(summary.remaining_balance, Decimal) - assert summary.remaining_balance == Decimal("3765.44") diff --git a/transaction_processor.py b/transaction_processor.py index deaac47..661e5ab 100644 --- a/transaction_processor.py +++ b/transaction_processor.py @@ -3,7 +3,6 @@ import asyncio import asyncpg from datetime import datetime, timedelta, timezone -from decimal import Decimal from typing import List, Optional, Dict, Any from loguru import logger import socket @@ -493,38 +492,28 @@ class LamassuTransactionProcessor: results = [] for row in reader: # Convert string values to appropriate types - # Use Decimal for monetary and percentage values processed_row = {} for key, value in row.items(): # Handle None/empty values consistently at data ingestion boundary if value == '' or value is None: - if key == 'crypto_amount': - processed_row[key] = 0 # Sats are always int - elif key == 'fiat_amount': - processed_row[key] = Decimal("0") # Fiat as Decimal + if key in ['fiat_amount', 'crypto_amount']: + processed_row[key] = 0 # Default numeric fields to 0 elif key in ['commission_percentage', 'discount']: - processed_row[key] = Decimal("0") # Percentages as Decimal + processed_row[key] = 0.0 # Default percentage fields to 0.0 else: processed_row[key] = None # Keep None for non-numeric fields elif key in ['transaction_id', 'device_id', 'crypto_code', 'fiat_code']: processed_row[key] = str(value) - elif key == 'crypto_amount': + elif key in ['fiat_amount', 'crypto_amount']: try: - processed_row[key] = int(float(value)) # Sats are always int + processed_row[key] = int(float(value)) except (ValueError, TypeError): - processed_row[key] = 0 - elif key == 'fiat_amount': - try: - # Convert via string to avoid float precision issues - processed_row[key] = Decimal(str(value)) - except (ValueError, TypeError): - processed_row[key] = Decimal("0") + processed_row[key] = 0 # Fallback to 0 for invalid values elif key in ['commission_percentage', 'discount']: try: - # Convert via string to avoid float precision issues - processed_row[key] = Decimal(str(value)) + processed_row[key] = float(value) except (ValueError, TypeError): - processed_row[key] = Decimal("0") + processed_row[key] = 0.0 # Fallback to 0.0 for invalid values elif key == 'transaction_time': from datetime import datetime # Parse PostgreSQL timestamp format and ensure it's in UTC for consistency @@ -680,23 +669,29 @@ class LamassuTransactionProcessor: logger.error(f"Error fetching transactions from Lamassu database: {e}") return [] - async def calculate_distribution_amounts(self, transaction: Dict[str, Any]) -> Dict[str, int]: - """Calculate how much each Flow Mode client should receive""" + async def calculate_distribution_amounts(self, transaction: Dict[str, Any]) -> tuple[Dict[str, Any], int]: + """Calculate how much each Flow Mode client should receive. + + Returns: + tuple: (distributions dict, orphan_sats int) + - distributions: {client_id: {fiat_amount, sats_amount, exchange_rate}} + - orphan_sats: sats that couldn't be distributed due to sync mismatch + """ try: # Get all active Flow Mode clients flow_clients = await get_flow_mode_clients() - + if not flow_clients: logger.info("No Flow Mode clients found - skipping distribution") - return {} + return {}, 0 - # Extract transaction details - guaranteed clean from data ingestion (Decimal types) + # Extract transaction details - guaranteed clean from data ingestion crypto_atoms = transaction.get("crypto_amount", 0) # Total sats with commission baked in - fiat_amount = transaction.get("fiat_amount", Decimal("0")) # Actual fiat dispensed (principal only) - commission_percentage = transaction.get("commission_percentage", Decimal("0")) # Already stored as Decimal (e.g., 0.045) - discount = transaction.get("discount", Decimal("0")) # Discount percentage + fiat_amount = transaction.get("fiat_amount", 0) # Actual fiat dispensed (principal only) + commission_percentage = transaction.get("commission_percentage", 0.0) # Already stored as decimal (e.g., 0.045) + discount = transaction.get("discount", 0.0) # Discount percentage transaction_time = transaction.get("transaction_time") # ATM transaction timestamp for temporal accuracy - + # Normalize transaction_time to UTC if present if transaction_time is not None: if transaction_time.tzinfo is None: @@ -708,20 +703,20 @@ class LamassuTransactionProcessor: original_tz = transaction_time.tzinfo transaction_time = transaction_time.astimezone(timezone.utc) logger.info(f"Converted transaction time from {original_tz} to UTC") - + # Validate required fields if crypto_atoms is None: logger.error(f"Missing crypto_amount in transaction: {transaction}") - return {} + return {}, 0 if fiat_amount is None: logger.error(f"Missing fiat_amount in transaction: {transaction}") - return {} + return {}, 0 if commission_percentage is None: logger.warning(f"Missing commission_percentage in transaction: {transaction}, defaulting to 0") - commission_percentage = Decimal("0") + commission_percentage = 0.0 if discount is None: logger.info(f"Missing discount in transaction: {transaction}, defaulting to 0") - discount = Decimal("0") + discount = 0.0 if transaction_time is None: logger.warning(f"Missing transaction_time in transaction: {transaction}") # Could use current time as fallback, but this indicates a data issue @@ -744,16 +739,15 @@ class LamassuTransactionProcessor: logger.warning("No transaction time available - using current balances (may be inaccurate)") # Get balance summaries for all clients to calculate proportions - client_balances: Dict[str, Decimal] = {} - total_confirmed_deposits = Decimal("0") - min_balance = Decimal("0.01") - + client_balances = {} + total_confirmed_deposits = 0 + for client in flow_clients: # Get balance as of the transaction time for temporal accuracy balance = await get_client_balance_summary(client.id, as_of_time=transaction_time) # Only include clients with positive remaining balance # NOTE: This works for fiat amounts that use cents - if balance.remaining_balance >= min_balance: + if balance.remaining_balance >= 0.01: client_balances[client.id] = balance.remaining_balance total_confirmed_deposits += balance.remaining_balance logger.debug(f"Client {client.id[:8]}... included with balance: {balance.remaining_balance:.2f} GTQ") @@ -762,40 +756,76 @@ class LamassuTransactionProcessor: if total_confirmed_deposits == 0: logger.info("No clients with remaining DCA balance - skipping distribution") - return {} + return {}, 0 - # Calculate sat allocations using the extracted pure function - sat_allocations = calculate_distribution(base_crypto_atoms, client_balances) + # Detect sync mismatch: more money in ATM than tracked client balances + sync_mismatch = total_confirmed_deposits < fiat_amount + if sync_mismatch: + orphan_fiat = fiat_amount - total_confirmed_deposits + logger.warning( + f"Sync mismatch detected: tracked balances ({total_confirmed_deposits:.2f} GTQ) " + f"< transaction ({fiat_amount} GTQ). Orphan amount: {orphan_fiat:.2f} GTQ" + ) - if not sat_allocations: - logger.info("No allocations calculated - skipping distribution") - return {} - - # Build final distributions dict with additional tracking fields + # Calculate distribution amounts distributions = {} - for client_id, client_sats_amount in sat_allocations.items(): - # Calculate proportion for logging - proportion = client_balances[client_id] / total_confirmed_deposits - # Calculate equivalent fiat value in GTQ for tracking purposes - if exchange_rate > 0: - client_fiat_amount = (Decimal(client_sats_amount) / exchange_rate).quantize(Decimal("0.01")) - else: - client_fiat_amount = Decimal("0") + if sync_mismatch: + # SYNC MISMATCH MODE: Cap each client's allocation to their remaining fiat balance + # Each client gets sats equivalent to their full remaining balance + for client_id, client_balance in client_balances.items(): + # Calculate sats equivalent to this client's remaining fiat balance + client_sats_amount = round(client_balance * exchange_rate) + proportion = client_balance / total_confirmed_deposits - distributions[client_id] = { - "fiat_amount": client_fiat_amount, - "sats_amount": client_sats_amount, - "exchange_rate": exchange_rate - } + # Calculate equivalent fiat value in GTQ for tracking purposes + client_fiat_amount = round(client_sats_amount / exchange_rate, 2) if exchange_rate > 0 else 0.0 - logger.info(f"Client {client_id[:8]}... gets {client_sats_amount} sats (≈{client_fiat_amount:.2f} GTQ, {proportion:.2%} share)") - - # Verification: ensure total distribution equals base amount - total_distributed = sum(dist["sats_amount"] for dist in distributions.values()) - if total_distributed != base_crypto_atoms: - logger.error(f"Distribution mismatch! Expected: {base_crypto_atoms} sats, Distributed: {total_distributed} sats") - raise ValueError(f"Satoshi distribution calculation error: {base_crypto_atoms} != {total_distributed}") + distributions[client_id] = { + "fiat_amount": client_fiat_amount, + "sats_amount": client_sats_amount, + "exchange_rate": exchange_rate + } + + logger.info(f"Client {client_id[:8]}... gets {client_sats_amount} sats (≈{client_fiat_amount:.2f} GTQ, {proportion:.2%} share)") + + # Calculate orphan sats (difference between base amount and distributed) + total_distributed = sum(dist["sats_amount"] for dist in distributions.values()) + orphan_sats = base_crypto_atoms - total_distributed + logger.info( + f"Sync mismatch distribution: {total_distributed} sats to clients, " + f"{orphan_sats} sats orphaned (staying in source wallet)" + ) + else: + # NORMAL MODE: Proportional distribution based on transaction amount + sat_allocations = calculate_distribution(base_crypto_atoms, client_balances) + + if not sat_allocations: + logger.info("No allocations calculated - skipping distribution") + return {}, 0 + + # Build final distributions dict with additional tracking fields + for client_id, client_sats_amount in sat_allocations.items(): + # Calculate proportion for logging + proportion = client_balances[client_id] / total_confirmed_deposits + + # Calculate equivalent fiat value in GTQ for tracking purposes + client_fiat_amount = round(client_sats_amount / exchange_rate, 2) if exchange_rate > 0 else 0.0 + + distributions[client_id] = { + "fiat_amount": client_fiat_amount, + "sats_amount": client_sats_amount, + "exchange_rate": exchange_rate + } + + logger.info(f"Client {client_id[:8]}... gets {client_sats_amount} sats (≈{client_fiat_amount:.2f} GTQ, {proportion:.2%} share)") + + # Verification: ensure total distribution equals base amount + total_distributed = sum(dist["sats_amount"] for dist in distributions.values()) + if total_distributed != base_crypto_atoms: + logger.error(f"Distribution mismatch! Expected: {base_crypto_atoms} sats, Distributed: {total_distributed} sats") + raise ValueError(f"Satoshi distribution calculation error: {base_crypto_atoms} != {total_distributed}") + orphan_sats = 0 # Safety check: Re-verify all clients still have positive balances before finalizing distributions # This prevents race conditions where balances changed during calculation @@ -815,18 +845,18 @@ class LamassuTransactionProcessor: # Recalculate proportions if some clients were rejected if len(final_distributions) == 0: logger.info("All clients rejected due to negative balances - no distributions") - return {} - + return {}, orphan_sats + # For simplicity, we'll still return the original distributions but log the warning # In a production system, you might want to recalculate the entire distribution logger.warning("Proceeding with original distribution despite balance warnings - manual review recommended") - + logger.info(f"Distribution verified: {total_distributed} sats distributed across {len(distributions)} clients (clients with positive allocations only)") - return distributions - + return distributions, orphan_sats + except Exception as e: logger.error(f"Error calculating distribution amounts: {e}") - return {} + return {}, 0 async def distribute_to_clients(self, transaction: Dict[str, Any], distributions: Dict[str, Dict[str, int]]) -> None: """Send Bitcoin payments to DCA clients""" @@ -935,7 +965,7 @@ class LamassuTransactionProcessor: } new_payment = await create_invoice( wallet_id=target_wallet.id, - amount=float(amount_sats), # LNBits expects float for amount + amount=float(amount_sats), # LNBits create_invoice expects float internal=True, # Internal transfer within LNBits memo=memo, extra=extra @@ -1018,20 +1048,20 @@ class LamassuTransactionProcessor: async def store_lamassu_transaction(self, transaction: Dict[str, Any]) -> Optional[str]: """Store the Lamassu transaction in our database for audit and UI""" try: - # Extract transaction data - guaranteed clean from data ingestion boundary (Decimal types) + # Extract transaction data - guaranteed clean from data ingestion boundary crypto_atoms = transaction.get("crypto_amount", 0) - fiat_amount = transaction.get("fiat_amount", Decimal("0")) - commission_percentage = transaction.get("commission_percentage", Decimal("0")) - discount = transaction.get("discount", Decimal("0")) + fiat_amount = transaction.get("fiat_amount", 0) + commission_percentage = transaction.get("commission_percentage", 0.0) + discount = transaction.get("discount", 0.0) transaction_time = transaction.get("transaction_time") - + # Normalize transaction_time to UTC if present if transaction_time is not None: if transaction_time.tzinfo is None: transaction_time = transaction_time.replace(tzinfo=timezone.utc) elif transaction_time.tzinfo != timezone.utc: transaction_time = transaction_time.astimezone(timezone.utc) - + # Calculate commission metrics using the extracted pure function base_crypto_atoms, commission_amount_sats, effective_commission = calculate_commission( crypto_atoms, commission_percentage, discount @@ -1039,13 +1069,11 @@ class LamassuTransactionProcessor: # Calculate exchange rate exchange_rate = calculate_exchange_rate(base_crypto_atoms, fiat_amount) - - # Create transaction data with GTQ amounts (Decimal already, just ensure 2 decimal places) - fiat_amount_rounded = fiat_amount.quantize(Decimal("0.01")) if isinstance(fiat_amount, Decimal) else Decimal(str(fiat_amount)).quantize(Decimal("0.01")) - + + # Create transaction data with GTQ amounts transaction_data = CreateLamassuTransactionData( lamassu_transaction_id=transaction["transaction_id"], - fiat_amount=fiat_amount_rounded, + fiat_amount=round(fiat_amount, 2), # Store GTQ with 2 decimal places crypto_amount=crypto_atoms, commission_percentage=commission_percentage, discount=discount, @@ -1102,7 +1130,7 @@ class LamassuTransactionProcessor: commission_payment = await create_invoice( wallet_id=admin_config.commission_wallet_id, - amount=float(commission_amount_sats), # LNBits expects float + amount=float(commission_amount_sats), # LNbits create_invoice expects float internal=True, memo=commission_memo, extra={ @@ -1157,17 +1185,23 @@ class LamassuTransactionProcessor: stored_transaction = await self.store_lamassu_transaction(transaction) # Calculate distribution amounts - distributions = await self.calculate_distribution_amounts(transaction) - + distributions, orphan_sats = await self.calculate_distribution_amounts(transaction) + if not distributions: - logger.info(f"No distributions calculated for transaction {transaction_id}") + if orphan_sats > 0: + logger.warning( + f"No client distributions for transaction {transaction_id}, " + f"but {orphan_sats} orphan sats remain in source wallet" + ) + else: + logger.info(f"No distributions calculated for transaction {transaction_id}") return # Calculate commission amount for sending to commission wallet crypto_atoms = transaction.get("crypto_amount", 0) - commission_percentage = transaction.get("commission_percentage", Decimal("0")) - discount = transaction.get("discount", Decimal("0")) - + commission_percentage = transaction.get("commission_percentage", 0.0) + discount = transaction.get("discount", 0.0) + # Calculate commission amount using the extracted pure function _, commission_amount_sats, _ = calculate_commission( crypto_atoms, commission_percentage, discount