diff --git a/crud.py b/crud.py index 94c1d20..2516f52 100644 --- a/crud.py +++ b/crud.py @@ -1,5 +1,6 @@ # Description: This file contains the CRUD operations for talking to the database. +from decimal import Decimal from typing import List, Optional, Union from datetime import datetime, timezone @@ -18,16 +19,33 @@ from .models import ( db = Database("ext_satoshimachine") +def prepare_for_db(values: dict) -> dict: + """ + Convert Decimal values to float for SQLite compatibility. + + SQLite doesn't support Decimal natively - it stores DECIMAL as REAL (float). + This function converts Decimal values to float before database writes. + The Pydantic models handle converting float back to Decimal on read. + """ + result = {} + for k, v in values.items(): + if isinstance(v, Decimal): + result[k] = float(v) + else: + result[k] = v + return result + + # DCA Client CRUD Operations async def create_dca_client(data: CreateDcaClientData) -> DcaClient: client_id = urlsafe_short_hash() await db.execute( """ - INSERT INTO satoshimachine.dca_clients + INSERT INTO satoshimachine.dca_clients (id, user_id, wallet_id, username, dca_mode, fixed_mode_daily_limit, status, created_at, updated_at) VALUES (:id, :user_id, :wallet_id, :username, :dca_mode, :fixed_mode_daily_limit, :status, :created_at, :updated_at) """, - { + prepare_for_db({ "id": client_id, "user_id": data.user_id, "wallet_id": data.wallet_id, @@ -37,7 +55,7 @@ async def create_dca_client(data: CreateDcaClientData) -> DcaClient: "status": "active", "created_at": datetime.now(), "updated_at": datetime.now() - } + }) ) return await get_dca_client(client_id) @@ -69,14 +87,14 @@ async def update_dca_client(client_id: str, data: UpdateDcaClientData) -> Option update_data = {k: v for k, v in data.dict().items() if v is not None} if not update_data: return await get_dca_client(client_id) - + update_data["updated_at"] = datetime.now() set_clause = ", ".join([f"{k} = :{k}" for k in update_data.keys()]) update_data["id"] = client_id - + await db.execute( f"UPDATE satoshimachine.dca_clients SET {set_clause} WHERE id = :id", - update_data + prepare_for_db(update_data) ) return await get_dca_client(client_id) @@ -93,11 +111,11 @@ async def create_deposit(data: CreateDepositData) -> DcaDeposit: deposit_id = urlsafe_short_hash() await db.execute( """ - INSERT INTO satoshimachine.dca_deposits + INSERT INTO satoshimachine.dca_deposits (id, client_id, amount, currency, status, notes, created_at) VALUES (:id, :client_id, :amount, :currency, :status, :notes, :created_at) """, - { + prepare_for_db({ "id": deposit_id, "client_id": data.client_id, "amount": data.amount, @@ -105,7 +123,7 @@ async def create_deposit(data: CreateDepositData) -> DcaDeposit: "status": "pending", "notes": data.notes, "created_at": datetime.now() - } + }) ) return await get_deposit(deposit_id) @@ -158,13 +176,13 @@ async def create_dca_payment(data: CreateDcaPaymentData) -> DcaPayment: payment_id = urlsafe_short_hash() await db.execute( """ - INSERT INTO satoshimachine.dca_payments - (id, client_id, amount_sats, amount_fiat, exchange_rate, transaction_type, + INSERT INTO satoshimachine.dca_payments + (id, client_id, amount_sats, amount_fiat, exchange_rate, transaction_type, lamassu_transaction_id, payment_hash, status, created_at, transaction_time) VALUES (:id, :client_id, :amount_sats, :amount_fiat, :exchange_rate, :transaction_type, :lamassu_transaction_id, :payment_hash, :status, :created_at, :transaction_time) """, - { + prepare_for_db({ "id": payment_id, "client_id": data.client_id, "amount_sats": data.amount_sats, @@ -176,7 +194,7 @@ async def create_dca_payment(data: CreateDcaPaymentData) -> DcaPayment: "status": "pending", "created_at": datetime.now(), "transaction_time": data.transaction_time - } + }) ) return await get_dca_payment(payment_id) @@ -295,22 +313,22 @@ async def get_fixed_mode_clients() -> List[DcaClient]: # Lamassu Configuration CRUD Operations async def create_lamassu_config(data: CreateLamassuConfigData) -> LamassuConfig: config_id = urlsafe_short_hash() - + # Deactivate any existing configs first (only one active config allowed) await db.execute( "UPDATE satoshimachine.lamassu_config SET is_active = false, updated_at = :updated_at", {"updated_at": datetime.now()} ) - + await db.execute( """ - INSERT INTO satoshimachine.lamassu_config + INSERT INTO satoshimachine.lamassu_config (id, host, port, database_name, username, password, source_wallet_id, commission_wallet_id, is_active, created_at, updated_at, use_ssh_tunnel, ssh_host, ssh_port, ssh_username, ssh_password, ssh_private_key, max_daily_limit_gtq) VALUES (:id, :host, :port, :database_name, :username, :password, :source_wallet_id, :commission_wallet_id, :is_active, :created_at, :updated_at, :use_ssh_tunnel, :ssh_host, :ssh_port, :ssh_username, :ssh_password, :ssh_private_key, :max_daily_limit_gtq) """, - { + prepare_for_db({ "id": config_id, "host": data.host, "port": data.port, @@ -329,7 +347,7 @@ async def create_lamassu_config(data: CreateLamassuConfigData) -> LamassuConfig: "ssh_password": data.ssh_password, "ssh_private_key": data.ssh_private_key, "max_daily_limit_gtq": data.max_daily_limit_gtq - } + }) ) return await get_lamassu_config(config_id) @@ -360,14 +378,14 @@ async def update_lamassu_config(config_id: str, data: UpdateLamassuConfigData) - update_data = {k: v for k, v in data.dict().items() if v is not None} if not update_data: return await get_lamassu_config(config_id) - + update_data["updated_at"] = datetime.now() set_clause = ", ".join([f"{k} = :{k}" for k in update_data.keys()]) update_data["id"] = config_id - + await db.execute( f"UPDATE satoshimachine.lamassu_config SET {set_clause} WHERE id = :id", - update_data + prepare_for_db(update_data) ) return await get_lamassu_config(config_id) @@ -436,9 +454,9 @@ async def create_lamassu_transaction(data: CreateLamassuTransactionData) -> Stor transaction_id = urlsafe_short_hash() await db.execute( """ - INSERT INTO satoshimachine.lamassu_transactions - (id, lamassu_transaction_id, fiat_amount, crypto_amount, commission_percentage, - discount, effective_commission, commission_amount_sats, base_amount_sats, + INSERT INTO satoshimachine.lamassu_transactions + (id, lamassu_transaction_id, fiat_amount, crypto_amount, commission_percentage, + discount, effective_commission, commission_amount_sats, base_amount_sats, exchange_rate, crypto_code, fiat_code, device_id, transaction_time, processed_at, clients_count, distributions_total_sats) VALUES (:id, :lamassu_transaction_id, :fiat_amount, :crypto_amount, :commission_percentage, @@ -446,7 +464,7 @@ async def create_lamassu_transaction(data: CreateLamassuTransactionData) -> Stor :exchange_rate, :crypto_code, :fiat_code, :device_id, :transaction_time, :processed_at, :clients_count, :distributions_total_sats) """, - { + prepare_for_db({ "id": transaction_id, "lamassu_transaction_id": data.lamassu_transaction_id, "fiat_amount": data.fiat_amount, @@ -464,7 +482,7 @@ async def create_lamassu_transaction(data: CreateLamassuTransactionData) -> Stor "processed_at": datetime.now(), "clients_count": 0, # Will be updated after distributions "distributions_total_sats": 0 # Will be updated after distributions - } + }) ) return await get_lamassu_transaction(transaction_id) diff --git a/models.py b/models.py index e7274c1..3e6aa2a 100644 --- a/models.py +++ b/models.py @@ -7,6 +7,16 @@ from typing import Optional from pydantic import BaseModel, validator +def _to_decimal(v): + """Convert a value to Decimal, handling floats from database.""" + if v is None: + return None + if isinstance(v, Decimal): + return v + # Convert via string to avoid float precision issues + return Decimal(str(v)) + + # DCA Client Models class CreateDcaClientData(BaseModel): user_id: str @@ -27,6 +37,10 @@ class DcaClient(BaseModel): created_at: datetime updated_at: datetime + @validator('fixed_mode_daily_limit', pre=True) + def convert_fixed_mode_daily_limit(cls, v): + return _to_decimal(v) + class Config: json_encoders = {Decimal: lambda v: float(v)} @@ -65,6 +79,10 @@ class DcaDeposit(BaseModel): created_at: datetime confirmed_at: Optional[datetime] + @validator('amount', pre=True) + def convert_amount(cls, v): + return _to_decimal(v) + class Config: json_encoders = {Decimal: lambda v: float(v)} @@ -99,6 +117,10 @@ class DcaPayment(BaseModel): created_at: datetime transaction_time: Optional[datetime] = None # Original ATM transaction time + @validator('amount_fiat', 'exchange_rate', pre=True) + def convert_decimals(cls, v): + return _to_decimal(v) + class Config: json_encoders = {Decimal: lambda v: float(v)} @@ -111,6 +133,10 @@ class ClientBalanceSummary(BaseModel): remaining_balance: Decimal # Available balance for DCA in GTQ currency: str + @validator('total_deposits', 'total_payments', 'remaining_balance', pre=True) + def convert_decimals(cls, v): + return _to_decimal(v) + class Config: json_encoders = {Decimal: lambda v: float(v)} @@ -165,6 +191,13 @@ class StoredLamassuTransaction(BaseModel): clients_count: int # Number of clients who received distributions distributions_total_sats: int # Total sats distributed to clients + @validator( + 'fiat_amount', 'commission_percentage', 'discount', + 'effective_commission', 'exchange_rate', pre=True + ) + def convert_decimals(cls, v): + return _to_decimal(v) + class Config: json_encoders = {Decimal: lambda v: float(v)} @@ -228,6 +261,10 @@ class LamassuConfig(BaseModel): # DCA Client Limits max_daily_limit_gtq: Decimal = Decimal("2000") # Maximum daily limit for Fixed mode clients + @validator('max_daily_limit_gtq', pre=True) + def convert_max_daily_limit(cls, v): + return _to_decimal(v) + class Config: json_encoders = {Decimal: lambda v: float(v)} diff --git a/tests/test_integration.py b/tests/test_integration.py index 9fe2623..fc1ff62 100644 --- a/tests/test_integration.py +++ b/tests/test_integration.py @@ -389,16 +389,16 @@ class TestExchangeRatePrecision: class TestCrudLayerDecimalHandling: """ - Test that CRUD operations correctly pass Decimal values to the database. + Test that CRUD operations handle Decimal→float conversion for SQLite. These tests mock the database layer to verify: - 1. Decimal values from models are passed correctly to db.execute() - 2. The correct SQL parameters include Decimal types + 1. Decimal values from models are converted to float via prepare_for_db() + 2. Float values from SQLite are converted back to Decimal via model validators """ @pytest.mark.asyncio - async def test_create_deposit_passes_decimal_amount(self): - """Verify create_deposit passes Decimal amount to database.""" + async def test_create_deposit_passes_float_amount_for_sqlite(self): + """Verify create_deposit passes float amount to database (SQLite compatibility).""" from unittest.mock import patch, AsyncMock # Create deposit data with Decimal @@ -419,7 +419,7 @@ class TestCrudLayerDecimalHandling: mock_db.fetchone = AsyncMock(return_value=DcaDeposit( id="deposit_123", client_id="test_client_123", - amount=Decimal("1500.75"), + amount=1500.75, # float from SQLite currency="GTQ", status="pending", notes="Test deposit", @@ -438,13 +438,13 @@ class TestCrudLayerDecimalHandling: call_args = mock_db.execute.call_args params = call_args[0][1] # Second positional arg is the params dict - # Verify the amount parameter is Decimal - assert isinstance(params["amount"], Decimal) - assert params["amount"] == Decimal("1500.75") + # Verify the amount parameter is float (for SQLite compatibility) + assert isinstance(params["amount"], float) + assert params["amount"] == 1500.75 @pytest.mark.asyncio - async def test_create_dca_payment_passes_decimal_values(self): - """Verify create_dca_payment passes Decimal fiat and exchange_rate.""" + async def test_create_dca_payment_passes_float_values_for_sqlite(self): + """Verify create_dca_payment passes float fiat and exchange_rate (SQLite compatibility).""" from unittest.mock import patch, AsyncMock # Parse a real transaction @@ -474,15 +474,15 @@ class TestCrudLayerDecimalHandling: assert isinstance(payment_data.amount_fiat, Decimal) assert isinstance(payment_data.exchange_rate, Decimal) - # Mock database + # Mock database - returns float like SQLite would mock_db = AsyncMock() mock_db.execute = AsyncMock() mock_db.fetchone = AsyncMock(return_value=DcaPayment( id="payment_123", client_id="test_client", amount_sats=client_sats, - amount_fiat=client_fiat, - exchange_rate=exchange_rate, + amount_fiat=float(client_fiat), # float from SQLite + exchange_rate=float(exchange_rate), # float from SQLite transaction_type="flow", lamassu_transaction_id="def456", payment_hash=None, @@ -502,13 +502,13 @@ class TestCrudLayerDecimalHandling: call_args = mock_db.execute.call_args params = call_args[0][1] - # Verify Decimal types in params - assert isinstance(params["amount_fiat"], Decimal) - assert isinstance(params["exchange_rate"], Decimal) + # Verify float types in params (SQLite compatibility) + assert isinstance(params["amount_fiat"], float) + assert isinstance(params["exchange_rate"], float) @pytest.mark.asyncio - async def test_create_lamassu_transaction_passes_all_decimals(self): - """Verify create_lamassu_transaction passes all Decimal fields.""" + async def test_create_lamassu_transaction_passes_floats_for_sqlite(self): + """Verify create_lamassu_transaction passes float fields (SQLite compatibility).""" from unittest.mock import patch, AsyncMock tx = parse_csv_like_transaction_processor(LAMASSU_CSV_DATA["8.75pct_large"]) @@ -535,27 +535,27 @@ class TestCrudLayerDecimalHandling: transaction_time=tx["transaction_time"], ) - # Verify all Decimal fields + # Verify all Decimal fields in the model assert isinstance(transaction_data.fiat_amount, Decimal) assert isinstance(transaction_data.commission_percentage, Decimal) assert isinstance(transaction_data.discount, Decimal) assert isinstance(transaction_data.effective_commission, Decimal) assert isinstance(transaction_data.exchange_rate, Decimal) - # Mock database + # Mock database - returns floats like SQLite would mock_db = AsyncMock() mock_db.execute = AsyncMock() mock_db.fetchone = AsyncMock(return_value=StoredLamassuTransaction( id="tx_123", lamassu_transaction_id=tx["transaction_id"], - fiat_amount=tx["fiat_amount"], + fiat_amount=float(tx["fiat_amount"]), # float from SQLite crypto_amount=tx["crypto_amount"], - commission_percentage=tx["commission_percentage"], - discount=tx["discount"], - effective_commission=effective, + commission_percentage=float(tx["commission_percentage"]), # float from SQLite + discount=float(tx["discount"]), # float from SQLite + effective_commission=float(effective), # float from SQLite commission_amount_sats=commission_sats, base_amount_sats=base_sats, - exchange_rate=exchange_rate, + exchange_rate=float(exchange_rate), # float from SQLite crypto_code=tx["crypto_code"], fiat_code=tx["fiat_code"], device_id=tx["device_id"], @@ -576,16 +576,16 @@ class TestCrudLayerDecimalHandling: call_args = mock_db.execute.call_args params = call_args[0][1] - # Verify all Decimal fields in params - assert isinstance(params["fiat_amount"], Decimal), f"fiat_amount is {type(params['fiat_amount'])}" - assert isinstance(params["commission_percentage"], Decimal) - assert isinstance(params["discount"], Decimal) - assert isinstance(params["effective_commission"], Decimal) - assert isinstance(params["exchange_rate"], Decimal) + # Verify all Decimal fields are converted to float for SQLite + assert isinstance(params["fiat_amount"], float), f"fiat_amount is {type(params['fiat_amount'])}" + assert isinstance(params["commission_percentage"], float) + assert isinstance(params["discount"], float) + assert isinstance(params["effective_commission"], float) + assert isinstance(params["exchange_rate"], float) # Verify values match - assert params["fiat_amount"] == Decimal("2000") - assert params["commission_percentage"] == Decimal("0.0875") + assert params["fiat_amount"] == 2000.0 + assert params["commission_percentage"] == 0.0875 assert params["base_amount_sats"] == 284322 assert params["commission_amount_sats"] == 24878 @@ -618,3 +618,211 @@ class TestCrudLayerDecimalHandling: assert result.total_deposits == Decimal("5000.00") assert result.total_payments == Decimal("1234.56") assert result.remaining_balance == Decimal("3765.44") + + +# ============================================================================= +# SQLITE ROUND-TRIP TESTS +# ============================================================================= + +class TestSqliteDecimalRoundTrip: + """ + Test that Decimal values survive the SQLite round-trip: + Decimal → float (prepare_for_db) → SQLite REAL → float → Decimal (validator) + + SQLite doesn't support Decimal natively - it stores DECIMAL as REAL (float). + We use prepare_for_db() to convert Decimal→float before writing, + and Pydantic validators with pre=True to convert float→Decimal on read. + """ + + def test_prepare_for_db_converts_decimals_to_float(self): + """Verify prepare_for_db converts Decimal to float.""" + from ..crud import prepare_for_db + + values = { + "amount": Decimal("1500.75"), + "exchange_rate": Decimal("146.6825"), + "commission": Decimal("0.0875"), + "name": "test", # Non-Decimal should pass through + "count": 42, # Non-Decimal should pass through + } + + result = prepare_for_db(values) + + # Decimals should be converted to float + assert isinstance(result["amount"], float) + assert isinstance(result["exchange_rate"], float) + assert isinstance(result["commission"], float) + + # Values should be preserved + assert result["amount"] == 1500.75 + assert result["exchange_rate"] == 146.6825 + assert result["commission"] == 0.0875 + + # Non-Decimals should pass through unchanged + assert result["name"] == "test" + assert result["count"] == 42 + + def test_model_validator_converts_float_to_decimal(self): + """Verify model validators convert float back to Decimal.""" + # Simulate what comes back from SQLite (floats) + db_row = { + "id": "deposit_123", + "client_id": "client_abc", + "amount": 1500.75, # float from SQLite + "currency": "GTQ", + "status": "pending", + "notes": "Test", + "created_at": datetime.now(timezone.utc), + "confirmed_at": None, + } + + # Create model from "database" data + deposit = DcaDeposit(**db_row) + + # Validator should have converted float → Decimal + assert isinstance(deposit.amount, Decimal) + assert deposit.amount == Decimal("1500.75") + + def test_payment_model_converts_multiple_floats(self): + """Verify DcaPayment converts all float fields to Decimal.""" + db_row = { + "id": "payment_123", + "client_id": "client_abc", + "amount_sats": 146682, + "amount_fiat": 1000.50, # float from SQLite + "exchange_rate": 146.6825, # float from SQLite + "transaction_type": "flow", + "lamassu_transaction_id": "tx_456", + "payment_hash": None, + "status": "confirmed", + "created_at": datetime.now(timezone.utc), + "transaction_time": datetime.now(timezone.utc), + } + + payment = DcaPayment(**db_row) + + assert isinstance(payment.amount_fiat, Decimal) + assert isinstance(payment.exchange_rate, Decimal) + assert payment.amount_fiat == Decimal("1000.50") + assert payment.exchange_rate == Decimal("146.6825") + + def test_stored_transaction_converts_all_decimal_fields(self): + """Verify StoredLamassuTransaction converts all float fields.""" + db_row = { + "id": "tx_123", + "lamassu_transaction_id": "lamassu_abc", + "fiat_amount": 2000.00, # float from SQLite + "crypto_amount": 309200, + "commission_percentage": 0.0875, # float from SQLite + "discount": 0.0, # float from SQLite + "effective_commission": 0.0875, # float from SQLite + "commission_amount_sats": 24878, + "base_amount_sats": 284322, + "exchange_rate": 142.161, # float from SQLite + "crypto_code": "BTC", + "fiat_code": "GTQ", + "device_id": "device1", + "transaction_time": datetime.now(timezone.utc), + "processed_at": datetime.now(timezone.utc), + "clients_count": 2, + "distributions_total_sats": 284322, + } + + tx = StoredLamassuTransaction(**db_row) + + # All Decimal fields should be converted + assert isinstance(tx.fiat_amount, Decimal) + assert isinstance(tx.commission_percentage, Decimal) + assert isinstance(tx.discount, Decimal) + assert isinstance(tx.effective_commission, Decimal) + assert isinstance(tx.exchange_rate, Decimal) + + # Values should be preserved + assert tx.fiat_amount == Decimal("2000.00") + assert tx.commission_percentage == Decimal("0.0875") + assert tx.discount == Decimal("0.0") + assert tx.effective_commission == Decimal("0.0875") + + def test_full_round_trip_preserves_precision(self): + """ + Test complete round-trip: Decimal → prepare_for_db → float → Decimal. + + This simulates what happens in production: + 1. We have Decimal values from calculations + 2. prepare_for_db converts them to float for SQLite + 3. SQLite stores as REAL + 4. We read back as float + 5. Model validators convert back to Decimal + """ + from ..crud import prepare_for_db + + # Original Decimal values (from calculations) + original = { + "fiat_amount": Decimal("1300.00"), + "commission_percentage": Decimal("0.055"), + "discount": Decimal("0"), + "effective_commission": Decimal("0.055"), + "exchange_rate": Decimal("149.90215490109"), # High precision + } + + # Step 1: Convert for database storage + for_db = prepare_for_db(original) + + # Verify all are floats + for key in original: + assert isinstance(for_db[key], float), f"{key} should be float" + + # Step 2: Simulate SQLite storage and retrieval + # (SQLite would store these as REAL and return as float) + from_db = for_db.copy() + + # Step 3: Create a model (simulating what happens when reading) + db_row = { + "id": "tx_test", + "lamassu_transaction_id": "test_123", + "fiat_amount": from_db["fiat_amount"], + "crypto_amount": 205600, + "commission_percentage": from_db["commission_percentage"], + "discount": from_db["discount"], + "effective_commission": from_db["effective_commission"], + "commission_amount_sats": 10718, + "base_amount_sats": 194882, + "exchange_rate": from_db["exchange_rate"], + "crypto_code": "BTC", + "fiat_code": "GTQ", + "device_id": "device1", + "transaction_time": datetime.now(timezone.utc), + "processed_at": datetime.now(timezone.utc), + "clients_count": 1, + "distributions_total_sats": 194882, + } + + tx = StoredLamassuTransaction(**db_row) + + # Step 4: Verify Decimal types restored + assert isinstance(tx.fiat_amount, Decimal) + assert isinstance(tx.commission_percentage, Decimal) + assert isinstance(tx.exchange_rate, Decimal) + + # Step 5: Verify precision for 2 decimal place values + assert tx.fiat_amount == Decimal("1300.0") # SQLite preserves this exactly + assert tx.commission_percentage == Decimal("0.055") + assert tx.discount == Decimal("0.0") + + def test_client_balance_summary_float_conversion(self): + """Verify ClientBalanceSummary converts floats from aggregation queries.""" + # SQLite SUM() returns float + db_row = { + "client_id": "client_123", + "total_deposits": 5000.00, # float from SUM() + "total_payments": 1234.56, # float from SUM() + "remaining_balance": 3765.44, # calculated float + "currency": "GTQ", + } + + summary = ClientBalanceSummary(**db_row) + + assert isinstance(summary.total_deposits, Decimal) + assert isinstance(summary.total_payments, Decimal) + assert isinstance(summary.remaining_balance, Decimal) + assert summary.remaining_balance == Decimal("3765.44")