fix: add SQLite compatibility for Decimal types

SQLite doesn't support Decimal natively - it stores DECIMAL columns as
REAL (float). This caused sqlite3.ProgrammingError when writing Decimal
values.

Changes:
- Add prepare_for_db() helper to convert Decimal→float before writes
- Add Pydantic validators to convert float→Decimal on model creation
- Update CRUD layer tests to verify float params for SQLite
- Add SQLite round-trip tests to verify precision is preserved

The data flow is now:
Decimal (calculations) → float (prepare_for_db) → SQLite → float → Decimal (validators)

🤖 Generated with [Claude Code](https://claude.com/claude-code)

Co-Authored-By: Claude Opus 4.5 <noreply@anthropic.com>
This commit is contained in:
padreug 2026-01-11 15:09:45 +01:00
parent d245047487
commit 904b3f1d61
3 changed files with 323 additions and 60 deletions

70
crud.py
View file

@ -1,5 +1,6 @@
# Description: This file contains the CRUD operations for talking to the database.
from decimal import Decimal
from typing import List, Optional, Union
from datetime import datetime, timezone
@ -18,16 +19,33 @@ from .models import (
db = Database("ext_satoshimachine")
def prepare_for_db(values: dict) -> dict:
"""
Convert Decimal values to float for SQLite compatibility.
SQLite doesn't support Decimal natively - it stores DECIMAL as REAL (float).
This function converts Decimal values to float before database writes.
The Pydantic models handle converting float back to Decimal on read.
"""
result = {}
for k, v in values.items():
if isinstance(v, Decimal):
result[k] = float(v)
else:
result[k] = v
return result
# DCA Client CRUD Operations
async def create_dca_client(data: CreateDcaClientData) -> DcaClient:
client_id = urlsafe_short_hash()
await db.execute(
"""
INSERT INTO satoshimachine.dca_clients
INSERT INTO satoshimachine.dca_clients
(id, user_id, wallet_id, username, dca_mode, fixed_mode_daily_limit, status, created_at, updated_at)
VALUES (:id, :user_id, :wallet_id, :username, :dca_mode, :fixed_mode_daily_limit, :status, :created_at, :updated_at)
""",
{
prepare_for_db({
"id": client_id,
"user_id": data.user_id,
"wallet_id": data.wallet_id,
@ -37,7 +55,7 @@ async def create_dca_client(data: CreateDcaClientData) -> DcaClient:
"status": "active",
"created_at": datetime.now(),
"updated_at": datetime.now()
}
})
)
return await get_dca_client(client_id)
@ -69,14 +87,14 @@ async def update_dca_client(client_id: str, data: UpdateDcaClientData) -> Option
update_data = {k: v for k, v in data.dict().items() if v is not None}
if not update_data:
return await get_dca_client(client_id)
update_data["updated_at"] = datetime.now()
set_clause = ", ".join([f"{k} = :{k}" for k in update_data.keys()])
update_data["id"] = client_id
await db.execute(
f"UPDATE satoshimachine.dca_clients SET {set_clause} WHERE id = :id",
update_data
prepare_for_db(update_data)
)
return await get_dca_client(client_id)
@ -93,11 +111,11 @@ async def create_deposit(data: CreateDepositData) -> DcaDeposit:
deposit_id = urlsafe_short_hash()
await db.execute(
"""
INSERT INTO satoshimachine.dca_deposits
INSERT INTO satoshimachine.dca_deposits
(id, client_id, amount, currency, status, notes, created_at)
VALUES (:id, :client_id, :amount, :currency, :status, :notes, :created_at)
""",
{
prepare_for_db({
"id": deposit_id,
"client_id": data.client_id,
"amount": data.amount,
@ -105,7 +123,7 @@ async def create_deposit(data: CreateDepositData) -> DcaDeposit:
"status": "pending",
"notes": data.notes,
"created_at": datetime.now()
}
})
)
return await get_deposit(deposit_id)
@ -158,13 +176,13 @@ async def create_dca_payment(data: CreateDcaPaymentData) -> DcaPayment:
payment_id = urlsafe_short_hash()
await db.execute(
"""
INSERT INTO satoshimachine.dca_payments
(id, client_id, amount_sats, amount_fiat, exchange_rate, transaction_type,
INSERT INTO satoshimachine.dca_payments
(id, client_id, amount_sats, amount_fiat, exchange_rate, transaction_type,
lamassu_transaction_id, payment_hash, status, created_at, transaction_time)
VALUES (:id, :client_id, :amount_sats, :amount_fiat, :exchange_rate, :transaction_type,
:lamassu_transaction_id, :payment_hash, :status, :created_at, :transaction_time)
""",
{
prepare_for_db({
"id": payment_id,
"client_id": data.client_id,
"amount_sats": data.amount_sats,
@ -176,7 +194,7 @@ async def create_dca_payment(data: CreateDcaPaymentData) -> DcaPayment:
"status": "pending",
"created_at": datetime.now(),
"transaction_time": data.transaction_time
}
})
)
return await get_dca_payment(payment_id)
@ -295,22 +313,22 @@ async def get_fixed_mode_clients() -> List[DcaClient]:
# Lamassu Configuration CRUD Operations
async def create_lamassu_config(data: CreateLamassuConfigData) -> LamassuConfig:
config_id = urlsafe_short_hash()
# Deactivate any existing configs first (only one active config allowed)
await db.execute(
"UPDATE satoshimachine.lamassu_config SET is_active = false, updated_at = :updated_at",
{"updated_at": datetime.now()}
)
await db.execute(
"""
INSERT INTO satoshimachine.lamassu_config
INSERT INTO satoshimachine.lamassu_config
(id, host, port, database_name, username, password, source_wallet_id, commission_wallet_id, is_active, created_at, updated_at,
use_ssh_tunnel, ssh_host, ssh_port, ssh_username, ssh_password, ssh_private_key, max_daily_limit_gtq)
VALUES (:id, :host, :port, :database_name, :username, :password, :source_wallet_id, :commission_wallet_id, :is_active, :created_at, :updated_at,
:use_ssh_tunnel, :ssh_host, :ssh_port, :ssh_username, :ssh_password, :ssh_private_key, :max_daily_limit_gtq)
""",
{
prepare_for_db({
"id": config_id,
"host": data.host,
"port": data.port,
@ -329,7 +347,7 @@ async def create_lamassu_config(data: CreateLamassuConfigData) -> LamassuConfig:
"ssh_password": data.ssh_password,
"ssh_private_key": data.ssh_private_key,
"max_daily_limit_gtq": data.max_daily_limit_gtq
}
})
)
return await get_lamassu_config(config_id)
@ -360,14 +378,14 @@ async def update_lamassu_config(config_id: str, data: UpdateLamassuConfigData) -
update_data = {k: v for k, v in data.dict().items() if v is not None}
if not update_data:
return await get_lamassu_config(config_id)
update_data["updated_at"] = datetime.now()
set_clause = ", ".join([f"{k} = :{k}" for k in update_data.keys()])
update_data["id"] = config_id
await db.execute(
f"UPDATE satoshimachine.lamassu_config SET {set_clause} WHERE id = :id",
update_data
prepare_for_db(update_data)
)
return await get_lamassu_config(config_id)
@ -436,9 +454,9 @@ async def create_lamassu_transaction(data: CreateLamassuTransactionData) -> Stor
transaction_id = urlsafe_short_hash()
await db.execute(
"""
INSERT INTO satoshimachine.lamassu_transactions
(id, lamassu_transaction_id, fiat_amount, crypto_amount, commission_percentage,
discount, effective_commission, commission_amount_sats, base_amount_sats,
INSERT INTO satoshimachine.lamassu_transactions
(id, lamassu_transaction_id, fiat_amount, crypto_amount, commission_percentage,
discount, effective_commission, commission_amount_sats, base_amount_sats,
exchange_rate, crypto_code, fiat_code, device_id, transaction_time, processed_at,
clients_count, distributions_total_sats)
VALUES (:id, :lamassu_transaction_id, :fiat_amount, :crypto_amount, :commission_percentage,
@ -446,7 +464,7 @@ async def create_lamassu_transaction(data: CreateLamassuTransactionData) -> Stor
:exchange_rate, :crypto_code, :fiat_code, :device_id, :transaction_time, :processed_at,
:clients_count, :distributions_total_sats)
""",
{
prepare_for_db({
"id": transaction_id,
"lamassu_transaction_id": data.lamassu_transaction_id,
"fiat_amount": data.fiat_amount,
@ -464,7 +482,7 @@ async def create_lamassu_transaction(data: CreateLamassuTransactionData) -> Stor
"processed_at": datetime.now(),
"clients_count": 0, # Will be updated after distributions
"distributions_total_sats": 0 # Will be updated after distributions
}
})
)
return await get_lamassu_transaction(transaction_id)

View file

@ -7,6 +7,16 @@ from typing import Optional
from pydantic import BaseModel, validator
def _to_decimal(v):
"""Convert a value to Decimal, handling floats from database."""
if v is None:
return None
if isinstance(v, Decimal):
return v
# Convert via string to avoid float precision issues
return Decimal(str(v))
# DCA Client Models
class CreateDcaClientData(BaseModel):
user_id: str
@ -27,6 +37,10 @@ class DcaClient(BaseModel):
created_at: datetime
updated_at: datetime
@validator('fixed_mode_daily_limit', pre=True)
def convert_fixed_mode_daily_limit(cls, v):
return _to_decimal(v)
class Config:
json_encoders = {Decimal: lambda v: float(v)}
@ -65,6 +79,10 @@ class DcaDeposit(BaseModel):
created_at: datetime
confirmed_at: Optional[datetime]
@validator('amount', pre=True)
def convert_amount(cls, v):
return _to_decimal(v)
class Config:
json_encoders = {Decimal: lambda v: float(v)}
@ -99,6 +117,10 @@ class DcaPayment(BaseModel):
created_at: datetime
transaction_time: Optional[datetime] = None # Original ATM transaction time
@validator('amount_fiat', 'exchange_rate', pre=True)
def convert_decimals(cls, v):
return _to_decimal(v)
class Config:
json_encoders = {Decimal: lambda v: float(v)}
@ -111,6 +133,10 @@ class ClientBalanceSummary(BaseModel):
remaining_balance: Decimal # Available balance for DCA in GTQ
currency: str
@validator('total_deposits', 'total_payments', 'remaining_balance', pre=True)
def convert_decimals(cls, v):
return _to_decimal(v)
class Config:
json_encoders = {Decimal: lambda v: float(v)}
@ -165,6 +191,13 @@ class StoredLamassuTransaction(BaseModel):
clients_count: int # Number of clients who received distributions
distributions_total_sats: int # Total sats distributed to clients
@validator(
'fiat_amount', 'commission_percentage', 'discount',
'effective_commission', 'exchange_rate', pre=True
)
def convert_decimals(cls, v):
return _to_decimal(v)
class Config:
json_encoders = {Decimal: lambda v: float(v)}
@ -228,6 +261,10 @@ class LamassuConfig(BaseModel):
# DCA Client Limits
max_daily_limit_gtq: Decimal = Decimal("2000") # Maximum daily limit for Fixed mode clients
@validator('max_daily_limit_gtq', pre=True)
def convert_max_daily_limit(cls, v):
return _to_decimal(v)
class Config:
json_encoders = {Decimal: lambda v: float(v)}

View file

@ -389,16 +389,16 @@ class TestExchangeRatePrecision:
class TestCrudLayerDecimalHandling:
"""
Test that CRUD operations correctly pass Decimal values to the database.
Test that CRUD operations handle Decimalfloat conversion for SQLite.
These tests mock the database layer to verify:
1. Decimal values from models are passed correctly to db.execute()
2. The correct SQL parameters include Decimal types
1. Decimal values from models are converted to float via prepare_for_db()
2. Float values from SQLite are converted back to Decimal via model validators
"""
@pytest.mark.asyncio
async def test_create_deposit_passes_decimal_amount(self):
"""Verify create_deposit passes Decimal amount to database."""
async def test_create_deposit_passes_float_amount_for_sqlite(self):
"""Verify create_deposit passes float amount to database (SQLite compatibility)."""
from unittest.mock import patch, AsyncMock
# Create deposit data with Decimal
@ -419,7 +419,7 @@ class TestCrudLayerDecimalHandling:
mock_db.fetchone = AsyncMock(return_value=DcaDeposit(
id="deposit_123",
client_id="test_client_123",
amount=Decimal("1500.75"),
amount=1500.75, # float from SQLite
currency="GTQ",
status="pending",
notes="Test deposit",
@ -438,13 +438,13 @@ class TestCrudLayerDecimalHandling:
call_args = mock_db.execute.call_args
params = call_args[0][1] # Second positional arg is the params dict
# Verify the amount parameter is Decimal
assert isinstance(params["amount"], Decimal)
assert params["amount"] == Decimal("1500.75")
# Verify the amount parameter is float (for SQLite compatibility)
assert isinstance(params["amount"], float)
assert params["amount"] == 1500.75
@pytest.mark.asyncio
async def test_create_dca_payment_passes_decimal_values(self):
"""Verify create_dca_payment passes Decimal fiat and exchange_rate."""
async def test_create_dca_payment_passes_float_values_for_sqlite(self):
"""Verify create_dca_payment passes float fiat and exchange_rate (SQLite compatibility)."""
from unittest.mock import patch, AsyncMock
# Parse a real transaction
@ -474,15 +474,15 @@ class TestCrudLayerDecimalHandling:
assert isinstance(payment_data.amount_fiat, Decimal)
assert isinstance(payment_data.exchange_rate, Decimal)
# Mock database
# Mock database - returns float like SQLite would
mock_db = AsyncMock()
mock_db.execute = AsyncMock()
mock_db.fetchone = AsyncMock(return_value=DcaPayment(
id="payment_123",
client_id="test_client",
amount_sats=client_sats,
amount_fiat=client_fiat,
exchange_rate=exchange_rate,
amount_fiat=float(client_fiat), # float from SQLite
exchange_rate=float(exchange_rate), # float from SQLite
transaction_type="flow",
lamassu_transaction_id="def456",
payment_hash=None,
@ -502,13 +502,13 @@ class TestCrudLayerDecimalHandling:
call_args = mock_db.execute.call_args
params = call_args[0][1]
# Verify Decimal types in params
assert isinstance(params["amount_fiat"], Decimal)
assert isinstance(params["exchange_rate"], Decimal)
# Verify float types in params (SQLite compatibility)
assert isinstance(params["amount_fiat"], float)
assert isinstance(params["exchange_rate"], float)
@pytest.mark.asyncio
async def test_create_lamassu_transaction_passes_all_decimals(self):
"""Verify create_lamassu_transaction passes all Decimal fields."""
async def test_create_lamassu_transaction_passes_floats_for_sqlite(self):
"""Verify create_lamassu_transaction passes float fields (SQLite compatibility)."""
from unittest.mock import patch, AsyncMock
tx = parse_csv_like_transaction_processor(LAMASSU_CSV_DATA["8.75pct_large"])
@ -535,27 +535,27 @@ class TestCrudLayerDecimalHandling:
transaction_time=tx["transaction_time"],
)
# Verify all Decimal fields
# Verify all Decimal fields in the model
assert isinstance(transaction_data.fiat_amount, Decimal)
assert isinstance(transaction_data.commission_percentage, Decimal)
assert isinstance(transaction_data.discount, Decimal)
assert isinstance(transaction_data.effective_commission, Decimal)
assert isinstance(transaction_data.exchange_rate, Decimal)
# Mock database
# Mock database - returns floats like SQLite would
mock_db = AsyncMock()
mock_db.execute = AsyncMock()
mock_db.fetchone = AsyncMock(return_value=StoredLamassuTransaction(
id="tx_123",
lamassu_transaction_id=tx["transaction_id"],
fiat_amount=tx["fiat_amount"],
fiat_amount=float(tx["fiat_amount"]), # float from SQLite
crypto_amount=tx["crypto_amount"],
commission_percentage=tx["commission_percentage"],
discount=tx["discount"],
effective_commission=effective,
commission_percentage=float(tx["commission_percentage"]), # float from SQLite
discount=float(tx["discount"]), # float from SQLite
effective_commission=float(effective), # float from SQLite
commission_amount_sats=commission_sats,
base_amount_sats=base_sats,
exchange_rate=exchange_rate,
exchange_rate=float(exchange_rate), # float from SQLite
crypto_code=tx["crypto_code"],
fiat_code=tx["fiat_code"],
device_id=tx["device_id"],
@ -576,16 +576,16 @@ class TestCrudLayerDecimalHandling:
call_args = mock_db.execute.call_args
params = call_args[0][1]
# Verify all Decimal fields in params
assert isinstance(params["fiat_amount"], Decimal), f"fiat_amount is {type(params['fiat_amount'])}"
assert isinstance(params["commission_percentage"], Decimal)
assert isinstance(params["discount"], Decimal)
assert isinstance(params["effective_commission"], Decimal)
assert isinstance(params["exchange_rate"], Decimal)
# Verify all Decimal fields are converted to float for SQLite
assert isinstance(params["fiat_amount"], float), f"fiat_amount is {type(params['fiat_amount'])}"
assert isinstance(params["commission_percentage"], float)
assert isinstance(params["discount"], float)
assert isinstance(params["effective_commission"], float)
assert isinstance(params["exchange_rate"], float)
# Verify values match
assert params["fiat_amount"] == Decimal("2000")
assert params["commission_percentage"] == Decimal("0.0875")
assert params["fiat_amount"] == 2000.0
assert params["commission_percentage"] == 0.0875
assert params["base_amount_sats"] == 284322
assert params["commission_amount_sats"] == 24878
@ -618,3 +618,211 @@ class TestCrudLayerDecimalHandling:
assert result.total_deposits == Decimal("5000.00")
assert result.total_payments == Decimal("1234.56")
assert result.remaining_balance == Decimal("3765.44")
# =============================================================================
# SQLITE ROUND-TRIP TESTS
# =============================================================================
class TestSqliteDecimalRoundTrip:
"""
Test that Decimal values survive the SQLite round-trip:
Decimal float (prepare_for_db) SQLite REAL float Decimal (validator)
SQLite doesn't support Decimal natively - it stores DECIMAL as REAL (float).
We use prepare_for_db() to convert Decimalfloat before writing,
and Pydantic validators with pre=True to convert floatDecimal on read.
"""
def test_prepare_for_db_converts_decimals_to_float(self):
"""Verify prepare_for_db converts Decimal to float."""
from ..crud import prepare_for_db
values = {
"amount": Decimal("1500.75"),
"exchange_rate": Decimal("146.6825"),
"commission": Decimal("0.0875"),
"name": "test", # Non-Decimal should pass through
"count": 42, # Non-Decimal should pass through
}
result = prepare_for_db(values)
# Decimals should be converted to float
assert isinstance(result["amount"], float)
assert isinstance(result["exchange_rate"], float)
assert isinstance(result["commission"], float)
# Values should be preserved
assert result["amount"] == 1500.75
assert result["exchange_rate"] == 146.6825
assert result["commission"] == 0.0875
# Non-Decimals should pass through unchanged
assert result["name"] == "test"
assert result["count"] == 42
def test_model_validator_converts_float_to_decimal(self):
"""Verify model validators convert float back to Decimal."""
# Simulate what comes back from SQLite (floats)
db_row = {
"id": "deposit_123",
"client_id": "client_abc",
"amount": 1500.75, # float from SQLite
"currency": "GTQ",
"status": "pending",
"notes": "Test",
"created_at": datetime.now(timezone.utc),
"confirmed_at": None,
}
# Create model from "database" data
deposit = DcaDeposit(**db_row)
# Validator should have converted float → Decimal
assert isinstance(deposit.amount, Decimal)
assert deposit.amount == Decimal("1500.75")
def test_payment_model_converts_multiple_floats(self):
"""Verify DcaPayment converts all float fields to Decimal."""
db_row = {
"id": "payment_123",
"client_id": "client_abc",
"amount_sats": 146682,
"amount_fiat": 1000.50, # float from SQLite
"exchange_rate": 146.6825, # float from SQLite
"transaction_type": "flow",
"lamassu_transaction_id": "tx_456",
"payment_hash": None,
"status": "confirmed",
"created_at": datetime.now(timezone.utc),
"transaction_time": datetime.now(timezone.utc),
}
payment = DcaPayment(**db_row)
assert isinstance(payment.amount_fiat, Decimal)
assert isinstance(payment.exchange_rate, Decimal)
assert payment.amount_fiat == Decimal("1000.50")
assert payment.exchange_rate == Decimal("146.6825")
def test_stored_transaction_converts_all_decimal_fields(self):
"""Verify StoredLamassuTransaction converts all float fields."""
db_row = {
"id": "tx_123",
"lamassu_transaction_id": "lamassu_abc",
"fiat_amount": 2000.00, # float from SQLite
"crypto_amount": 309200,
"commission_percentage": 0.0875, # float from SQLite
"discount": 0.0, # float from SQLite
"effective_commission": 0.0875, # float from SQLite
"commission_amount_sats": 24878,
"base_amount_sats": 284322,
"exchange_rate": 142.161, # float from SQLite
"crypto_code": "BTC",
"fiat_code": "GTQ",
"device_id": "device1",
"transaction_time": datetime.now(timezone.utc),
"processed_at": datetime.now(timezone.utc),
"clients_count": 2,
"distributions_total_sats": 284322,
}
tx = StoredLamassuTransaction(**db_row)
# All Decimal fields should be converted
assert isinstance(tx.fiat_amount, Decimal)
assert isinstance(tx.commission_percentage, Decimal)
assert isinstance(tx.discount, Decimal)
assert isinstance(tx.effective_commission, Decimal)
assert isinstance(tx.exchange_rate, Decimal)
# Values should be preserved
assert tx.fiat_amount == Decimal("2000.00")
assert tx.commission_percentage == Decimal("0.0875")
assert tx.discount == Decimal("0.0")
assert tx.effective_commission == Decimal("0.0875")
def test_full_round_trip_preserves_precision(self):
"""
Test complete round-trip: Decimal prepare_for_db float Decimal.
This simulates what happens in production:
1. We have Decimal values from calculations
2. prepare_for_db converts them to float for SQLite
3. SQLite stores as REAL
4. We read back as float
5. Model validators convert back to Decimal
"""
from ..crud import prepare_for_db
# Original Decimal values (from calculations)
original = {
"fiat_amount": Decimal("1300.00"),
"commission_percentage": Decimal("0.055"),
"discount": Decimal("0"),
"effective_commission": Decimal("0.055"),
"exchange_rate": Decimal("149.90215490109"), # High precision
}
# Step 1: Convert for database storage
for_db = prepare_for_db(original)
# Verify all are floats
for key in original:
assert isinstance(for_db[key], float), f"{key} should be float"
# Step 2: Simulate SQLite storage and retrieval
# (SQLite would store these as REAL and return as float)
from_db = for_db.copy()
# Step 3: Create a model (simulating what happens when reading)
db_row = {
"id": "tx_test",
"lamassu_transaction_id": "test_123",
"fiat_amount": from_db["fiat_amount"],
"crypto_amount": 205600,
"commission_percentage": from_db["commission_percentage"],
"discount": from_db["discount"],
"effective_commission": from_db["effective_commission"],
"commission_amount_sats": 10718,
"base_amount_sats": 194882,
"exchange_rate": from_db["exchange_rate"],
"crypto_code": "BTC",
"fiat_code": "GTQ",
"device_id": "device1",
"transaction_time": datetime.now(timezone.utc),
"processed_at": datetime.now(timezone.utc),
"clients_count": 1,
"distributions_total_sats": 194882,
}
tx = StoredLamassuTransaction(**db_row)
# Step 4: Verify Decimal types restored
assert isinstance(tx.fiat_amount, Decimal)
assert isinstance(tx.commission_percentage, Decimal)
assert isinstance(tx.exchange_rate, Decimal)
# Step 5: Verify precision for 2 decimal place values
assert tx.fiat_amount == Decimal("1300.0") # SQLite preserves this exactly
assert tx.commission_percentage == Decimal("0.055")
assert tx.discount == Decimal("0.0")
def test_client_balance_summary_float_conversion(self):
"""Verify ClientBalanceSummary converts floats from aggregation queries."""
# SQLite SUM() returns float
db_row = {
"client_id": "client_123",
"total_deposits": 5000.00, # float from SUM()
"total_payments": 1234.56, # float from SUM()
"remaining_balance": 3765.44, # calculated float
"currency": "GTQ",
}
summary = ClientBalanceSummary(**db_row)
assert isinstance(summary.total_deposits, Decimal)
assert isinstance(summary.total_payments, Decimal)
assert isinstance(summary.remaining_balance, Decimal)
assert summary.remaining_balance == Decimal("3765.44")