From c83ebf43ab676b068ed6714e0762ed3347916c65 Mon Sep 17 00:00:00 2001 From: padreug Date: Sun, 6 Jul 2025 00:00:30 +0200 Subject: [PATCH 01/10] 01 Refactor currency handling to store amounts in GTQ: Removed currency conversion utilities, updated models and API endpoints to directly handle GTQ amounts, and modified transaction processing logic for consistency. Enhanced frontend to reflect these changes, ensuring accurate display and submission of GTQ values across the application. Refactor GTQ storage migration: Moved the conversion logic for centavo amounts to GTQ into a new migration function, m004_convert_to_gtq_storage, ensuring proper data type changes and updates across relevant tables. This enhances clarity and maintains the integrity of the migration process. --- crud.py | 2 +- currency_utils.py | 45 --------------- migrations.py | 84 ++++++++++++++++++++++++++++ models.py | 82 ++++++++++----------------- static/js/index.js | 12 ++-- templates/satmachineadmin/index.html | 8 +-- transaction_processor.py | 36 ++++++------ views_api.py | 50 +++++------------ 8 files changed, 157 insertions(+), 162 deletions(-) delete mode 100644 currency_utils.py diff --git a/crud.py b/crud.py index 60bad39..94c1d20 100644 --- a/crud.py +++ b/crud.py @@ -267,7 +267,7 @@ async def get_client_balance_summary(client_id: str, as_of_time: Optional[dateti from loguru import logger # Verify timezone consistency for temporal filtering tz_info = "UTC" if as_of_time.tzinfo == timezone.utc else f"TZ: {as_of_time.tzinfo}" - logger.info(f"Client {client_id[:8]}... balance as of {as_of_time} ({tz_info}): deposits.confirmed_at <= cutoff, payments.transaction_time <= cutoff → {total_deposits - total_payments} centavos remaining") + logger.info(f"Client {client_id[:8]}... balance as of {as_of_time} ({tz_info}): deposits.confirmed_at <= cutoff, payments.transaction_time <= cutoff → {total_deposits - total_payments:.2f} GTQ remaining") return ClientBalanceSummary( client_id=client_id, diff --git a/currency_utils.py b/currency_utils.py deleted file mode 100644 index 3241134..0000000 --- a/currency_utils.py +++ /dev/null @@ -1,45 +0,0 @@ -# Currency conversion utilities for API boundary -from decimal import Decimal -from typing import Union - - -def gtq_to_centavos(gtq_amount: Union[float, int, str]) -> int: - """Convert GTQ to centavos for database storage""" - return int(Decimal(str(gtq_amount)) * 100) - - -def centavos_to_gtq(centavos: int) -> float: - """Convert centavos to GTQ for API responses""" - return float(centavos) / 100 - - -def format_gtq_currency(centavos: int) -> str: - """Format centavos as GTQ currency string""" - gtq_amount = centavos_to_gtq(centavos) - return f"Q{gtq_amount:.2f}" - - -# Conversion helpers for API responses -def deposit_db_to_api(deposit_db) -> dict: - """Convert database deposit model to API response""" - return { - "id": deposit_db.id, - "client_id": deposit_db.client_id, - "amount_gtq": centavos_to_gtq(deposit_db.amount), - "currency": deposit_db.currency, - "status": deposit_db.status, - "notes": deposit_db.notes, - "created_at": deposit_db.created_at, - "confirmed_at": deposit_db.confirmed_at - } - - -def balance_summary_db_to_api(balance_db) -> dict: - """Convert database balance summary to API response""" - return { - "client_id": balance_db.client_id, - "total_deposits_gtq": centavos_to_gtq(balance_db.total_deposits), - "total_payments_gtq": centavos_to_gtq(balance_db.total_payments), - "remaining_balance_gtq": centavos_to_gtq(balance_db.remaining_balance), - "currency": balance_db.currency - } \ No newline at end of file diff --git a/migrations.py b/migrations.py index 0456f43..6654c71 100644 --- a/migrations.py +++ b/migrations.py @@ -134,4 +134,88 @@ async def m003_add_max_daily_limit_config(db): ALTER TABLE satoshimachine.lamassu_config ADD COLUMN max_daily_limit_gtq INTEGER NOT NULL DEFAULT 2000 """ + ) + + +async def m004_convert_to_gtq_storage(db): + """ + Convert centavo storage to GTQ storage by changing data types and converting existing data. + """ + # Convert dca_deposits amounts from centavos to GTQ + await db.execute( + """ + UPDATE satoshimachine.dca_deposits + SET amount = CAST(amount AS DECIMAL(10,2)) / 100.0 + WHERE currency = 'GTQ' + """ + ) + + # Convert dca_payments amounts from centavos to GTQ + await db.execute( + """ + UPDATE satoshimachine.dca_payments + SET amount_fiat = CAST(amount_fiat AS DECIMAL(10,2)) / 100.0 + """ + ) + + # Convert lamassu_transactions amounts from centavos to GTQ + await db.execute( + """ + UPDATE satoshimachine.lamassu_transactions + SET fiat_amount = CAST(fiat_amount AS DECIMAL(10,2)) / 100.0 + """ + ) + + # Convert fixed_mode_daily_limit from centavos to GTQ + await db.execute( + """ + UPDATE satoshimachine.dca_clients + SET fixed_mode_daily_limit = CAST(fixed_mode_daily_limit AS DECIMAL(10,2)) / 100.0 + WHERE fixed_mode_daily_limit IS NOT NULL + """ + ) + + # Convert max_daily_limit_gtq in config (if already in centavos) + await db.execute( + """ + UPDATE satoshimachine.lamassu_config + SET max_daily_limit_gtq = CAST(max_daily_limit_gtq AS DECIMAL(10,2)) / 100.0 + WHERE max_daily_limit_gtq > 1000 + """ + ) + + # Change column types to DECIMAL + await db.execute( + """ + ALTER TABLE satoshimachine.dca_deposits + ALTER COLUMN amount TYPE DECIMAL(10,2) + """ + ) + + await db.execute( + """ + ALTER TABLE satoshimachine.dca_payments + ALTER COLUMN amount_fiat TYPE DECIMAL(10,2) + """ + ) + + await db.execute( + """ + ALTER TABLE satoshimachine.lamassu_transactions + ALTER COLUMN fiat_amount TYPE DECIMAL(10,2) + """ + ) + + await db.execute( + """ + ALTER TABLE satoshimachine.dca_clients + ALTER COLUMN fixed_mode_daily_limit TYPE DECIMAL(10,2) + """ + ) + + await db.execute( + """ + ALTER TABLE satoshimachine.lamassu_config + ALTER COLUMN max_daily_limit_gtq TYPE DECIMAL(10,2) + """ ) \ No newline at end of file diff --git a/models.py b/models.py index 7727bee..e4bf64d 100644 --- a/models.py +++ b/models.py @@ -3,7 +3,7 @@ from datetime import datetime from typing import Optional -from pydantic import BaseModel +from pydantic import BaseModel, validator # DCA Client Models @@ -12,7 +12,7 @@ class CreateDcaClientData(BaseModel): wallet_id: str username: str dca_mode: str = "flow" # 'flow' or 'fixed' - fixed_mode_daily_limit: Optional[int] = None + fixed_mode_daily_limit: Optional[float] = None class DcaClient(BaseModel): @@ -30,45 +30,29 @@ class DcaClient(BaseModel): class UpdateDcaClientData(BaseModel): username: Optional[str] = None dca_mode: Optional[str] = None - fixed_mode_daily_limit: Optional[int] = None + fixed_mode_daily_limit: Optional[float] = None status: Optional[str] = None -# API Models for Deposits (Frontend <-> Backend communication) -class CreateDepositAPI(BaseModel): - """API model - frontend sends GTQ amounts""" - client_id: str - amount_gtq: float # Amount in GTQ (e.g., 150.75) - currency: str = "GTQ" - notes: Optional[str] = None - - -class DepositAPI(BaseModel): - """API model - backend returns GTQ amounts""" - id: str - client_id: str - amount_gtq: float # Amount in GTQ (e.g., 150.75) - currency: str - status: str # 'pending' or 'confirmed' - notes: Optional[str] - created_at: datetime - confirmed_at: Optional[datetime] - - -# Database Models for Deposits (Internal storage in centavos) +# Deposit Models (Now storing GTQ directly) class CreateDepositData(BaseModel): - """Internal model - database stores centavos""" client_id: str - amount: int # Amount in smallest currency unit (centavos for GTQ) + amount: float # Amount in GTQ (e.g., 150.75) currency: str = "GTQ" notes: Optional[str] = None + + @validator('amount') + def round_amount_to_cents(cls, v): + """Ensure amount is rounded to 2 decimal places for DECIMAL(10,2) storage""" + if v is not None: + return round(float(v), 2) + return v class DcaDeposit(BaseModel): - """Internal model - database stores centavos""" id: str client_id: str - amount: int + amount: float # Amount in GTQ (e.g., 150.75) currency: str status: str # 'pending' or 'confirmed' notes: Optional[str] @@ -85,7 +69,7 @@ class UpdateDepositStatusData(BaseModel): class CreateDcaPaymentData(BaseModel): client_id: str amount_sats: int - amount_fiat: int # Stored in centavos (GTQ * 100) for precision + amount_fiat: float # Amount in GTQ (e.g., 150.75) exchange_rate: float transaction_type: str # 'flow', 'fixed', 'manual', 'commission' lamassu_transaction_id: Optional[str] = None @@ -97,7 +81,7 @@ class DcaPayment(BaseModel): id: str client_id: str amount_sats: int - amount_fiat: int # Stored in centavos (GTQ * 100) for precision + amount_fiat: float # Amount in GTQ (e.g., 150.75) exchange_rate: float transaction_type: str lamassu_transaction_id: Optional[str] @@ -107,30 +91,19 @@ class DcaPayment(BaseModel): transaction_time: Optional[datetime] = None # Original ATM transaction time -# API Models for Client Balance Summary -class ClientBalanceSummaryAPI(BaseModel): - """API model - returns GTQ amounts""" - client_id: str - total_deposits_gtq: float # Total confirmed deposits in GTQ - total_payments_gtq: float # Total payments made in GTQ - remaining_balance_gtq: float # Available balance for DCA in GTQ - currency: str - - -# Internal Models for Client Balance Summary +# Client Balance Summary (Now storing GTQ directly) class ClientBalanceSummary(BaseModel): - """Internal model - stores centavos""" client_id: str - total_deposits: int # Total confirmed deposits - total_payments: int # Total payments made - remaining_balance: int # Available balance for DCA + total_deposits: float # Total confirmed deposits in GTQ + total_payments: float # Total payments made in GTQ + remaining_balance: float # Available balance for DCA in GTQ currency: str # Transaction Processing Models class LamassuTransaction(BaseModel): transaction_id: str - amount_fiat: int # Stored in centavos (GTQ * 100) for precision + amount_fiat: float # Amount in GTQ (e.g., 150.75) amount_crypto: int exchange_rate: float transaction_type: str # 'cash_in' or 'cash_out' @@ -141,7 +114,7 @@ class LamassuTransaction(BaseModel): # Lamassu Transaction Storage Models class CreateLamassuTransactionData(BaseModel): lamassu_transaction_id: str - fiat_amount: int # Stored in centavos (GTQ * 100) for precision + fiat_amount: float # Amount in GTQ (e.g., 150.75) crypto_amount: int commission_percentage: float discount: float = 0.0 @@ -158,7 +131,7 @@ class CreateLamassuTransactionData(BaseModel): class StoredLamassuTransaction(BaseModel): id: str lamassu_transaction_id: str - fiat_amount: int + fiat_amount: float # Amount in GTQ (e.g., 150.75) crypto_amount: int commission_percentage: float discount: float @@ -194,7 +167,14 @@ class CreateLamassuConfigData(BaseModel): ssh_password: Optional[str] = None ssh_private_key: Optional[str] = None # Path to private key file or key content # DCA Client Limits - max_daily_limit_gtq: int = 2000 # Maximum daily limit for Fixed mode clients + max_daily_limit_gtq: float = 2000.0 # Maximum daily limit for Fixed mode clients + + @validator('max_daily_limit_gtq') + def round_max_daily_limit(cls, v): + """Ensure max daily limit is rounded to 2 decimal places""" + if v is not None: + return round(float(v), 2) + return v class LamassuConfig(BaseModel): @@ -224,7 +204,7 @@ class LamassuConfig(BaseModel): last_poll_time: Optional[datetime] = None last_successful_poll: Optional[datetime] = None # DCA Client Limits - max_daily_limit_gtq: int = 2000 # Maximum daily limit for Fixed mode clients + max_daily_limit_gtq: float = 2000.0 # Maximum daily limit for Fixed mode clients class UpdateLamassuConfigData(BaseModel): diff --git a/static/js/index.js b/static/js/index.js index 6178a02..1989e9b 100644 --- a/static/js/index.js +++ b/static/js/index.js @@ -27,7 +27,7 @@ window.app = Vue.createApp({ depositsTable: { columns: [ { name: 'client_id', align: 'left', label: 'Client', field: 'client_id' }, - { name: 'amount_gtq', align: 'left', label: 'Amount', field: 'amount_gtq' }, + { name: 'amount', align: 'left', label: 'Amount', field: 'amount' }, { name: 'currency', align: 'left', label: 'Currency', field: 'currency' }, { name: 'status', align: 'left', label: 'Status', field: 'status' }, { name: 'created_at', align: 'left', label: 'Created', field: 'created_at' }, @@ -130,11 +130,11 @@ window.app = Vue.createApp({ /////////////////////////////////////////////////// methods: { - // Utility Methods - Simplified since API handles conversion + // Utility Methods formatCurrency(amount) { if (!amount) return 'Q 0.00'; - // Amount is already in GTQ from API + // Amount is now stored as GTQ directly in database return new Intl.NumberFormat('es-GT', { style: 'currency', currency: 'GTQ', @@ -279,7 +279,7 @@ window.app = Vue.createApp({ ) return { ...client, - remaining_balance: balance.remaining_balance_gtq + remaining_balance: balance.remaining_balance } } catch (error) { console.error(`Error fetching balance for client ${client.id}:`, error) @@ -303,7 +303,7 @@ window.app = Vue.createApp({ try { const data = { client_id: this.quickDepositForm.selectedClient?.value, - amount_gtq: this.quickDepositForm.amount, // Send GTQ directly - API handles conversion + amount: this.quickDepositForm.amount, // Send GTQ directly - now stored as GTQ currency: 'GTQ', notes: this.quickDepositForm.notes } @@ -376,7 +376,7 @@ window.app = Vue.createApp({ try { const data = { client_id: this.depositFormDialog.data.client_id, - amount_gtq: this.depositFormDialog.data.amount, // Send GTQ directly - API handles conversion + amount: this.depositFormDialog.data.amount, // Send GTQ directly - now stored as GTQ currency: this.depositFormDialog.data.currency, notes: this.depositFormDialog.data.notes } diff --git a/templates/satmachineadmin/index.html b/templates/satmachineadmin/index.html index 8901460..61099ce 100644 --- a/templates/satmachineadmin/index.html +++ b/templates/satmachineadmin/index.html @@ -173,7 +173,7 @@
${ getClientUsername(col.value) }
-
${ formatCurrency(col.value) }
+
${ formatCurrency(col.value) }
${ col.value } @@ -470,9 +470,9 @@ Balance Summary - Deposits: ${ formatCurrency(clientDetailsDialog.balance.total_deposits_gtq) } | - Payments: ${ formatCurrency(clientDetailsDialog.balance.total_payments_gtq) } | - Remaining: ${ formatCurrency(clientDetailsDialog.balance.remaining_balance_gtq) } + Deposits: ${ formatCurrency(clientDetailsDialog.balance.total_deposits) } | + Payments: ${ formatCurrency(clientDetailsDialog.balance.total_payments) } | + Remaining: ${ formatCurrency(clientDetailsDialog.balance.remaining_balance) } diff --git a/transaction_processor.py b/transaction_processor.py index c9c4f84..895f8d7 100644 --- a/transaction_processor.py +++ b/transaction_processor.py @@ -671,7 +671,7 @@ class LamassuTransactionProcessor: # Since crypto_atoms already includes commission, we need to extract the base amount # Formula: crypto_atoms = base_amount * (1 + effective_commission) # Therefore: base_amount = crypto_atoms / (1 + effective_commission) - base_crypto_atoms = int(crypto_atoms / (1 + effective_commission)) + base_crypto_atoms = round(crypto_atoms / (1 + effective_commission)) commission_amount_sats = crypto_atoms - base_crypto_atoms else: effective_commission = 0.0 @@ -755,9 +755,8 @@ class LamassuTransactionProcessor: client_sats_amount = calc['allocated_sats'] proportion = calc['proportion'] - # Calculate equivalent fiat value in centavos for tracking purposes (industry standard) - # Store as centavos to maintain precision and avoid floating-point errors - client_fiat_amount = round(client_sats_amount * 100 / exchange_rate) if exchange_rate > 0 else 0 + # Calculate equivalent fiat value in GTQ for tracking purposes + client_fiat_amount = round(client_sats_amount / exchange_rate, 2) if exchange_rate > 0 else 0.0 distributions[client_id] = { "fiat_amount": client_fiat_amount, @@ -765,7 +764,7 @@ class LamassuTransactionProcessor: "exchange_rate": exchange_rate } - logger.info(f"Client {client_id[:8]}... gets {client_sats_amount} sats (≈{client_fiat_amount/100:.2f} GTQ, {proportion:.2%} share)") + logger.info(f"Client {client_id[:8]}... gets {client_sats_amount} sats (≈{client_fiat_amount:.2f} GTQ, {proportion:.2%} share)") # Verification: ensure total distribution equals base amount total_distributed = sum(dist["sats_amount"] for dist in distributions.values()) @@ -781,9 +780,9 @@ class LamassuTransactionProcessor: current_balance = await get_client_balance_summary(client_id) if current_balance.remaining_balance > 0: final_distributions[client_id] = distribution - logger.info(f"Client {client_id[:8]}... final balance check: {current_balance.remaining_balance} centavos - APPROVED for {distribution['sats_amount']} sats") + logger.info(f"Client {client_id[:8]}... final balance check: {current_balance.remaining_balance:.2f} GTQ - APPROVED for {distribution['sats_amount']} sats") else: - logger.warning(f"Client {client_id[:8]}... final balance check: {current_balance.remaining_balance} centavos - REJECTED (negative balance)") + logger.warning(f"Client {client_id[:8]}... final balance check: {current_balance.remaining_balance:.2f} GTQ - REJECTED (negative balance)") if len(final_distributions) != len(distributions): logger.warning(f"Rejected {len(distributions) - len(final_distributions)} clients due to negative balances during final check") @@ -830,22 +829,22 @@ class LamassuTransactionProcessor: # Final safety check: Verify client still has positive balance before payment current_balance = await get_client_balance_summary(client_id) if current_balance.remaining_balance <= 0: - logger.error(f"CRITICAL: Client {client_id[:8]}... has negative balance ({current_balance.remaining_balance} centavos) - REFUSING payment of {distribution['sats_amount']} sats") + logger.error(f"CRITICAL: Client {client_id[:8]}... has negative balance ({current_balance.remaining_balance:.2f} GTQ) - REFUSING payment of {distribution['sats_amount']} sats") continue # Verify balance is sufficient for this distribution - fiat_equivalent = distribution["fiat_amount"] # Already in centavos + fiat_equivalent = distribution["fiat_amount"] # Amount in GTQ if current_balance.remaining_balance < fiat_equivalent: - logger.error(f"CRITICAL: Client {client_id[:8]}... insufficient balance ({current_balance.remaining_balance} < {fiat_equivalent} centavos) - REFUSING payment") + logger.error(f"CRITICAL: Client {client_id[:8]}... insufficient balance ({current_balance.remaining_balance:.2f} < {fiat_equivalent:.2f} GTQ) - REFUSING payment") continue - logger.info(f"Client {client_id[:8]}... pre-payment balance check: {current_balance.remaining_balance} centavos - SUFFICIENT for {fiat_equivalent} centavos payment") + logger.info(f"Client {client_id[:8]}... pre-payment balance check: {current_balance.remaining_balance:.2f} GTQ - SUFFICIENT for {fiat_equivalent:.2f} GTQ payment") # Create DCA payment record payment_data = CreateDcaPaymentData( client_id=client_id, amount_sats=distribution["sats_amount"], - amount_fiat=distribution["fiat_amount"], # Still store centavos in DB + amount_fiat=distribution["fiat_amount"], # Amount in GTQ exchange_rate=distribution["exchange_rate"], transaction_type="flow", lamassu_transaction_id=transaction_id, @@ -888,12 +887,9 @@ class LamassuTransactionProcessor: return False # Create descriptive memo with DCA metrics - fiat_amount_centavos = distribution.get("fiat_amount", 0) + fiat_amount_gtq = distribution.get("fiat_amount", 0.0) exchange_rate = distribution.get("exchange_rate", 0) - # Convert centavos to GTQ for display - fiat_amount_gtq = fiat_amount_centavos / 100 - # Calculate cost basis (fiat per BTC) if exchange_rate > 0: # exchange_rate is sats per fiat unit, so convert to fiat per BTC @@ -1011,7 +1007,7 @@ class LamassuTransactionProcessor: # Calculate commission metrics if commission_percentage > 0: effective_commission = commission_percentage * (100 - discount) / 100 - base_crypto_atoms = int(crypto_atoms / (1 + effective_commission)) + base_crypto_atoms = round(crypto_atoms / (1 + effective_commission)) commission_amount_sats = crypto_atoms - base_crypto_atoms else: effective_commission = 0.0 @@ -1021,10 +1017,10 @@ class LamassuTransactionProcessor: # Calculate exchange rate exchange_rate = base_crypto_atoms / fiat_amount if fiat_amount > 0 else 0 - # Create transaction data (store fiat_amount in centavos for consistency) + # Create transaction data with GTQ amounts transaction_data = CreateLamassuTransactionData( lamassu_transaction_id=transaction["transaction_id"], - fiat_amount=int(fiat_amount * 100), # Convert GTQ to centavos + fiat_amount=round(fiat_amount, 2), # Store GTQ with 2 decimal places crypto_amount=crypto_atoms, commission_percentage=commission_percentage, discount=discount, @@ -1137,7 +1133,7 @@ class LamassuTransactionProcessor: if commission_percentage and commission_percentage > 0: effective_commission = commission_percentage * (100 - discount) / 100 - base_crypto_atoms = int(crypto_atoms / (1 + effective_commission)) + base_crypto_atoms = round(crypto_atoms / (1 + effective_commission)) commission_amount_sats = crypto_atoms - base_crypto_atoms else: commission_amount_sats = 0 diff --git a/views_api.py b/views_api.py index 9d3c41a..3700497 100644 --- a/views_api.py +++ b/views_api.py @@ -38,23 +38,14 @@ from .models import ( DcaClient, UpdateDcaClientData, CreateDepositData, - CreateDepositAPI, - DepositAPI, DcaDeposit, UpdateDepositStatusData, ClientBalanceSummary, - ClientBalanceSummaryAPI, CreateLamassuConfigData, LamassuConfig, UpdateLamassuConfigData, StoredLamassuTransaction, ) -from .currency_utils import ( - gtq_to_centavos, - centavos_to_gtq, - deposit_db_to_api, - balance_summary_db_to_api, -) satmachineadmin_api_router = APIRouter() @@ -96,7 +87,7 @@ async def api_get_dca_client( async def api_get_client_balance( client_id: str, wallet: WalletTypeInfo = Depends(check_super_user), -) -> ClientBalanceSummaryAPI: +) -> ClientBalanceSummary: """Get client balance summary""" client = await get_dca_client(client_id) if not client: @@ -104,8 +95,7 @@ async def api_get_client_balance( status_code=HTTPStatus.NOT_FOUND, detail="DCA client not found." ) - balance_db = await get_client_balance_summary(client_id) - return ClientBalanceSummaryAPI(**balance_summary_db_to_api(balance_db)) + return await get_client_balance_summary(client_id) # DCA Deposit Endpoints @@ -114,31 +104,30 @@ async def api_get_client_balance( @satmachineadmin_api_router.get("/api/v1/dca/deposits") async def api_get_deposits( wallet: WalletTypeInfo = Depends(check_super_user), -) -> list[DepositAPI]: +) -> list[DcaDeposit]: """Get all deposits""" - deposits_db = await get_all_deposits() - return [DepositAPI(**deposit_db_to_api(deposit)) for deposit in deposits_db] + return await get_all_deposits() @satmachineadmin_api_router.get("/api/v1/dca/deposits/{deposit_id}") async def api_get_deposit( deposit_id: str, wallet: WalletTypeInfo = Depends(check_super_user), -) -> DepositAPI: +) -> DcaDeposit: """Get a specific deposit""" - deposit_db = await get_deposit(deposit_id) - if not deposit_db: + deposit = await get_deposit(deposit_id) + if not deposit: raise HTTPException( status_code=HTTPStatus.NOT_FOUND, detail="Deposit not found." ) - return DepositAPI(**deposit_db_to_api(deposit_db)) + return deposit @satmachineadmin_api_router.post("/api/v1/dca/deposits", status_code=HTTPStatus.CREATED) async def api_create_deposit( - data: CreateDepositAPI, + data: CreateDepositData, user: User = Depends(check_super_user), -) -> DepositAPI: +) -> DcaDeposit: """Create a new deposit""" # Verify client exists client = await get_dca_client(data.client_id) @@ -147,16 +136,7 @@ async def api_create_deposit( status_code=HTTPStatus.NOT_FOUND, detail="DCA client not found." ) - # Convert GTQ to centavos at API boundary - deposit_data = CreateDepositData( - client_id=data.client_id, - amount=gtq_to_centavos(data.amount_gtq), - currency=data.currency, - notes=data.notes - ) - - deposit_db = await create_deposit(deposit_data) - return DepositAPI(**deposit_db_to_api(deposit_db)) + return await create_deposit(data) @satmachineadmin_api_router.put("/api/v1/dca/deposits/{deposit_id}/status") @@ -164,7 +144,7 @@ async def api_update_deposit_status( deposit_id: str, data: UpdateDepositStatusData, user: User = Depends(check_super_user), -) -> DepositAPI: +) -> DcaDeposit: """Update deposit status (e.g., confirm deposit)""" deposit = await get_deposit(deposit_id) if not deposit: @@ -172,13 +152,13 @@ async def api_update_deposit_status( status_code=HTTPStatus.NOT_FOUND, detail="Deposit not found." ) - updated_deposit_db = await update_deposit_status(deposit_id, data) - if not updated_deposit_db: + updated_deposit = await update_deposit_status(deposit_id, data) + if not updated_deposit: raise HTTPException( status_code=HTTPStatus.INTERNAL_SERVER_ERROR, detail="Failed to update deposit.", ) - return DepositAPI(**deposit_db_to_api(updated_deposit_db)) + return updated_deposit # Transaction Polling Endpoints From a864f285e42dc1648f771cccf88058a4505059bc Mon Sep 17 00:00:00 2001 From: padreug Date: Sun, 6 Jul 2025 00:24:49 +0200 Subject: [PATCH 02/10] Refactor m004_convert_to_gtq_storage migration: Streamlined the conversion of centavo amounts to GTQ by detecting the database type (PostgreSQL or SQLite) and applying appropriate data type changes and updates. This enhances clarity and ensures proper handling of data conversions across relevant tables. --- migrations.py | 105 ++++++++++++++------------------------------------ 1 file changed, 28 insertions(+), 77 deletions(-) diff --git a/migrations.py b/migrations.py index 6654c71..c8147f5 100644 --- a/migrations.py +++ b/migrations.py @@ -140,82 +140,33 @@ async def m003_add_max_daily_limit_config(db): async def m004_convert_to_gtq_storage(db): """ Convert centavo storage to GTQ storage by changing data types and converting existing data. + Handles both SQLite (data conversion only) and PostgreSQL (data + schema changes). """ - # Convert dca_deposits amounts from centavos to GTQ - await db.execute( - """ - UPDATE satoshimachine.dca_deposits - SET amount = CAST(amount AS DECIMAL(10,2)) / 100.0 - WHERE currency = 'GTQ' - """ - ) + # Detect database type + db_type = str(type(db)).lower() + is_postgres = 'postgres' in db_type or 'asyncpg' in db_type - # Convert dca_payments amounts from centavos to GTQ - await db.execute( - """ - UPDATE satoshimachine.dca_payments - SET amount_fiat = CAST(amount_fiat AS DECIMAL(10,2)) / 100.0 - """ - ) - - # Convert lamassu_transactions amounts from centavos to GTQ - await db.execute( - """ - UPDATE satoshimachine.lamassu_transactions - SET fiat_amount = CAST(fiat_amount AS DECIMAL(10,2)) / 100.0 - """ - ) - - # Convert fixed_mode_daily_limit from centavos to GTQ - await db.execute( - """ - UPDATE satoshimachine.dca_clients - SET fixed_mode_daily_limit = CAST(fixed_mode_daily_limit AS DECIMAL(10,2)) / 100.0 - WHERE fixed_mode_daily_limit IS NOT NULL - """ - ) - - # Convert max_daily_limit_gtq in config (if already in centavos) - await db.execute( - """ - UPDATE satoshimachine.lamassu_config - SET max_daily_limit_gtq = CAST(max_daily_limit_gtq AS DECIMAL(10,2)) / 100.0 - WHERE max_daily_limit_gtq > 1000 - """ - ) - - # Change column types to DECIMAL - await db.execute( - """ - ALTER TABLE satoshimachine.dca_deposits - ALTER COLUMN amount TYPE DECIMAL(10,2) - """ - ) - - await db.execute( - """ - ALTER TABLE satoshimachine.dca_payments - ALTER COLUMN amount_fiat TYPE DECIMAL(10,2) - """ - ) - - await db.execute( - """ - ALTER TABLE satoshimachine.lamassu_transactions - ALTER COLUMN fiat_amount TYPE DECIMAL(10,2) - """ - ) - - await db.execute( - """ - ALTER TABLE satoshimachine.dca_clients - ALTER COLUMN fixed_mode_daily_limit TYPE DECIMAL(10,2) - """ - ) - - await db.execute( - """ - ALTER TABLE satoshimachine.lamassu_config - ALTER COLUMN max_daily_limit_gtq TYPE DECIMAL(10,2) - """ - ) \ No newline at end of file + if is_postgres: + # PostgreSQL: Need to change column types first, then convert data + + # Change column types to DECIMAL(10,2) + await db.execute("ALTER TABLE satoshimachine.dca_deposits ALTER COLUMN amount TYPE DECIMAL(10,2)") + await db.execute("ALTER TABLE satoshimachine.dca_payments ALTER COLUMN amount_fiat TYPE DECIMAL(10,2)") + await db.execute("ALTER TABLE satoshimachine.lamassu_transactions ALTER COLUMN fiat_amount TYPE DECIMAL(10,2)") + await db.execute("ALTER TABLE satoshimachine.dca_clients ALTER COLUMN fixed_mode_daily_limit TYPE DECIMAL(10,2)") + await db.execute("ALTER TABLE satoshimachine.lamassu_config ALTER COLUMN max_daily_limit_gtq TYPE DECIMAL(10,2)") + + # Convert data from centavos to GTQ + await db.execute("UPDATE satoshimachine.dca_deposits SET amount = amount / 100.0 WHERE currency = 'GTQ'") + await db.execute("UPDATE satoshimachine.dca_payments SET amount_fiat = amount_fiat / 100.0") + await db.execute("UPDATE satoshimachine.lamassu_transactions SET fiat_amount = fiat_amount / 100.0") + await db.execute("UPDATE satoshimachine.dca_clients SET fixed_mode_daily_limit = fixed_mode_daily_limit / 100.0 WHERE fixed_mode_daily_limit IS NOT NULL") + await db.execute("UPDATE satoshimachine.lamassu_config SET max_daily_limit_gtq = max_daily_limit_gtq / 100.0 WHERE max_daily_limit_gtq > 1000") + + else: + # SQLite: Data conversion only (dynamic typing handles the rest) + await db.execute("UPDATE satoshimachine.dca_deposits SET amount = CAST(amount AS REAL) / 100.0 WHERE currency = 'GTQ'") + await db.execute("UPDATE satoshimachine.dca_payments SET amount_fiat = CAST(amount_fiat AS REAL) / 100.0") + await db.execute("UPDATE satoshimachine.lamassu_transactions SET fiat_amount = CAST(fiat_amount AS REAL) / 100.0") + await db.execute("UPDATE satoshimachine.dca_clients SET fixed_mode_daily_limit = CAST(fixed_mode_daily_limit AS REAL) / 100.0 WHERE fixed_mode_daily_limit IS NOT NULL") + await db.execute("UPDATE satoshimachine.lamassu_config SET max_daily_limit_gtq = CAST(max_daily_limit_gtq AS REAL) / 100.0 WHERE max_daily_limit_gtq > 1000") \ No newline at end of file From bca39b91cd13eb1d76fe37dc8d371cf8eab38083 Mon Sep 17 00:00:00 2001 From: padreug Date: Sun, 6 Jul 2025 00:54:19 +0200 Subject: [PATCH 03/10] Enhance commission memo generation in transaction processing: Added discount handling to the commission memo in LamassuTransactionProcessor, allowing for the calculation of effective commission percentages when discounts are applied. This improves clarity in transaction details by providing a more accurate representation of commissions after discounts. --- transaction_processor.py | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/transaction_processor.py b/transaction_processor.py index 895f8d7..a32a32c 100644 --- a/transaction_processor.py +++ b/transaction_processor.py @@ -1061,7 +1061,19 @@ class LamassuTransactionProcessor: # Create invoice in commission wallet with DCA metrics fiat_amount = transaction.get("fiat_amount", 0) commission_percentage = transaction.get("commission_percentage", 0) * 100 # Convert to percentage - commission_memo = f"DCA Commission: {commission_amount_sats:,} sats • {commission_percentage:.1f}% • {fiat_amount:,} GTQ transaction" + discount = transaction.get("discount", 0.0) # Discount percentage + + # Calculate effective commission for display + if commission_percentage > 0: + effective_commission_percentage = commission_percentage * (100 - discount) / 100 + else: + effective_commission_percentage = 0.0 + + # Create detailed memo showing discount if applied + if discount > 0: + commission_memo = f"DCA Commission: {commission_amount_sats:,} sats • {commission_percentage:.1f}% - {discount:.1f}% discount = {effective_commission_percentage:.1f}% effective • {fiat_amount:,} GTQ transaction" + else: + commission_memo = f"DCA Commission: {commission_amount_sats:,} sats • {commission_percentage:.1f}% • {fiat_amount:,} GTQ transaction" commission_payment = await create_invoice( wallet_id=admin_config.commission_wallet_id, From 5d41e0c50e3b8c7d2c10298b1610c43440a8bd68 Mon Sep 17 00:00:00 2001 From: padreug Date: Sun, 6 Jul 2025 01:15:26 +0200 Subject: [PATCH 04/10] Refine distribution logic in transaction processing: Updated the LamassuTransactionProcessor to only create distributions for clients with positive allocated sats. Enhanced logging to indicate when clients are skipped due to zero amounts, improving clarity and accuracy in transaction reporting. --- transaction_processor.py | 30 +++++++++++++++++------------- 1 file changed, 17 insertions(+), 13 deletions(-) diff --git a/transaction_processor.py b/transaction_processor.py index a32a32c..7b08ac9 100644 --- a/transaction_processor.py +++ b/transaction_processor.py @@ -749,22 +749,26 @@ class LamassuTransactionProcessor: else: client_calculations[i % len(client_calculations)]['allocated_sats'] -= 1 - # Second pass: create distributions with final amounts + # Second pass: create distributions with final amounts (only for clients with positive allocations) for calc in client_calculations: client_id = calc['client_id'] client_sats_amount = calc['allocated_sats'] proportion = calc['proportion'] - # Calculate equivalent fiat value in GTQ for tracking purposes - client_fiat_amount = round(client_sats_amount / exchange_rate, 2) if exchange_rate > 0 else 0.0 - - distributions[client_id] = { - "fiat_amount": client_fiat_amount, - "sats_amount": client_sats_amount, - "exchange_rate": exchange_rate - } - - logger.info(f"Client {client_id[:8]}... gets {client_sats_amount} sats (≈{client_fiat_amount:.2f} GTQ, {proportion:.2%} share)") + # Only create distributions for clients with positive sats amounts + if client_sats_amount > 0: + # Calculate equivalent fiat value in GTQ for tracking purposes + client_fiat_amount = round(client_sats_amount / exchange_rate, 2) if exchange_rate > 0 else 0.0 + + distributions[client_id] = { + "fiat_amount": client_fiat_amount, + "sats_amount": client_sats_amount, + "exchange_rate": exchange_rate + } + + logger.info(f"Client {client_id[:8]}... gets {client_sats_amount} sats (≈{client_fiat_amount:.2f} GTQ, {proportion:.2%} share)") + else: + logger.info(f"Client {client_id[:8]}... gets {client_sats_amount} sats (≈0.00 GTQ, {proportion:.2%} share) - SKIPPED (zero amount)") # Verification: ensure total distribution equals base amount total_distributed = sum(dist["sats_amount"] for dist in distributions.values()) @@ -796,8 +800,8 @@ class LamassuTransactionProcessor: # In a production system, you might want to recalculate the entire distribution logger.warning("Proceeding with original distribution despite balance warnings - manual review recommended") - logger.info(f"Distribution verified: {total_distributed} sats distributed across {len(final_distributions)} clients") - return final_distributions + logger.info(f"Distribution verified: {total_distributed} sats distributed across {len(distributions)} clients (clients with positive allocations only)") + return distributions except Exception as e: logger.error(f"Error calculating distribution amounts: {e}") From 4843b431478022dc83bab479e1e4571682f3e5e3 Mon Sep 17 00:00:00 2001 From: padreug Date: Sun, 6 Jul 2025 02:04:11 +0200 Subject: [PATCH 05/10] FIX: exclude flow_clients remaining_balance values less than 0.01 --- transaction_processor.py | 33 +++++++++++++++++---------------- 1 file changed, 17 insertions(+), 16 deletions(-) diff --git a/transaction_processor.py b/transaction_processor.py index 7b08ac9..22c4edd 100644 --- a/transaction_processor.py +++ b/transaction_processor.py @@ -696,9 +696,14 @@ class LamassuTransactionProcessor: for client in flow_clients: # Get balance as of the transaction time for temporal accuracy balance = await get_client_balance_summary(client.id, as_of_time=transaction_time) - if balance.remaining_balance > 0: # Only include clients with remaining balance + # Only include clients with positive remaining balance + # NOTE: This works for fiat amounts that use cents + if balance.remaining_balance >= 0.01: client_balances[client.id] = balance.remaining_balance total_confirmed_deposits += balance.remaining_balance + logger.debug(f"Client {client.id[:8]}... included with balance: {balance.remaining_balance:.2f} GTQ") + else: + logger.info(f"Client {client.id[:8]}... excluded - zero/negative balance: {balance.remaining_balance:.2f} GTQ") if total_confirmed_deposits == 0: logger.info("No clients with remaining DCA balance - skipping distribution") @@ -749,26 +754,22 @@ class LamassuTransactionProcessor: else: client_calculations[i % len(client_calculations)]['allocated_sats'] -= 1 - # Second pass: create distributions with final amounts (only for clients with positive allocations) + # Second pass: create distributions with final amounts for calc in client_calculations: client_id = calc['client_id'] client_sats_amount = calc['allocated_sats'] proportion = calc['proportion'] - # Only create distributions for clients with positive sats amounts - if client_sats_amount > 0: - # Calculate equivalent fiat value in GTQ for tracking purposes - client_fiat_amount = round(client_sats_amount / exchange_rate, 2) if exchange_rate > 0 else 0.0 - - distributions[client_id] = { - "fiat_amount": client_fiat_amount, - "sats_amount": client_sats_amount, - "exchange_rate": exchange_rate - } - - logger.info(f"Client {client_id[:8]}... gets {client_sats_amount} sats (≈{client_fiat_amount:.2f} GTQ, {proportion:.2%} share)") - else: - logger.info(f"Client {client_id[:8]}... gets {client_sats_amount} sats (≈0.00 GTQ, {proportion:.2%} share) - SKIPPED (zero amount)") + # Calculate equivalent fiat value in GTQ for tracking purposes + client_fiat_amount = round(client_sats_amount / exchange_rate, 2) if exchange_rate > 0 else 0.0 + + distributions[client_id] = { + "fiat_amount": client_fiat_amount, + "sats_amount": client_sats_amount, + "exchange_rate": exchange_rate + } + + logger.info(f"Client {client_id[:8]}... gets {client_sats_amount} sats (≈{client_fiat_amount:.2f} GTQ, {proportion:.2%} share)") # Verification: ensure total distribution equals base amount total_distributed = sum(dist["sats_amount"] for dist in distributions.values()) From 077e097fc22d21fbfda7b8caf70191160a9bf00e Mon Sep 17 00:00:00 2001 From: padreug Date: Tue, 8 Jul 2025 06:14:52 +0200 Subject: [PATCH 06/10] Refactor transaction data handling in LamassuTransactionProcessor: Improved consistency in processing None and empty values during data ingestion. Defaulted numeric fields to 0 and percentage fields to 0.0 for better error handling. Ensured clean extraction of transaction details, enhancing reliability in transaction processing. --- transaction_processor.py | 40 ++++++++++++++++++++++++++-------------- 1 file changed, 26 insertions(+), 14 deletions(-) diff --git a/transaction_processor.py b/transaction_processor.py index 22c4edd..a07be52 100644 --- a/transaction_processor.py +++ b/transaction_processor.py @@ -493,14 +493,26 @@ class LamassuTransactionProcessor: # Convert string values to appropriate types processed_row = {} for key, value in row.items(): - if value == '': - processed_row[key] = None + # Handle None/empty values consistently at data ingestion boundary + if value == '' or value is None: + if key in ['fiat_amount', 'crypto_amount']: + processed_row[key] = 0 # Default numeric fields to 0 + elif key in ['commission_percentage', 'discount']: + processed_row[key] = 0.0 # Default percentage fields to 0.0 + else: + processed_row[key] = None # Keep None for non-numeric fields elif key in ['transaction_id', 'device_id', 'crypto_code', 'fiat_code']: processed_row[key] = str(value) elif key in ['fiat_amount', 'crypto_amount']: - processed_row[key] = int(float(value)) if value else 0 + try: + processed_row[key] = int(float(value)) + except (ValueError, TypeError): + processed_row[key] = 0 # Fallback to 0 for invalid values elif key in ['commission_percentage', 'discount']: - processed_row[key] = float(value) if value else 0.0 + try: + processed_row[key] = float(value) + except (ValueError, TypeError): + processed_row[key] = 0.0 # Fallback to 0.0 for invalid values elif key == 'transaction_time': from datetime import datetime # Parse PostgreSQL timestamp format and ensure it's in UTC for consistency @@ -628,11 +640,11 @@ class LamassuTransactionProcessor: logger.info("No Flow Mode clients found - skipping distribution") return {} - # Extract transaction details with None-safe defaults - crypto_atoms = transaction.get("crypto_amount") # Total sats with commission baked in - fiat_amount = transaction.get("fiat_amount") # Actual fiat dispensed (principal only) - commission_percentage = transaction.get("commission_percentage") # Already stored as decimal (e.g., 0.045) - discount = transaction.get("discount") # Discount percentage + # Extract transaction details - guaranteed clean from data ingestion + crypto_atoms = transaction.get("crypto_amount", 0) # Total sats with commission baked in + fiat_amount = transaction.get("fiat_amount", 0) # Actual fiat dispensed (principal only) + commission_percentage = transaction.get("commission_percentage", 0.0) # Already stored as decimal (e.g., 0.045) + discount = transaction.get("discount", 0.0) # Discount percentage transaction_time = transaction.get("transaction_time") # ATM transaction timestamp for temporal accuracy # Normalize transaction_time to UTC if present @@ -995,11 +1007,11 @@ class LamassuTransactionProcessor: async def store_lamassu_transaction(self, transaction: Dict[str, Any]) -> Optional[str]: """Store the Lamassu transaction in our database for audit and UI""" try: - # Extract and validate transaction data + # Extract transaction data - guaranteed clean from data ingestion boundary crypto_atoms = transaction.get("crypto_amount", 0) fiat_amount = transaction.get("fiat_amount", 0) - commission_percentage = transaction.get("commission_percentage") or 0.0 - discount = transaction.get("discount") or 0.0 + commission_percentage = transaction.get("commission_percentage", 0.0) + discount = transaction.get("discount", 0.0) transaction_time = transaction.get("transaction_time") # Normalize transaction_time to UTC if present @@ -1145,8 +1157,8 @@ class LamassuTransactionProcessor: # Calculate commission amount for sending to commission wallet crypto_atoms = transaction.get("crypto_amount", 0) - commission_percentage = transaction.get("commission_percentage") or 0.0 - discount = transaction.get("discount") or 0.0 + commission_percentage = transaction.get("commission_percentage", 0.0) + discount = transaction.get("discount", 0.0) if commission_percentage and commission_percentage > 0: effective_commission = commission_percentage * (100 - discount) / 100 From 230beccc37ee39e10df9dff91f1e56ec5f71de1b Mon Sep 17 00:00:00 2001 From: padreug Date: Sat, 19 Jul 2025 00:21:13 +0200 Subject: [PATCH 07/10] Improve balance verification in LamassuTransactionProcessor: Added rounding to two decimal places for balance and fiat amounts to ensure precision in comparisons. Enhanced logging for insufficient balance scenarios, improving clarity in transaction processing and error reporting. --- transaction_processor.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/transaction_processor.py b/transaction_processor.py index a07be52..9a58408 100644 --- a/transaction_processor.py +++ b/transaction_processor.py @@ -849,10 +849,13 @@ class LamassuTransactionProcessor: logger.error(f"CRITICAL: Client {client_id[:8]}... has negative balance ({current_balance.remaining_balance:.2f} GTQ) - REFUSING payment of {distribution['sats_amount']} sats") continue - # Verify balance is sufficient for this distribution + # Verify balance is sufficient for this distribution (round to 2 decimal places to match DECIMAL(10,2) precision) fiat_equivalent = distribution["fiat_amount"] # Amount in GTQ - if current_balance.remaining_balance < fiat_equivalent: - logger.error(f"CRITICAL: Client {client_id[:8]}... insufficient balance ({current_balance.remaining_balance:.2f} < {fiat_equivalent:.2f} GTQ) - REFUSING payment") + # Round both values to 2 decimal places to match database precision and avoid floating point comparison issues + balance_rounded = round(current_balance.remaining_balance, 2) + amount_rounded = round(fiat_equivalent, 2) + if balance_rounded < amount_rounded: + logger.error(f"CRITICAL: Client {client_id[:8]}... insufficient balance ({balance_rounded:.2f} < {amount_rounded:.2f} GTQ) - REFUSING payment") continue logger.info(f"Client {client_id[:8]}... pre-payment balance check: {current_balance.remaining_balance:.2f} GTQ - SUFFICIENT for {fiat_equivalent:.2f} GTQ payment") From fe38e08d4e9faa14282ba3fb6f095f0a08d40272 Mon Sep 17 00:00:00 2001 From: padreug Date: Fri, 24 Oct 2025 00:15:22 +0200 Subject: [PATCH 08/10] Adds manual transaction processing feature Implements functionality to manually process specific Lamassu transactions by ID, bypassing dispense checks. This allows administrators to handle transactions that may have failed due to dispense issues or were settled manually outside of the automated process. The feature includes a new UI dialog for entering the transaction ID and an API endpoint to fetch and process the transaction, crediting wallets and distributing funds according to the DCA configuration. --- static/js/index.js | 80 +++++++++++++++++++++ templates/satmachineadmin/index.html | 100 +++++++++++++++++++++++---- transaction_processor.py | 42 ++++++++++- views_api.py | 67 ++++++++++++++++++ 4 files changed, 272 insertions(+), 17 deletions(-) diff --git a/static/js/index.js b/static/js/index.js index 1989e9b..ee3e587 100644 --- a/static/js/index.js +++ b/static/js/index.js @@ -92,8 +92,15 @@ window.app = Vue.createApp({ testingConnection: false, runningManualPoll: false, runningTestTransaction: false, + processingSpecificTransaction: false, lamassuConfig: null, + // Manual transaction processing + manualTransactionDialog: { + show: false, + transactionId: '' + }, + // Config dialog configDialog: { show: false, @@ -586,6 +593,79 @@ window.app = Vue.createApp({ await this.getDeposits() await this.getLamassuTransactions() await this.getLamassuConfig() + } catch (error) { + LNbits.utils.notifyApiError(error) + } finally { + this.runningTestTransaction = false + } + }, + + openManualTransactionDialog() { + this.manualTransactionDialog.transactionId = '' + this.manualTransactionDialog.show = true + }, + + async processSpecificTransaction() { + if (!this.manualTransactionDialog.transactionId) { + this.$q.notify({ + type: 'warning', + message: 'Please enter a transaction ID', + timeout: 3000 + }) + return + } + + this.processingSpecificTransaction = true + try { + const { data } = await LNbits.api.request( + 'POST', + `/satmachineadmin/api/v1/dca/process-transaction/${this.manualTransactionDialog.transactionId}`, + null + ) + + if (data.already_processed) { + this.$q.notify({ + type: 'warning', + message: `Transaction already processed with ${data.payment_count} distributions`, + timeout: 5000 + }) + this.manualTransactionDialog.show = false + return + } + + // Show detailed results + const details = data.transaction_details + let dialogContent = `Manual Transaction Processing Results

` + dialogContent += `Transaction ID: ${details.transaction_id}
` + dialogContent += `Status: ${details.status}
` + dialogContent += `Dispense: ${details.dispense ? 'Yes' : 'No'}
` + dialogContent += `Dispense Confirmed: ${details.dispense_confirmed ? 'Yes' : 'No'}
` + dialogContent += `Crypto Amount: ${details.crypto_amount} sats
` + dialogContent += `Fiat Amount: ${details.fiat_amount}
` + dialogContent += `
Transaction processed successfully!` + + this.$q.dialog({ + title: 'Transaction Processed', + message: dialogContent, + html: true, + ok: { + color: 'positive', + label: 'Great!' + } + }) + + this.$q.notify({ + type: 'positive', + message: `Transaction ${details.transaction_id} processed successfully`, + timeout: 5000 + }) + + // Close dialog and refresh data + this.manualTransactionDialog.show = false + await this.getDcaClients() + await this.getDeposits() + await this.getLamassuTransactions() + await this.getLamassuConfig() } catch (error) { LNbits.utils.notifyApiError(error) diff --git a/templates/satmachineadmin/index.html b/templates/satmachineadmin/index.html index 61099ce..87f0a41 100644 --- a/templates/satmachineadmin/index.html +++ b/templates/satmachineadmin/index.html @@ -335,21 +335,32 @@ > Test Connection - Manual Poll - + Process specific transaction by ID (bypasses dispense checks) + Manual TX + + @@ -690,7 +701,7 @@
Transaction Distribution Details
- +
@@ -709,7 +720,7 @@ Total Amount - ${ formatCurrency(distributionDialog.transaction.fiat_amount) } + ${ formatCurrency(distributionDialog.transaction.fiat_amount) } (${ formatSats(distributionDialog.transaction.crypto_amount) }) @@ -718,7 +729,7 @@ Commission - ${ (distributionDialog.transaction.commission_percentage * 100).toFixed(1) }% + ${ (distributionDialog.transaction.commission_percentage * 100).toFixed(1) }% (with ${ distributionDialog.transaction.discount }% discount = ${ (distributionDialog.transaction.effective_commission * 100).toFixed(1) }% effective) @@ -740,11 +751,11 @@
- + - +
Client Distributions
- + - +
Close
+ + + + + + +
Process Specific Transaction
+ + + +
+ Use with caution: This bypasses all dispense status checks and will process the transaction even if dispense_confirmed is false. Only use this for manually settled transactions. +
+ + + + + + + +
+ This will: +
    +
  • Fetch the transaction from Lamassu regardless of dispense status
    • +
  • Process it through the normal DCA distribution flow
    • +
  • Credit the source wallet and distribute to clients
    • +
  • Send commission to the commission wallet (if configured)
    • +
+
+ +
+ + Process Transaction + + + Cancel + +
+ + + +
{% endblock %} diff --git a/transaction_processor.py b/transaction_processor.py index 9a58408..bd6ae77 100644 --- a/transaction_processor.py +++ b/transaction_processor.py @@ -560,6 +560,44 @@ class LamassuTransactionProcessor: logger.error(f"Error executing SSH query: {e}") return [] + async def fetch_transaction_by_id(self, db_config: Dict[str, Any], transaction_id: str) -> Optional[Dict[str, Any]]: + """Fetch a specific transaction by ID from Lamassu database, bypassing all status filters""" + try: + logger.info(f"Fetching transaction {transaction_id} from Lamassu database (bypass all filters)") + + # Query for specific transaction ID without any status/dispense filters + lamassu_query = f""" + SELECT + co.id as transaction_id, + co.fiat as fiat_amount, + co.crypto_atoms as crypto_amount, + co.confirmed_at as transaction_time, + co.device_id, + co.status, + co.commission_percentage, + co.discount, + co.crypto_code, + co.fiat_code, + co.dispense, + co.dispense_confirmed + FROM cash_out_txs co + WHERE co.id = '{transaction_id}' + """ + + results = await self.execute_ssh_query(db_config, lamassu_query) + + if not results: + logger.warning(f"Transaction {transaction_id} not found in Lamassu database") + return None + + transaction = results[0] + logger.info(f"Found transaction {transaction_id}: status={transaction.get('status')}, dispense={transaction.get('dispense')}, dispense_confirmed={transaction.get('dispense_confirmed')}") + return transaction + + except Exception as e: + logger.error(f"Error fetching transaction {transaction_id} from Lamassu database: {e}") + return None + async def fetch_new_transactions(self, db_config: Dict[str, Any]) -> List[Dict[str, Any]]: """Fetch new successful transactions from Lamassu database since last poll""" try: @@ -573,13 +611,13 @@ class LamassuTransactionProcessor: # Fallback to last 24 hours for first run or if no previous poll time_threshold = datetime.now(timezone.utc) - timedelta(hours=24) logger.info(f"No previous poll found, checking last 24 hours since: {time_threshold}") - + # Convert to UTC if not already timezone-aware if time_threshold.tzinfo is None: time_threshold = time_threshold.replace(tzinfo=timezone.utc) elif time_threshold.tzinfo != timezone.utc: time_threshold = time_threshold.astimezone(timezone.utc) - + # Format as UTC for database query time_threshold_str = time_threshold.strftime('%Y-%m-%d %H:%M:%S UTC') diff --git a/views_api.py b/views_api.py index 3700497..42ae53d 100644 --- a/views_api.py +++ b/views_api.py @@ -233,6 +233,73 @@ async def api_manual_poll( ) +@satmachineadmin_api_router.post("/api/v1/dca/process-transaction/{transaction_id}") +async def api_process_specific_transaction( + transaction_id: str, + user: User = Depends(check_super_user), +): + """ + Manually process a specific Lamassu transaction by ID, bypassing all status filters. + + This endpoint is useful for processing transactions that were manually settled + or had dispense issues but need to be included in DCA distribution. + """ + try: + from .transaction_processor import transaction_processor + from .crud import get_payments_by_lamassu_transaction + + # Get database configuration + db_config = await transaction_processor.connect_to_lamassu_db() + if not db_config: + raise HTTPException( + status_code=HTTPStatus.SERVICE_UNAVAILABLE, + detail="Could not get Lamassu database configuration", + ) + + # Check if transaction was already processed + existing_payments = await get_payments_by_lamassu_transaction(transaction_id) + if existing_payments: + return { + "success": False, + "already_processed": True, + "message": f"Transaction {transaction_id} was already processed with {len(existing_payments)} distributions", + "payment_count": len(existing_payments), + } + + # Fetch the specific transaction from Lamassu (bypassing all filters) + transaction = await transaction_processor.fetch_transaction_by_id(db_config, transaction_id) + + if not transaction: + raise HTTPException( + status_code=HTTPStatus.NOT_FOUND, + detail=f"Transaction {transaction_id} not found in Lamassu database", + ) + + # Process the transaction through normal DCA flow + await transaction_processor.process_transaction(transaction) + + return { + "success": True, + "message": f"Transaction {transaction_id} processed successfully", + "transaction_details": { + "transaction_id": transaction_id, + "status": transaction.get("status"), + "dispense": transaction.get("dispense"), + "dispense_confirmed": transaction.get("dispense_confirmed"), + "crypto_amount": transaction.get("crypto_amount"), + "fiat_amount": transaction.get("fiat_amount"), + }, + } + + except HTTPException: + raise + except Exception as e: + raise HTTPException( + status_code=HTTPStatus.INTERNAL_SERVER_ERROR, + detail=f"Error processing transaction {transaction_id}: {str(e)}", + ) + + @satmachineadmin_api_router.post("/api/v1/dca/test-transaction") async def api_test_transaction( user: User = Depends(check_super_user), From 1b7374fa70ee4bc2a010f0edf6d5e689c8b7a2a2 Mon Sep 17 00:00:00 2001 From: padreug Date: Fri, 24 Oct 2025 00:30:34 +0200 Subject: [PATCH 09/10] Removes test transaction UI button Removes the test transaction button from the admin UI. The test transaction endpoint is still available in the API for development and debugging purposes. --- templates/satmachineadmin/index.html | 10 --- views_api.py | 125 ++++++++++++++------------- 2 files changed, 64 insertions(+), 71 deletions(-) diff --git a/templates/satmachineadmin/index.html b/templates/satmachineadmin/index.html index 87f0a41..46eef37 100644 --- a/templates/satmachineadmin/index.html +++ b/templates/satmachineadmin/index.html @@ -356,16 +356,6 @@ Process specific transaction by ID (bypasses dispense checks) Manual TX - - Test Transaction - diff --git a/views_api.py b/views_api.py index 42ae53d..a4a8b94 100644 --- a/views_api.py +++ b/views_api.py @@ -300,67 +300,70 @@ async def api_process_specific_transaction( ) -@satmachineadmin_api_router.post("/api/v1/dca/test-transaction") -async def api_test_transaction( - user: User = Depends(check_super_user), - crypto_atoms: int = 103, - commission_percentage: float = 0.03, - discount: float = 0.0, -) -> dict: - """Test transaction processing with simulated Lamassu transaction data""" - try: - from .transaction_processor import transaction_processor - import uuid - from datetime import datetime, timezone - - # Create a mock transaction that mimics Lamassu database structure - mock_transaction = { - "transaction_id": str(uuid.uuid4())[:8], # Short ID for testing - "crypto_amount": crypto_atoms, # Total sats including commission - "fiat_amount": 100, # Mock fiat amount (100 centavos = 1 GTQ) - "commission_percentage": commission_percentage, # Already as decimal - "discount": discount, - "transaction_time": datetime.now(timezone.utc), - "crypto_code": "BTC", - "fiat_code": "GTQ", - "device_id": "test_device", - "status": "confirmed", - } - - # Process the mock transaction through the complete DCA flow - await transaction_processor.process_transaction(mock_transaction) - - # Calculate commission for response - if commission_percentage > 0: - effective_commission = commission_percentage * (100 - discount) / 100 - base_crypto_atoms = int(crypto_atoms / (1 + effective_commission)) - commission_amount_sats = crypto_atoms - base_crypto_atoms - else: - base_crypto_atoms = crypto_atoms - commission_amount_sats = 0 - - return { - "success": True, - "message": "Test transaction processed successfully", - "transaction_details": { - "transaction_id": mock_transaction["transaction_id"], - "total_amount_sats": crypto_atoms, - "base_amount_sats": base_crypto_atoms, - "commission_amount_sats": commission_amount_sats, - "commission_percentage": commission_percentage - * 100, # Show as percentage - "effective_commission": effective_commission * 100 - if commission_percentage > 0 - else 0, - "discount": discount, - }, - } - - except Exception as e: - raise HTTPException( - status_code=HTTPStatus.INTERNAL_SERVER_ERROR, - detail=f"Error processing test transaction: {str(e)}", - ) +# COMMENTED OUT FOR PRODUCTION - Test transaction endpoint disabled +# Uncomment only for development/debugging purposes +# +# @satmachineadmin_api_router.post("/api/v1/dca/test-transaction") +# async def api_test_transaction( +# user: User = Depends(check_super_user), +# crypto_atoms: int = 103, +# commission_percentage: float = 0.03, +# discount: float = 0.0, +# ) -> dict: +# """Test transaction processing with simulated Lamassu transaction data""" +# try: +# from .transaction_processor import transaction_processor +# import uuid +# from datetime import datetime, timezone +# +# # Create a mock transaction that mimics Lamassu database structure +# mock_transaction = { +# "transaction_id": str(uuid.uuid4())[:8], # Short ID for testing +# "crypto_amount": crypto_atoms, # Total sats including commission +# "fiat_amount": 100, # Mock fiat amount (100 centavos = 1 GTQ) +# "commission_percentage": commission_percentage, # Already as decimal +# "discount": discount, +# "transaction_time": datetime.now(timezone.utc), +# "crypto_code": "BTC", +# "fiat_code": "GTQ", +# "device_id": "test_device", +# "status": "confirmed", +# } +# +# # Process the mock transaction through the complete DCA flow +# await transaction_processor.process_transaction(mock_transaction) +# +# # Calculate commission for response +# if commission_percentage > 0: +# effective_commission = commission_percentage * (100 - discount) / 100 +# base_crypto_atoms = int(crypto_atoms / (1 + effective_commission)) +# commission_amount_sats = crypto_atoms - base_crypto_atoms +# else: +# base_crypto_atoms = crypto_atoms +# commission_amount_sats = 0 +# +# return { +# "success": True, +# "message": "Test transaction processed successfully", +# "transaction_details": { +# "transaction_id": mock_transaction["transaction_id"], +# "total_amount_sats": crypto_atoms, +# "base_amount_sats": base_crypto_atoms, +# "commission_amount_sats": commission_amount_sats, +# "commission_percentage": commission_percentage +# * 100, # Show as percentage +# "effective_commission": effective_commission * 100 +# if commission_percentage > 0 +# else 0, +# "discount": discount, +# }, +# } +# +# except Exception as e: +# raise HTTPException( +# status_code=HTTPStatus.INTERNAL_SERVER_ERROR, +# detail=f"Error processing test transaction: {str(e)}", +# ) # Lamassu Transaction Endpoints From cd0d958c2cc0b599c75dbead396e90aad84829f6 Mon Sep 17 00:00:00 2001 From: padreug Date: Mon, 3 Nov 2025 22:22:55 +0100 Subject: [PATCH 10/10] consolidate docs --- misc-docs/EMERGENCY_PROTOCOLS.md | 1362 +++++++++++++++++ misc-docs/EMERGENCY_PROTOCOLS_PRINT.md | 1362 +++++++++++++++++ .../Lamassu-Database-Analysis.md | 0 3 files changed, 2724 insertions(+) create mode 100644 misc-docs/EMERGENCY_PROTOCOLS.md create mode 100644 misc-docs/EMERGENCY_PROTOCOLS_PRINT.md rename Lamassu-Database-Analysis.md => misc-docs/Lamassu-Database-Analysis.md (100%) diff --git a/misc-docs/EMERGENCY_PROTOCOLS.md b/misc-docs/EMERGENCY_PROTOCOLS.md new file mode 100644 index 0000000..e408774 --- /dev/null +++ b/misc-docs/EMERGENCY_PROTOCOLS.md @@ -0,0 +1,1362 @@ +# Satoshi Machine Admin - Emergency Protocols +## DCA System Failure Recovery Guide + +**Document Version**: 1.0 +**Last Updated**: 2025-10-19 +**Extension Version**: v0.0.1 +**Status**: Production + +--- + +## Table of Contents + +1. [Critical Failure Scenarios](#critical-failure-scenarios) +2. [Emergency Protocol Checklist](#emergency-protocol-checklist) +3. [Recovery Procedures](#recovery-procedures) +4. [Prevention Measures](#prevention-measures) +5. [Monitoring & Alerts](#monitoring--alerts) +6. [Contact Information](#contact-information) + +--- + +## Critical Failure Scenarios + +### 1. Duplicate Transaction Processing ⚠️ CRITICAL + +**Risk Level**: 🔴 **CRITICAL** +**Impact**: Same Lamassu transaction processed twice → double distribution to clients → financial loss + +#### Detection Methods + +1. **Dashboard Monitoring**: + - Sudden large balance deductions from client accounts + - Multiple distribution entries for same timestamp + - Commission wallet receiving duplicate amounts + +2. **Database Query**: +```sql +-- Find duplicate transactions +SELECT transaction_id, COUNT(*) as count, + STRING_AGG(id::text, ', ') as record_ids +FROM satoshimachine.lamassu_transactions +GROUP BY transaction_id +HAVING COUNT(*) > 1; +``` + +3. **Automated Alert Triggers**: + - Same `txn_id` appears in multiple processing cycles + - Client balance drops more than expected based on deposit ratios + +#### Immediate Response + +1. ✅ **STOP POLLING IMMEDIATELY** - Disable automatic background task +2. ✅ Document all duplicate entries with screenshots +3. ✅ Identify affected clients and amounts +4. ✅ Calculate total over-distribution amount + +#### Recovery Steps + +```sql +-- Step 1: Identify duplicate distributions +SELECT lt.transaction_id, lt.id, lt.created_at, lt.base_amount, + COUNT(dp.id) as distribution_count +FROM satoshimachine.lamassu_transactions lt +LEFT JOIN satoshimachine.dca_payments dp ON dp.lamassu_transaction_id = lt.id +GROUP BY lt.id +HAVING COUNT(dp.id) > (SELECT COUNT(*) FROM satoshimachine.dca_clients WHERE remaining_balance > 0); + +-- Step 2: Calculate over-distributed amounts per client +SELECT client_id, + SUM(amount_sats) as total_received, + -- Manual calculation of expected amount needed here +FROM satoshimachine.dca_payments +WHERE lamassu_transaction_id IN (SELECT id FROM duplicates_table) +GROUP BY client_id; +``` + +**Manual Correction**: +1. Calculate correct distribution amounts +2. Create compensating negative adjustments (if supported) OR +3. Deduct from future distributions until balanced +4. Document all corrections in audit log +5. Notify affected clients if material amount + +#### Prevention Measures + +**Required Code Changes**: +```python +# Add to transaction_processor.py BEFORE processing +existing = await get_lamassu_transaction_by_txid(txn_id) +if existing: + logger.warning(f"⚠️ Transaction {txn_id} already processed, skipping") + return None +``` + +**Required Database Change**: +```sql +ALTER TABLE satoshimachine.lamassu_transactions +ADD CONSTRAINT unique_transaction_id UNIQUE (transaction_id); +``` + +--- + +### 2. SSH/Database Connection Loss + +**Risk Level**: 🟡 **MEDIUM** +**Impact**: Polling stops → transactions not processed → clients not receiving Bitcoin on time + +#### Detection Methods + +1. **Dashboard Indicators**: + - No new records in transaction history for > 24 hours + - "Test Connection" button fails in admin configuration + - Background task logs show SSH connection errors + +2. **Database Query**: +```sql +-- Check last successful poll +SELECT MAX(created_at) as last_transaction, + EXTRACT(EPOCH FROM (NOW() - MAX(created_at)))/3600 as hours_since_last +FROM satoshimachine.lamassu_transactions; + +-- If hours_since_last > 24, investigate immediately +``` + +3. **Log File Check**: +```bash +# Check LNBits logs for SSH errors +tail -n 100 /home/padreug/AioLabs/Git/lnbits-extensions/lnbits/data/logs/lnbits.log | grep -i "ssh\|connection" +``` + +#### Immediate Response + +1. ✅ Verify network connectivity to Lamassu server +2. ✅ Test SSH credentials manually: +```bash +ssh postgres@ -p -i +``` +3. ✅ Check firewall rules and network changes +4. ✅ Verify Lamassu server is running and accessible + +#### Recovery Steps + +**Option A: Credential Issues** +1. Regenerate SSH keys if compromised +2. Update authorized_keys on Lamassu server +3. Update configuration in admin dashboard +4. Test connection before re-enabling polling + +**Option B: Network Issues** +1. Coordinate with network admin to restore connectivity +2. Verify IP whitelisting if applicable +3. Test connection stability before resuming + +**Option C: Lamassu Server Issues** +1. Contact Lamassu administrator +2. Verify PostgreSQL service is running +3. Check database is accessible + +**Post-Recovery**: +```python +# System automatically catches up using last_polled_at timestamp +# Run manual poll to process missed transactions +POST /api/v1/dca/manual-poll +``` + +#### Prevention Measures + +1. **SSH Key Authentication** (more reliable than password): +```bash +# Generate dedicated key for this service +ssh-keygen -t ed25519 -f ~/.ssh/satmachine_lamassu -C "satmachine-polling" +``` + +2. **Connection Monitoring**: Implement daily health check +3. **Retry Logic**: Add exponential backoff in polling code +4. **Alert System**: Email/SMS when polling fails for > 2 hours + +--- + +### 3. Payment Distribution Failures + +**Risk Level**: 🔴 **CRITICAL** +**Impact**: Commission deducted and client balances reduced, but transfers fail → money stuck in limbo + +#### Detection Methods + +1. **Dashboard Monitoring**: + - Client balances decrease but payment status shows "failed" + - Commission wallet balance doesn't increase as expected + - Error notifications in payment processing + +2. **Database Query**: +```sql +-- Find stuck/failed payments (older than 1 hour, not completed) +SELECT dp.id, dp.client_id, dp.amount_sats, dp.status, dp.created_at, + c.username, dp.payment_hash +FROM satoshimachine.dca_payments dp +JOIN satoshimachine.dca_clients c ON dp.client_id = c.id +WHERE dp.status != 'completed' + AND dp.created_at < NOW() - INTERVAL '1 hour' +ORDER BY dp.created_at DESC; +``` + +3. **Wallet Balance Check**: +```sql +-- Compare expected vs actual commission wallet balance +SELECT + SUM(commission_amount) as total_commission_expected, + -- Manually check actual wallet balance in LNBits +FROM satoshimachine.lamassu_transactions; +``` + +#### Immediate Response + +1. ✅ **STOP POLLING** - Prevent more transactions from failing +2. ✅ Identify root cause: + - Insufficient balance in source wallet? + - Invalid wallet adminkey? + - LNBits API issues? + - Network connectivity to LNBits? + +3. ✅ Document all failed payments with IDs and amounts + +#### Recovery Steps + +**Step 1: Verify Wallet Configuration** +```bash +# Test that commission wallet is accessible +curl -X GET https:///api/v1/wallet \ + -H "X-Api-Key: " +``` + +**Step 2: Check Wallet Balance** +- Ensure commission wallet has sufficient balance +- Verify no wallet locks or restrictions + +**Step 3: Manual Retry Process** + +**Option A: Retry via API** (if retry endpoint exists) +```python +# Retry failed payment +POST /api/v1/dca/payments/{payment_id}/retry +``` + +**Option B: Manual Recreation** (if no retry available) +1. Query failed payment details +2. Mark original payment as "cancelled" +3. Create new payment entry with same parameters +4. Process through normal payment flow +5. Update client records + +**Step 4: Verify Reconciliation** +```sql +-- After recovery, verify all clients balance correctly +SELECT + c.id, + c.username, + c.remaining_balance, + COALESCE(SUM(d.amount), 0) as total_deposits, + COALESCE(SUM(CASE WHEN p.status = 'completed' THEN p.amount_sats ELSE 0 END), 0) as total_payments +FROM satoshimachine.dca_clients c +LEFT JOIN satoshimachine.dca_deposits d ON c.id = d.client_id AND d.status = 'confirmed' +LEFT JOIN satoshimachine.dca_payments p ON c.id = p.client_id +GROUP BY c.id; +``` + +#### Prevention Measures + +**Required Code Changes**: +```python +# Add pre-flight check in transaction_processor.py +async def verify_commission_wallet_accessible(): + """Verify commission wallet exists and is accessible before processing""" + try: + response = await wallet_api_call(commission_wallet_id) + if not response.ok: + raise Exception("Commission wallet not accessible") + return True + except Exception as e: + logger.error(f"Pre-flight check failed: {e}") + return False + +# Add transaction rollback on distribution failure +async def process_transaction_with_rollback(): + transaction_record = None + try: + # Deduct balances + # Create distributions + # Transfer funds + # Mark complete + except Exception as e: + # ROLLBACK: Restore client balances + # Mark transaction as failed + # Alert admin +``` + +**Monitoring**: +1. Alert if any payment remains in non-completed status > 15 minutes +2. Daily reconciliation of commission wallet balance +3. Automated testing of wallet accessibility + +--- + +### 4. Balance Discrepancies + +**Risk Level**: 🟡 **MEDIUM** +**Impact**: Client balances don't match deposit/payment records → accounting errors, audit failures + +#### Detection Methods + +**Full Reconciliation Query**: +```sql +-- Complete balance reconciliation report +SELECT + c.id, + c.username, + c.remaining_balance as current_balance, + COALESCE(SUM(d.amount), 0) as total_deposits, + COALESCE(SUM(CASE WHEN p.status = 'completed' THEN p.amount_sats ELSE 0 END), 0) as total_distributed, + COALESCE(SUM(d.amount), 0) - COALESCE(SUM(CASE WHEN p.status = 'completed' THEN p.amount_sats ELSE 0 END), 0) as calculated_balance, + c.remaining_balance - (COALESCE(SUM(d.amount), 0) - COALESCE(SUM(CASE WHEN p.status = 'completed' THEN p.amount_sats ELSE 0 END), 0)) as discrepancy +FROM satoshimachine.dca_clients c +LEFT JOIN satoshimachine.dca_deposits d ON c.id = d.client_id AND d.status = 'confirmed' +LEFT JOIN satoshimachine.dca_payments p ON c.id = p.client_id +GROUP BY c.id, c.username, c.remaining_balance +HAVING ABS(c.remaining_balance - (COALESCE(SUM(d.amount), 0) - COALESCE(SUM(CASE WHEN p.status = 'completed' THEN p.amount_sats ELSE 0 END), 0))) > 1; +``` + +#### Immediate Response + +1. ✅ Run full reconciliation query above +2. ✅ Export complete audit trail to CSV +3. ✅ Document discrepancy amounts per client +4. ✅ Identify pattern (all clients affected? specific time period?) + +#### Recovery Steps + +**For Each Discrepancy**: + +1. **Trace Transaction History**: +```sql +-- Get complete transaction history for client +SELECT 'DEPOSIT' as type, id, amount, status, created_at, confirmed_at +FROM satoshimachine.dca_deposits +WHERE client_id = +UNION ALL +SELECT 'PAYMENT' as type, id, amount_sats, status, created_at, NULL +FROM satoshimachine.dca_payments +WHERE client_id = +ORDER BY created_at; +``` + +2. **Manual Recalculation**: + - Sum all confirmed deposits + - Subtract all completed payments + - Compare to current balance + - Identify missing/extra transactions + +3. **Correction Methods**: + +**Option A: Adjustment Entry** (Recommended) +```sql +-- Create compensating deposit for positive discrepancy +INSERT INTO satoshimachine.dca_deposits (client_id, amount, status, note) +VALUES (, , 'confirmed', 'Balance correction - reconciliation 2025-10-19'); + +-- OR Create compensating payment for negative discrepancy +INSERT INTO satoshimachine.dca_payments (client_id, amount_sats, status, note) +VALUES (, , 'completed', 'Balance correction - reconciliation 2025-10-19'); +``` + +**Option B: Direct Balance Update** (Use with extreme caution) +```sql +-- ONLY if audit trail is complete and discrepancy is unexplained +UPDATE satoshimachine.dca_clients +SET remaining_balance = , + updated_at = NOW() +WHERE id = ; + +-- MUST document in separate audit log +``` + +#### Prevention Measures + +**Daily Automated Reconciliation**: +```python +# Add to tasks.py +async def daily_reconciliation_check(): + """Run daily at 00:00 UTC""" + discrepancies = await find_balance_discrepancies() + if discrepancies: + await send_alert_to_admin(discrepancies) + await log_reconciliation_report(discrepancies) +``` + +**Database Constraints**: +```sql +-- Prevent negative balances +ALTER TABLE satoshimachine.dca_clients +ADD CONSTRAINT positive_balance CHECK (remaining_balance >= 0); + +-- Prevent confirmed deposits with zero amount +ALTER TABLE satoshimachine.dca_deposits +ADD CONSTRAINT positive_deposit CHECK (amount > 0); +``` + +**Audit Enhancements**: +- Store before/after balance with each transaction +- Implement change log table for all balance modifications +- Automated snapshots of all balances daily + +--- + +### 5. Commission Calculation Errors + +**Risk Level**: 🟠 **HIGH** +**Impact**: Wrong commission rate applied → over/under collection → revenue loss or client overcharge + +#### Detection Methods + +**Verification Query**: +```sql +-- Verify all commission calculations are mathematically correct +SELECT + id, + transaction_id, + crypto_atoms, + commission_percentage, + discount, + base_amount, + commission_amount, + -- Recalculate expected values + ROUND(crypto_atoms / (1 + (commission_percentage * (100 - discount) / 100))) as expected_base, + ROUND(crypto_atoms - (crypto_atoms / (1 + (commission_percentage * (100 - discount) / 100)))) as expected_commission, + -- Calculate differences + base_amount - ROUND(crypto_atoms / (1 + (commission_percentage * (100 - discount) / 100))) as base_difference, + commission_amount - ROUND(crypto_atoms - (crypto_atoms / (1 + (commission_percentage * (100 - discount) / 100)))) as commission_difference +FROM satoshimachine.lamassu_transactions +WHERE ABS(base_amount - ROUND(crypto_atoms / (1 + (commission_percentage * (100 - discount) / 100)))) > 1 + OR ABS(commission_amount - ROUND(crypto_atoms - (crypto_atoms / (1 + (commission_percentage * (100 - discount) / 100))))) > 1; +``` + +**Manual Spot Check**: +``` +Example: 2000 GTQ → 266,800 sats (3% commission, 0% discount) + +Expected calculation: +- Effective commission = 0.03 * (100 - 0) / 100 = 0.03 +- Base amount = 266800 / (1 + 0.03) = 258,835 sats +- Commission = 266800 - 258835 = 7,965 sats + +Verify these match database values. +``` + +#### Immediate Response + +1. ✅ Run verification query to identify all affected transactions +2. ✅ Calculate total under/over collection amount +3. ✅ Determine if pattern (all transactions? specific time period? specific commission rate?) +4. ✅ **STOP POLLING** if active miscalculation detected + +#### Recovery Steps + +**Step 1: Quantify Impact** +```sql +-- Total revenue impact +SELECT + COUNT(*) as affected_transactions, + SUM(commission_difference) as total_revenue_impact, + MIN(created_at) as first_occurrence, + MAX(created_at) as last_occurrence +FROM ( + -- Use verification query from above +) as calc_check +WHERE ABS(commission_difference) > 1; +``` + +**Step 2: Client Impact Assessment** +```sql +-- Which clients were affected and by how much +SELECT + c.id, + c.username, + COUNT(lt.id) as affected_transactions, + SUM(lt.base_amount) as total_distributed, + SUM(expected_base) as should_have_distributed, + SUM(expected_base - lt.base_amount) as client_impact +FROM satoshimachine.lamassu_transactions lt +JOIN satoshimachine.dca_payments dp ON dp.lamassu_transaction_id = lt.id +JOIN satoshimachine.dca_clients c ON dp.client_id = c.id +WHERE -- filter for affected transactions +GROUP BY c.id; +``` + +**Step 3: Correction** + +**If Under-Collected Commission**: +- Revenue lost to business +- Client received correct amount +- No client-facing correction needed +- Document for accounting + +**If Over-Collected Commission**: +- Client under-distributed +- Create compensating payments to affected clients: +```sql +-- Add to client balances +UPDATE satoshimachine.dca_clients c +SET remaining_balance = remaining_balance + adjustment.amount +FROM ( + -- Calculate adjustment per client + SELECT client_id, SUM(should_have_distributed - actual_distributed) as amount + FROM affected_transactions_analysis + GROUP BY client_id +) adjustment +WHERE c.id = adjustment.client_id; +``` + +**Step 4: Fix Code Bug** +- Identify root cause in `transaction_processor.py` +- Add unit test for failed scenario +- Deploy fix +- Verify with test transaction + +#### Prevention Measures + +**Unit Tests** (Add to test suite): +```python +def test_commission_calculation_scenarios(): + """Test edge cases for commission calculation""" + test_cases = [ + # (crypto_atoms, commission%, discount%, expected_base, expected_commission) + (266800, 0.03, 0.0, 258835, 7965), + (100000, 0.05, 10.0, 95694, 4306), # With discount + (1, 0.03, 0.0, 0, 1), # Minimum amount + # Add more edge cases + ] + for case in test_cases: + result = calculate_commission(*case[:3]) + assert result.base_amount == case[3] + assert result.commission_amount == case[4] +``` + +**Calculation Verification** (Add to processing): +```python +# After calculation, verify math is correct +calculated_total = base_amount + commission_amount +assert abs(calculated_total - crypto_atoms) <= 1, "Commission calculation error detected" +``` + +**Audit Trail**: +- Store all calculation parameters with each transaction +- Log formula used and intermediate values +- Enable post-processing verification + +--- + +### 6. Wallet Key Rotation/Invalidation + +**Risk Level**: 🟠 **HIGH** +**Impact**: Commission wallet adminkey changes → can't receive commission payments → processing halts + +#### Detection Methods + +1. **API Error Responses**: + - Payment API returns 401/403 authentication errors + - "Invalid API key" messages in logs + +2. **Wallet Balance Check**: +```sql +-- Commission not accumulating despite transactions +SELECT + SUM(commission_amount) as expected_total_commission, + -- Compare to actual wallet balance in LNBits dashboard +FROM satoshimachine.lamassu_transactions +WHERE created_at > ''; +``` + +3. **Manual Test**: +```bash +# Test commission wallet adminkey +curl -X GET https:///api/v1/wallet \ + -H "X-Api-Key: " + +# Should return wallet details, not auth error +``` + +#### Immediate Response + +1. ✅ **STOP POLLING** - No point processing if can't distribute +2. ✅ Identify if key was intentionally rotated or compromised +3. ✅ Obtain new valid adminkey for commission wallet +4. ✅ Verify source wallet adminkey is also still valid + +#### Recovery Steps + +**Step 1: Update Configuration** +1. Access admin dashboard +2. Navigate to Configuration section +3. Update commission wallet adminkey +4. Update source wallet adminkey if also affected +5. **Test Connection** before saving + +**Step 2: Verify Configuration Persisted** +```sql +-- Check configuration was saved correctly +SELECT commission_wallet_id, + LEFT(commission_wallet_adminkey, 10) || '...' as key_preview, + updated_at +FROM satoshimachine.lamassu_config +ORDER BY updated_at DESC +LIMIT 1; +``` + +**Step 3: Reprocess Failed Transactions** +- Identify transactions that failed due to auth errors +- Mark for retry or manually reprocess +- Verify commission payments complete successfully + +**Step 4: Resume Operations** +1. Test with manual poll first +2. Verify single transaction processes completely +3. Re-enable automatic polling +4. Monitor for 24 hours + +#### Prevention Measures + +**Key Management Documentation**: +- Document which LNBits wallets are used for commission/source +- Store backup admin keys in secure location (password manager) +- Define key rotation procedure with testing steps +- Require testing in staging before production changes + +**Configuration Validation**: +```python +# Add to config save endpoint in views_api.py +async def validate_wallet_keys(commission_key, source_key): + """Test wallet keys before saving configuration""" + # Test commission wallet + commission_valid = await test_wallet_access(commission_key) + if not commission_valid: + raise ValueError("Commission wallet key is invalid") + + # Test source wallet (if applicable) + if source_key: + source_valid = await test_wallet_access(source_key) + if not source_valid: + raise ValueError("Source wallet key is invalid") + + return True +``` + +**Automated Monitoring**: +- Daily health check of wallet accessibility +- Alert if wallet API calls start failing +- Backup key verification in secure environment + +--- + +## Emergency Protocol Checklist + +Use this checklist when ANY error is detected in the DCA system. + +### Phase 1: Immediate Actions (First 5 Minutes) + +- [ ] **Stop Automatic Processing** + - Disable background polling task + - Verify no transactions are currently processing + - Document time polling was stopped + +- [ ] **Assess Severity** + - Is money at risk? (duplicate processing, failed payments) + - Are clients affected? (missing distributions, balance errors) + - Is this blocking operations? (connection loss, wallet issues) + +- [ ] **Initial Documentation** + - Take screenshots of error messages + - Note exact timestamp of error detection + - Record current system state (balances, last transaction, etc.) + +- [ ] **Notify Stakeholders** + - Alert system administrator + - Notify clients if distributions will be delayed > 24 hours + - Escalate if financial impact > threshold + +### Phase 2: Investigation (15-30 Minutes) + +- [ ] **Collect Diagnostic Information** + - Run relevant SQL queries from scenarios above + - Check LNBits logs: `/lnbits/data/logs/` + - Review recent configuration changes + - Test external connections (SSH, wallets) + +- [ ] **Identify Root Cause** + - Match symptoms to failure scenarios above + - Determine if human error, system failure, or external issue + - Estimate scope of impact (time range, # clients, # transactions) + +- [ ] **Document Findings** + - Record root cause analysis + - List all affected records (transaction IDs, client IDs) + - Calculate financial impact (over/under distributed amounts) + - Take database snapshots for audit trail + +### Phase 3: Recovery (30 Minutes - 2 Hours) + +- [ ] **Fix Root Cause** + - Apply code fix if bug + - Update configuration if settings issue + - Restore connection if network issue + - Refer to specific recovery steps in scenarios above + +- [ ] **Data Correction** + - Run reconciliation queries + - Execute correction SQL statements + - Verify all client balances are accurate + - Ensure audit trail is complete + +- [ ] **Verification** + - Test fix with single transaction + - Verify wallets are accessible + - Confirm connections are stable + - Run full reconciliation report + +### Phase 4: Resumption (After Verification) + +- [ ] **Gradual Restart** + - Process one manual poll successfully + - Monitor for errors during processing + - Verify distributions complete end-to-end + - Check commission payments arrive correctly + +- [ ] **Re-enable Automation** + - Turn on background polling task + - Set monitoring alerts + - Document in system log + +- [ ] **Enhanced Monitoring** + - Watch closely for 24 hours + - Run reconciliation after next 3-5 transactions + - Verify no recurrence of issue + +### Phase 5: Post-Incident (24-48 Hours After) + +- [ ] **Complete Post-Mortem** + - Document full timeline of incident + - Record exact root cause and fix applied + - Calculate total impact (financial, time, clients affected) + - Identify what went well and what could improve + +- [ ] **Implement Safeguards** + - Add prevention measures from scenario sections + - Implement new monitoring/alerts + - Add automated tests for this failure mode + - Update runbooks and documentation + +- [ ] **Stakeholder Communication** + - Send incident report to management + - Notify affected clients if applicable + - Document lessons learned + - Update emergency contact procedures if needed + +--- + +## Recovery Procedures + +### Full System Reconciliation + +Run this complete reconciliation procedure weekly or after any incident: + +```sql +-- ============================================ +-- FULL SYSTEM RECONCILIATION REPORT +-- ============================================ + +-- 1. Client Balance Reconciliation +WITH client_financials AS ( + SELECT + c.id, + c.username, + c.remaining_balance as current_balance, + COALESCE(SUM(d.amount), 0) as total_deposits, + COALESCE(SUM(CASE WHEN p.status = 'completed' THEN p.amount_sats ELSE 0 END), 0) as total_payments, + COUNT(DISTINCT d.id) as deposit_count, + COUNT(DISTINCT p.id) as payment_count + FROM satoshimachine.dca_clients c + LEFT JOIN satoshimachine.dca_deposits d ON c.id = d.client_id AND d.status = 'confirmed' + LEFT JOIN satoshimachine.dca_payments p ON c.id = p.client_id + GROUP BY c.id +) +SELECT + id, + username, + current_balance, + total_deposits, + total_payments, + (total_deposits - total_payments) as calculated_balance, + (current_balance - (total_deposits - total_payments)) as discrepancy, + deposit_count, + payment_count, + CASE + WHEN ABS(current_balance - (total_deposits - total_payments)) <= 1 THEN '✅ OK' + ELSE '⚠️ MISMATCH' + END as status +FROM client_financials +ORDER BY ABS(current_balance - (total_deposits - total_payments)) DESC; + +-- 2. Transaction Processing Verification +SELECT + COUNT(*) as total_transactions, + SUM(crypto_atoms) as total_sats_processed, + SUM(base_amount) as total_distributed, + SUM(commission_amount) as total_commission, + MIN(created_at) as first_transaction, + MAX(created_at) as last_transaction +FROM satoshimachine.lamassu_transactions; + +-- 3. Failed/Pending Payments Check +SELECT + status, + COUNT(*) as count, + SUM(amount_sats) as total_amount, + MIN(created_at) as oldest, + MAX(created_at) as newest +FROM satoshimachine.dca_payments +GROUP BY status +ORDER BY + CASE status + WHEN 'failed' THEN 1 + WHEN 'pending' THEN 2 + WHEN 'completed' THEN 3 + END; + +-- 4. Unconfirmed Deposits Check (sitting too long) +SELECT + id, + client_id, + amount, + status, + created_at, + EXTRACT(EPOCH FROM (NOW() - created_at))/3600 as hours_pending +FROM satoshimachine.dca_deposits +WHERE status = 'pending' + AND created_at < NOW() - INTERVAL '48 hours' +ORDER BY created_at; + +-- 5. Commission Calculation Verification (sample check) +SELECT + id, + transaction_id, + crypto_atoms, + commission_percentage, + discount, + base_amount, + commission_amount, + ROUND(crypto_atoms / (1 + (commission_percentage * (100 - discount) / 100))) as expected_base, + base_amount - ROUND(crypto_atoms / (1 + (commission_percentage * (100 - discount) / 100))) as difference +FROM satoshimachine.lamassu_transactions +ORDER BY created_at DESC +LIMIT 20; +``` + +### Emergency Access Procedures + +**If Admin Dashboard Inaccessible**: + +1. **Direct Database Access**: +```bash +# Connect to LNBits database +sqlite3 /home/padreug/AioLabs/Git/lnbits-extensions/lnbits/data/database.sqlite + +# Or PostgreSQL if used +psql -h localhost -U lnbits -d lnbits +``` + +2. **Direct Configuration Update**: +```sql +-- Update Lamassu config directly +UPDATE satoshimachine.lamassu_config +SET polling_enabled = false +WHERE id = (SELECT MAX(id) FROM satoshimachine.lamassu_config); +``` + +3. **Manual Client Balance Update**: +```sql +-- ONLY in emergency when dashboard unavailable +UPDATE satoshimachine.dca_clients +SET remaining_balance = +WHERE id = ; +-- MUST document this action in incident log +``` + +**If Background Task Won't Stop**: + +```bash +# Find LNBits process +ps aux | grep lnbits + +# Restart LNBits service (will stop all background tasks) +systemctl restart lnbits +# OR if running manually: +pkill -f lnbits +uv run lnbits +``` + +### Data Export for Audit + +**Complete Audit Trail Export**: + +```bash +# Export all DCA-related tables to CSV +sqlite3 -header -csv /path/to/lnbits/database.sqlite \ + "SELECT * FROM satoshimachine.lamassu_transactions;" \ + > lamassu_transactions_export_$(date +%Y%m%d_%H%M%S).csv + +sqlite3 -header -csv /path/to/lnbits/database.sqlite \ + "SELECT * FROM satoshimachine.dca_payments;" \ + > dca_payments_export_$(date +%Y%m%d_%H%M%S).csv + +sqlite3 -header -csv /path/to/lnbits/database.sqlite \ + "SELECT * FROM satoshimachine.dca_deposits;" \ + > dca_deposits_export_$(date +%Y%m%d_%H%M%S).csv + +sqlite3 -header -csv /path/to/lnbits/database.sqlite \ + "SELECT * FROM satoshimachine.dca_clients;" \ + > dca_clients_export_$(date +%Y%m%d_%H%M%S).csv +``` + +**Combined Audit Report**: + +```sql +-- Complete transaction-to-distribution audit trail +SELECT + lt.id as transaction_id, + lt.transaction_id as lamassu_txn_id, + lt.created_at as transaction_time, + lt.crypto_atoms as total_sats, + lt.fiat_code, + lt.fiat_amount, + lt.commission_percentage, + lt.discount, + lt.base_amount as distributable_sats, + lt.commission_amount, + c.id as client_id, + c.username as client_name, + dp.amount_sats as client_received, + dp.status as payment_status, + dp.payment_hash +FROM satoshimachine.lamassu_transactions lt +LEFT JOIN satoshimachine.dca_payments dp ON dp.lamassu_transaction_id = lt.id +LEFT JOIN satoshimachine.dca_clients c ON dp.client_id = c.id +ORDER BY lt.created_at DESC, c.username; +``` + +--- + +## Prevention Measures + +### Required Immediate Implementations + +#### 1. Idempotency Protection (CRITICAL) + +**File**: `transaction_processor.py` + +```python +async def process_lamassu_transaction(txn_data: dict) -> Optional[LamassuTransaction]: + """Process transaction with idempotency check""" + + # CRITICAL: Check if already processed + existing = await get_lamassu_transaction_by_txid(txn_data['id']) + if existing: + logger.warning(f"⚠️ Transaction {txn_data['id']} already processed at {existing.created_at}, skipping") + return None + + # Continue with processing... +``` + +#### 2. Database Constraints + +**File**: `migrations.py` + +```sql +-- Add unique constraint on transaction_id +ALTER TABLE satoshimachine.lamassu_transactions +ADD CONSTRAINT unique_transaction_id UNIQUE (transaction_id); + +-- Prevent negative balances +ALTER TABLE satoshimachine.dca_clients +ADD CONSTRAINT positive_balance CHECK (remaining_balance >= 0); + +-- Ensure positive amounts +ALTER TABLE satoshimachine.dca_deposits +ADD CONSTRAINT positive_deposit CHECK (amount > 0); + +ALTER TABLE satoshimachine.dca_payments +ADD CONSTRAINT positive_payment CHECK (amount_sats > 0); +``` + +#### 3. Transaction Status Tracking + +**File**: `models.py` + +```python +class LamassuTransaction(BaseModel): + # ... existing fields ... + status: str = "pending" # pending, processing, completed, failed + error_message: Optional[str] = None + processed_at: Optional[datetime] = None +``` + +#### 4. Pre-flight Wallet Validation + +**File**: `transaction_processor.py` + +```python +async def validate_system_ready() -> Tuple[bool, str]: + """Validate system is ready to process transactions""" + + # Check commission wallet accessible + try: + commission_wallet = await get_wallet(config.commission_wallet_id) + if not commission_wallet: + return False, "Commission wallet not accessible" + except Exception as e: + return False, f"Commission wallet error: {str(e)}" + + # Check for stuck payments + stuck_payments = await get_stuck_payments(hours=2) + if stuck_payments: + return False, f"{len(stuck_payments)} payments stuck for >2 hours" + + # Check database connectivity + try: + await db_health_check() + except Exception as e: + return False, f"Database health check failed: {str(e)}" + + return True, "System ready" + +# Call before processing +ready, message = await validate_system_ready() +if not ready: + logger.error(f"System not ready: {message}") + await send_alert_to_admin(message) + return +``` + +### Automated Monitoring Implementation + +#### Daily Reconciliation Task + +**File**: `tasks.py` + +```python +@scheduler.scheduled_job("cron", hour=0, minute=0) # Daily at midnight UTC +async def daily_reconciliation(): + """Run daily balance reconciliation and report discrepancies""" + + logger.info("Starting daily reconciliation...") + + discrepancies = await find_balance_discrepancies() + + if discrepancies: + report = generate_reconciliation_report(discrepancies) + await send_alert_to_admin("⚠️ Balance Discrepancies Detected", report) + await log_reconciliation_issue(discrepancies) + else: + logger.info("✅ Daily reconciliation passed - all balances match") + + # Check for stuck payments + stuck_payments = await get_stuck_payments(hours=24) + if stuck_payments: + await send_alert_to_admin( + f"⚠️ {len(stuck_payments)} Stuck Payments Detected", + format_stuck_payments_report(stuck_payments) + ) +``` + +#### Connection Health Monitor + +```python +@scheduler.scheduled_job("interval", hours=2) # Every 2 hours +async def check_system_health(): + """Monitor system health and alert on issues""" + + issues = [] + + # Check last successful poll + last_poll = await get_last_successful_poll_time() + if last_poll and (datetime.utcnow() - last_poll).total_seconds() > 86400: # 24 hours + issues.append(f"No successful poll in {(datetime.utcnow() - last_poll).total_seconds() / 3600:.1f} hours") + + # Check wallet accessibility + try: + await test_wallet_access(config.commission_wallet_adminkey) + except Exception as e: + issues.append(f"Commission wallet inaccessible: {str(e)}") + + # Check database connectivity + try: + await test_lamassu_connection() + except Exception as e: + issues.append(f"Lamassu database connection failed: {str(e)}") + + if issues: + await send_alert_to_admin("⚠️ System Health Check Failed", "\n".join(issues)) +``` + +### Alert Configuration + +**File**: `config.py` or environment variables + +```python +# Alert settings +ALERT_EMAIL = "admin@yourdomain.com" +ALERT_WEBHOOK = "https://hooks.slack.com/..." # Slack/Discord webhook +ALERT_PHONE = "+1234567890" # For critical alerts (optional) + +# Alert thresholds +MAX_STUCK_PAYMENT_HOURS = 2 +MAX_POLL_DELAY_HOURS = 24 +MAX_BALANCE_DISCREPANCY_SATS = 100 +``` + +--- + +## Monitoring & Alerts + +### Dashboard Indicators to Watch + +#### Critical Indicators (Check Daily) +- ✅ **Last Successful Poll Time** - Should be within last 2-4 hours (based on polling interval) +- ✅ **Failed Payment Count** - Should be 0; investigate immediately if > 0 +- ✅ **Commission Wallet Balance** - Should increase proportionally with transactions +- ✅ **Active Clients with Balance** - Cross-reference with expected DCA participants + +#### Warning Indicators (Check Weekly) +- ⚠️ **Pending Deposits > 48 hours** - May indicate confirmation workflow issue +- ⚠️ **Client Balance Reconciliation** - Run full reconciliation report +- ⚠️ **Average Commission %** - Verify matches configured rates +- ⚠️ **Transaction Processing Time** - Should complete within minutes, not hours + +### Automated Alert Triggers + +Implement these alerts in production: + +| Alert | Severity | Trigger Condition | Response Time | +|-------|----------|-------------------|---------------| +| Duplicate Transaction Detected | 🔴 CRITICAL | Same `transaction_id` inserted twice | Immediate | +| Payment Stuck > 15 minutes | 🔴 CRITICAL | Payment status not "completed" after 15min | < 30 minutes | +| Polling Failed > 24 hours | 🟠 HIGH | No new transactions in 24 hours | < 2 hours | +| Balance Discrepancy > 100 sats | 🟠 HIGH | Reconciliation finds error > threshold | < 4 hours | +| Wallet Inaccessible | 🟠 HIGH | Commission wallet returns auth error | < 1 hour | +| Database Connection Failed | 🟡 MEDIUM | Cannot connect to Lamassu DB | < 4 hours | +| Commission Calculation Anomaly | 🟡 MEDIUM | Calculated amount differs from formula | < 24 hours | + +### Log Files to Monitor + +**Location**: `/home/padreug/AioLabs/Git/lnbits-extensions/lnbits/data/logs/` + +**Key Log Patterns**: +```bash +# Critical errors +grep -i "error\|exception\|failed" lnbits.log | tail -100 + +# Transaction processing +grep "LamassuTransactionProcessor" lnbits.log | tail -50 + +# Payment distribution +grep "dca_payment\|distribution" lnbits.log | tail -50 + +# SSH connection issues +grep -i "ssh\|connection\|timeout" lnbits.log | tail -50 + +# Wallet API calls +grep "wallet.*api\|payment_hash" lnbits.log | tail -50 +``` + +### Manual Checks (Weekly) + +**Sunday 00:00 UTC - Weekly Audit**: + +1. [ ] Run full reconciliation SQL report +2. [ ] Export all tables to CSV for backup +3. [ ] Verify commission wallet balance matches sum of commission_amount +4. [ ] Check for any pending deposits > 7 days old +5. [ ] Review last 20 transactions for calculation correctness +6. [ ] Test database connection from admin dashboard +7. [ ] Test manual poll to verify end-to-end flow +8. [ ] Review error logs for any concerning patterns + +--- + +## Contact Information + +### System Access + +**LNBits Admin Dashboard**: +- URL: `https:///satoshimachine` +- Requires superuser authentication + +**Database Access**: +```bash +# LNBits database +sqlite3 /home/padreug/AioLabs/Git/lnbits-extensions/lnbits/data/database.sqlite + +# Direct table access +sqlite3 /path/to/db "SELECT * FROM satoshimachine.;" +``` + +**Log Files**: +```bash +# Main logs +tail -f /home/padreug/AioLabs/Git/lnbits-extensions/lnbits/data/logs/lnbits.log + +# Error logs only +tail -f /home/padreug/AioLabs/Git/lnbits-extensions/lnbits/data/logs/lnbits.log | grep -i error +``` + +### Emergency Escalation + +**Level 1 - System Administrator** (First Contact): +- Name: _______________________ +- Email: _______________________ +- Phone: _______________________ +- Availability: _______________________ + +**Level 2 - Technical Lead** (If L1 unavailable): +- Name: _______________________ +- Email: _______________________ +- Phone: _______________________ +- Availability: _______________________ + +**Level 3 - Business Owner** (Financial impact > $X): +- Name: _______________________ +- Email: _______________________ +- Phone: _______________________ +- Availability: _______________________ + +### External Contacts + +**Lamassu Administrator**: +- Name: _______________________ +- Email: _______________________ +- Phone: _______________________ +- SSH access issues, database access, ATM questions + +**LNBits Infrastructure**: +- Name: _______________________ +- Email: _______________________ +- Phone: _______________________ +- Wallet issues, API problems, system downtime + +**Accountant/Auditor**: +- Name: _______________________ +- Email: _______________________ +- For balance discrepancies requiring financial reconciliation + +--- + +## Appendix: Quick Reference Commands + +### Emergency Stop + +```bash +# Stop LNBits service (stops all background tasks) +systemctl stop lnbits + +# Or kill process +pkill -f lnbits +``` + +### Emergency Database Disable Polling + +```sql +-- Disable automatic polling +UPDATE satoshimachine.lamassu_config +SET polling_enabled = false; +``` + +### Quick Balance Check + +```sql +-- All client balances summary +SELECT id, username, remaining_balance, created_at +FROM satoshimachine.dca_clients +ORDER BY remaining_balance DESC; +``` + +### Last 10 Transactions + +```sql +SELECT id, transaction_id, created_at, crypto_atoms, base_amount, commission_amount +FROM satoshimachine.lamassu_transactions +ORDER BY created_at DESC +LIMIT 10; +``` + +### Failed Payments + +```sql +SELECT * FROM satoshimachine.dca_payments +WHERE status != 'completed' +ORDER BY created_at DESC; +``` + +### Export Everything (Backup) + +```bash +#!/bin/bash +# Emergency full backup +DATE=$(date +%Y%m%d_%H%M%S) +BACKUP_DIR="emergency_backup_${DATE}" +mkdir -p $BACKUP_DIR + +sqlite3 -header -csv /path/to/database.sqlite \ + "SELECT * FROM satoshimachine.lamassu_transactions;" \ + > ${BACKUP_DIR}/lamassu_transactions.csv + +sqlite3 -header -csv /path/to/database.sqlite \ + "SELECT * FROM satoshimachine.dca_payments;" \ + > ${BACKUP_DIR}/dca_payments.csv + +sqlite3 -header -csv /path/to/database.sqlite \ + "SELECT * FROM satoshimachine.dca_deposits;" \ + > ${BACKUP_DIR}/dca_deposits.csv + +sqlite3 -header -csv /path/to/database.sqlite \ + "SELECT * FROM satoshimachine.dca_clients;" \ + > ${BACKUP_DIR}/dca_clients.csv + +echo "Backup complete in ${BACKUP_DIR}/" +``` + +--- + +## Document Change Log + +| Version | Date | Author | Changes | +|---------|------|--------|---------| +| 1.0 | 2025-10-19 | Claude Code | Initial emergency protocols document | +| | | | | +| | | | | + +--- + +## Sign-Off + +This document has been reviewed and approved for use in production emergency response: + +**System Administrator**: _____________________ Date: _______ + +**Technical Lead**: _____________________ Date: _______ + +**Business Owner**: _____________________ Date: _______ + +--- + +**END OF DOCUMENT** + +*Keep this document accessible at all times. Print and store in emergency response binder.* +*Review and update quarterly or after any major incident.* diff --git a/misc-docs/EMERGENCY_PROTOCOLS_PRINT.md b/misc-docs/EMERGENCY_PROTOCOLS_PRINT.md new file mode 100644 index 0000000..eb5b6b9 --- /dev/null +++ b/misc-docs/EMERGENCY_PROTOCOLS_PRINT.md @@ -0,0 +1,1362 @@ +# Satoshi Machine Admin - Emergency Protocols +## DCA System Failure Recovery Guide + +**Document Version**: 1.0 +**Last Updated**: 2025-10-19 +**Extension Version**: v0.0.1 +**Status**: Production + +--- + +## Table of Contents + +1. [Critical Failure Scenarios](#critical-failure-scenarios) +2. [Emergency Protocol Checklist](#emergency-protocol-checklist) +3. [Recovery Procedures](#recovery-procedures) +4. [Prevention Measures](#prevention-measures) +5. [Monitoring & Alerts](#monitoring--alerts) +6. [Contact Information](#contact-information) + +--- + +## Critical Failure Scenarios + +### 1. Duplicate Transaction Processing WARNING CRITICAL + +**Risk Level**: CRITICAL **CRITICAL** +**Impact**: Same Lamassu transaction processed twice → double distribution to clients → financial loss + +#### Detection Methods + +1. **Dashboard Monitoring**: + - Sudden large balance deductions from client accounts + - Multiple distribution entries for same timestamp + - Commission wallet receiving duplicate amounts + +2. **Database Query**: +```sql +-- Find duplicate transactions +SELECT transaction_id, COUNT(*) as count, + STRING_AGG(id::text, ', ') as record_ids +FROM satoshimachine.lamassu_transactions +GROUP BY transaction_id +HAVING COUNT(*) > 1; +``` + +3. **Automated Alert Triggers**: + - Same `txn_id` appears in multiple processing cycles + - Client balance drops more than expected based on deposit ratios + +#### Immediate Response + +1. [OK] **STOP POLLING IMMEDIATELY** - Disable automatic background task +2. [OK] Document all duplicate entries with screenshots +3. [OK] Identify affected clients and amounts +4. [OK] Calculate total over-distribution amount + +#### Recovery Steps + +```sql +-- Step 1: Identify duplicate distributions +SELECT lt.transaction_id, lt.id, lt.created_at, lt.base_amount, + COUNT(dp.id) as distribution_count +FROM satoshimachine.lamassu_transactions lt +LEFT JOIN satoshimachine.dca_payments dp ON dp.lamassu_transaction_id = lt.id +GROUP BY lt.id +HAVING COUNT(dp.id) > (SELECT COUNT(*) FROM satoshimachine.dca_clients WHERE remaining_balance > 0); + +-- Step 2: Calculate over-distributed amounts per client +SELECT client_id, + SUM(amount_sats) as total_received, + -- Manual calculation of expected amount needed here +FROM satoshimachine.dca_payments +WHERE lamassu_transaction_id IN (SELECT id FROM duplicates_table) +GROUP BY client_id; +``` + +**Manual Correction**: +1. Calculate correct distribution amounts +2. Create compensating negative adjustments (if supported) OR +3. Deduct from future distributions until balanced +4. Document all corrections in audit log +5. Notify affected clients if material amount + +#### Prevention Measures + +**Required Code Changes**: +```python +# Add to transaction_processor.py BEFORE processing +existing = await get_lamassu_transaction_by_txid(txn_id) +if existing: + logger.warning(f"WARNING Transaction {txn_id} already processed, skipping") + return None +``` + +**Required Database Change**: +```sql +ALTER TABLE satoshimachine.lamassu_transactions +ADD CONSTRAINT unique_transaction_id UNIQUE (transaction_id); +``` + +--- + +### 2. SSH/Database Connection Loss + +**Risk Level**: MEDIUM **MEDIUM** +**Impact**: Polling stops → transactions not processed → clients not receiving Bitcoin on time + +#### Detection Methods + +1. **Dashboard Indicators**: + - No new records in transaction history for > 24 hours + - "Test Connection" button fails in admin configuration + - Background task logs show SSH connection errors + +2. **Database Query**: +```sql +-- Check last successful poll +SELECT MAX(created_at) as last_transaction, + EXTRACT(EPOCH FROM (NOW() - MAX(created_at)))/3600 as hours_since_last +FROM satoshimachine.lamassu_transactions; + +-- If hours_since_last > 24, investigate immediately +``` + +3. **Log File Check**: +```bash +# Check LNBits logs for SSH errors +tail -n 100 /home/padreug/AioLabs/Git/lnbits-extensions/lnbits/data/logs/lnbits.log | grep -i "ssh\|connection" +``` + +#### Immediate Response + +1. [OK] Verify network connectivity to Lamassu server +2. [OK] Test SSH credentials manually: +```bash +ssh postgres@ -p -i +``` +3. [OK] Check firewall rules and network changes +4. [OK] Verify Lamassu server is running and accessible + +#### Recovery Steps + +**Option A: Credential Issues** +1. Regenerate SSH keys if compromised +2. Update authorized_keys on Lamassu server +3. Update configuration in admin dashboard +4. Test connection before re-enabling polling + +**Option B: Network Issues** +1. Coordinate with network admin to restore connectivity +2. Verify IP whitelisting if applicable +3. Test connection stability before resuming + +**Option C: Lamassu Server Issues** +1. Contact Lamassu administrator +2. Verify PostgreSQL service is running +3. Check database is accessible + +**Post-Recovery**: +```python +# System automatically catches up using last_polled_at timestamp +# Run manual poll to process missed transactions +POST /api/v1/dca/manual-poll +``` + +#### Prevention Measures + +1. **SSH Key Authentication** (more reliable than password): +```bash +# Generate dedicated key for this service +ssh-keygen -t ed25519 -f ~/.ssh/satmachine_lamassu -C "satmachine-polling" +``` + +2. **Connection Monitoring**: Implement daily health check +3. **Retry Logic**: Add exponential backoff in polling code +4. **Alert System**: Email/SMS when polling fails for > 2 hours + +--- + +### 3. Payment Distribution Failures + +**Risk Level**: CRITICAL **CRITICAL** +**Impact**: Commission deducted and client balances reduced, but transfers fail → money stuck in limbo + +#### Detection Methods + +1. **Dashboard Monitoring**: + - Client balances decrease but payment status shows "failed" + - Commission wallet balance doesn't increase as expected + - Error notifications in payment processing + +2. **Database Query**: +```sql +-- Find stuck/failed payments (older than 1 hour, not completed) +SELECT dp.id, dp.client_id, dp.amount_sats, dp.status, dp.created_at, + c.username, dp.payment_hash +FROM satoshimachine.dca_payments dp +JOIN satoshimachine.dca_clients c ON dp.client_id = c.id +WHERE dp.status != 'completed' + AND dp.created_at < NOW() - INTERVAL '1 hour' +ORDER BY dp.created_at DESC; +``` + +3. **Wallet Balance Check**: +```sql +-- Compare expected vs actual commission wallet balance +SELECT + SUM(commission_amount) as total_commission_expected, + -- Manually check actual wallet balance in LNBits +FROM satoshimachine.lamassu_transactions; +``` + +#### Immediate Response + +1. [OK] **STOP POLLING** - Prevent more transactions from failing +2. [OK] Identify root cause: + - Insufficient balance in source wallet? + - Invalid wallet adminkey? + - LNBits API issues? + - Network connectivity to LNBits? + +3. [OK] Document all failed payments with IDs and amounts + +#### Recovery Steps + +**Step 1: Verify Wallet Configuration** +```bash +# Test that commission wallet is accessible +curl -X GET https:///api/v1/wallet \ + -H "X-Api-Key: " +``` + +**Step 2: Check Wallet Balance** +- Ensure commission wallet has sufficient balance +- Verify no wallet locks or restrictions + +**Step 3: Manual Retry Process** + +**Option A: Retry via API** (if retry endpoint exists) +```python +# Retry failed payment +POST /api/v1/dca/payments/{payment_id}/retry +``` + +**Option B: Manual Recreation** (if no retry available) +1. Query failed payment details +2. Mark original payment as "cancelled" +3. Create new payment entry with same parameters +4. Process through normal payment flow +5. Update client records + +**Step 4: Verify Reconciliation** +```sql +-- After recovery, verify all clients balance correctly +SELECT + c.id, + c.username, + c.remaining_balance, + COALESCE(SUM(d.amount), 0) as total_deposits, + COALESCE(SUM(CASE WHEN p.status = 'completed' THEN p.amount_sats ELSE 0 END), 0) as total_payments +FROM satoshimachine.dca_clients c +LEFT JOIN satoshimachine.dca_deposits d ON c.id = d.client_id AND d.status = 'confirmed' +LEFT JOIN satoshimachine.dca_payments p ON c.id = p.client_id +GROUP BY c.id; +``` + +#### Prevention Measures + +**Required Code Changes**: +```python +# Add pre-flight check in transaction_processor.py +async def verify_commission_wallet_accessible(): + """Verify commission wallet exists and is accessible before processing""" + try: + response = await wallet_api_call(commission_wallet_id) + if not response.ok: + raise Exception("Commission wallet not accessible") + return True + except Exception as e: + logger.error(f"Pre-flight check failed: {e}") + return False + +# Add transaction rollback on distribution failure +async def process_transaction_with_rollback(): + transaction_record = None + try: + # Deduct balances + # Create distributions + # Transfer funds + # Mark complete + except Exception as e: + # ROLLBACK: Restore client balances + # Mark transaction as failed + # Alert admin +``` + +**Monitoring**: +1. Alert if any payment remains in non-completed status > 15 minutes +2. Daily reconciliation of commission wallet balance +3. Automated testing of wallet accessibility + +--- + +### 4. Balance Discrepancies + +**Risk Level**: MEDIUM **MEDIUM** +**Impact**: Client balances don't match deposit/payment records → accounting errors, audit failures + +#### Detection Methods + +**Full Reconciliation Query**: +```sql +-- Complete balance reconciliation report +SELECT + c.id, + c.username, + c.remaining_balance as current_balance, + COALESCE(SUM(d.amount), 0) as total_deposits, + COALESCE(SUM(CASE WHEN p.status = 'completed' THEN p.amount_sats ELSE 0 END), 0) as total_distributed, + COALESCE(SUM(d.amount), 0) - COALESCE(SUM(CASE WHEN p.status = 'completed' THEN p.amount_sats ELSE 0 END), 0) as calculated_balance, + c.remaining_balance - (COALESCE(SUM(d.amount), 0) - COALESCE(SUM(CASE WHEN p.status = 'completed' THEN p.amount_sats ELSE 0 END), 0)) as discrepancy +FROM satoshimachine.dca_clients c +LEFT JOIN satoshimachine.dca_deposits d ON c.id = d.client_id AND d.status = 'confirmed' +LEFT JOIN satoshimachine.dca_payments p ON c.id = p.client_id +GROUP BY c.id, c.username, c.remaining_balance +HAVING ABS(c.remaining_balance - (COALESCE(SUM(d.amount), 0) - COALESCE(SUM(CASE WHEN p.status = 'completed' THEN p.amount_sats ELSE 0 END), 0))) > 1; +``` + +#### Immediate Response + +1. [OK] Run full reconciliation query above +2. [OK] Export complete audit trail to CSV +3. [OK] Document discrepancy amounts per client +4. [OK] Identify pattern (all clients affected? specific time period?) + +#### Recovery Steps + +**For Each Discrepancy**: + +1. **Trace Transaction History**: +```sql +-- Get complete transaction history for client +SELECT 'DEPOSIT' as type, id, amount, status, created_at, confirmed_at +FROM satoshimachine.dca_deposits +WHERE client_id = +UNION ALL +SELECT 'PAYMENT' as type, id, amount_sats, status, created_at, NULL +FROM satoshimachine.dca_payments +WHERE client_id = +ORDER BY created_at; +``` + +2. **Manual Recalculation**: + - Sum all confirmed deposits + - Subtract all completed payments + - Compare to current balance + - Identify missing/extra transactions + +3. **Correction Methods**: + +**Option A: Adjustment Entry** (Recommended) +```sql +-- Create compensating deposit for positive discrepancy +INSERT INTO satoshimachine.dca_deposits (client_id, amount, status, note) +VALUES (, , 'confirmed', 'Balance correction - reconciliation 2025-10-19'); + +-- OR Create compensating payment for negative discrepancy +INSERT INTO satoshimachine.dca_payments (client_id, amount_sats, status, note) +VALUES (, , 'completed', 'Balance correction - reconciliation 2025-10-19'); +``` + +**Option B: Direct Balance Update** (Use with extreme caution) +```sql +-- ONLY if audit trail is complete and discrepancy is unexplained +UPDATE satoshimachine.dca_clients +SET remaining_balance = , + updated_at = NOW() +WHERE id = ; + +-- MUST document in separate audit log +``` + +#### Prevention Measures + +**Daily Automated Reconciliation**: +```python +# Add to tasks.py +async def daily_reconciliation_check(): + """Run daily at 00:00 UTC""" + discrepancies = await find_balance_discrepancies() + if discrepancies: + await send_alert_to_admin(discrepancies) + await log_reconciliation_report(discrepancies) +``` + +**Database Constraints**: +```sql +-- Prevent negative balances +ALTER TABLE satoshimachine.dca_clients +ADD CONSTRAINT positive_balance CHECK (remaining_balance >= 0); + +-- Prevent confirmed deposits with zero amount +ALTER TABLE satoshimachine.dca_deposits +ADD CONSTRAINT positive_deposit CHECK (amount > 0); +``` + +**Audit Enhancements**: +- Store before/after balance with each transaction +- Implement change log table for all balance modifications +- Automated snapshots of all balances daily + +--- + +### 5. Commission Calculation Errors + +**Risk Level**: HIGH **HIGH** +**Impact**: Wrong commission rate applied → over/under collection → revenue loss or client overcharge + +#### Detection Methods + +**Verification Query**: +```sql +-- Verify all commission calculations are mathematically correct +SELECT + id, + transaction_id, + crypto_atoms, + commission_percentage, + discount, + base_amount, + commission_amount, + -- Recalculate expected values + ROUND(crypto_atoms / (1 + (commission_percentage * (100 - discount) / 100))) as expected_base, + ROUND(crypto_atoms - (crypto_atoms / (1 + (commission_percentage * (100 - discount) / 100)))) as expected_commission, + -- Calculate differences + base_amount - ROUND(crypto_atoms / (1 + (commission_percentage * (100 - discount) / 100))) as base_difference, + commission_amount - ROUND(crypto_atoms - (crypto_atoms / (1 + (commission_percentage * (100 - discount) / 100)))) as commission_difference +FROM satoshimachine.lamassu_transactions +WHERE ABS(base_amount - ROUND(crypto_atoms / (1 + (commission_percentage * (100 - discount) / 100)))) > 1 + OR ABS(commission_amount - ROUND(crypto_atoms - (crypto_atoms / (1 + (commission_percentage * (100 - discount) / 100))))) > 1; +``` + +**Manual Spot Check**: +``` +Example: 2000 GTQ → 266,800 sats (3% commission, 0% discount) + +Expected calculation: +- Effective commission = 0.03 * (100 - 0) / 100 = 0.03 +- Base amount = 266800 / (1 + 0.03) = 258,835 sats +- Commission = 266800 - 258835 = 7,965 sats + +Verify these match database values. +``` + +#### Immediate Response + +1. [OK] Run verification query to identify all affected transactions +2. [OK] Calculate total under/over collection amount +3. [OK] Determine if pattern (all transactions? specific time period? specific commission rate?) +4. [OK] **STOP POLLING** if active miscalculation detected + +#### Recovery Steps + +**Step 1: Quantify Impact** +```sql +-- Total revenue impact +SELECT + COUNT(*) as affected_transactions, + SUM(commission_difference) as total_revenue_impact, + MIN(created_at) as first_occurrence, + MAX(created_at) as last_occurrence +FROM ( + -- Use verification query from above +) as calc_check +WHERE ABS(commission_difference) > 1; +``` + +**Step 2: Client Impact Assessment** +```sql +-- Which clients were affected and by how much +SELECT + c.id, + c.username, + COUNT(lt.id) as affected_transactions, + SUM(lt.base_amount) as total_distributed, + SUM(expected_base) as should_have_distributed, + SUM(expected_base - lt.base_amount) as client_impact +FROM satoshimachine.lamassu_transactions lt +JOIN satoshimachine.dca_payments dp ON dp.lamassu_transaction_id = lt.id +JOIN satoshimachine.dca_clients c ON dp.client_id = c.id +WHERE -- filter for affected transactions +GROUP BY c.id; +``` + +**Step 3: Correction** + +**If Under-Collected Commission**: +- Revenue lost to business +- Client received correct amount +- No client-facing correction needed +- Document for accounting + +**If Over-Collected Commission**: +- Client under-distributed +- Create compensating payments to affected clients: +```sql +-- Add to client balances +UPDATE satoshimachine.dca_clients c +SET remaining_balance = remaining_balance + adjustment.amount +FROM ( + -- Calculate adjustment per client + SELECT client_id, SUM(should_have_distributed - actual_distributed) as amount + FROM affected_transactions_analysis + GROUP BY client_id +) adjustment +WHERE c.id = adjustment.client_id; +``` + +**Step 4: Fix Code Bug** +- Identify root cause in `transaction_processor.py` +- Add unit test for failed scenario +- Deploy fix +- Verify with test transaction + +#### Prevention Measures + +**Unit Tests** (Add to test suite): +```python +def test_commission_calculation_scenarios(): + """Test edge cases for commission calculation""" + test_cases = [ + # (crypto_atoms, commission%, discount%, expected_base, expected_commission) + (266800, 0.03, 0.0, 258835, 7965), + (100000, 0.05, 10.0, 95694, 4306), # With discount + (1, 0.03, 0.0, 0, 1), # Minimum amount + # Add more edge cases + ] + for case in test_cases: + result = calculate_commission(*case[:3]) + assert result.base_amount == case[3] + assert result.commission_amount == case[4] +``` + +**Calculation Verification** (Add to processing): +```python +# After calculation, verify math is correct +calculated_total = base_amount + commission_amount +assert abs(calculated_total - crypto_atoms) <= 1, "Commission calculation error detected" +``` + +**Audit Trail**: +- Store all calculation parameters with each transaction +- Log formula used and intermediate values +- Enable post-processing verification + +--- + +### 6. Wallet Key Rotation/Invalidation + +**Risk Level**: HIGH **HIGH** +**Impact**: Commission wallet adminkey changes → can't receive commission payments → processing halts + +#### Detection Methods + +1. **API Error Responses**: + - Payment API returns 401/403 authentication errors + - "Invalid API key" messages in logs + +2. **Wallet Balance Check**: +```sql +-- Commission not accumulating despite transactions +SELECT + SUM(commission_amount) as expected_total_commission, + -- Compare to actual wallet balance in LNBits dashboard +FROM satoshimachine.lamassu_transactions +WHERE created_at > ''; +``` + +3. **Manual Test**: +```bash +# Test commission wallet adminkey +curl -X GET https:///api/v1/wallet \ + -H "X-Api-Key: " + +# Should return wallet details, not auth error +``` + +#### Immediate Response + +1. [OK] **STOP POLLING** - No point processing if can't distribute +2. [OK] Identify if key was intentionally rotated or compromised +3. [OK] Obtain new valid adminkey for commission wallet +4. [OK] Verify source wallet adminkey is also still valid + +#### Recovery Steps + +**Step 1: Update Configuration** +1. Access admin dashboard +2. Navigate to Configuration section +3. Update commission wallet adminkey +4. Update source wallet adminkey if also affected +5. **Test Connection** before saving + +**Step 2: Verify Configuration Persisted** +```sql +-- Check configuration was saved correctly +SELECT commission_wallet_id, + LEFT(commission_wallet_adminkey, 10) || '...' as key_preview, + updated_at +FROM satoshimachine.lamassu_config +ORDER BY updated_at DESC +LIMIT 1; +``` + +**Step 3: Reprocess Failed Transactions** +- Identify transactions that failed due to auth errors +- Mark for retry or manually reprocess +- Verify commission payments complete successfully + +**Step 4: Resume Operations** +1. Test with manual poll first +2. Verify single transaction processes completely +3. Re-enable automatic polling +4. Monitor for 24 hours + +#### Prevention Measures + +**Key Management Documentation**: +- Document which LNBits wallets are used for commission/source +- Store backup admin keys in secure location (password manager) +- Define key rotation procedure with testing steps +- Require testing in staging before production changes + +**Configuration Validation**: +```python +# Add to config save endpoint in views_api.py +async def validate_wallet_keys(commission_key, source_key): + """Test wallet keys before saving configuration""" + # Test commission wallet + commission_valid = await test_wallet_access(commission_key) + if not commission_valid: + raise ValueError("Commission wallet key is invalid") + + # Test source wallet (if applicable) + if source_key: + source_valid = await test_wallet_access(source_key) + if not source_valid: + raise ValueError("Source wallet key is invalid") + + return True +``` + +**Automated Monitoring**: +- Daily health check of wallet accessibility +- Alert if wallet API calls start failing +- Backup key verification in secure environment + +--- + +## Emergency Protocol Checklist + +Use this checklist when ANY error is detected in the DCA system. + +### Phase 1: Immediate Actions (First 5 Minutes) + +- [ ] **Stop Automatic Processing** + - Disable background polling task + - Verify no transactions are currently processing + - Document time polling was stopped + +- [ ] **Assess Severity** + - Is money at risk? (duplicate processing, failed payments) + - Are clients affected? (missing distributions, balance errors) + - Is this blocking operations? (connection loss, wallet issues) + +- [ ] **Initial Documentation** + - Take screenshots of error messages + - Note exact timestamp of error detection + - Record current system state (balances, last transaction, etc.) + +- [ ] **Notify Stakeholders** + - Alert system administrator + - Notify clients if distributions will be delayed > 24 hours + - Escalate if financial impact > threshold + +### Phase 2: Investigation (15-30 Minutes) + +- [ ] **Collect Diagnostic Information** + - Run relevant SQL queries from scenarios above + - Check LNBits logs: `/lnbits/data/logs/` + - Review recent configuration changes + - Test external connections (SSH, wallets) + +- [ ] **Identify Root Cause** + - Match symptoms to failure scenarios above + - Determine if human error, system failure, or external issue + - Estimate scope of impact (time range, # clients, # transactions) + +- [ ] **Document Findings** + - Record root cause analysis + - List all affected records (transaction IDs, client IDs) + - Calculate financial impact (over/under distributed amounts) + - Take database snapshots for audit trail + +### Phase 3: Recovery (30 Minutes - 2 Hours) + +- [ ] **Fix Root Cause** + - Apply code fix if bug + - Update configuration if settings issue + - Restore connection if network issue + - Refer to specific recovery steps in scenarios above + +- [ ] **Data Correction** + - Run reconciliation queries + - Execute correction SQL statements + - Verify all client balances are accurate + - Ensure audit trail is complete + +- [ ] **Verification** + - Test fix with single transaction + - Verify wallets are accessible + - Confirm connections are stable + - Run full reconciliation report + +### Phase 4: Resumption (After Verification) + +- [ ] **Gradual Restart** + - Process one manual poll successfully + - Monitor for errors during processing + - Verify distributions complete end-to-end + - Check commission payments arrive correctly + +- [ ] **Re-enable Automation** + - Turn on background polling task + - Set monitoring alerts + - Document in system log + +- [ ] **Enhanced Monitoring** + - Watch closely for 24 hours + - Run reconciliation after next 3-5 transactions + - Verify no recurrence of issue + +### Phase 5: Post-Incident (24-48 Hours After) + +- [ ] **Complete Post-Mortem** + - Document full timeline of incident + - Record exact root cause and fix applied + - Calculate total impact (financial, time, clients affected) + - Identify what went well and what could improve + +- [ ] **Implement Safeguards** + - Add prevention measures from scenario sections + - Implement new monitoring/alerts + - Add automated tests for this failure mode + - Update runbooks and documentation + +- [ ] **Stakeholder Communication** + - Send incident report to management + - Notify affected clients if applicable + - Document lessons learned + - Update emergency contact procedures if needed + +--- + +## Recovery Procedures + +### Full System Reconciliation + +Run this complete reconciliation procedure weekly or after any incident: + +```sql +-- ============================================ +-- FULL SYSTEM RECONCILIATION REPORT +-- ============================================ + +-- 1. Client Balance Reconciliation +WITH client_financials AS ( + SELECT + c.id, + c.username, + c.remaining_balance as current_balance, + COALESCE(SUM(d.amount), 0) as total_deposits, + COALESCE(SUM(CASE WHEN p.status = 'completed' THEN p.amount_sats ELSE 0 END), 0) as total_payments, + COUNT(DISTINCT d.id) as deposit_count, + COUNT(DISTINCT p.id) as payment_count + FROM satoshimachine.dca_clients c + LEFT JOIN satoshimachine.dca_deposits d ON c.id = d.client_id AND d.status = 'confirmed' + LEFT JOIN satoshimachine.dca_payments p ON c.id = p.client_id + GROUP BY c.id +) +SELECT + id, + username, + current_balance, + total_deposits, + total_payments, + (total_deposits - total_payments) as calculated_balance, + (current_balance - (total_deposits - total_payments)) as discrepancy, + deposit_count, + payment_count, + CASE + WHEN ABS(current_balance - (total_deposits - total_payments)) <= 1 THEN '[OK] OK' + ELSE 'WARNING MISMATCH' + END as status +FROM client_financials +ORDER BY ABS(current_balance - (total_deposits - total_payments)) DESC; + +-- 2. Transaction Processing Verification +SELECT + COUNT(*) as total_transactions, + SUM(crypto_atoms) as total_sats_processed, + SUM(base_amount) as total_distributed, + SUM(commission_amount) as total_commission, + MIN(created_at) as first_transaction, + MAX(created_at) as last_transaction +FROM satoshimachine.lamassu_transactions; + +-- 3. Failed/Pending Payments Check +SELECT + status, + COUNT(*) as count, + SUM(amount_sats) as total_amount, + MIN(created_at) as oldest, + MAX(created_at) as newest +FROM satoshimachine.dca_payments +GROUP BY status +ORDER BY + CASE status + WHEN 'failed' THEN 1 + WHEN 'pending' THEN 2 + WHEN 'completed' THEN 3 + END; + +-- 4. Unconfirmed Deposits Check (sitting too long) +SELECT + id, + client_id, + amount, + status, + created_at, + EXTRACT(EPOCH FROM (NOW() - created_at))/3600 as hours_pending +FROM satoshimachine.dca_deposits +WHERE status = 'pending' + AND created_at < NOW() - INTERVAL '48 hours' +ORDER BY created_at; + +-- 5. Commission Calculation Verification (sample check) +SELECT + id, + transaction_id, + crypto_atoms, + commission_percentage, + discount, + base_amount, + commission_amount, + ROUND(crypto_atoms / (1 + (commission_percentage * (100 - discount) / 100))) as expected_base, + base_amount - ROUND(crypto_atoms / (1 + (commission_percentage * (100 - discount) / 100))) as difference +FROM satoshimachine.lamassu_transactions +ORDER BY created_at DESC +LIMIT 20; +``` + +### Emergency Access Procedures + +**If Admin Dashboard Inaccessible**: + +1. **Direct Database Access**: +```bash +# Connect to LNBits database +sqlite3 /home/padreug/AioLabs/Git/lnbits-extensions/lnbits/data/database.sqlite + +# Or PostgreSQL if used +psql -h localhost -U lnbits -d lnbits +``` + +2. **Direct Configuration Update**: +```sql +-- Update Lamassu config directly +UPDATE satoshimachine.lamassu_config +SET polling_enabled = false +WHERE id = (SELECT MAX(id) FROM satoshimachine.lamassu_config); +``` + +3. **Manual Client Balance Update**: +```sql +-- ONLY in emergency when dashboard unavailable +UPDATE satoshimachine.dca_clients +SET remaining_balance = +WHERE id = ; +-- MUST document this action in incident log +``` + +**If Background Task Won't Stop**: + +```bash +# Find LNBits process +ps aux | grep lnbits + +# Restart LNBits service (will stop all background tasks) +systemctl restart lnbits +# OR if running manually: +pkill -f lnbits +uv run lnbits +``` + +### Data Export for Audit + +**Complete Audit Trail Export**: + +```bash +# Export all DCA-related tables to CSV +sqlite3 -header -csv /path/to/lnbits/database.sqlite \ + "SELECT * FROM satoshimachine.lamassu_transactions;" \ + > lamassu_transactions_export_$(date +%Y%m%d_%H%M%S).csv + +sqlite3 -header -csv /path/to/lnbits/database.sqlite \ + "SELECT * FROM satoshimachine.dca_payments;" \ + > dca_payments_export_$(date +%Y%m%d_%H%M%S).csv + +sqlite3 -header -csv /path/to/lnbits/database.sqlite \ + "SELECT * FROM satoshimachine.dca_deposits;" \ + > dca_deposits_export_$(date +%Y%m%d_%H%M%S).csv + +sqlite3 -header -csv /path/to/lnbits/database.sqlite \ + "SELECT * FROM satoshimachine.dca_clients;" \ + > dca_clients_export_$(date +%Y%m%d_%H%M%S).csv +``` + +**Combined Audit Report**: + +```sql +-- Complete transaction-to-distribution audit trail +SELECT + lt.id as transaction_id, + lt.transaction_id as lamassu_txn_id, + lt.created_at as transaction_time, + lt.crypto_atoms as total_sats, + lt.fiat_code, + lt.fiat_amount, + lt.commission_percentage, + lt.discount, + lt.base_amount as distributable_sats, + lt.commission_amount, + c.id as client_id, + c.username as client_name, + dp.amount_sats as client_received, + dp.status as payment_status, + dp.payment_hash +FROM satoshimachine.lamassu_transactions lt +LEFT JOIN satoshimachine.dca_payments dp ON dp.lamassu_transaction_id = lt.id +LEFT JOIN satoshimachine.dca_clients c ON dp.client_id = c.id +ORDER BY lt.created_at DESC, c.username; +``` + +--- + +## Prevention Measures + +### Required Immediate Implementations + +#### 1. Idempotency Protection (CRITICAL) + +**File**: `transaction_processor.py` + +```python +async def process_lamassu_transaction(txn_data: dict) -> Optional[LamassuTransaction]: + """Process transaction with idempotency check""" + + # CRITICAL: Check if already processed + existing = await get_lamassu_transaction_by_txid(txn_data['id']) + if existing: + logger.warning(f"WARNING Transaction {txn_data['id']} already processed at {existing.created_at}, skipping") + return None + + # Continue with processing... +``` + +#### 2. Database Constraints + +**File**: `migrations.py` + +```sql +-- Add unique constraint on transaction_id +ALTER TABLE satoshimachine.lamassu_transactions +ADD CONSTRAINT unique_transaction_id UNIQUE (transaction_id); + +-- Prevent negative balances +ALTER TABLE satoshimachine.dca_clients +ADD CONSTRAINT positive_balance CHECK (remaining_balance >= 0); + +-- Ensure positive amounts +ALTER TABLE satoshimachine.dca_deposits +ADD CONSTRAINT positive_deposit CHECK (amount > 0); + +ALTER TABLE satoshimachine.dca_payments +ADD CONSTRAINT positive_payment CHECK (amount_sats > 0); +``` + +#### 3. Transaction Status Tracking + +**File**: `models.py` + +```python +class LamassuTransaction(BaseModel): + # ... existing fields ... + status: str = "pending" # pending, processing, completed, failed + error_message: Optional[str] = None + processed_at: Optional[datetime] = None +``` + +#### 4. Pre-flight Wallet Validation + +**File**: `transaction_processor.py` + +```python +async def validate_system_ready() -> Tuple[bool, str]: + """Validate system is ready to process transactions""" + + # Check commission wallet accessible + try: + commission_wallet = await get_wallet(config.commission_wallet_id) + if not commission_wallet: + return False, "Commission wallet not accessible" + except Exception as e: + return False, f"Commission wallet error: {str(e)}" + + # Check for stuck payments + stuck_payments = await get_stuck_payments(hours=2) + if stuck_payments: + return False, f"{len(stuck_payments)} payments stuck for >2 hours" + + # Check database connectivity + try: + await db_health_check() + except Exception as e: + return False, f"Database health check failed: {str(e)}" + + return True, "System ready" + +# Call before processing +ready, message = await validate_system_ready() +if not ready: + logger.error(f"System not ready: {message}") + await send_alert_to_admin(message) + return +``` + +### Automated Monitoring Implementation + +#### Daily Reconciliation Task + +**File**: `tasks.py` + +```python +@scheduler.scheduled_job("cron", hour=0, minute=0) # Daily at midnight UTC +async def daily_reconciliation(): + """Run daily balance reconciliation and report discrepancies""" + + logger.info("Starting daily reconciliation...") + + discrepancies = await find_balance_discrepancies() + + if discrepancies: + report = generate_reconciliation_report(discrepancies) + await send_alert_to_admin("WARNING Balance Discrepancies Detected", report) + await log_reconciliation_issue(discrepancies) + else: + logger.info("[OK] Daily reconciliation passed - all balances match") + + # Check for stuck payments + stuck_payments = await get_stuck_payments(hours=24) + if stuck_payments: + await send_alert_to_admin( + f"WARNING {len(stuck_payments)} Stuck Payments Detected", + format_stuck_payments_report(stuck_payments) + ) +``` + +#### Connection Health Monitor + +```python +@scheduler.scheduled_job("interval", hours=2) # Every 2 hours +async def check_system_health(): + """Monitor system health and alert on issues""" + + issues = [] + + # Check last successful poll + last_poll = await get_last_successful_poll_time() + if last_poll and (datetime.utcnow() - last_poll).total_seconds() > 86400: # 24 hours + issues.append(f"No successful poll in {(datetime.utcnow() - last_poll).total_seconds() / 3600:.1f} hours") + + # Check wallet accessibility + try: + await test_wallet_access(config.commission_wallet_adminkey) + except Exception as e: + issues.append(f"Commission wallet inaccessible: {str(e)}") + + # Check database connectivity + try: + await test_lamassu_connection() + except Exception as e: + issues.append(f"Lamassu database connection failed: {str(e)}") + + if issues: + await send_alert_to_admin("WARNING System Health Check Failed", "\n".join(issues)) +``` + +### Alert Configuration + +**File**: `config.py` or environment variables + +```python +# Alert settings +ALERT_EMAIL = "admin@yourdomain.com" +ALERT_WEBHOOK = "https://hooks.slack.com/..." # Slack/Discord webhook +ALERT_PHONE = "+1234567890" # For critical alerts (optional) + +# Alert thresholds +MAX_STUCK_PAYMENT_HOURS = 2 +MAX_POLL_DELAY_HOURS = 24 +MAX_BALANCE_DISCREPANCY_SATS = 100 +``` + +--- + +## Monitoring & Alerts + +### Dashboard Indicators to Watch + +#### Critical Indicators (Check Daily) +- [OK] **Last Successful Poll Time** - Should be within last 2-4 hours (based on polling interval) +- [OK] **Failed Payment Count** - Should be 0; investigate immediately if > 0 +- [OK] **Commission Wallet Balance** - Should increase proportionally with transactions +- [OK] **Active Clients with Balance** - Cross-reference with expected DCA participants + +#### Warning Indicators (Check Weekly) +- WARNING **Pending Deposits > 48 hours** - May indicate confirmation workflow issue +- WARNING **Client Balance Reconciliation** - Run full reconciliation report +- WARNING **Average Commission %** - Verify matches configured rates +- WARNING **Transaction Processing Time** - Should complete within minutes, not hours + +### Automated Alert Triggers + +Implement these alerts in production: + +| Alert | Severity | Trigger Condition | Response Time | +|-------|----------|-------------------|---------------| +| Duplicate Transaction Detected | CRITICAL CRITICAL | Same `transaction_id` inserted twice | Immediate | +| Payment Stuck > 15 minutes | CRITICAL CRITICAL | Payment status not "completed" after 15min | < 30 minutes | +| Polling Failed > 24 hours | HIGH HIGH | No new transactions in 24 hours | < 2 hours | +| Balance Discrepancy > 100 sats | HIGH HIGH | Reconciliation finds error > threshold | < 4 hours | +| Wallet Inaccessible | HIGH HIGH | Commission wallet returns auth error | < 1 hour | +| Database Connection Failed | MEDIUM MEDIUM | Cannot connect to Lamassu DB | < 4 hours | +| Commission Calculation Anomaly | MEDIUM MEDIUM | Calculated amount differs from formula | < 24 hours | + +### Log Files to Monitor + +**Location**: `/home/padreug/AioLabs/Git/lnbits-extensions/lnbits/data/logs/` + +**Key Log Patterns**: +```bash +# Critical errors +grep -i "error\|exception\|failed" lnbits.log | tail -100 + +# Transaction processing +grep "LamassuTransactionProcessor" lnbits.log | tail -50 + +# Payment distribution +grep "dca_payment\|distribution" lnbits.log | tail -50 + +# SSH connection issues +grep -i "ssh\|connection\|timeout" lnbits.log | tail -50 + +# Wallet API calls +grep "wallet.*api\|payment_hash" lnbits.log | tail -50 +``` + +### Manual Checks (Weekly) + +**Sunday 00:00 UTC - Weekly Audit**: + +1. [ ] Run full reconciliation SQL report +2. [ ] Export all tables to CSV for backup +3. [ ] Verify commission wallet balance matches sum of commission_amount +4. [ ] Check for any pending deposits > 7 days old +5. [ ] Review last 20 transactions for calculation correctness +6. [ ] Test database connection from admin dashboard +7. [ ] Test manual poll to verify end-to-end flow +8. [ ] Review error logs for any concerning patterns + +--- + +## Contact Information + +### System Access + +**LNBits Admin Dashboard**: +- URL: `https:///satoshimachine` +- Requires superuser authentication + +**Database Access**: +```bash +# LNBits database +sqlite3 /home/padreug/AioLabs/Git/lnbits-extensions/lnbits/data/database.sqlite + +# Direct table access +sqlite3 /path/to/db "SELECT * FROM satoshimachine.;" +``` + +**Log Files**: +```bash +# Main logs +tail -f /home/padreug/AioLabs/Git/lnbits-extensions/lnbits/data/logs/lnbits.log + +# Error logs only +tail -f /home/padreug/AioLabs/Git/lnbits-extensions/lnbits/data/logs/lnbits.log | grep -i error +``` + +### Emergency Escalation + +**Level 1 - System Administrator** (First Contact): +- Name: _______________________ +- Email: _______________________ +- Phone: _______________________ +- Availability: _______________________ + +**Level 2 - Technical Lead** (If L1 unavailable): +- Name: _______________________ +- Email: _______________________ +- Phone: _______________________ +- Availability: _______________________ + +**Level 3 - Business Owner** (Financial impact > $X): +- Name: _______________________ +- Email: _______________________ +- Phone: _______________________ +- Availability: _______________________ + +### External Contacts + +**Lamassu Administrator**: +- Name: _______________________ +- Email: _______________________ +- Phone: _______________________ +- SSH access issues, database access, ATM questions + +**LNBits Infrastructure**: +- Name: _______________________ +- Email: _______________________ +- Phone: _______________________ +- Wallet issues, API problems, system downtime + +**Accountant/Auditor**: +- Name: _______________________ +- Email: _______________________ +- For balance discrepancies requiring financial reconciliation + +--- + +## Appendix: Quick Reference Commands + +### Emergency Stop + +```bash +# Stop LNBits service (stops all background tasks) +systemctl stop lnbits + +# Or kill process +pkill -f lnbits +``` + +### Emergency Database Disable Polling + +```sql +-- Disable automatic polling +UPDATE satoshimachine.lamassu_config +SET polling_enabled = false; +``` + +### Quick Balance Check + +```sql +-- All client balances summary +SELECT id, username, remaining_balance, created_at +FROM satoshimachine.dca_clients +ORDER BY remaining_balance DESC; +``` + +### Last 10 Transactions + +```sql +SELECT id, transaction_id, created_at, crypto_atoms, base_amount, commission_amount +FROM satoshimachine.lamassu_transactions +ORDER BY created_at DESC +LIMIT 10; +``` + +### Failed Payments + +```sql +SELECT * FROM satoshimachine.dca_payments +WHERE status != 'completed' +ORDER BY created_at DESC; +``` + +### Export Everything (Backup) + +```bash +#!/bin/bash +# Emergency full backup +DATE=$(date +%Y%m%d_%H%M%S) +BACKUP_DIR="emergency_backup_${DATE}" +mkdir -p $BACKUP_DIR + +sqlite3 -header -csv /path/to/database.sqlite \ + "SELECT * FROM satoshimachine.lamassu_transactions;" \ + > ${BACKUP_DIR}/lamassu_transactions.csv + +sqlite3 -header -csv /path/to/database.sqlite \ + "SELECT * FROM satoshimachine.dca_payments;" \ + > ${BACKUP_DIR}/dca_payments.csv + +sqlite3 -header -csv /path/to/database.sqlite \ + "SELECT * FROM satoshimachine.dca_deposits;" \ + > ${BACKUP_DIR}/dca_deposits.csv + +sqlite3 -header -csv /path/to/database.sqlite \ + "SELECT * FROM satoshimachine.dca_clients;" \ + > ${BACKUP_DIR}/dca_clients.csv + +echo "Backup complete in ${BACKUP_DIR}/" +``` + +--- + +## Document Change Log + +| Version | Date | Author | Changes | +|---------|------|--------|---------| +| 1.0 | 2025-10-19 | Claude Code | Initial emergency protocols document | +| | | | | +| | | | | + +--- + +## Sign-Off + +This document has been reviewed and approved for use in production emergency response: + +**System Administrator**: _____________________ Date: _______ + +**Technical Lead**: _____________________ Date: _______ + +**Business Owner**: _____________________ Date: _______ + +--- + +**END OF DOCUMENT** + +*Keep this document accessible at all times. Print and store in emergency response binder.* +*Review and update quarterly or after any major incident.* diff --git a/Lamassu-Database-Analysis.md b/misc-docs/Lamassu-Database-Analysis.md similarity index 100% rename from Lamassu-Database-Analysis.md rename to misc-docs/Lamassu-Database-Analysis.md