From 9d7a3af9d7df135b3344ca629b7510c102f878ca Mon Sep 17 00:00:00 2001 From: hatim boufnichel Date: Sat, 23 Mar 2024 00:02:26 +0100 Subject: [PATCH] watchdog v1 --- src/index.ts | 3 +- src/services/lnd/watchdog.ts | 112 ++++++++++++++++++++++++++-- src/services/main/index.ts | 18 +++-- src/services/main/paymentManager.ts | 4 +- 4 files changed, 119 insertions(+), 18 deletions(-) diff --git a/src/index.ts b/src/index.ts index a9651b57..276b3a69 100644 --- a/src/index.ts +++ b/src/index.ts @@ -26,7 +26,8 @@ const start = async () => { const mainHandler = new Main(mainSettings, storageManager) await mainHandler.lnd.Warmup() if (!mainSettings.skipSanityCheck) { - await mainHandler.VerifyEventsLog() + const totalUsersBalance = await mainHandler.VerifyEventsLog() + await mainHandler.paymentManager.watchDog.SeedLndBalance(totalUsersBalance) } const serverMethods = GetServerMethods(mainHandler) const nostrSettings = LoadNosrtSettingsFromEnv() diff --git a/src/services/lnd/watchdog.ts b/src/services/lnd/watchdog.ts index a3a554a1..d5f70865 100644 --- a/src/services/lnd/watchdog.ts +++ b/src/services/lnd/watchdog.ts @@ -4,14 +4,20 @@ import { LightningHandler } from "./index.js"; export type WatchdogSettings = { maxDiffBps: number maxDiffSats: number + maxUpdateDiffSats: number } export const LoadWatchdogSettingsFromEnv = (test = false): WatchdogSettings => { return { maxDiffBps: EnvMustBeInteger("WATCHDOG_MAX_DIFF_BPS"), - maxDiffSats: EnvMustBeInteger("WATCHDOG_MAX_DIFF_SATS") + maxDiffSats: EnvMustBeInteger("WATCHDOG_MAX_DIFF_SATS"), + maxUpdateDiffSats: EnvMustBeInteger("WATCHDOG_MAX_UPDATE_DIFF_SATS") } } export class Watchdog { + initialLndBalance: number; + initialUsersBalance: number; + lastLndBalance: number; + lastUsersBalance: number; lnd: LightningHandler; settings: WatchdogSettings; log = getLogger({ appName: "watchdog" }) @@ -20,15 +26,105 @@ export class Watchdog { this.settings = settings; } + SeedLndBalance = async (totalUsersBalance: number) => { + this.initialLndBalance = await this.getTotalLndBalance() + this.lastLndBalance = this.initialLndBalance + + this.initialUsersBalance = totalUsersBalance + this.lastUsersBalance = this.initialUsersBalance + } + + getTotalLndBalance = async () => { + const { channelsBalance, confirmedBalance } = await this.lnd.GetBalance() + return confirmedBalance + channelsBalance.reduce((acc, { localBalanceSats }) => acc + localBalanceSats, 0) + } + + checkBalanceUpdate = (deltaLnd: number, deltaUsers: number, type: 'incremental' | 'absolute', threshold: number) => { + this.log("LND balance update:", deltaLnd, "sats", type === 'incremental' ? "since last balance check" : "since app startup") + this.log("Users balance update:", deltaUsers, "sats", type === 'incremental' ? "since last balance check" : "since app startup") + + const result = this.checkDeltas(deltaLnd, deltaUsers) + switch (result.type) { + case 'mismatch': + if (deltaLnd < 0) { + this.log("WARNING! LND balance decreased while users balance increased creating a difference of", result.absoluteDiff, "sats") + if (result.absoluteDiff > threshold) { + this.log("Difference is too big for an update, locking outgoing operations") + return true + } + } else { + this.log("LND balance increased while users balance decreased creating a difference of", result.absoluteDiff, "sats, could be caused by data loss, or liquidity injection") + return false + } + break + case 'negative': + if (Math.abs(deltaLnd) > Math.abs(deltaUsers)) { + this.log("WARNING! LND balance decreased more than users balance with a difference of", result.absoluteDiff, "sats") + if (result.absoluteDiff > threshold) { + this.log("Difference is too big for an update, locking outgoing operations") + return true + } + } else { + this.log("LND balance decreased less than users balance with a difference of", result.absoluteDiff, "sats, could be caused by data loss, or liquidity injection") + return false + } + break + case 'positive': + if (deltaLnd < deltaUsers) { + this.log("WARNING! LND balance increased less than users balance with a difference of", result.absoluteDiff, "sats") + if (result.absoluteDiff > threshold) { + this.log("Difference is too big for an update, locking outgoing operations") + return true + } + } else { + this.log("LND balance increased more than users balance with a difference of", result.absoluteDiff, "sats, could be caused by data loss, or liquidity injection") + return false + } + } + return false + } + PaymentRequested = async (totalUsersBalance: number) => { this.log("Payment requested, checking balance") - const { channelsBalance, confirmedBalance } = await this.lnd.GetBalance() - const totalLndBalance = confirmedBalance + channelsBalance.reduce((acc, { localBalanceSats }) => acc + localBalanceSats, 0) - const diffSats = Math.abs(totalLndBalance - totalUsersBalance) - const diffBps = (diffSats / Math.max(totalLndBalance, totalUsersBalance)) * 10_000 - if (diffSats > this.settings.maxDiffSats || diffBps > this.settings.maxDiffBps) { - this.log(`LND balance ${totalLndBalance} is too different from users balance ${totalUsersBalance}`) + const totalLndBalance = await this.getTotalLndBalance() + const IncDeltaLnd = totalLndBalance - this.lastLndBalance + const IncDeltaUsers = totalUsersBalance - this.lastUsersBalance + const denyIncremental = this.checkBalanceUpdate(IncDeltaLnd, IncDeltaUsers, 'incremental', this.settings.maxUpdateDiffSats) + if (denyIncremental) { + this.log("Balance mismatch detected in incremental update, locking outgoing operations") this.lnd.LockOutgoingOperations() + return + } + const AbsDeltaLnd = totalLndBalance - this.initialLndBalance + const AbsDeltaUsers = totalUsersBalance - this.initialUsersBalance + const denyAbsolute = this.checkBalanceUpdate(AbsDeltaLnd, AbsDeltaUsers, 'absolute', this.settings.maxDiffSats) + if (denyAbsolute) { + this.log("Balance mismatch detected in absolute update, locking outgoing operations") + this.lnd.LockOutgoingOperations() + return + } + this.lastLndBalance = totalLndBalance + this.lastUsersBalance = totalUsersBalance + } + + checkDeltas = (deltaLnd: number, deltaUsers: number): DeltaCheckResult => { + if (deltaLnd < 0) { + if (deltaUsers < 0) { + const diff = Math.abs(deltaLnd - deltaUsers) + return { type: 'negative', absoluteDiff: diff, relativeDiff: diff / Math.max(deltaLnd, deltaUsers) } + } else { + const diff = Math.abs(deltaLnd) + deltaUsers + return { type: 'mismatch', absoluteDiff: diff } + } + } else { + if (deltaUsers < 0) { + const diff = deltaLnd + Math.abs(deltaUsers) + return { type: 'mismatch', absoluteDiff: diff } + } else { + const diff = Math.abs(deltaLnd - deltaUsers) + return { type: 'positive', absoluteDiff: diff, relativeDiff: diff / Math.max(deltaLnd, deltaUsers) } + } } } -} \ No newline at end of file +} +type DeltaCheckResult = { type: 'negative' | 'positive', absoluteDiff: number, relativeDiff: number } | { type: 'mismatch', absoluteDiff: number } \ No newline at end of file diff --git a/src/services/main/index.ts b/src/services/main/index.ts index c95e11e7..05f9d103 100644 --- a/src/services/main/index.ts +++ b/src/services/main/index.ts @@ -113,12 +113,12 @@ export default class { if (!updateResult.affected) { throw new Error("unable to flag chain transaction as paid") } - await this.storage.userStorage.IncrementUserBalance(userAddress.user.user_id, amount - serviceFee, userAddress.address, tx) + const addressData = `${userAddress.address}:${tx_hash}` + this.storage.eventsLog.LogEvent({ type: 'address_paid', userId: userAddress.user.user_id, appId: userAddress.linkedApplication.app_id, appUserId: "", balance: userAddress.user.balance_sats, data: addressData, amount }) + await this.storage.userStorage.IncrementUserBalance(userAddress.user.user_id, amount - serviceFee, addressData, tx) if (serviceFee > 0) { await this.storage.userStorage.IncrementUserBalance(userAddress.linkedApplication.owner.user_id, serviceFee, 'fees', tx) } - const addressData = `${userAddress.address}:${tx_hash}` - this.storage.eventsLog.LogEvent({ type: 'address_paid', userId: userAddress.user.user_id, appId: userAddress.linkedApplication.app_id, appUserId: "", balance: userAddress.user.balance_sats, data: addressData, amount }) const operationId = `${Types.UserOperationType.INCOMING_TX}-${serialId}` const op = { amount, paidAtUnix: Date.now() / 1000, inbound: true, type: Types.UserOperationType.INCOMING_TX, identifier: userAddress.address, operationId, network_fee: 0, service_fee: serviceFee, confirmed: true, tx_hash: c.tx.tx_hash, internal: c.tx.internal } this.sendOperationToNostr(userAddress.linkedApplication!, userAddress.user.user_id, op) @@ -148,12 +148,13 @@ export default class { // This call will fail if the transaction is already registered const addedTx = await this.storage.paymentStorage.AddAddressReceivingTransaction(userAddress, txOutput.hash, txOutput.index, amount, fee, internal, blockHeight, tx) if (internal) { - await this.storage.userStorage.IncrementUserBalance(userAddress.user.user_id, addedTx.paid_amount - fee, userAddress.address, tx) + const addressData = `${address}:${txOutput.hash}` + this.storage.eventsLog.LogEvent({ type: 'address_paid', userId: userAddress.user.user_id, appId: userAddress.linkedApplication.app_id, appUserId: "", balance: userAddress.user.balance_sats, data: addressData, amount }) + await this.storage.userStorage.IncrementUserBalance(userAddress.user.user_id, addedTx.paid_amount - fee, addressData, tx) if (fee > 0) { await this.storage.userStorage.IncrementUserBalance(userAddress.linkedApplication.owner.user_id, fee, 'fees', tx) } - const addressData = `${address}:${txOutput.hash}` - this.storage.eventsLog.LogEvent({ type: 'address_paid', userId: userAddress.user.user_id, appId: userAddress.linkedApplication.app_id, appUserId: "", balance: userAddress.user.balance_sats, data: addressData, amount }) + } const operationId = `${Types.UserOperationType.INCOMING_TX}-${addedTx.serial_id}` const op = { amount, paidAtUnix: Date.now() / 1000, inbound: true, type: Types.UserOperationType.INCOMING_TX, identifier: userAddress.address, operationId, network_fee: 0, service_fee: fee, confirmed: internal, tx_hash: txOutput.hash, internal: false } @@ -183,11 +184,11 @@ export default class { } try { await this.storage.paymentStorage.FlagInvoiceAsPaid(userInvoice, amount, fee, internal, tx) + this.storage.eventsLog.LogEvent({ type: 'invoice_paid', userId: userInvoice.user.user_id, appId: userInvoice.linkedApplication.app_id, appUserId: "", balance: userInvoice.user.balance_sats, data: paymentRequest, amount }) await this.storage.userStorage.IncrementUserBalance(userInvoice.user.user_id, amount - fee, userInvoice.invoice, tx) if (fee > 0) { await this.storage.userStorage.IncrementUserBalance(userInvoice.linkedApplication.owner.user_id, fee, 'fees', tx) } - this.storage.eventsLog.LogEvent({ type: 'invoice_paid', userId: userInvoice.user.user_id, appId: userInvoice.linkedApplication.app_id, appUserId: "", balance: userInvoice.user.balance_sats, data: paymentRequest, amount }) await this.triggerPaidCallback(log, userInvoice.callbackUrl) const operationId = `${Types.UserOperationType.INCOMING_INVOICE}-${userInvoice.serial_id}` const op = { amount, paidAtUnix: Date.now() / 1000, inbound: true, type: Types.UserOperationType.INCOMING_INVOICE, identifier: userInvoice.invoice, operationId, network_fee: 0, service_fee: fee, confirmed: true, tx_hash: "", internal } @@ -305,6 +306,9 @@ export default class { throw new Error("sanity check on balance failed, expected: " + u.updatedBalance + " found: " + user.balance_sats) } })) + + const total = await this.storage.paymentStorage.GetTotalUsersBalance() + return total || 0 } checkUserEntry(e: LoggedEvent, u: { ts: number, updatedBalance: number } | undefined) { diff --git a/src/services/main/paymentManager.ts b/src/services/main/paymentManager.ts index 77e7aa7f..34a6270e 100644 --- a/src/services/main/paymentManager.ts +++ b/src/services/main/paymentManager.ts @@ -144,7 +144,7 @@ export default class { async PayInvoice(userId: string, req: Types.PayInvoiceRequest, linkedApplication: Application): Promise { this.log("paying invoice", req.invoice, "for user", userId, "with amount", req.amount) - this.WatchdogCheck() + await this.WatchdogCheck() const decoded = await this.lnd.DecodeInvoice(req.invoice) if (decoded.numSatoshis !== 0 && req.amount !== 0) { throw new Error("invoice has value, do not provide amount the the request") @@ -201,7 +201,7 @@ export default class { async PayAddress(ctx: Types.UserContext, req: Types.PayAddressRequest): Promise { throw new Error("address payment currently disabled, use Lightning instead") - this.WatchdogCheck() + await this.WatchdogCheck() const { blockHeight } = await this.lnd.GetInfo() const app = await this.storage.applicationStorage.GetApplication(ctx.app_id) const serviceFee = this.getServiceFee(Types.UserOperationType.OUTGOING_TX, req.amoutSats, false)