From f20abdd5f479ac7c3ad14c077f1c88e477556001 Mon Sep 17 00:00:00 2001 From: hatim boufnichel Date: Tue, 19 Mar 2024 21:17:18 +0100 Subject: [PATCH 1/4] watchdog v0 --- env.example | 2 ++ src/services/lnd/index.ts | 2 ++ src/services/lnd/lnd.ts | 17 +++++++++++++ src/services/lnd/mock.ts | 6 +++++ src/services/lnd/watchdog.ts | 34 ++++++++++++++++++++++++++ src/services/main/index.ts | 2 ++ src/services/main/paymentManager.ts | 10 ++++++++ src/services/main/settings.ts | 2 ++ src/services/storage/paymentStorage.ts | 4 +++ 9 files changed, 79 insertions(+) create mode 100644 src/services/lnd/watchdog.ts diff --git a/env.example b/env.example index 721db7e9..37cc237c 100644 --- a/env.example +++ b/env.example @@ -44,3 +44,5 @@ MIGRATE_DB=false RECORD_PERFORMANCE=true SKIP_SANITY_CHECK=false DISABLE_EXTERNAL_PAYMENTS=false +WATCHDOG_MAX_DIFF_BPS=100 +WATCHDOG_MAX_DIFF_SATS=10000 \ No newline at end of file diff --git a/src/services/lnd/index.ts b/src/services/lnd/index.ts index bc1309cb..9cda8da4 100644 --- a/src/services/lnd/index.ts +++ b/src/services/lnd/index.ts @@ -38,6 +38,8 @@ export interface LightningHandler { GetForwardingHistory(indexOffset: number): Promise<{ fee: number, chanIdIn: string, chanIdOut: string, timestampNs: number, offset: number }[]> GetAllPaidInvoices(max: number): Promise GetAllPayments(max: number): Promise + LockOutgoingOperations(): void + UnlockOutgoingOperations(): void } export default (settings: LndSettings, addressPaidCb: AddressPaidCb, invoicePaidCb: InvoicePaidCb, newBlockCb: NewBlockCb, htlcCb: HtlcCb): LightningHandler => { diff --git a/src/services/lnd/lnd.ts b/src/services/lnd/lnd.ts index a43e1fcf..e1128e1b 100644 --- a/src/services/lnd/lnd.ts +++ b/src/services/lnd/lnd.ts @@ -33,6 +33,7 @@ export default class { newBlockCb: NewBlockCb htlcCb: HtlcCb log = getLogger({ appName: 'lndManager' }) + outgoingOpsLocked = false constructor(settings: LndSettings, addressPaidCb: AddressPaidCb, invoicePaidCb: InvoicePaidCb, newBlockCb: NewBlockCb, htlcCb: HtlcCb) { this.settings = settings this.addressPaidCb = addressPaidCb @@ -60,6 +61,14 @@ export default class { this.router = new RouterClient(transport) this.chainNotifier = new ChainNotifierClient(transport) } + + LockOutgoingOperations(): void { + this.outgoingOpsLocked = true + } + UnlockOutgoingOperations(): void { + this.outgoingOpsLocked = false + } + SetMockInvoiceAsPaid(invoice: string, amount: number): Promise { throw new Error("SetMockInvoiceAsPaid only available in mock mode") } @@ -251,6 +260,10 @@ export default class { return { local: r.localBalance ? Number(r.localBalance.sat) : 0, remote: r.remoteBalance ? Number(r.remoteBalance.sat) : 0 } } async PayInvoice(invoice: string, amount: number, feeLimit: number): Promise { + if (this.outgoingOpsLocked) { + this.log("outgoing ops locked, rejecting payment request") + throw new Error("lnd node is currently out of sync") + } await this.Health() this.log("paying invoice", invoice, "for", amount, "sats") const abortController = new AbortController() @@ -287,6 +300,10 @@ export default class { } async PayAddress(address: string, amount: number, satPerVByte: number, label = ""): Promise { + if (this.outgoingOpsLocked) { + this.log("outgoing ops locked, rejecting payment request") + throw new Error("lnd node is currently out of sync") + } await this.Health() this.log("sending chain TX for", amount, "sats", "to", address) const res = await this.lightning.sendCoins(SendCoinsReq(address, amount, satPerVByte, label), DeadLineMetadata()) diff --git a/src/services/lnd/mock.ts b/src/services/lnd/mock.ts index 44b2e6cb..17fd70c1 100644 --- a/src/services/lnd/mock.ts +++ b/src/services/lnd/mock.ts @@ -131,6 +131,12 @@ export default class { async GetAllPayments(max: number): Promise { throw new Error("not implemented") } + LockOutgoingOperations() { + throw new Error("not implemented") + } + UnlockOutgoingOperations() { + throw new Error("not implemented") + } } diff --git a/src/services/lnd/watchdog.ts b/src/services/lnd/watchdog.ts new file mode 100644 index 00000000..a3a554a1 --- /dev/null +++ b/src/services/lnd/watchdog.ts @@ -0,0 +1,34 @@ +import { EnvMustBeInteger } from "../helpers/envParser.js"; +import { getLogger } from "../helpers/logger.js"; +import { LightningHandler } from "./index.js"; +export type WatchdogSettings = { + maxDiffBps: number + maxDiffSats: number +} +export const LoadWatchdogSettingsFromEnv = (test = false): WatchdogSettings => { + return { + maxDiffBps: EnvMustBeInteger("WATCHDOG_MAX_DIFF_BPS"), + maxDiffSats: EnvMustBeInteger("WATCHDOG_MAX_DIFF_SATS") + } +} +export class Watchdog { + lnd: LightningHandler; + settings: WatchdogSettings; + log = getLogger({ appName: "watchdog" }) + constructor(settings: WatchdogSettings, lnd: LightningHandler) { + this.lnd = lnd; + this.settings = settings; + } + + PaymentRequested = async (totalUsersBalance: number) => { + this.log("Payment requested, checking balance") + const { channelsBalance, confirmedBalance } = await this.lnd.GetBalance() + const totalLndBalance = confirmedBalance + channelsBalance.reduce((acc, { localBalanceSats }) => acc + localBalanceSats, 0) + const diffSats = Math.abs(totalLndBalance - totalUsersBalance) + const diffBps = (diffSats / Math.max(totalLndBalance, totalUsersBalance)) * 10_000 + if (diffSats > this.settings.maxDiffSats || diffBps > this.settings.maxDiffBps) { + this.log(`LND balance ${totalLndBalance} is too different from users balance ${totalUsersBalance}`) + this.lnd.LockOutgoingOperations() + } + } +} \ No newline at end of file diff --git a/src/services/main/index.ts b/src/services/main/index.ts index bbf03146..c95e11e7 100644 --- a/src/services/main/index.ts +++ b/src/services/main/index.ts @@ -17,8 +17,10 @@ import { UnsignedEvent } from '../nostr/tools/event.js' import { NostrSend } from '../nostr/handler.js' import MetricsManager from '../metrics/index.js' import EventsLogManager, { LoggedEvent } from '../storage/eventsLog.js' +import { LoadWatchdogSettingsFromEnv } from '../lnd/watchdog.js' export const LoadMainSettingsFromEnv = (test = false): MainSettings => { return { + watchDogSettings: LoadWatchdogSettingsFromEnv(test), lndSettings: LoadLndSettingsFromEnv(test), storageSettings: LoadStorageSettingsFromEnv(test), jwtSecret: EnvMustBeNonEmptyString("JWT_SECRET"), diff --git a/src/services/main/paymentManager.ts b/src/services/main/paymentManager.ts index a119e563..77e7aa7f 100644 --- a/src/services/main/paymentManager.ts +++ b/src/services/main/paymentManager.ts @@ -14,6 +14,7 @@ import { SendCoinsResponse } from '../../../proto/lnd/lightning.js' import { Event, verifiedSymbol, verifySignature } from '../nostr/tools/event.js' import { AddressReceivingTransaction } from '../storage/entity/AddressReceivingTransaction.js' import { UserTransactionPayment } from '../storage/entity/UserTransactionPayment.js' +import { Watchdog } from '../lnd/watchdog.js' interface UserOperationInfo { serial_id: number paid_amount: number @@ -45,10 +46,12 @@ export default class { addressPaidCb: AddressPaidCb invoicePaidCb: InvoicePaidCb log = getLogger({ appName: "PaymentManager" }) + watchDog: Watchdog constructor(storage: Storage, lnd: LightningHandler, settings: MainSettings, addressPaidCb: AddressPaidCb, invoicePaidCb: InvoicePaidCb) { this.storage = storage this.settings = settings this.lnd = lnd + this.watchDog = new Watchdog(settings.watchDogSettings, lnd) this.addressPaidCb = addressPaidCb this.invoicePaidCb = invoicePaidCb } @@ -134,8 +137,14 @@ export default class { } } + async WatchdogCheck() { + const total = await this.storage.paymentStorage.GetTotalUsersBalance() + await this.watchDog.PaymentRequested(total || 0) + } + async PayInvoice(userId: string, req: Types.PayInvoiceRequest, linkedApplication: Application): Promise { this.log("paying invoice", req.invoice, "for user", userId, "with amount", req.amount) + this.WatchdogCheck() const decoded = await this.lnd.DecodeInvoice(req.invoice) if (decoded.numSatoshis !== 0 && req.amount !== 0) { throw new Error("invoice has value, do not provide amount the the request") @@ -192,6 +201,7 @@ export default class { async PayAddress(ctx: Types.UserContext, req: Types.PayAddressRequest): Promise { throw new Error("address payment currently disabled, use Lightning instead") + this.WatchdogCheck() const { blockHeight } = await this.lnd.GetInfo() const app = await this.storage.applicationStorage.GetApplication(ctx.app_id) const serviceFee = this.getServiceFee(Types.UserOperationType.OUTGOING_TX, req.amoutSats, false) diff --git a/src/services/main/settings.ts b/src/services/main/settings.ts index 7397e97b..cbd26f7b 100644 --- a/src/services/main/settings.ts +++ b/src/services/main/settings.ts @@ -1,8 +1,10 @@ import { StorageSettings } from '../storage/index.js' import { LndSettings } from '../lnd/settings.js' +import { WatchdogSettings } from '../lnd/watchdog.js' export type MainSettings = { storageSettings: StorageSettings, lndSettings: LndSettings, + watchDogSettings: WatchdogSettings, jwtSecret: string incomingTxFee: number outgoingTxFee: number diff --git a/src/services/storage/paymentStorage.ts b/src/services/storage/paymentStorage.ts index 3fceb8a7..03281232 100644 --- a/src/services/storage/paymentStorage.ts +++ b/src/services/storage/paymentStorage.ts @@ -360,4 +360,8 @@ export default class { break; } } + + async GetTotalUsersBalance(entityManager = this.DB) { + return entityManager.getRepository(User).sum("balance_sats") + } } \ No newline at end of file From 9d7a3af9d7df135b3344ca629b7510c102f878ca Mon Sep 17 00:00:00 2001 From: hatim boufnichel Date: Sat, 23 Mar 2024 00:02:26 +0100 Subject: [PATCH 2/4] watchdog v1 --- src/index.ts | 3 +- src/services/lnd/watchdog.ts | 112 ++++++++++++++++++++++++++-- src/services/main/index.ts | 18 +++-- src/services/main/paymentManager.ts | 4 +- 4 files changed, 119 insertions(+), 18 deletions(-) diff --git a/src/index.ts b/src/index.ts index a9651b57..276b3a69 100644 --- a/src/index.ts +++ b/src/index.ts @@ -26,7 +26,8 @@ const start = async () => { const mainHandler = new Main(mainSettings, storageManager) await mainHandler.lnd.Warmup() if (!mainSettings.skipSanityCheck) { - await mainHandler.VerifyEventsLog() + const totalUsersBalance = await mainHandler.VerifyEventsLog() + await mainHandler.paymentManager.watchDog.SeedLndBalance(totalUsersBalance) } const serverMethods = GetServerMethods(mainHandler) const nostrSettings = LoadNosrtSettingsFromEnv() diff --git a/src/services/lnd/watchdog.ts b/src/services/lnd/watchdog.ts index a3a554a1..d5f70865 100644 --- a/src/services/lnd/watchdog.ts +++ b/src/services/lnd/watchdog.ts @@ -4,14 +4,20 @@ import { LightningHandler } from "./index.js"; export type WatchdogSettings = { maxDiffBps: number maxDiffSats: number + maxUpdateDiffSats: number } export const LoadWatchdogSettingsFromEnv = (test = false): WatchdogSettings => { return { maxDiffBps: EnvMustBeInteger("WATCHDOG_MAX_DIFF_BPS"), - maxDiffSats: EnvMustBeInteger("WATCHDOG_MAX_DIFF_SATS") + maxDiffSats: EnvMustBeInteger("WATCHDOG_MAX_DIFF_SATS"), + maxUpdateDiffSats: EnvMustBeInteger("WATCHDOG_MAX_UPDATE_DIFF_SATS") } } export class Watchdog { + initialLndBalance: number; + initialUsersBalance: number; + lastLndBalance: number; + lastUsersBalance: number; lnd: LightningHandler; settings: WatchdogSettings; log = getLogger({ appName: "watchdog" }) @@ -20,15 +26,105 @@ export class Watchdog { this.settings = settings; } + SeedLndBalance = async (totalUsersBalance: number) => { + this.initialLndBalance = await this.getTotalLndBalance() + this.lastLndBalance = this.initialLndBalance + + this.initialUsersBalance = totalUsersBalance + this.lastUsersBalance = this.initialUsersBalance + } + + getTotalLndBalance = async () => { + const { channelsBalance, confirmedBalance } = await this.lnd.GetBalance() + return confirmedBalance + channelsBalance.reduce((acc, { localBalanceSats }) => acc + localBalanceSats, 0) + } + + checkBalanceUpdate = (deltaLnd: number, deltaUsers: number, type: 'incremental' | 'absolute', threshold: number) => { + this.log("LND balance update:", deltaLnd, "sats", type === 'incremental' ? "since last balance check" : "since app startup") + this.log("Users balance update:", deltaUsers, "sats", type === 'incremental' ? "since last balance check" : "since app startup") + + const result = this.checkDeltas(deltaLnd, deltaUsers) + switch (result.type) { + case 'mismatch': + if (deltaLnd < 0) { + this.log("WARNING! LND balance decreased while users balance increased creating a difference of", result.absoluteDiff, "sats") + if (result.absoluteDiff > threshold) { + this.log("Difference is too big for an update, locking outgoing operations") + return true + } + } else { + this.log("LND balance increased while users balance decreased creating a difference of", result.absoluteDiff, "sats, could be caused by data loss, or liquidity injection") + return false + } + break + case 'negative': + if (Math.abs(deltaLnd) > Math.abs(deltaUsers)) { + this.log("WARNING! LND balance decreased more than users balance with a difference of", result.absoluteDiff, "sats") + if (result.absoluteDiff > threshold) { + this.log("Difference is too big for an update, locking outgoing operations") + return true + } + } else { + this.log("LND balance decreased less than users balance with a difference of", result.absoluteDiff, "sats, could be caused by data loss, or liquidity injection") + return false + } + break + case 'positive': + if (deltaLnd < deltaUsers) { + this.log("WARNING! LND balance increased less than users balance with a difference of", result.absoluteDiff, "sats") + if (result.absoluteDiff > threshold) { + this.log("Difference is too big for an update, locking outgoing operations") + return true + } + } else { + this.log("LND balance increased more than users balance with a difference of", result.absoluteDiff, "sats, could be caused by data loss, or liquidity injection") + return false + } + } + return false + } + PaymentRequested = async (totalUsersBalance: number) => { this.log("Payment requested, checking balance") - const { channelsBalance, confirmedBalance } = await this.lnd.GetBalance() - const totalLndBalance = confirmedBalance + channelsBalance.reduce((acc, { localBalanceSats }) => acc + localBalanceSats, 0) - const diffSats = Math.abs(totalLndBalance - totalUsersBalance) - const diffBps = (diffSats / Math.max(totalLndBalance, totalUsersBalance)) * 10_000 - if (diffSats > this.settings.maxDiffSats || diffBps > this.settings.maxDiffBps) { - this.log(`LND balance ${totalLndBalance} is too different from users balance ${totalUsersBalance}`) + const totalLndBalance = await this.getTotalLndBalance() + const IncDeltaLnd = totalLndBalance - this.lastLndBalance + const IncDeltaUsers = totalUsersBalance - this.lastUsersBalance + const denyIncremental = this.checkBalanceUpdate(IncDeltaLnd, IncDeltaUsers, 'incremental', this.settings.maxUpdateDiffSats) + if (denyIncremental) { + this.log("Balance mismatch detected in incremental update, locking outgoing operations") this.lnd.LockOutgoingOperations() + return + } + const AbsDeltaLnd = totalLndBalance - this.initialLndBalance + const AbsDeltaUsers = totalUsersBalance - this.initialUsersBalance + const denyAbsolute = this.checkBalanceUpdate(AbsDeltaLnd, AbsDeltaUsers, 'absolute', this.settings.maxDiffSats) + if (denyAbsolute) { + this.log("Balance mismatch detected in absolute update, locking outgoing operations") + this.lnd.LockOutgoingOperations() + return + } + this.lastLndBalance = totalLndBalance + this.lastUsersBalance = totalUsersBalance + } + + checkDeltas = (deltaLnd: number, deltaUsers: number): DeltaCheckResult => { + if (deltaLnd < 0) { + if (deltaUsers < 0) { + const diff = Math.abs(deltaLnd - deltaUsers) + return { type: 'negative', absoluteDiff: diff, relativeDiff: diff / Math.max(deltaLnd, deltaUsers) } + } else { + const diff = Math.abs(deltaLnd) + deltaUsers + return { type: 'mismatch', absoluteDiff: diff } + } + } else { + if (deltaUsers < 0) { + const diff = deltaLnd + Math.abs(deltaUsers) + return { type: 'mismatch', absoluteDiff: diff } + } else { + const diff = Math.abs(deltaLnd - deltaUsers) + return { type: 'positive', absoluteDiff: diff, relativeDiff: diff / Math.max(deltaLnd, deltaUsers) } + } } } -} \ No newline at end of file +} +type DeltaCheckResult = { type: 'negative' | 'positive', absoluteDiff: number, relativeDiff: number } | { type: 'mismatch', absoluteDiff: number } \ No newline at end of file diff --git a/src/services/main/index.ts b/src/services/main/index.ts index c95e11e7..05f9d103 100644 --- a/src/services/main/index.ts +++ b/src/services/main/index.ts @@ -113,12 +113,12 @@ export default class { if (!updateResult.affected) { throw new Error("unable to flag chain transaction as paid") } - await this.storage.userStorage.IncrementUserBalance(userAddress.user.user_id, amount - serviceFee, userAddress.address, tx) + const addressData = `${userAddress.address}:${tx_hash}` + this.storage.eventsLog.LogEvent({ type: 'address_paid', userId: userAddress.user.user_id, appId: userAddress.linkedApplication.app_id, appUserId: "", balance: userAddress.user.balance_sats, data: addressData, amount }) + await this.storage.userStorage.IncrementUserBalance(userAddress.user.user_id, amount - serviceFee, addressData, tx) if (serviceFee > 0) { await this.storage.userStorage.IncrementUserBalance(userAddress.linkedApplication.owner.user_id, serviceFee, 'fees', tx) } - const addressData = `${userAddress.address}:${tx_hash}` - this.storage.eventsLog.LogEvent({ type: 'address_paid', userId: userAddress.user.user_id, appId: userAddress.linkedApplication.app_id, appUserId: "", balance: userAddress.user.balance_sats, data: addressData, amount }) const operationId = `${Types.UserOperationType.INCOMING_TX}-${serialId}` const op = { amount, paidAtUnix: Date.now() / 1000, inbound: true, type: Types.UserOperationType.INCOMING_TX, identifier: userAddress.address, operationId, network_fee: 0, service_fee: serviceFee, confirmed: true, tx_hash: c.tx.tx_hash, internal: c.tx.internal } this.sendOperationToNostr(userAddress.linkedApplication!, userAddress.user.user_id, op) @@ -148,12 +148,13 @@ export default class { // This call will fail if the transaction is already registered const addedTx = await this.storage.paymentStorage.AddAddressReceivingTransaction(userAddress, txOutput.hash, txOutput.index, amount, fee, internal, blockHeight, tx) if (internal) { - await this.storage.userStorage.IncrementUserBalance(userAddress.user.user_id, addedTx.paid_amount - fee, userAddress.address, tx) + const addressData = `${address}:${txOutput.hash}` + this.storage.eventsLog.LogEvent({ type: 'address_paid', userId: userAddress.user.user_id, appId: userAddress.linkedApplication.app_id, appUserId: "", balance: userAddress.user.balance_sats, data: addressData, amount }) + await this.storage.userStorage.IncrementUserBalance(userAddress.user.user_id, addedTx.paid_amount - fee, addressData, tx) if (fee > 0) { await this.storage.userStorage.IncrementUserBalance(userAddress.linkedApplication.owner.user_id, fee, 'fees', tx) } - const addressData = `${address}:${txOutput.hash}` - this.storage.eventsLog.LogEvent({ type: 'address_paid', userId: userAddress.user.user_id, appId: userAddress.linkedApplication.app_id, appUserId: "", balance: userAddress.user.balance_sats, data: addressData, amount }) + } const operationId = `${Types.UserOperationType.INCOMING_TX}-${addedTx.serial_id}` const op = { amount, paidAtUnix: Date.now() / 1000, inbound: true, type: Types.UserOperationType.INCOMING_TX, identifier: userAddress.address, operationId, network_fee: 0, service_fee: fee, confirmed: internal, tx_hash: txOutput.hash, internal: false } @@ -183,11 +184,11 @@ export default class { } try { await this.storage.paymentStorage.FlagInvoiceAsPaid(userInvoice, amount, fee, internal, tx) + this.storage.eventsLog.LogEvent({ type: 'invoice_paid', userId: userInvoice.user.user_id, appId: userInvoice.linkedApplication.app_id, appUserId: "", balance: userInvoice.user.balance_sats, data: paymentRequest, amount }) await this.storage.userStorage.IncrementUserBalance(userInvoice.user.user_id, amount - fee, userInvoice.invoice, tx) if (fee > 0) { await this.storage.userStorage.IncrementUserBalance(userInvoice.linkedApplication.owner.user_id, fee, 'fees', tx) } - this.storage.eventsLog.LogEvent({ type: 'invoice_paid', userId: userInvoice.user.user_id, appId: userInvoice.linkedApplication.app_id, appUserId: "", balance: userInvoice.user.balance_sats, data: paymentRequest, amount }) await this.triggerPaidCallback(log, userInvoice.callbackUrl) const operationId = `${Types.UserOperationType.INCOMING_INVOICE}-${userInvoice.serial_id}` const op = { amount, paidAtUnix: Date.now() / 1000, inbound: true, type: Types.UserOperationType.INCOMING_INVOICE, identifier: userInvoice.invoice, operationId, network_fee: 0, service_fee: fee, confirmed: true, tx_hash: "", internal } @@ -305,6 +306,9 @@ export default class { throw new Error("sanity check on balance failed, expected: " + u.updatedBalance + " found: " + user.balance_sats) } })) + + const total = await this.storage.paymentStorage.GetTotalUsersBalance() + return total || 0 } checkUserEntry(e: LoggedEvent, u: { ts: number, updatedBalance: number } | undefined) { diff --git a/src/services/main/paymentManager.ts b/src/services/main/paymentManager.ts index 77e7aa7f..34a6270e 100644 --- a/src/services/main/paymentManager.ts +++ b/src/services/main/paymentManager.ts @@ -144,7 +144,7 @@ export default class { async PayInvoice(userId: string, req: Types.PayInvoiceRequest, linkedApplication: Application): Promise { this.log("paying invoice", req.invoice, "for user", userId, "with amount", req.amount) - this.WatchdogCheck() + await this.WatchdogCheck() const decoded = await this.lnd.DecodeInvoice(req.invoice) if (decoded.numSatoshis !== 0 && req.amount !== 0) { throw new Error("invoice has value, do not provide amount the the request") @@ -201,7 +201,7 @@ export default class { async PayAddress(ctx: Types.UserContext, req: Types.PayAddressRequest): Promise { throw new Error("address payment currently disabled, use Lightning instead") - this.WatchdogCheck() + await this.WatchdogCheck() const { blockHeight } = await this.lnd.GetInfo() const app = await this.storage.applicationStorage.GetApplication(ctx.app_id) const serviceFee = this.getServiceFee(Types.UserOperationType.OUTGOING_TX, req.amoutSats, false) From 34cebc800ec57f30738bf043b115c46b0ee36006 Mon Sep 17 00:00:00 2001 From: boufni95 Date: Tue, 26 Mar 2024 18:53:28 +0100 Subject: [PATCH 3/4] fix --- env.example | 8 ++++++-- src/services/lnd/watchdog.ts | 2 -- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/env.example b/env.example index 37cc237c..4a709cd8 100644 --- a/env.example +++ b/env.example @@ -44,5 +44,9 @@ MIGRATE_DB=false RECORD_PERFORMANCE=true SKIP_SANITY_CHECK=false DISABLE_EXTERNAL_PAYMENTS=false -WATCHDOG_MAX_DIFF_BPS=100 -WATCHDOG_MAX_DIFF_SATS=10000 \ No newline at end of file + +# Max difference between users balance and LND balance since beginning of app execution +WATCHDOG_MAX_DIFF_SATS=10000 + +# Max difference between users balance and LND balance after each payment +WATCHDOG_MAX_UPDATE_DIFF_SATS=1000 \ No newline at end of file diff --git a/src/services/lnd/watchdog.ts b/src/services/lnd/watchdog.ts index d5f70865..681a7a34 100644 --- a/src/services/lnd/watchdog.ts +++ b/src/services/lnd/watchdog.ts @@ -2,13 +2,11 @@ import { EnvMustBeInteger } from "../helpers/envParser.js"; import { getLogger } from "../helpers/logger.js"; import { LightningHandler } from "./index.js"; export type WatchdogSettings = { - maxDiffBps: number maxDiffSats: number maxUpdateDiffSats: number } export const LoadWatchdogSettingsFromEnv = (test = false): WatchdogSettings => { return { - maxDiffBps: EnvMustBeInteger("WATCHDOG_MAX_DIFF_BPS"), maxDiffSats: EnvMustBeInteger("WATCHDOG_MAX_DIFF_SATS"), maxUpdateDiffSats: EnvMustBeInteger("WATCHDOG_MAX_UPDATE_DIFF_SATS") } From ad599dd5651dfec9af6295174d9d7e8e836a6d58 Mon Sep 17 00:00:00 2001 From: boufni95 Date: Tue, 26 Mar 2024 19:52:24 +0100 Subject: [PATCH 4/4] no incremental check, optional env --- src/services/helpers/envParser.ts | 11 +++++++++ src/services/lnd/watchdog.ts | 41 +++++++++---------------------- 2 files changed, 23 insertions(+), 29 deletions(-) diff --git a/src/services/helpers/envParser.ts b/src/services/helpers/envParser.ts index 87c44d81..fa8a4e92 100644 --- a/src/services/helpers/envParser.ts +++ b/src/services/helpers/envParser.ts @@ -10,6 +10,17 @@ export const EnvMustBeInteger = (name: string): number => { } return +env } +export const EnvCanBeInteger = (name: string, defaultValue = 0): number => { + const env = process.env[name] + if (!env) { + return defaultValue + } + const envNum = +env + if (isNaN(envNum) || !Number.isInteger(envNum)) { + throw new Error(`${name} ENV must be an integer number or nothing`); + } + return envNum +} export const EnvCanBeBoolean = (name: string): boolean => { const env = process.env[name] if (!env) return false diff --git a/src/services/lnd/watchdog.ts b/src/services/lnd/watchdog.ts index 681a7a34..c054e4c7 100644 --- a/src/services/lnd/watchdog.ts +++ b/src/services/lnd/watchdog.ts @@ -1,21 +1,17 @@ -import { EnvMustBeInteger } from "../helpers/envParser.js"; +import { EnvCanBeInteger } from "../helpers/envParser.js"; import { getLogger } from "../helpers/logger.js"; import { LightningHandler } from "./index.js"; export type WatchdogSettings = { maxDiffSats: number - maxUpdateDiffSats: number } export const LoadWatchdogSettingsFromEnv = (test = false): WatchdogSettings => { return { - maxDiffSats: EnvMustBeInteger("WATCHDOG_MAX_DIFF_SATS"), - maxUpdateDiffSats: EnvMustBeInteger("WATCHDOG_MAX_UPDATE_DIFF_SATS") + maxDiffSats: EnvCanBeInteger("WATCHDOG_MAX_DIFF_SATS") } } export class Watchdog { initialLndBalance: number; initialUsersBalance: number; - lastLndBalance: number; - lastUsersBalance: number; lnd: LightningHandler; settings: WatchdogSettings; log = getLogger({ appName: "watchdog" }) @@ -26,10 +22,7 @@ export class Watchdog { SeedLndBalance = async (totalUsersBalance: number) => { this.initialLndBalance = await this.getTotalLndBalance() - this.lastLndBalance = this.initialLndBalance - this.initialUsersBalance = totalUsersBalance - this.lastUsersBalance = this.initialUsersBalance } getTotalLndBalance = async () => { @@ -37,16 +30,16 @@ export class Watchdog { return confirmedBalance + channelsBalance.reduce((acc, { localBalanceSats }) => acc + localBalanceSats, 0) } - checkBalanceUpdate = (deltaLnd: number, deltaUsers: number, type: 'incremental' | 'absolute', threshold: number) => { - this.log("LND balance update:", deltaLnd, "sats", type === 'incremental' ? "since last balance check" : "since app startup") - this.log("Users balance update:", deltaUsers, "sats", type === 'incremental' ? "since last balance check" : "since app startup") + checkBalanceUpdate = (deltaLnd: number, deltaUsers: number) => { + this.log("LND balance update:", deltaLnd, "sats since app startup") + this.log("Users balance update:", deltaUsers, "sats since app startup") const result = this.checkDeltas(deltaLnd, deltaUsers) switch (result.type) { case 'mismatch': if (deltaLnd < 0) { this.log("WARNING! LND balance decreased while users balance increased creating a difference of", result.absoluteDiff, "sats") - if (result.absoluteDiff > threshold) { + if (result.absoluteDiff > this.settings.maxDiffSats) { this.log("Difference is too big for an update, locking outgoing operations") return true } @@ -58,7 +51,7 @@ export class Watchdog { case 'negative': if (Math.abs(deltaLnd) > Math.abs(deltaUsers)) { this.log("WARNING! LND balance decreased more than users balance with a difference of", result.absoluteDiff, "sats") - if (result.absoluteDiff > threshold) { + if (result.absoluteDiff > this.settings.maxDiffSats) { this.log("Difference is too big for an update, locking outgoing operations") return true } @@ -70,7 +63,7 @@ export class Watchdog { case 'positive': if (deltaLnd < deltaUsers) { this.log("WARNING! LND balance increased less than users balance with a difference of", result.absoluteDiff, "sats") - if (result.absoluteDiff > threshold) { + if (result.absoluteDiff > this.settings.maxDiffSats) { this.log("Difference is too big for an update, locking outgoing operations") return true } @@ -85,24 +78,14 @@ export class Watchdog { PaymentRequested = async (totalUsersBalance: number) => { this.log("Payment requested, checking balance") const totalLndBalance = await this.getTotalLndBalance() - const IncDeltaLnd = totalLndBalance - this.lastLndBalance - const IncDeltaUsers = totalUsersBalance - this.lastUsersBalance - const denyIncremental = this.checkBalanceUpdate(IncDeltaLnd, IncDeltaUsers, 'incremental', this.settings.maxUpdateDiffSats) - if (denyIncremental) { - this.log("Balance mismatch detected in incremental update, locking outgoing operations") - this.lnd.LockOutgoingOperations() - return - } - const AbsDeltaLnd = totalLndBalance - this.initialLndBalance - const AbsDeltaUsers = totalUsersBalance - this.initialUsersBalance - const denyAbsolute = this.checkBalanceUpdate(AbsDeltaLnd, AbsDeltaUsers, 'absolute', this.settings.maxDiffSats) - if (denyAbsolute) { + const deltaLnd = totalLndBalance - this.initialLndBalance + const deltaUsers = totalUsersBalance - this.initialUsersBalance + const deny = this.checkBalanceUpdate(deltaLnd, deltaUsers) + if (deny) { this.log("Balance mismatch detected in absolute update, locking outgoing operations") this.lnd.LockOutgoingOperations() return } - this.lastLndBalance = totalLndBalance - this.lastUsersBalance = totalUsersBalance } checkDeltas = (deltaLnd: number, deltaUsers: number): DeltaCheckResult => {