From f20abdd5f479ac7c3ad14c077f1c88e477556001 Mon Sep 17 00:00:00 2001 From: hatim boufnichel Date: Tue, 19 Mar 2024 21:17:18 +0100 Subject: [PATCH] watchdog v0 --- env.example | 2 ++ src/services/lnd/index.ts | 2 ++ src/services/lnd/lnd.ts | 17 +++++++++++++ src/services/lnd/mock.ts | 6 +++++ src/services/lnd/watchdog.ts | 34 ++++++++++++++++++++++++++ src/services/main/index.ts | 2 ++ src/services/main/paymentManager.ts | 10 ++++++++ src/services/main/settings.ts | 2 ++ src/services/storage/paymentStorage.ts | 4 +++ 9 files changed, 79 insertions(+) create mode 100644 src/services/lnd/watchdog.ts diff --git a/env.example b/env.example index 721db7e9..37cc237c 100644 --- a/env.example +++ b/env.example @@ -44,3 +44,5 @@ MIGRATE_DB=false RECORD_PERFORMANCE=true SKIP_SANITY_CHECK=false DISABLE_EXTERNAL_PAYMENTS=false +WATCHDOG_MAX_DIFF_BPS=100 +WATCHDOG_MAX_DIFF_SATS=10000 \ No newline at end of file diff --git a/src/services/lnd/index.ts b/src/services/lnd/index.ts index bc1309cb..9cda8da4 100644 --- a/src/services/lnd/index.ts +++ b/src/services/lnd/index.ts @@ -38,6 +38,8 @@ export interface LightningHandler { GetForwardingHistory(indexOffset: number): Promise<{ fee: number, chanIdIn: string, chanIdOut: string, timestampNs: number, offset: number }[]> GetAllPaidInvoices(max: number): Promise GetAllPayments(max: number): Promise + LockOutgoingOperations(): void + UnlockOutgoingOperations(): void } export default (settings: LndSettings, addressPaidCb: AddressPaidCb, invoicePaidCb: InvoicePaidCb, newBlockCb: NewBlockCb, htlcCb: HtlcCb): LightningHandler => { diff --git a/src/services/lnd/lnd.ts b/src/services/lnd/lnd.ts index a43e1fcf..e1128e1b 100644 --- a/src/services/lnd/lnd.ts +++ b/src/services/lnd/lnd.ts @@ -33,6 +33,7 @@ export default class { newBlockCb: NewBlockCb htlcCb: HtlcCb log = getLogger({ appName: 'lndManager' }) + outgoingOpsLocked = false constructor(settings: LndSettings, addressPaidCb: AddressPaidCb, invoicePaidCb: InvoicePaidCb, newBlockCb: NewBlockCb, htlcCb: HtlcCb) { this.settings = settings this.addressPaidCb = addressPaidCb @@ -60,6 +61,14 @@ export default class { this.router = new RouterClient(transport) this.chainNotifier = new ChainNotifierClient(transport) } + + LockOutgoingOperations(): void { + this.outgoingOpsLocked = true + } + UnlockOutgoingOperations(): void { + this.outgoingOpsLocked = false + } + SetMockInvoiceAsPaid(invoice: string, amount: number): Promise { throw new Error("SetMockInvoiceAsPaid only available in mock mode") } @@ -251,6 +260,10 @@ export default class { return { local: r.localBalance ? Number(r.localBalance.sat) : 0, remote: r.remoteBalance ? Number(r.remoteBalance.sat) : 0 } } async PayInvoice(invoice: string, amount: number, feeLimit: number): Promise { + if (this.outgoingOpsLocked) { + this.log("outgoing ops locked, rejecting payment request") + throw new Error("lnd node is currently out of sync") + } await this.Health() this.log("paying invoice", invoice, "for", amount, "sats") const abortController = new AbortController() @@ -287,6 +300,10 @@ export default class { } async PayAddress(address: string, amount: number, satPerVByte: number, label = ""): Promise { + if (this.outgoingOpsLocked) { + this.log("outgoing ops locked, rejecting payment request") + throw new Error("lnd node is currently out of sync") + } await this.Health() this.log("sending chain TX for", amount, "sats", "to", address) const res = await this.lightning.sendCoins(SendCoinsReq(address, amount, satPerVByte, label), DeadLineMetadata()) diff --git a/src/services/lnd/mock.ts b/src/services/lnd/mock.ts index 44b2e6cb..17fd70c1 100644 --- a/src/services/lnd/mock.ts +++ b/src/services/lnd/mock.ts @@ -131,6 +131,12 @@ export default class { async GetAllPayments(max: number): Promise { throw new Error("not implemented") } + LockOutgoingOperations() { + throw new Error("not implemented") + } + UnlockOutgoingOperations() { + throw new Error("not implemented") + } } diff --git a/src/services/lnd/watchdog.ts b/src/services/lnd/watchdog.ts new file mode 100644 index 00000000..a3a554a1 --- /dev/null +++ b/src/services/lnd/watchdog.ts @@ -0,0 +1,34 @@ +import { EnvMustBeInteger } from "../helpers/envParser.js"; +import { getLogger } from "../helpers/logger.js"; +import { LightningHandler } from "./index.js"; +export type WatchdogSettings = { + maxDiffBps: number + maxDiffSats: number +} +export const LoadWatchdogSettingsFromEnv = (test = false): WatchdogSettings => { + return { + maxDiffBps: EnvMustBeInteger("WATCHDOG_MAX_DIFF_BPS"), + maxDiffSats: EnvMustBeInteger("WATCHDOG_MAX_DIFF_SATS") + } +} +export class Watchdog { + lnd: LightningHandler; + settings: WatchdogSettings; + log = getLogger({ appName: "watchdog" }) + constructor(settings: WatchdogSettings, lnd: LightningHandler) { + this.lnd = lnd; + this.settings = settings; + } + + PaymentRequested = async (totalUsersBalance: number) => { + this.log("Payment requested, checking balance") + const { channelsBalance, confirmedBalance } = await this.lnd.GetBalance() + const totalLndBalance = confirmedBalance + channelsBalance.reduce((acc, { localBalanceSats }) => acc + localBalanceSats, 0) + const diffSats = Math.abs(totalLndBalance - totalUsersBalance) + const diffBps = (diffSats / Math.max(totalLndBalance, totalUsersBalance)) * 10_000 + if (diffSats > this.settings.maxDiffSats || diffBps > this.settings.maxDiffBps) { + this.log(`LND balance ${totalLndBalance} is too different from users balance ${totalUsersBalance}`) + this.lnd.LockOutgoingOperations() + } + } +} \ No newline at end of file diff --git a/src/services/main/index.ts b/src/services/main/index.ts index bbf03146..c95e11e7 100644 --- a/src/services/main/index.ts +++ b/src/services/main/index.ts @@ -17,8 +17,10 @@ import { UnsignedEvent } from '../nostr/tools/event.js' import { NostrSend } from '../nostr/handler.js' import MetricsManager from '../metrics/index.js' import EventsLogManager, { LoggedEvent } from '../storage/eventsLog.js' +import { LoadWatchdogSettingsFromEnv } from '../lnd/watchdog.js' export const LoadMainSettingsFromEnv = (test = false): MainSettings => { return { + watchDogSettings: LoadWatchdogSettingsFromEnv(test), lndSettings: LoadLndSettingsFromEnv(test), storageSettings: LoadStorageSettingsFromEnv(test), jwtSecret: EnvMustBeNonEmptyString("JWT_SECRET"), diff --git a/src/services/main/paymentManager.ts b/src/services/main/paymentManager.ts index a119e563..77e7aa7f 100644 --- a/src/services/main/paymentManager.ts +++ b/src/services/main/paymentManager.ts @@ -14,6 +14,7 @@ import { SendCoinsResponse } from '../../../proto/lnd/lightning.js' import { Event, verifiedSymbol, verifySignature } from '../nostr/tools/event.js' import { AddressReceivingTransaction } from '../storage/entity/AddressReceivingTransaction.js' import { UserTransactionPayment } from '../storage/entity/UserTransactionPayment.js' +import { Watchdog } from '../lnd/watchdog.js' interface UserOperationInfo { serial_id: number paid_amount: number @@ -45,10 +46,12 @@ export default class { addressPaidCb: AddressPaidCb invoicePaidCb: InvoicePaidCb log = getLogger({ appName: "PaymentManager" }) + watchDog: Watchdog constructor(storage: Storage, lnd: LightningHandler, settings: MainSettings, addressPaidCb: AddressPaidCb, invoicePaidCb: InvoicePaidCb) { this.storage = storage this.settings = settings this.lnd = lnd + this.watchDog = new Watchdog(settings.watchDogSettings, lnd) this.addressPaidCb = addressPaidCb this.invoicePaidCb = invoicePaidCb } @@ -134,8 +137,14 @@ export default class { } } + async WatchdogCheck() { + const total = await this.storage.paymentStorage.GetTotalUsersBalance() + await this.watchDog.PaymentRequested(total || 0) + } + async PayInvoice(userId: string, req: Types.PayInvoiceRequest, linkedApplication: Application): Promise { this.log("paying invoice", req.invoice, "for user", userId, "with amount", req.amount) + this.WatchdogCheck() const decoded = await this.lnd.DecodeInvoice(req.invoice) if (decoded.numSatoshis !== 0 && req.amount !== 0) { throw new Error("invoice has value, do not provide amount the the request") @@ -192,6 +201,7 @@ export default class { async PayAddress(ctx: Types.UserContext, req: Types.PayAddressRequest): Promise { throw new Error("address payment currently disabled, use Lightning instead") + this.WatchdogCheck() const { blockHeight } = await this.lnd.GetInfo() const app = await this.storage.applicationStorage.GetApplication(ctx.app_id) const serviceFee = this.getServiceFee(Types.UserOperationType.OUTGOING_TX, req.amoutSats, false) diff --git a/src/services/main/settings.ts b/src/services/main/settings.ts index 7397e97b..cbd26f7b 100644 --- a/src/services/main/settings.ts +++ b/src/services/main/settings.ts @@ -1,8 +1,10 @@ import { StorageSettings } from '../storage/index.js' import { LndSettings } from '../lnd/settings.js' +import { WatchdogSettings } from '../lnd/watchdog.js' export type MainSettings = { storageSettings: StorageSettings, lndSettings: LndSettings, + watchDogSettings: WatchdogSettings, jwtSecret: string incomingTxFee: number outgoingTxFee: number diff --git a/src/services/storage/paymentStorage.ts b/src/services/storage/paymentStorage.ts index 3fceb8a7..03281232 100644 --- a/src/services/storage/paymentStorage.ts +++ b/src/services/storage/paymentStorage.ts @@ -360,4 +360,8 @@ export default class { break; } } + + async GetTotalUsersBalance(entityManager = this.DB) { + return entityManager.getRepository(User).sum("balance_sats") + } } \ No newline at end of file