watchdog v0

env.example

@@ -44,3 +44,5 @@ MIGRATE_DB=false
 RECORD_PERFORMANCE=true
 SKIP_SANITY_CHECK=false
 DISABLE_EXTERNAL_PAYMENTS=false
+WATCHDOG_MAX_DIFF_BPS=100
+WATCHDOG_MAX_DIFF_SATS=10000

src/services/lnd/index.ts

@@ -38,6 +38,8 @@ export interface LightningHandler {
     GetForwardingHistory(indexOffset: number): Promise<{ fee: number, chanIdIn: string, chanIdOut: string, timestampNs: number, offset: number }[]>
     GetAllPaidInvoices(max: number): Promise<ListInvoiceResponse>
     GetAllPayments(max: number): Promise<ListPaymentsResponse>
+    LockOutgoingOperations(): void
+    UnlockOutgoingOperations(): void
 }
 
 export default (settings: LndSettings, addressPaidCb: AddressPaidCb, invoicePaidCb: InvoicePaidCb, newBlockCb: NewBlockCb, htlcCb: HtlcCb): LightningHandler => {

src/services/lnd/lnd.ts

@@ -33,6 +33,7 @@ export default class {
     newBlockCb: NewBlockCb
     htlcCb: HtlcCb
     log = getLogger({ appName: 'lndManager' })
+    outgoingOpsLocked = false
     constructor(settings: LndSettings, addressPaidCb: AddressPaidCb, invoicePaidCb: InvoicePaidCb, newBlockCb: NewBlockCb, htlcCb: HtlcCb) {
         this.settings = settings
         this.addressPaidCb = addressPaidCb
@@ -60,6 +61,14 @@ export default class {
         this.router = new RouterClient(transport)
         this.chainNotifier = new ChainNotifierClient(transport)
     }
+
+    LockOutgoingOperations(): void {
+        this.outgoingOpsLocked = true
+    }
+    UnlockOutgoingOperations(): void {
+        this.outgoingOpsLocked = false
+    }
+
     SetMockInvoiceAsPaid(invoice: string, amount: number): Promise<void> {
         throw new Error("SetMockInvoiceAsPaid only available in mock mode")
     }
@@ -251,6 +260,10 @@ export default class {
         return { local: r.localBalance ? Number(r.localBalance.sat) : 0, remote: r.remoteBalance ? Number(r.remoteBalance.sat) : 0 }
     }
     async PayInvoice(invoice: string, amount: number, feeLimit: number): Promise<PaidInvoice> {
+        if (this.outgoingOpsLocked) {
+            this.log("outgoing ops locked, rejecting payment request")
+            throw new Error("lnd node is currently out of sync")
+        }
         await this.Health()
         this.log("paying invoice", invoice, "for", amount, "sats")
         const abortController = new AbortController()
@@ -287,6 +300,10 @@ export default class {
     }
 
     async PayAddress(address: string, amount: number, satPerVByte: number, label = ""): Promise<SendCoinsResponse> {
+        if (this.outgoingOpsLocked) {
+            this.log("outgoing ops locked, rejecting payment request")
+            throw new Error("lnd node is currently out of sync")
+        }
         await this.Health()
         this.log("sending chain TX for", amount, "sats", "to", address)
         const res = await this.lightning.sendCoins(SendCoinsReq(address, amount, satPerVByte, label), DeadLineMetadata())

src/services/lnd/mock.ts

@@ -131,6 +131,12 @@ export default class {
     async GetAllPayments(max: number): Promise<ListPaymentsResponse> {
         throw new Error("not implemented")
     }
+    LockOutgoingOperations() {
+        throw new Error("not implemented")
+    }
+    UnlockOutgoingOperations() {
+        throw new Error("not implemented")
+    }
 }
 
 

src/services/lnd/watchdog.ts

@@ -0,0 +1,34 @@
+import { EnvMustBeInteger } from "../helpers/envParser.js";
+import { getLogger } from "../helpers/logger.js";
+import { LightningHandler } from "./index.js";
+export type WatchdogSettings = {
+    maxDiffBps: number
+    maxDiffSats: number
+}
+export const LoadWatchdogSettingsFromEnv = (test = false): WatchdogSettings => {
+    return {
+        maxDiffBps: EnvMustBeInteger("WATCHDOG_MAX_DIFF_BPS"),
+        maxDiffSats: EnvMustBeInteger("WATCHDOG_MAX_DIFF_SATS")
+    }
+}
+export class Watchdog {
+    lnd: LightningHandler;
+    settings: WatchdogSettings;
+    log = getLogger({ appName: "watchdog" })
+    constructor(settings: WatchdogSettings, lnd: LightningHandler) {
+        this.lnd = lnd;
+        this.settings = settings;
+    }
+
+    PaymentRequested = async (totalUsersBalance: number) => {
+        this.log("Payment requested, checking balance")
+        const { channelsBalance, confirmedBalance } = await this.lnd.GetBalance()
+        const totalLndBalance = confirmedBalance + channelsBalance.reduce((acc, { localBalanceSats }) => acc + localBalanceSats, 0)
+        const diffSats = Math.abs(totalLndBalance - totalUsersBalance)
+        const diffBps = (diffSats / Math.max(totalLndBalance, totalUsersBalance)) * 10_000
+        if (diffSats > this.settings.maxDiffSats || diffBps > this.settings.maxDiffBps) {
+            this.log(`LND balance ${totalLndBalance} is too different from users balance ${totalUsersBalance}`)
+            this.lnd.LockOutgoingOperations()
+        }
+    }
+}

src/services/main/index.ts

@@ -17,8 +17,10 @@ import { UnsignedEvent } from '../nostr/tools/event.js'
 import { NostrSend } from '../nostr/handler.js'
 import MetricsManager from '../metrics/index.js'
 import EventsLogManager, { LoggedEvent } from '../storage/eventsLog.js'
+import { LoadWatchdogSettingsFromEnv } from '../lnd/watchdog.js'
 export const LoadMainSettingsFromEnv = (test = false): MainSettings => {
     return {
+        watchDogSettings: LoadWatchdogSettingsFromEnv(test),
         lndSettings: LoadLndSettingsFromEnv(test),
         storageSettings: LoadStorageSettingsFromEnv(test),
         jwtSecret: EnvMustBeNonEmptyString("JWT_SECRET"),

src/services/main/paymentManager.ts

@@ -14,6 +14,7 @@ import { SendCoinsResponse } from '../../../proto/lnd/lightning.js'
 import { Event, verifiedSymbol, verifySignature } from '../nostr/tools/event.js'
 import { AddressReceivingTransaction } from '../storage/entity/AddressReceivingTransaction.js'
 import { UserTransactionPayment } from '../storage/entity/UserTransactionPayment.js'
+import { Watchdog } from '../lnd/watchdog.js'
 interface UserOperationInfo {
     serial_id: number
     paid_amount: number
@@ -45,10 +46,12 @@ export default class {
     addressPaidCb: AddressPaidCb
     invoicePaidCb: InvoicePaidCb
     log = getLogger({ appName: "PaymentManager" })
+    watchDog: Watchdog
     constructor(storage: Storage, lnd: LightningHandler, settings: MainSettings, addressPaidCb: AddressPaidCb, invoicePaidCb: InvoicePaidCb) {
         this.storage = storage
         this.settings = settings
         this.lnd = lnd
+        this.watchDog = new Watchdog(settings.watchDogSettings, lnd)
         this.addressPaidCb = addressPaidCb
         this.invoicePaidCb = invoicePaidCb
     }
@@ -134,8 +137,14 @@ export default class {
         }
     }
 
+    async WatchdogCheck() {
+        const total = await this.storage.paymentStorage.GetTotalUsersBalance()
+        await this.watchDog.PaymentRequested(total || 0)
+    }
+
     async PayInvoice(userId: string, req: Types.PayInvoiceRequest, linkedApplication: Application): Promise<Types.PayInvoiceResponse> {
         this.log("paying invoice", req.invoice, "for user", userId, "with amount", req.amount)
+        this.WatchdogCheck()
         const decoded = await this.lnd.DecodeInvoice(req.invoice)
         if (decoded.numSatoshis !== 0 && req.amount !== 0) {
             throw new Error("invoice has value, do not provide amount the the request")
@@ -192,6 +201,7 @@ export default class {
 
     async PayAddress(ctx: Types.UserContext, req: Types.PayAddressRequest): Promise<Types.PayAddressResponse> {
         throw new Error("address payment currently disabled, use Lightning instead")
+        this.WatchdogCheck()
         const { blockHeight } = await this.lnd.GetInfo()
         const app = await this.storage.applicationStorage.GetApplication(ctx.app_id)
         const serviceFee = this.getServiceFee(Types.UserOperationType.OUTGOING_TX, req.amoutSats, false)

src/services/main/settings.ts

@@ -1,8 +1,10 @@
 import { StorageSettings } from '../storage/index.js'
 import { LndSettings } from '../lnd/settings.js'
+import { WatchdogSettings } from '../lnd/watchdog.js'
 export type MainSettings = {
     storageSettings: StorageSettings,
     lndSettings: LndSettings,
+    watchDogSettings: WatchdogSettings,
     jwtSecret: string
     incomingTxFee: number
     outgoingTxFee: number

src/services/storage/paymentStorage.ts

@@ -360,4 +360,8 @@ export default class {
                 break;
         }
     }
+
+    async GetTotalUsersBalance(entityManager = this.DB) {
+        return entityManager.getRepository(User).sum("balance_sats")
+    }
 }