watchdog v1

src/index.ts

@@ -26,7 +26,8 @@ const start = async () => {
     const mainHandler = new Main(mainSettings, storageManager)
     await mainHandler.lnd.Warmup()
     if (!mainSettings.skipSanityCheck) {
-        await mainHandler.VerifyEventsLog()
+        const totalUsersBalance = await mainHandler.VerifyEventsLog()
+        await mainHandler.paymentManager.watchDog.SeedLndBalance(totalUsersBalance)
     }
     const serverMethods = GetServerMethods(mainHandler)
     const nostrSettings = LoadNosrtSettingsFromEnv()

src/services/lnd/watchdog.ts

@@ -4,14 +4,20 @@ import { LightningHandler } from "./index.js";
 export type WatchdogSettings = {
     maxDiffBps: number
     maxDiffSats: number
+    maxUpdateDiffSats: number
 }
 export const LoadWatchdogSettingsFromEnv = (test = false): WatchdogSettings => {
     return {
         maxDiffBps: EnvMustBeInteger("WATCHDOG_MAX_DIFF_BPS"),
-        maxDiffSats: EnvMustBeInteger("WATCHDOG_MAX_DIFF_SATS")
+        maxDiffSats: EnvMustBeInteger("WATCHDOG_MAX_DIFF_SATS"),
+        maxUpdateDiffSats: EnvMustBeInteger("WATCHDOG_MAX_UPDATE_DIFF_SATS")
     }
 }
 export class Watchdog {
+    initialLndBalance: number;
+    initialUsersBalance: number;
+    lastLndBalance: number;
+    lastUsersBalance: number;
     lnd: LightningHandler;
     settings: WatchdogSettings;
     log = getLogger({ appName: "watchdog" })
@@ -20,15 +26,105 @@ export class Watchdog {
         this.settings = settings;
     }
 
+    SeedLndBalance = async (totalUsersBalance: number) => {
+        this.initialLndBalance = await this.getTotalLndBalance()
+        this.lastLndBalance = this.initialLndBalance
+
+        this.initialUsersBalance = totalUsersBalance
+        this.lastUsersBalance = this.initialUsersBalance
+    }
+
+    getTotalLndBalance = async () => {
+        const { channelsBalance, confirmedBalance } = await this.lnd.GetBalance()
+        return confirmedBalance + channelsBalance.reduce((acc, { localBalanceSats }) => acc + localBalanceSats, 0)
+    }
+
+    checkBalanceUpdate = (deltaLnd: number, deltaUsers: number, type: 'incremental' | 'absolute', threshold: number) => {
+        this.log("LND balance update:", deltaLnd, "sats", type === 'incremental' ? "since last balance check" : "since app startup")
+        this.log("Users balance update:", deltaUsers, "sats", type === 'incremental' ? "since last balance check" : "since app startup")
+
+        const result = this.checkDeltas(deltaLnd, deltaUsers)
+        switch (result.type) {
+            case 'mismatch':
+                if (deltaLnd < 0) {
+                    this.log("WARNING! LND balance decreased while users balance increased creating a difference of", result.absoluteDiff, "sats")
+                    if (result.absoluteDiff > threshold) {
+                        this.log("Difference is too big for an update, locking outgoing operations")
+                        return true
+                    }
+                } else {
+                    this.log("LND balance increased while users balance decreased creating a difference of", result.absoluteDiff, "sats, could be caused by data loss, or liquidity injection")
+                    return false
+                }
+                break
+            case 'negative':
+                if (Math.abs(deltaLnd) > Math.abs(deltaUsers)) {
+                    this.log("WARNING! LND balance decreased more than users balance with a difference of", result.absoluteDiff, "sats")
+                    if (result.absoluteDiff > threshold) {
+                        this.log("Difference is too big for an update, locking outgoing operations")
+                        return true
+                    }
+                } else {
+                    this.log("LND balance decreased less than users balance with a difference of", result.absoluteDiff, "sats, could be caused by data loss, or liquidity injection")
+                    return false
+                }
+                break
+            case 'positive':
+                if (deltaLnd < deltaUsers) {
+                    this.log("WARNING! LND balance increased less than users balance with a difference of", result.absoluteDiff, "sats")
+                    if (result.absoluteDiff > threshold) {
+                        this.log("Difference is too big for an update, locking outgoing operations")
+                        return true
+                    }
+                } else {
+                    this.log("LND balance increased more than users balance with a difference of", result.absoluteDiff, "sats, could be caused by data loss, or liquidity injection")
+                    return false
+                }
+        }
+        return false
+    }
+
     PaymentRequested = async (totalUsersBalance: number) => {
         this.log("Payment requested, checking balance")
-        const { channelsBalance, confirmedBalance } = await this.lnd.GetBalance()
+        const totalLndBalance = await this.getTotalLndBalance()
-        const totalLndBalance = confirmedBalance + channelsBalance.reduce((acc, { localBalanceSats }) => acc + localBalanceSats, 0)
+        const IncDeltaLnd = totalLndBalance - this.lastLndBalance
-        const diffSats = Math.abs(totalLndBalance - totalUsersBalance)
+        const IncDeltaUsers = totalUsersBalance - this.lastUsersBalance
-        const diffBps = (diffSats / Math.max(totalLndBalance, totalUsersBalance)) * 10_000
+        const denyIncremental = this.checkBalanceUpdate(IncDeltaLnd, IncDeltaUsers, 'incremental', this.settings.maxUpdateDiffSats)
-        if (diffSats > this.settings.maxDiffSats || diffBps > this.settings.maxDiffBps) {
+        if (denyIncremental) {
-            this.log(`LND balance ${totalLndBalance} is too different from users balance ${totalUsersBalance}`)
+            this.log("Balance mismatch detected in incremental update, locking outgoing operations")
             this.lnd.LockOutgoingOperations()
+            return
+        }
+        const AbsDeltaLnd = totalLndBalance - this.initialLndBalance
+        const AbsDeltaUsers = totalUsersBalance - this.initialUsersBalance
+        const denyAbsolute = this.checkBalanceUpdate(AbsDeltaLnd, AbsDeltaUsers, 'absolute', this.settings.maxDiffSats)
+        if (denyAbsolute) {
+            this.log("Balance mismatch detected in absolute update, locking outgoing operations")
+            this.lnd.LockOutgoingOperations()
+            return
+        }
+        this.lastLndBalance = totalLndBalance
+        this.lastUsersBalance = totalUsersBalance
+    }
+
+    checkDeltas = (deltaLnd: number, deltaUsers: number): DeltaCheckResult => {
+        if (deltaLnd < 0) {
+            if (deltaUsers < 0) {
+                const diff = Math.abs(deltaLnd - deltaUsers)
+                return { type: 'negative', absoluteDiff: diff, relativeDiff: diff / Math.max(deltaLnd, deltaUsers) }
+            } else {
+                const diff = Math.abs(deltaLnd) + deltaUsers
+                return { type: 'mismatch', absoluteDiff: diff }
+            }
+        } else {
+            if (deltaUsers < 0) {
+                const diff = deltaLnd + Math.abs(deltaUsers)
+                return { type: 'mismatch', absoluteDiff: diff }
+            } else {
+                const diff = Math.abs(deltaLnd - deltaUsers)
+                return { type: 'positive', absoluteDiff: diff, relativeDiff: diff / Math.max(deltaLnd, deltaUsers) }
+            }
         }
     }
-}
+}
+type DeltaCheckResult = { type: 'negative' | 'positive', absoluteDiff: number, relativeDiff: number } | { type: 'mismatch', absoluteDiff: number }

src/services/main/index.ts

@@ -113,12 +113,12 @@ export default class {
                     if (!updateResult.affected) {
                         throw new Error("unable to flag chain transaction as paid")
                     }
-                    await this.storage.userStorage.IncrementUserBalance(userAddress.user.user_id, amount - serviceFee, userAddress.address, tx)
+                    const addressData = `${userAddress.address}:${tx_hash}`
+                    this.storage.eventsLog.LogEvent({ type: 'address_paid', userId: userAddress.user.user_id, appId: userAddress.linkedApplication.app_id, appUserId: "", balance: userAddress.user.balance_sats, data: addressData, amount })
+                    await this.storage.userStorage.IncrementUserBalance(userAddress.user.user_id, amount - serviceFee, addressData, tx)
                     if (serviceFee > 0) {
                         await this.storage.userStorage.IncrementUserBalance(userAddress.linkedApplication.owner.user_id, serviceFee, 'fees', tx)
                     }
-                    const addressData = `${userAddress.address}:${tx_hash}`
-                    this.storage.eventsLog.LogEvent({ type: 'address_paid', userId: userAddress.user.user_id, appId: userAddress.linkedApplication.app_id, appUserId: "", balance: userAddress.user.balance_sats, data: addressData, amount })
                     const operationId = `${Types.UserOperationType.INCOMING_TX}-${serialId}`
                     const op = { amount, paidAtUnix: Date.now() / 1000, inbound: true, type: Types.UserOperationType.INCOMING_TX, identifier: userAddress.address, operationId, network_fee: 0, service_fee: serviceFee, confirmed: true, tx_hash: c.tx.tx_hash, internal: c.tx.internal }
                     this.sendOperationToNostr(userAddress.linkedApplication!, userAddress.user.user_id, op)
@@ -148,12 +148,13 @@ export default class {
                 // This call will fail if the transaction is already registered
                 const addedTx = await this.storage.paymentStorage.AddAddressReceivingTransaction(userAddress, txOutput.hash, txOutput.index, amount, fee, internal, blockHeight, tx)
                 if (internal) {
-                    await this.storage.userStorage.IncrementUserBalance(userAddress.user.user_id, addedTx.paid_amount - fee, userAddress.address, tx)
+                    const addressData = `${address}:${txOutput.hash}`
+                    this.storage.eventsLog.LogEvent({ type: 'address_paid', userId: userAddress.user.user_id, appId: userAddress.linkedApplication.app_id, appUserId: "", balance: userAddress.user.balance_sats, data: addressData, amount })
+                    await this.storage.userStorage.IncrementUserBalance(userAddress.user.user_id, addedTx.paid_amount - fee, addressData, tx)
                     if (fee > 0) {
                         await this.storage.userStorage.IncrementUserBalance(userAddress.linkedApplication.owner.user_id, fee, 'fees', tx)
                     }
-                    const addressData = `${address}:${txOutput.hash}`
+
-                    this.storage.eventsLog.LogEvent({ type: 'address_paid', userId: userAddress.user.user_id, appId: userAddress.linkedApplication.app_id, appUserId: "", balance: userAddress.user.balance_sats, data: addressData, amount })
                 }
                 const operationId = `${Types.UserOperationType.INCOMING_TX}-${addedTx.serial_id}`
                 const op = { amount, paidAtUnix: Date.now() / 1000, inbound: true, type: Types.UserOperationType.INCOMING_TX, identifier: userAddress.address, operationId, network_fee: 0, service_fee: fee, confirmed: internal, tx_hash: txOutput.hash, internal: false }
@@ -183,11 +184,11 @@ export default class {
             }
             try {
                 await this.storage.paymentStorage.FlagInvoiceAsPaid(userInvoice, amount, fee, internal, tx)
+                this.storage.eventsLog.LogEvent({ type: 'invoice_paid', userId: userInvoice.user.user_id, appId: userInvoice.linkedApplication.app_id, appUserId: "", balance: userInvoice.user.balance_sats, data: paymentRequest, amount })
                 await this.storage.userStorage.IncrementUserBalance(userInvoice.user.user_id, amount - fee, userInvoice.invoice, tx)
                 if (fee > 0) {
                     await this.storage.userStorage.IncrementUserBalance(userInvoice.linkedApplication.owner.user_id, fee, 'fees', tx)
                 }
-                this.storage.eventsLog.LogEvent({ type: 'invoice_paid', userId: userInvoice.user.user_id, appId: userInvoice.linkedApplication.app_id, appUserId: "", balance: userInvoice.user.balance_sats, data: paymentRequest, amount })
                 await this.triggerPaidCallback(log, userInvoice.callbackUrl)
                 const operationId = `${Types.UserOperationType.INCOMING_INVOICE}-${userInvoice.serial_id}`
                 const op = { amount, paidAtUnix: Date.now() / 1000, inbound: true, type: Types.UserOperationType.INCOMING_INVOICE, identifier: userInvoice.invoice, operationId, network_fee: 0, service_fee: fee, confirmed: true, tx_hash: "", internal }
@@ -305,6 +306,9 @@ export default class {
                 throw new Error("sanity check on balance failed, expected: " + u.updatedBalance + " found: " + user.balance_sats)
             }
         }))
+
+        const total = await this.storage.paymentStorage.GetTotalUsersBalance()
+        return total || 0
     }
 
     checkUserEntry(e: LoggedEvent, u: { ts: number, updatedBalance: number } | undefined) {

src/services/main/paymentManager.ts

@@ -144,7 +144,7 @@ export default class {
 
     async PayInvoice(userId: string, req: Types.PayInvoiceRequest, linkedApplication: Application): Promise<Types.PayInvoiceResponse> {
         this.log("paying invoice", req.invoice, "for user", userId, "with amount", req.amount)
-        this.WatchdogCheck()
+        await this.WatchdogCheck()
         const decoded = await this.lnd.DecodeInvoice(req.invoice)
         if (decoded.numSatoshis !== 0 && req.amount !== 0) {
             throw new Error("invoice has value, do not provide amount the the request")
@@ -201,7 +201,7 @@ export default class {
 
     async PayAddress(ctx: Types.UserContext, req: Types.PayAddressRequest): Promise<Types.PayAddressResponse> {
         throw new Error("address payment currently disabled, use Lightning instead")
-        this.WatchdogCheck()
+        await this.WatchdogCheck()
         const { blockHeight } = await this.lnd.GetInfo()
         const app = await this.storage.applicationStorage.GetApplication(ctx.app_id)
         const serviceFee = this.getServiceFee(Types.UserOperationType.OUTGOING_TX, req.amoutSats, false)