Merge pull request #850 from shocknet/fix-lnd-crash

fix lnd crash

src/services/lnd/lnd.ts

@@ -24,7 +24,7 @@ import SettingsManager from '../main/settingsManager.js';
 import { LndNodeSettings, LndSettings } from '../main/settings.js';
 
 const DeadLineMetadata = (deadline = 10 * 1000) => ({ deadline: Date.now() + deadline })
-const deadLndRetrySeconds = 5
+const deadLndRetrySeconds = 20
 type TxActionOptions = { useProvider: boolean, from: 'user' | 'system' }
 type NodeSettingsOverride = {
     lndAddr: string
@@ -51,9 +51,11 @@ export default class {
     outgoingOpsLocked = false
     liquidProvider: LiquidityProvider
     utils: Utils
-    constructor(getSettings: () => { lndSettings: LndSettings, lndNodeSettings: LndNodeSettings }, liquidProvider: LiquidityProvider, utils: Utils, addressPaidCb: AddressPaidCb, invoicePaidCb: InvoicePaidCb, newBlockCb: NewBlockCb, htlcCb: HtlcCb, channelEventCb: ChannelEventCb) {
+    unlockLnd: () => Promise<void>
+    constructor(getSettings: () => { lndSettings: LndSettings, lndNodeSettings: LndNodeSettings }, liquidProvider: LiquidityProvider, unlockLnd: () => Promise<any>, utils: Utils, addressPaidCb: AddressPaidCb, invoicePaidCb: InvoicePaidCb, newBlockCb: NewBlockCb, htlcCb: HtlcCb, channelEventCb: ChannelEventCb) {
         this.getSettings = getSettings
         this.utils = utils
+        this.unlockLnd = unlockLnd
         this.addressPaidCb = addressPaidCb
         this.invoicePaidCb = invoicePaidCb
         this.newBlockCb = newBlockCb
@@ -103,9 +105,10 @@ export default class {
     }
 
     async Warmup() {
+        // console.log("Warming up LND")
         this.SubscribeAddressPaid()
         this.SubscribeInvoicePaid()
-        this.SubscribeNewBlock()
+        await this.SubscribeNewBlock()
         this.SubscribeHtlcEvents()
         this.SubscribeChannelEvents()
         const now = Date.now()
@@ -127,20 +130,24 @@ export default class {
     }
 
     async GetInfo(): Promise<NodeInfo> {
+        // console.log("Getting info")
         const res = await this.lightning.getInfo({}, DeadLineMetadata())
         return res.response
     }
     async ListPendingChannels(): Promise<PendingChannelsResponse> {
+        // console.log("Listing pending channels")
         const res = await this.lightning.pendingChannels({ includeRawTx: false }, DeadLineMetadata())
         return res.response
     }
     async ListChannels(peerLookup = false): Promise<ListChannelsResponse> {
+        // console.log("Listing channels")
         const res = await this.lightning.listChannels({
             activeOnly: false, inactiveOnly: false, privateOnly: false, publicOnly: false, peer: Buffer.alloc(0), peerAliasLookup: peerLookup
         }, DeadLineMetadata())
         return res.response
     }
     async ListClosedChannels(): Promise<ClosedChannelsResponse> {
+        // console.log("Listing closed channels")
         const res = await this.lightning.closedChannels({
             abandoned: true,
             breach: true,
@@ -153,6 +160,7 @@ export default class {
     }
 
     async Health(): Promise<void> {
+        // console.log("Checking health")
         if (!this.ready) {
             throw new Error("not ready")
         }
@@ -163,16 +171,17 @@ export default class {
     }
 
     RestartStreams() {
-        if (!this.ready) {
+        // console.log("Restarting streams")
+        if (!this.ready || this.abortController.signal.aborted) {
             return
         }
         this.log("LND is dead, will try to reconnect in", deadLndRetrySeconds, "seconds")
         const interval = setInterval(async () => {
             try {
-                await this.Health()
+                await this.unlockLnd()
                 this.log("LND is back online")
                 clearInterval(interval)
-                this.Warmup()
+                await this.Warmup()
             } catch (err) {
                 this.log("LND still dead, will try again in", deadLndRetrySeconds, "seconds")
             }
@@ -180,6 +189,7 @@ export default class {
     }
 
     async SubscribeChannelEvents() {
+        // console.log("Subscribing to channel events")
         const stream = this.lightning.subscribeChannelEvents({}, { abort: this.abortController.signal })
         stream.responses.onMessage(async channel => {
             const channels = await this.ListChannels()
@@ -194,6 +204,7 @@ export default class {
     }
 
     async SubscribeHtlcEvents() {
+        // console.log("Subscribing to htlc events")
         const stream = this.router.subscribeHtlcEvents({}, { abort: this.abortController.signal })
         stream.responses.onMessage(htlc => {
             this.htlcCb(htlc)
@@ -207,20 +218,22 @@ export default class {
     }
 
     async SubscribeNewBlock() {
+        // console.log("Subscribing to new block")
         const { blockHeight } = await this.GetInfo()
         const stream = this.chainNotifier.registerBlockEpochNtfn({ height: blockHeight, hash: Buffer.alloc(0) }, { abort: this.abortController.signal })
         stream.responses.onMessage(block => {
             this.newBlockCb(block.height)
         })
         stream.responses.onError(error => {
-            this.log("Error with onchain tx stream")
+            this.log("Error with new block stream")
         })
         stream.responses.onComplete(() => {
-            this.log("onchain tx stream closed")
+            this.log("new block stream closed")
         })
     }
 
     SubscribeAddressPaid(): void {
+        // console.log("Subscribing to address paid")
         const stream = this.lightning.subscribeTransactions({
             account: "",
             endHeight: 0,
@@ -247,6 +260,7 @@ export default class {
     }
 
     SubscribeInvoicePaid(): void {
+        // console.log("Subscribing to invoice paid")
         const stream = this.lightning.subscribeInvoices({
             settleIndex: BigInt(this.latestKnownSettleIndex),
             addIndex: 0n,
@@ -257,17 +271,25 @@ export default class {
                 this.invoicePaidCb(invoice.paymentRequest, Number(invoice.amtPaidSat), 'lnd')
             }
         })
+        let restarted = false
         stream.responses.onError(error => {
             this.log("Error with invoice stream")
+            if (!restarted) {
+                restarted = true
+                this.RestartStreams()
+            }
         })
         stream.responses.onComplete(() => {
             this.log("invoice stream closed")
-            this.RestartStreams()
+            if (!restarted) {
+                restarted = true
+                this.RestartStreams()
+            }
         })
     }
 
     async NewAddress(addressType: Types.AddressType, { useProvider, from }: TxActionOptions): Promise<NewAddressResponse> {
-
+        //  console.log("Creating new address")
         let lndAddressType: AddressType
         switch (addressType) {
             case Types.AddressType.NESTED_PUBKEY_HASH:
@@ -297,6 +319,7 @@ export default class {
     }
 
     async NewInvoice(value: number, memo: string, expiry: number, { useProvider, from }: TxActionOptions, blind = false): Promise<Invoice> {
+        // console.log("Creating new invoice")
         if (useProvider) {
             console.log("using provider")
             const invoice = await this.liquidProvider.AddInvoice(value, memo, from, expiry)
@@ -314,6 +337,7 @@ export default class {
     }
 
     async DecodeInvoice(paymentRequest: string): Promise<DecodedInvoice> {
+        // console.log("Decoding invoice")
         const res = await this.lightning.decodePayReq({ payReq: paymentRequest }, DeadLineMetadata())
         return { numSatoshis: Number(res.response.numSatoshis), paymentHash: res.response.paymentHash }
     }
@@ -327,11 +351,13 @@ export default class {
     }
 
     async ChannelBalance(): Promise<{ local: number, remote: number }> {
+        // console.log("Getting channel balance")
         const res = await this.lightning.channelBalance({})
         const r = res.response
         return { local: r.localBalance ? Number(r.localBalance.sat) : 0, remote: r.remoteBalance ? Number(r.remoteBalance.sat) : 0 }
     }
     async PayInvoice(invoice: string, amount: number, feeLimit: number, decodedAmount: number, { useProvider, from }: TxActionOptions, paymentIndexCb?: (index: number) => void): Promise<PaidInvoice> {
+        // console.log("Paying invoice")
         if (this.outgoingOpsLocked) {
             this.log("outgoing ops locked, rejecting payment request")
             throw new Error("lnd node is currently out of sync")
@@ -378,6 +404,7 @@ export default class {
     }
 
     async EstimateChainFees(address: string, amount: number, targetConf: number): Promise<EstimateFeeResponse> {
+        // console.log("Estimating chain fees")
         await this.Health()
         const res = await this.lightning.estimateFee({
             addrToAmount: { [address]: BigInt(amount) },
@@ -390,6 +417,7 @@ export default class {
     }
 
     async PayAddress(address: string, amount: number, satPerVByte: number, label = "", { useProvider, from }: TxActionOptions): Promise<SendCoinsResponse> {
+        // console.log("Paying address")
         if (this.outgoingOpsLocked) {
             this.log("outgoing ops locked, rejecting payment request")
             throw new Error("lnd node is currently out of sync")
@@ -409,16 +437,19 @@ export default class {
     }
 
     async GetTransactions(startHeight: number): Promise<TransactionDetails> {
+        // console.log("Getting transactions")
         const res = await this.lightning.getTransactions({ startHeight, endHeight: 0, account: "" }, DeadLineMetadata())
         return res.response
     }
 
     async GetChannelInfo(chanId: string) {
+        // console.log("Getting channel info")
         const res = await this.lightning.getChanInfo({ chanId, chanPoint: "" }, DeadLineMetadata())
         return res.response
     }
 
     async UpdateChannelPolicy(chanPoint: string, policy: Types.ChannelPolicy) {
+        // console.log("Updating channel policy")
         const split = chanPoint.split(':')
 
         const res = await this.lightning.updateChannelPolicy({
@@ -436,16 +467,19 @@ export default class {
     }
 
     async GetChannelBalance() {
+        // console.log("Getting channel balance")
         const res = await this.lightning.channelBalance({}, DeadLineMetadata())
         return res.response
     }
 
     async GetWalletBalance() {
+        // console.log("Getting wallet balance")
         const res = await this.lightning.walletBalance({ account: "", minConfs: 1 }, DeadLineMetadata())
         return res.response
     }
 
     async GetTotalBalace() {
+        // console.log("Getting total balance")
         const walletBalance = await this.GetWalletBalance()
         const confirmedWalletBalance = Number(walletBalance.confirmedBalance)
         this.utils.stateBundler.AddBalancePoint('walletBalance', confirmedWalletBalance)
@@ -460,6 +494,7 @@ export default class {
     }
 
     async GetBalance(): Promise<BalanceInfo> { // TODO: remove this
+        //      console.log("Getting balance")
         const wRes = await this.lightning.walletBalance({ account: "", minConfs: 1 }, DeadLineMetadata())
         const { confirmedBalance, unconfirmedBalance, totalBalance } = wRes.response
         const { response } = await this.lightning.listChannels({
@@ -475,20 +510,24 @@ export default class {
     }
 
     async GetForwardingHistory(indexOffset: number, startTime = 0, endTime = 0): Promise<ForwardingHistoryResponse> {
+        // console.log("Getting forwarding history")
         const { response } = await this.lightning.forwardingHistory({ indexOffset, numMaxEvents: 0, startTime: BigInt(startTime), endTime: BigInt(endTime), peerAliasLookup: false }, DeadLineMetadata())
         return response
     }
 
     async GetAllPaidInvoices(max: number) {
+        // console.log("Getting all paid invoices")
         const res = await this.lightning.listInvoices({ indexOffset: 0n, numMaxInvoices: BigInt(max), pendingOnly: false, reversed: true, creationDateEnd: 0n, creationDateStart: 0n }, DeadLineMetadata())
         return res.response
     }
     async GetAllPayments(max: number) {
+        // console.log("Getting all payments")
         const res = await this.lightning.listPayments({ countTotalPayments: false, includeIncomplete: false, indexOffset: 0n, maxPayments: BigInt(max), reversed: true, creationDateEnd: 0n, creationDateStart: 0n })
         return res.response
     }
 
     async GetPayment(paymentIndex: number) {
+        // console.log("Getting payment")
         if (paymentIndex === 0) {
             throw new Error("payment index starts from 1")
         }
@@ -500,6 +539,7 @@ export default class {
     }
 
     async GetLatestPaymentIndex(from = 0) {
+        // console.log("Getting latest payment index")
         let indexOffset = BigInt(from)
         while (true) {
             const res = await this.lightning.listPayments({ countTotalPayments: false, includeIncomplete: false, indexOffset, maxPayments: 0n, reversed: false, creationDateEnd: 0n, creationDateStart: 0n }, DeadLineMetadata())
@@ -511,6 +551,7 @@ export default class {
     }
 
     async ConnectPeer(addr: { pubkey: string, host: string }) {
+        // console.log("Connecting to peer")
         const res = await this.lightning.connectPeer({
             addr,
             perm: true,
@@ -520,6 +561,7 @@ export default class {
     }
 
     async GetPaymentFromHash(paymentHash: string): Promise<Payment | null> {
+        // console.log("Getting payment from hash")
         const abortController = new AbortController()
         const stream = this.router.trackPaymentV2({
             paymentHash: Buffer.from(paymentHash, 'hex'),
@@ -541,11 +583,13 @@ export default class {
     }
 
     async GetTx(txid: string) {
+        // console.log("Getting transaction")
         const res = await this.walletKit.getTransaction({ txid }, DeadLineMetadata())
         return res.response
     }
 
     async AddPeer(pub: string, host: string, port: number) {
+        // console.log("Adding peer")
         const res = await this.lightning.connectPeer({
             addr: {
                 pubkey: pub,
@@ -558,11 +602,13 @@ export default class {
     }
 
     async ListPeers() {
+        // console.log("Listing peers")
         const res = await this.lightning.listPeers({ latestError: true }, DeadLineMetadata())
         return res.response
     }
 
     async OpenChannel(destination: string, closeAddress: string, fundingAmount: number, pushSats: number, satsPerVByte: number): Promise<OpenStatusUpdate> {
+        // console.log("Opening channel")
         const abortController = new AbortController()
         const req = OpenChannelReq(destination, closeAddress, fundingAmount, pushSats, satsPerVByte)
         const stream = this.lightning.openChannel(req, { abort: abortController.signal })
@@ -583,6 +629,7 @@ export default class {
     }
 
     async CloseChannel(fundingTx: string, outputIndex: number, force: boolean, satPerVByte: number): Promise<PendingUpdate> {
+        // console.log("Closing channel")
         const stream = this.lightning.closeChannel({
             deliveryAddress: "",
             force: force,

src/services/main/index.ts

@@ -79,7 +79,7 @@ export default class {
             lndSettings: settings.getSettings().lndSettings,
             lndNodeSettings: settings.getSettings().lndNodeSettings
         })
-        this.lnd = new LND(lndGetSettings, this.liquidityProvider, this.utils, this.addressPaidCb, this.invoicePaidCb, this.newBlockCb, this.htlcCb, this.channelEventCb)
+        this.lnd = new LND(lndGetSettings, this.liquidityProvider, () => this.unlocker.Unlock(), this.utils, this.addressPaidCb, this.invoicePaidCb, this.newBlockCb, this.htlcCb, this.channelEventCb)
         this.liquidityManager = new LiquidityManager(this.settings, this.storage, this.utils, this.liquidityProvider, this.lnd, this.rugPullTracker)
         this.metricsManager = new MetricsManager(this.storage, this.lnd)
 

src/services/main/settingsManager.ts

@@ -44,7 +44,7 @@ export default class SettingsManager {
             toAdd[key] = value
         }
         this.settings = this.loadEnvs(dbSettings, addToDb)
-        this.log("adding", toAdd.length, "settings to db")
+        this.log("adding", Object.keys(toAdd).length, "settings to db")
         for (const key in toAdd) {
             await this.storage.settingsStorage.setDbEnvIFNeeded(key, toAdd[key])
         }

src/services/main/unlocker.ts

@@ -301,12 +301,12 @@ export class Unlocker {
 
     GetWalletPassword = () => {
         const path = this.settings.getStorageSettings().walletPasswordPath
-        let password = Buffer.alloc(0)
+        let password: Buffer | null = null
         try {
             password = fs.readFileSync(path)
         } catch {
         }
-        if (password.length === 0) {
+        if (!password || password.length === 0) {
             this.log("no wallet password configured, using wallet secret")
             const secret = this.GetWalletSecret(false)
             if (secret === "") {

src/services/main/watchdog.ts

@@ -173,7 +173,12 @@ export class Watchdog {
 
     StartCheck = async () => {
         this.latestCheckStart = Date.now()
-        await this.updateAccumulatedHtlcFees()
+        try {
+            await this.updateAccumulatedHtlcFees()
+        } catch (err: any) {
+            this.log("Error updating accumulated htlc fees", err.message || err)
+            return
+        }
         const totalUsersBalance = await this.storage.paymentStorage.GetTotalUsersBalance()
         this.utils.stateBundler.AddBalancePoint('usersBalance', totalUsersBalance)
         const { totalExternal, otherExternal } = await this.getAggregatedExternalBalance()

src/services/storage/paymentStorage.ts

@@ -90,7 +90,7 @@ export default class {
                 take
             }, txId);
             items.push(...firstBatch);
-        } 
+        }
 
         const needMore = take - items.length
         // If need more, fetch higher paid_at_unix
@@ -134,7 +134,7 @@ export default class {
     }
 
     async RemoveUserInvoices(userId: string, txId?: string) {
-        return this.dbs.Delete<UserReceivingInvoice>('UserReceivingInvoice',  { user: { user_id: userId } }, txId)
+        return this.dbs.Delete<UserReceivingInvoice>('UserReceivingInvoice', { user: { user_id: userId } }, txId)
     }
 
     async GetAddressOwner(address: string, txId?: string): Promise<UserReceivingAddress | null> {
@@ -401,7 +401,7 @@ export default class {
         ])
         const receivingTransactions = await Promise.all(receivingAddresses.map(addr =>
             this.dbs.Find<AddressReceivingTransaction>('AddressReceivingTransaction', { where: { user_address: { serial_id: addr.serial_id }, ...time } })))
-            return {
+        return {
             receivingInvoices, receivingAddresses, receivingTransactions,
             outgoingInvoices, outgoingTransactions,
             userToUser
@@ -419,8 +419,8 @@ export default class {
 
     async VerifyDbEvent(e: LoggedEvent) {
         switch (e.type) {
-            case "new_invoice":
+            /*             case "new_invoice":
-                return orFail(this.dbs.FindOne<UserReceivingInvoice>('UserReceivingInvoice', { where: { invoice: e.data, user: { user_id: e.userId } } }))
+                            return orFail(this.dbs.FindOne<UserReceivingInvoice>('UserReceivingInvoice', { where: { invoice: e.data, user: { user_id: e.userId } } })) */
             case 'new_address':
                 return orFail(this.dbs.FindOne<UserReceivingAddress>('UserReceivingAddress', { where: { address: e.data, user: { user_id: e.userId } } }))
             case 'invoice_paid':

src/tests/networkSetup.ts

@@ -24,8 +24,8 @@ export const setupNetwork = async (): Promise<ChainTools> => {
     const lndNodeSettings = LoadLndNodeSettingsFromEnv({})
     const secondLndNodeSettings = LoadSecondLndSettingsFromEnv()
     const liquiditySettings: LiquiditySettings = { disableLiquidityProvider: true, liquidityProviderPub: "", useOnlyLiquidityProvider: false }
-    const alice = new LND(() => ({ lndSettings, lndNodeSettings }), new LiquidityProvider(() => liquiditySettings, setupUtils, async () => { }, async () => { }), setupUtils, async () => { }, async () => { }, () => { }, () => { }, () => { })
+    const alice = new LND(() => ({ lndSettings, lndNodeSettings }), new LiquidityProvider(() => liquiditySettings, setupUtils, async () => { }, async () => { }), async () => { }, setupUtils, async () => { }, async () => { }, () => { }, () => { }, () => { })
-    const bob = new LND(() => ({ lndSettings, lndNodeSettings: secondLndNodeSettings }), new LiquidityProvider(() => liquiditySettings, setupUtils, async () => { }, async () => { }), setupUtils, async () => { }, async () => { }, () => { }, () => { }, () => { })
+    const bob = new LND(() => ({ lndSettings, lndNodeSettings: secondLndNodeSettings }), new LiquidityProvider(() => liquiditySettings, setupUtils, async () => { }, async () => { }), async () => { }, setupUtils, async () => { }, async () => { }, () => { }, () => { }, () => { })
     await tryUntil<void>(async i => {
         const peers = await alice.ListPeers()
         if (peers.peers.length > 0) {

src/tests/testBase.ts

@@ -87,12 +87,12 @@ export const SetupTest = async (d: Describe, chainTools: ChainTools): Promise<Te
     const lndSettings = LoadLndSettingsFromEnv({})
     const secondLndNodeSettings = LoadSecondLndSettingsFromEnv()
     const otherLndSetting = () => ({ lndSettings, lndNodeSettings: secondLndNodeSettings })
-    const externalAccessToOtherLnd = new LND(otherLndSetting, new LiquidityProvider(() => liquiditySettings, extermnalUtils, async () => { }, async () => { }), extermnalUtils, async () => { }, async () => { }, () => { }, () => { }, () => { })
+    const externalAccessToOtherLnd = new LND(otherLndSetting, new LiquidityProvider(() => liquiditySettings, extermnalUtils, async () => { }, async () => { }), async () => { }, extermnalUtils, async () => { }, async () => { }, () => { }, () => { }, () => { })
     await externalAccessToOtherLnd.Warmup()
 
     const thirdLndNodeSettings = LoadThirdLndSettingsFromEnv()
     const thirdLndSetting = () => ({ lndSettings, lndNodeSettings: thirdLndNodeSettings })
-    const externalAccessToThirdLnd = new LND(thirdLndSetting, new LiquidityProvider(() => liquiditySettings, extermnalUtils, async () => { }, async () => { }), extermnalUtils, async () => { }, async () => { }, () => { }, () => { }, () => { })
+    const externalAccessToThirdLnd = new LND(thirdLndSetting, new LiquidityProvider(() => liquiditySettings, extermnalUtils, async () => { }, async () => { }), async () => { }, extermnalUtils, async () => { }, async () => { }, () => { }, () => { }, () => { })
     await externalAccessToThirdLnd.Warmup()