From 1623777c1f3ffe7608dc1795107b1c09c5b7ad68 Mon Sep 17 00:00:00 2001 From: boufni95 Date: Fri, 31 Oct 2025 17:34:40 +0000 Subject: [PATCH 1/4] fix lnd crash --- src/services/lnd/lnd.ts | 57 +++++++++++++++++++++++++++++++---- src/services/main/watchdog.ts | 7 ++++- 2 files changed, 57 insertions(+), 7 deletions(-) diff --git a/src/services/lnd/lnd.ts b/src/services/lnd/lnd.ts index 254ccb7b..b68975b5 100644 --- a/src/services/lnd/lnd.ts +++ b/src/services/lnd/lnd.ts @@ -95,9 +95,10 @@ export default class { } async Warmup() { + // console.log("Warming up LND") this.SubscribeAddressPaid() this.SubscribeInvoicePaid() - this.SubscribeNewBlock() + await this.SubscribeNewBlock() this.SubscribeHtlcEvents() this.SubscribeChannelEvents() const now = Date.now() @@ -119,20 +120,24 @@ export default class { } async GetInfo(): Promise { + // console.log("Getting info") const res = await this.lightning.getInfo({}, DeadLineMetadata()) return res.response } async ListPendingChannels(): Promise { + // console.log("Listing pending channels") const res = await this.lightning.pendingChannels({ includeRawTx: false }, DeadLineMetadata()) return res.response } async ListChannels(peerLookup = false): Promise { + // console.log("Listing channels") const res = await this.lightning.listChannels({ activeOnly: false, inactiveOnly: false, privateOnly: false, publicOnly: false, peer: Buffer.alloc(0), peerAliasLookup: peerLookup }, DeadLineMetadata()) return res.response } async ListClosedChannels(): Promise { + // console.log("Listing closed channels") const res = await this.lightning.closedChannels({ abandoned: true, breach: true, @@ -145,6 +150,7 @@ export default class { } async Health(): Promise { + // console.log("Checking health") if (!this.ready) { throw new Error("not ready") } @@ -155,6 +161,7 @@ export default class { } RestartStreams() { + // console.log("Restarting streams") if (!this.ready) { return } @@ -164,7 +171,7 @@ export default class { await this.Health() this.log("LND is back online") clearInterval(interval) - this.Warmup() + await this.Warmup() } catch (err) { this.log("LND still dead, will try again in", deadLndRetrySeconds, "seconds") } @@ -172,6 +179,7 @@ export default class { } async SubscribeChannelEvents() { + // console.log("Subscribing to channel events") const stream = this.lightning.subscribeChannelEvents({}, { abort: this.abortController.signal }) stream.responses.onMessage(async channel => { const channels = await this.ListChannels() @@ -186,6 +194,7 @@ export default class { } async SubscribeHtlcEvents() { + // console.log("Subscribing to htlc events") const stream = this.router.subscribeHtlcEvents({}, { abort: this.abortController.signal }) stream.responses.onMessage(htlc => { this.htlcCb(htlc) @@ -199,20 +208,22 @@ export default class { } async SubscribeNewBlock() { + // console.log("Subscribing to new block") const { blockHeight } = await this.GetInfo() const stream = this.chainNotifier.registerBlockEpochNtfn({ height: blockHeight, hash: Buffer.alloc(0) }, { abort: this.abortController.signal }) stream.responses.onMessage(block => { this.newBlockCb(block.height) }) stream.responses.onError(error => { - this.log("Error with onchain tx stream") + this.log("Error with new block stream") }) stream.responses.onComplete(() => { - this.log("onchain tx stream closed") + this.log("new block stream closed") }) } SubscribeAddressPaid(): void { + // console.log("Subscribing to address paid") const stream = this.lightning.subscribeTransactions({ account: "", endHeight: 0, @@ -239,6 +250,7 @@ export default class { } SubscribeInvoicePaid(): void { + // console.log("Subscribing to invoice paid") const stream = this.lightning.subscribeInvoices({ settleIndex: BigInt(this.latestKnownSettleIndex), addIndex: 0n, @@ -249,17 +261,25 @@ export default class { this.invoicePaidCb(invoice.paymentRequest, Number(invoice.amtPaidSat), 'lnd') } }) + let restarted = false stream.responses.onError(error => { this.log("Error with invoice stream") + if (!restarted) { + restarted = true + this.RestartStreams() + } }) stream.responses.onComplete(() => { this.log("invoice stream closed") - this.RestartStreams() + if (!restarted) { + restarted = true + this.RestartStreams() + } }) } async NewAddress(addressType: Types.AddressType, { useProvider, from }: TxActionOptions): Promise { - + // console.log("Creating new address") let lndAddressType: AddressType switch (addressType) { case Types.AddressType.NESTED_PUBKEY_HASH: @@ -289,6 +309,7 @@ export default class { } async NewInvoice(value: number, memo: string, expiry: number, { useProvider, from }: TxActionOptions, blind = false): Promise { + // console.log("Creating new invoice") if (useProvider) { console.log("using provider") const invoice = await this.liquidProvider.AddInvoice(value, memo, from, expiry) @@ -306,6 +327,7 @@ export default class { } async DecodeInvoice(paymentRequest: string): Promise { + // console.log("Decoding invoice") const res = await this.lightning.decodePayReq({ payReq: paymentRequest }, DeadLineMetadata()) return { numSatoshis: Number(res.response.numSatoshis), paymentHash: res.response.paymentHash } } @@ -319,11 +341,13 @@ export default class { } async ChannelBalance(): Promise<{ local: number, remote: number }> { + // console.log("Getting channel balance") const res = await this.lightning.channelBalance({}) const r = res.response return { local: r.localBalance ? Number(r.localBalance.sat) : 0, remote: r.remoteBalance ? Number(r.remoteBalance.sat) : 0 } } async PayInvoice(invoice: string, amount: number, feeLimit: number, decodedAmount: number, { useProvider, from }: TxActionOptions, paymentIndexCb?: (index: number) => void): Promise { + // console.log("Paying invoice") if (this.outgoingOpsLocked) { this.log("outgoing ops locked, rejecting payment request") throw new Error("lnd node is currently out of sync") @@ -370,6 +394,7 @@ export default class { } async EstimateChainFees(address: string, amount: number, targetConf: number): Promise { + // console.log("Estimating chain fees") await this.Health() const res = await this.lightning.estimateFee({ addrToAmount: { [address]: BigInt(amount) }, @@ -382,6 +407,7 @@ export default class { } async PayAddress(address: string, amount: number, satPerVByte: number, label = "", { useProvider, from }: TxActionOptions): Promise { + // console.log("Paying address") if (this.outgoingOpsLocked) { this.log("outgoing ops locked, rejecting payment request") throw new Error("lnd node is currently out of sync") @@ -401,16 +427,19 @@ export default class { } async GetTransactions(startHeight: number): Promise { + // console.log("Getting transactions") const res = await this.lightning.getTransactions({ startHeight, endHeight: 0, account: "" }, DeadLineMetadata()) return res.response } async GetChannelInfo(chanId: string) { + // console.log("Getting channel info") const res = await this.lightning.getChanInfo({ chanId, chanPoint: "" }, DeadLineMetadata()) return res.response } async UpdateChannelPolicy(chanPoint: string, policy: Types.ChannelPolicy) { + // console.log("Updating channel policy") const split = chanPoint.split(':') const res = await this.lightning.updateChannelPolicy({ @@ -428,16 +457,19 @@ export default class { } async GetChannelBalance() { + // console.log("Getting channel balance") const res = await this.lightning.channelBalance({}, DeadLineMetadata()) return res.response } async GetWalletBalance() { + // console.log("Getting wallet balance") const res = await this.lightning.walletBalance({ account: "", minConfs: 1 }, DeadLineMetadata()) return res.response } async GetTotalBalace() { + // console.log("Getting total balance") const walletBalance = await this.GetWalletBalance() const confirmedWalletBalance = Number(walletBalance.confirmedBalance) this.utils.stateBundler.AddBalancePoint('walletBalance', confirmedWalletBalance) @@ -452,6 +484,7 @@ export default class { } async GetBalance(): Promise { // TODO: remove this + // console.log("Getting balance") const wRes = await this.lightning.walletBalance({ account: "", minConfs: 1 }, DeadLineMetadata()) const { confirmedBalance, unconfirmedBalance, totalBalance } = wRes.response const { response } = await this.lightning.listChannels({ @@ -467,20 +500,24 @@ export default class { } async GetForwardingHistory(indexOffset: number, startTime = 0, endTime = 0): Promise { + // console.log("Getting forwarding history") const { response } = await this.lightning.forwardingHistory({ indexOffset, numMaxEvents: 0, startTime: BigInt(startTime), endTime: BigInt(endTime), peerAliasLookup: false }, DeadLineMetadata()) return response } async GetAllPaidInvoices(max: number) { + // console.log("Getting all paid invoices") const res = await this.lightning.listInvoices({ indexOffset: 0n, numMaxInvoices: BigInt(max), pendingOnly: false, reversed: true, creationDateEnd: 0n, creationDateStart: 0n }, DeadLineMetadata()) return res.response } async GetAllPayments(max: number) { + // console.log("Getting all payments") const res = await this.lightning.listPayments({ countTotalPayments: false, includeIncomplete: false, indexOffset: 0n, maxPayments: BigInt(max), reversed: true, creationDateEnd: 0n, creationDateStart: 0n }) return res.response } async GetPayment(paymentIndex: number) { + // console.log("Getting payment") if (paymentIndex === 0) { throw new Error("payment index starts from 1") } @@ -492,6 +529,7 @@ export default class { } async GetLatestPaymentIndex(from = 0) { + // console.log("Getting latest payment index") let indexOffset = BigInt(from) while (true) { const res = await this.lightning.listPayments({ countTotalPayments: false, includeIncomplete: false, indexOffset, maxPayments: 0n, reversed: false, creationDateEnd: 0n, creationDateStart: 0n }, DeadLineMetadata()) @@ -503,6 +541,7 @@ export default class { } async ConnectPeer(addr: { pubkey: string, host: string }) { + // console.log("Connecting to peer") const res = await this.lightning.connectPeer({ addr, perm: true, @@ -512,6 +551,7 @@ export default class { } async GetPaymentFromHash(paymentHash: string): Promise { + // console.log("Getting payment from hash") const abortController = new AbortController() const stream = this.router.trackPaymentV2({ paymentHash: Buffer.from(paymentHash, 'hex'), @@ -533,11 +573,13 @@ export default class { } async GetTx(txid: string) { + // console.log("Getting transaction") const res = await this.walletKit.getTransaction({ txid }, DeadLineMetadata()) return res.response } async AddPeer(pub: string, host: string, port: number) { + // console.log("Adding peer") const res = await this.lightning.connectPeer({ addr: { pubkey: pub, @@ -550,11 +592,13 @@ export default class { } async ListPeers() { + // console.log("Listing peers") const res = await this.lightning.listPeers({ latestError: true }, DeadLineMetadata()) return res.response } async OpenChannel(destination: string, closeAddress: string, fundingAmount: number, pushSats: number, satsPerVByte: number): Promise { + // console.log("Opening channel") const abortController = new AbortController() const req = OpenChannelReq(destination, closeAddress, fundingAmount, pushSats, satsPerVByte) const stream = this.lightning.openChannel(req, { abort: abortController.signal }) @@ -575,6 +619,7 @@ export default class { } async CloseChannel(fundingTx: string, outputIndex: number, force: boolean, satPerVByte: number): Promise { + // console.log("Closing channel") const stream = this.lightning.closeChannel({ deliveryAddress: "", force: force, diff --git a/src/services/main/watchdog.ts b/src/services/main/watchdog.ts index 07e59662..50215045 100644 --- a/src/services/main/watchdog.ts +++ b/src/services/main/watchdog.ts @@ -179,7 +179,12 @@ export class Watchdog { StartCheck = async () => { this.latestCheckStart = Date.now() - await this.updateAccumulatedHtlcFees() + try { + await this.updateAccumulatedHtlcFees() + } catch (err: any) { + this.log("Error updating accumulated htlc fees", err.message || err) + return + } const totalUsersBalance = await this.storage.paymentStorage.GetTotalUsersBalance() this.utils.stateBundler.AddBalancePoint('usersBalance', totalUsersBalance) const { totalExternal, otherExternal } = await this.getAggregatedExternalBalance() From 89154933b9cc62df42dcf46a44671ca007b1d081 Mon Sep 17 00:00:00 2001 From: boufni95 Date: Fri, 31 Oct 2025 18:25:40 +0000 Subject: [PATCH 2/4] unlock on reconnect --- src/services/lnd/lnd.ts | 8 +++++--- src/services/main/index.ts | 2 +- src/services/main/unlocker.ts | 4 ++-- src/tests/networkSetup.ts | 4 ++-- src/tests/testBase.ts | 4 ++-- 5 files changed, 12 insertions(+), 10 deletions(-) diff --git a/src/services/lnd/lnd.ts b/src/services/lnd/lnd.ts index b68975b5..6aaa7f9b 100644 --- a/src/services/lnd/lnd.ts +++ b/src/services/lnd/lnd.ts @@ -21,7 +21,7 @@ import { Utils } from '../helpers/utilsWrapper.js'; import { TxPointSettings } from '../storage/tlv/stateBundler.js'; import { WalletKitClient } from '../../../proto/lnd/walletkit.client.js'; const DeadLineMetadata = (deadline = 10 * 1000) => ({ deadline: Date.now() + deadline }) -const deadLndRetrySeconds = 5 +const deadLndRetrySeconds = 20 type TxActionOptions = { useProvider: boolean, from: 'user' | 'system' } export default class { lightning: LightningClient @@ -43,9 +43,11 @@ export default class { outgoingOpsLocked = false liquidProvider: LiquidityProvider utils: Utils - constructor(settings: LndSettings, liquidProvider: LiquidityProvider, utils: Utils, addressPaidCb: AddressPaidCb, invoicePaidCb: InvoicePaidCb, newBlockCb: NewBlockCb, htlcCb: HtlcCb, channelEventCb: ChannelEventCb) { + unlockLnd: () => Promise + constructor(settings: LndSettings, liquidProvider: LiquidityProvider, unlockLnd: () => Promise, utils: Utils, addressPaidCb: AddressPaidCb, invoicePaidCb: InvoicePaidCb, newBlockCb: NewBlockCb, htlcCb: HtlcCb, channelEventCb: ChannelEventCb) { this.settings = settings this.utils = utils + this.unlockLnd = unlockLnd this.addressPaidCb = addressPaidCb this.invoicePaidCb = invoicePaidCb this.newBlockCb = newBlockCb @@ -168,7 +170,7 @@ export default class { this.log("LND is dead, will try to reconnect in", deadLndRetrySeconds, "seconds") const interval = setInterval(async () => { try { - await this.Health() + await this.unlockLnd() this.log("LND is back online") clearInterval(interval) await this.Warmup() diff --git a/src/services/main/index.ts b/src/services/main/index.ts index 71c64592..5c5fef16 100644 --- a/src/services/main/index.ts +++ b/src/services/main/index.ts @@ -74,7 +74,7 @@ export default class { const updateProviderBalance = (b: number) => this.storage.liquidityStorage.IncrementTrackedProviderBalance('lnPub', settings.liquiditySettings.liquidityProviderPub, b) this.liquidityProvider = new LiquidityProvider(settings.liquiditySettings.liquidityProviderPub, this.utils, this.invoicePaidCb, updateProviderBalance) this.rugPullTracker = new RugPullTracker(this.storage, this.liquidityProvider) - this.lnd = new LND(settings.lndSettings, this.liquidityProvider, this.utils, this.addressPaidCb, this.invoicePaidCb, this.newBlockCb, this.htlcCb, this.channelEventCb) + this.lnd = new LND(settings.lndSettings, this.liquidityProvider, () => this.unlocker.Unlock(), this.utils, this.addressPaidCb, this.invoicePaidCb, this.newBlockCb, this.htlcCb, this.channelEventCb) this.liquidityManager = new LiquidityManager(this.settings.liquiditySettings, this.storage, this.utils, this.liquidityProvider, this.lnd, this.rugPullTracker) this.metricsManager = new MetricsManager(this.storage, this.lnd) diff --git a/src/services/main/unlocker.ts b/src/services/main/unlocker.ts index b104ffd3..7221d183 100644 --- a/src/services/main/unlocker.ts +++ b/src/services/main/unlocker.ts @@ -301,12 +301,12 @@ export class Unlocker { GetWalletPassword = () => { const path = this.settings.walletPasswordPath - let password = Buffer.alloc(0) + let password: Buffer | null = null try { password = fs.readFileSync(path) } catch { } - if (password.length === 0) { + if (!password || password.length === 0) { this.log("no wallet password configured, using wallet secret") const secret = this.GetWalletSecret(false) if (secret === "") { diff --git a/src/tests/networkSetup.ts b/src/tests/networkSetup.ts index 5944a1fc..40f0e58b 100644 --- a/src/tests/networkSetup.ts +++ b/src/tests/networkSetup.ts @@ -14,8 +14,8 @@ export const setupNetwork = async (): Promise => { await core.InitAddress() await core.Mine(1) const setupUtils = new Utils({ dataDir: settings.storageSettings.dataDir, allowResetMetricsStorages: settings.allowResetMetricsStorages }) - const alice = new LND(settings.lndSettings, new LiquidityProvider("", setupUtils, async () => { }, async () => { }), setupUtils, async () => { }, async () => { }, () => { }, () => { }, () => { }) - const bob = new LND({ ...settings.lndSettings, mainNode: settings.lndSettings.otherNode }, new LiquidityProvider("", setupUtils, async () => { }, async () => { }), setupUtils, async () => { }, async () => { }, () => { }, () => { }, () => { }) + const alice = new LND(settings.lndSettings, new LiquidityProvider("", setupUtils, async () => { }, async () => { }), async () => { }, setupUtils, async () => { }, async () => { }, () => { }, () => { }, () => { }) + const bob = new LND({ ...settings.lndSettings, mainNode: settings.lndSettings.otherNode }, new LiquidityProvider("", setupUtils, async () => { }, async () => { }), async () => { }, setupUtils, async () => { }, async () => { }, () => { }, () => { }, () => { }) await tryUntil(async i => { const peers = await alice.ListPeers() if (peers.peers.length > 0) { diff --git a/src/tests/testBase.ts b/src/tests/testBase.ts index 95454a95..fdd599b9 100644 --- a/src/tests/testBase.ts +++ b/src/tests/testBase.ts @@ -78,11 +78,11 @@ export const SetupTest = async (d: Describe, chainTools: ChainTools): Promise { }, async () => { }), extermnalUtils, async () => { }, async () => { }, () => { }, () => { }, () => { }) + const externalAccessToOtherLnd = new LND(otherLndSetting, new LiquidityProvider("", extermnalUtils, async () => { }, async () => { }), async () => { }, extermnalUtils, async () => { }, async () => { }, () => { }, () => { }, () => { }) await externalAccessToOtherLnd.Warmup() const thirdLndSetting = { ...settings.lndSettings, mainNode: settings.lndSettings.thirdNode } - const externalAccessToThirdLnd = new LND(thirdLndSetting, new LiquidityProvider("", extermnalUtils, async () => { }, async () => { }), extermnalUtils, async () => { }, async () => { }, () => { }, () => { }, () => { }) + const externalAccessToThirdLnd = new LND(thirdLndSetting, new LiquidityProvider("", extermnalUtils, async () => { }, async () => { }), async () => { }, extermnalUtils, async () => { }, async () => { }, () => { }, () => { }, () => { }) await externalAccessToThirdLnd.Warmup() From 203dde3d389f5810e00f08b5fbc9b06cb6ec7a9e Mon Sep 17 00:00:00 2001 From: boufni95 Date: Fri, 31 Oct 2025 18:33:28 +0000 Subject: [PATCH 3/4] dont reconnect after abort --- src/services/lnd/lnd.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/services/lnd/lnd.ts b/src/services/lnd/lnd.ts index 6aaa7f9b..0f9daaa9 100644 --- a/src/services/lnd/lnd.ts +++ b/src/services/lnd/lnd.ts @@ -164,7 +164,7 @@ export default class { RestartStreams() { // console.log("Restarting streams") - if (!this.ready) { + if (!this.ready || this.abortController.signal.aborted) { return } this.log("LND is dead, will try to reconnect in", deadLndRetrySeconds, "seconds") From 3e45e4ec90e5b77a73166aac372b0d241d6bc456 Mon Sep 17 00:00:00 2001 From: boufni95 Date: Fri, 31 Oct 2025 19:37:21 +0000 Subject: [PATCH 4/4] disable new invoice check --- src/services/storage/paymentStorage.ts | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/src/services/storage/paymentStorage.ts b/src/services/storage/paymentStorage.ts index 5f3d15a9..ef29957c 100644 --- a/src/services/storage/paymentStorage.ts +++ b/src/services/storage/paymentStorage.ts @@ -90,7 +90,7 @@ export default class { take }, txId); items.push(...firstBatch); - } + } const needMore = take - items.length // If need more, fetch higher paid_at_unix @@ -134,7 +134,7 @@ export default class { } async RemoveUserInvoices(userId: string, txId?: string) { - return this.dbs.Delete('UserReceivingInvoice', { user: { user_id: userId } }, txId) + return this.dbs.Delete('UserReceivingInvoice', { user: { user_id: userId } }, txId) } async GetAddressOwner(address: string, txId?: string): Promise { @@ -401,7 +401,7 @@ export default class { ]) const receivingTransactions = await Promise.all(receivingAddresses.map(addr => this.dbs.Find('AddressReceivingTransaction', { where: { user_address: { serial_id: addr.serial_id }, ...time } }))) - return { + return { receivingInvoices, receivingAddresses, receivingTransactions, outgoingInvoices, outgoingTransactions, userToUser @@ -419,8 +419,8 @@ export default class { async VerifyDbEvent(e: LoggedEvent) { switch (e.type) { - case "new_invoice": - return orFail(this.dbs.FindOne('UserReceivingInvoice', { where: { invoice: e.data, user: { user_id: e.userId } } })) + /* case "new_invoice": + return orFail(this.dbs.FindOne('UserReceivingInvoice', { where: { invoice: e.data, user: { user_id: e.userId } } })) */ case 'new_address': return orFail(this.dbs.FindOne('UserReceivingAddress', { where: { address: e.data, user: { user_id: e.userId } } })) case 'invoice_paid':