logs and handle restarts

.gitignore

@@ -8,3 +8,4 @@ temp/
 build/
 db.sqlite
 .key/
+logs

src/auth.ts

@@ -1,7 +1,7 @@
 import { ServerOptions } from "../proto/autogenerated/ts/express_server";
 import { AdminContext } from "../proto/autogenerated/ts/types";
 import Main from './services/main'
-import { getLogger } from './services/helpers/logger'
+import { getLogger } from './services/helpers/logger.js'
 const serverOptions = (mainHandler: Main): ServerOptions => {
     const log = getLogger({})
     return {

src/services/lnd/index.ts

@@ -4,6 +4,7 @@ import { EnvMustBeNonEmptyString, EnvMustBeInteger, EnvCanBeBoolean } from '../h
 import { AddressPaidCb, DecodedInvoice, Invoice, InvoicePaidCb, LndSettings, NodeInfo, PaidInvoice } from './settings.js'
 import LND from './lnd.js'
 import MockLnd from './mock.js'
+import { getLogger } from '../helpers/logger.js'
 export const LoadLndSettingsFromEnv = (test = false): LndSettings => {
     const lndAddr = EnvMustBeNonEmptyString("LND_ADDRESS")
     const lndCertPath = EnvMustBeNonEmptyString("LND_CERT_PATH")
@@ -32,10 +33,10 @@ export interface LightningHandler {
 
 export default (settings: LndSettings, addressPaidCb: AddressPaidCb, invoicePaidCb: InvoicePaidCb): LightningHandler => {
     if (settings.mockLnd) {
-        console.log("registering mock lnd handler")
+        getLogger({})("registering mock lnd handler")
         return new MockLnd(settings, addressPaidCb, invoicePaidCb)
     } else {
-        console.log("registering prod lnd handler")
+        getLogger({})("registering prod lnd handler")
         return new LND(settings, addressPaidCb, invoicePaidCb)
     }
 }

src/services/lnd/lnd.ts

@@ -15,7 +15,7 @@ import { SendCoinsReq } from './sendCoinsReq.js';
 import { LndSettings, AddressPaidCb, InvoicePaidCb, NodeInfo, Invoice, DecodedInvoice, PaidInvoice } from './settings.js';
 import { getLogger } from '../helpers/logger.js';
 const DeadLineMetadata = (deadline = 10 * 1000) => ({ deadline: Date.now() + deadline })
-
+const deadLndRetrySeconds = 5
 export default class {
     lightning: LightningClient
     invoices: InvoicesClient
@@ -51,8 +51,6 @@ export default class {
         this.lightning = new LightningClient(transport)
         this.invoices = new InvoicesClient(transport)
         this.router = new RouterClient(transport)
-        this.SubscribeAddressPaid()
-        this.SubscribeInvoicePaid()
     }
     SetMockInvoiceAsPaid(invoice: string, amount: number): Promise<void> {
         throw new Error("SetMockInvoiceAsPaid only available in mock mode")
@@ -60,7 +58,11 @@ export default class {
     Stop() {
         this.abortController.abort()
     }
-    async Warmup() { this.ready = true }
+    async Warmup() {
+        this.SubscribeAddressPaid()
+        this.SubscribeInvoicePaid()
+        this.ready = true
+    }
 
     async GetInfo(): Promise<NodeInfo> {
         const res = await this.lightning.getInfo({}, DeadLineMetadata())
@@ -73,12 +75,27 @@ export default class {
         }
         const info = await this.GetInfo()
         if (!info.syncedToChain || !info.syncedToGraph) {
-            throw new Error("not ready")
+            throw new Error("not synced")
         }
     }
-    checkReady(): void {
+
-        if (!this.ready) throw new Error("lnd not ready, warmup required before usage")
+    RestartStreams() {
+        if (!this.ready) {
+            return
         }
+        this.log("LND is dead, will try to reconnect in", deadLndRetrySeconds, "seconds")
+        const interval = setInterval(async () => {
+            try {
+                await this.Health()
+                this.log("LND is back online")
+                clearInterval(interval)
+                this.Warmup()
+            } catch (err) {
+                this.log("LND still dead, will try again in", deadLndRetrySeconds, "seconds")
+            }
+        }, deadLndRetrySeconds * 1000)
+    }
+
     SubscribeAddressPaid(): void {
         const stream = this.lightning.subscribeTransactions({
             account: "",
@@ -100,7 +117,10 @@ export default class {
             }
         })
         stream.responses.onError(error => {
-            this.log("Error with invoice stream")
+            this.log("Error with onchain tx stream")
+        })
+        stream.responses.onComplete(() => {
+            this.log("onchain tx stream closed")
         })
     }
 
@@ -119,9 +139,14 @@ export default class {
         stream.responses.onError(error => {
             this.log("Error with invoice stream")
         })
+        stream.responses.onComplete(() => {
+            this.log("invoice stream closed")
+            this.RestartStreams()
+        })
     }
+
     async NewAddress(addressType: Types.AddressType): Promise<NewAddressResponse> {
-        this.checkReady()
+        await this.Health()
         let lndAddressType: AddressType
         switch (addressType) {
             case Types.AddressType.NESTED_PUBKEY_HASH:
@@ -141,7 +166,7 @@ export default class {
     }
 
     async NewInvoice(value: number, memo: string, expiry: number): Promise<Invoice> {
-        this.checkReady()
+        await this.Health()
         const encoder = new TextEncoder()
         const ecoded = encoder.encode(memo)
         const hashed = crypto.createHash('sha256').update(ecoded).digest();
@@ -163,7 +188,7 @@ export default class {
     }
 
     async PayInvoice(invoice: string, amount: number, feeLimit: number): Promise<PaidInvoice> {
-        this.checkReady()
+        await this.Health()
         const abortController = new AbortController()
         const req = PayInvoiceReq(invoice, amount, feeLimit)
         const stream = this.router.sendPaymentV2(req, { abort: abortController.signal })
@@ -172,7 +197,6 @@ export default class {
                 rej(error)
             })
             stream.responses.onMessage(payment => {
-                console.log(payment)
                 switch (payment.status) {
                     case Payment_PaymentStatus.FAILED:
                         this.log("invoice payment failed", payment.failureReason)
@@ -187,7 +211,7 @@ export default class {
     }
 
     async EstimateChainFees(address: string, amount: number, targetConf: number): Promise<EstimateFeeResponse> {
-        this.checkReady()
+        await this.Health()
         const res = await this.lightning.estimateFee({
             addrToAmount: { [address]: BigInt(amount) },
             minConfs: 1,
@@ -198,7 +222,7 @@ export default class {
     }
 
     async PayAddress(address: string, amount: number, satPerVByte: number, label = ""): Promise<SendCoinsResponse> {
-        this.checkReady()
+        await this.Health()
         const res = await this.lightning.sendCoins(SendCoinsReq(address, amount, satPerVByte, label), DeadLineMetadata())
         this.log("sent chain TX for", amount, "sats")
         return res.response
@@ -206,7 +230,7 @@ export default class {
 
 
     async OpenChannel(destination: string, closeAddress: string, fundingAmount: number, pushSats: number): Promise<string> {
-        this.checkReady()
+        await this.Health()
         const abortController = new AbortController()
         const req = OpenChannelReq(destination, closeAddress, fundingAmount, pushSats)
         const stream = this.lightning.openChannel(req, { abort: abortController.signal })

src/services/lnd/mock.ts

@@ -13,6 +13,7 @@ import { AddInvoiceReq } from './addInvoiceReq.js';
 import { PayInvoiceReq } from './payInvoiceReq.js';
 import { SendCoinsReq } from './sendCoinsReq.js';
 import { LndSettings, AddressPaidCb, InvoicePaidCb, NodeInfo, Invoice, DecodedInvoice, PaidInvoice } from './settings.js';
+import { getLogger } from '../helpers/logger.js';
 
 export default class {
     invoicesAwaiting: Record<string /* invoice */, { value: number, memo: string, expiryUnix: number }> = {}
@@ -86,10 +87,11 @@ export default class {
     }
 
     async PayInvoice(invoice: string, amount: number, feeLimit: number): Promise<PaidInvoice> {
-        console.log('payng', invoice)
+        const log = getLogger({})
+        log('payng', invoice)
         await new Promise(res => setTimeout(res, 200))
         const amt = this.decodeOutboundInvoice(invoice)
-        console.log('paid', invoice)
+        log('paid', invoice)
         return { feeSat: 1, paymentPreimage: "all_good", valueSat: amt || amount }
     }
 

src/services/main/index.ts

@@ -9,6 +9,7 @@ import PaymentManager from './paymentManager.js'
 import { MainSettings } from './settings.js'
 import NewLightningHandler, { LoadLndSettingsFromEnv, LightningHandler } from "../lnd/index.js"
 import { AddressPaidCb, InvoicePaidCb } from "../lnd/settings.js"
+import { getLogger, PubLogger } from "../helpers/logger.js"
 export const LoadMainSettingsFromEnv = (test = false): MainSettings => {
     return {
         lndSettings: LoadLndSettingsFromEnv(test),
@@ -75,8 +76,9 @@ export default class {
         this.storage.StartTransaction(async tx => {
             const userInvoice = await this.storage.paymentStorage.GetInvoiceOwner(paymentRequest, tx)
             if (!userInvoice || userInvoice.paid_at_unix > 0) { return }
+            const log = getLogger({})
             if (!userInvoice.linkedApplication) {
-                console.error("an invoice was paid, that has no linked application")
+                log("ERROR", "an invoice was paid, that has no linked application")
                 return
             }
             const isAppUserPayment = userInvoice.user.user_id !== userInvoice.linkedApplication.owner.user_id
@@ -92,22 +94,23 @@ export default class {
                 if (isAppUserPayment && fee > 0) {
                     await this.storage.userStorage.IncrementUserBalance(userInvoice.linkedApplication.owner.user_id, fee, tx)
                 }
-                await this.triggerPaidCallback(userInvoice.callbackUrl)
+
-            } catch {
+                await this.triggerPaidCallback(log, userInvoice.callbackUrl)
-                //TODO
+                log("paid invoice processed successfully")
+            } catch (err: any) {
+                log("ERROR", "cannot process paid invoice", err.message || "")
             }
         })
     }
 
-    async triggerPaidCallback(url: string) {
+    async triggerPaidCallback(log: PubLogger, url: string) {
-        console.log(url)
         if (!url) {
             return
         }
         try {
             await fetch(url + "&ok=true")
         } catch (err: any) {
-            console.log("error sending cb", err)
+            log("error sending paid callback for invoice", err.message || "")
         }
     }
 }

src/services/main/paymentManager.ts

@@ -5,6 +5,7 @@ import { MainSettings } from './settings.js'
 import { InboundOptionals, defaultInvoiceExpiry } from '../storage/paymentStorage.js'
 import { LightningHandler } from '../lnd/index.js'
 import { Application } from '../storage/entity/Application.js'
+import { getLogger } from '../helpers/logger.js'
 interface UserOperationInfo {
     serial_id: number
     paid_amount: number
@@ -59,7 +60,7 @@ export default class {
         if (!this.settings.lndSettings.mockLnd) {
             throw new Error("mock disabled, cannot set invoice as paid")
         }
-        console.log("setting mock balance...")
+        getLogger({})("setting mock balance...")
         await this.storage.userStorage.UpdateUser(userId, { balance_sats: balance })
     }
 
@@ -81,7 +82,6 @@ export default class {
     }
 
     async lockUserWithMinBalance(userId: string, minBalance: number) {
-        console.log("locking", userId)
         return this.storage.StartTransaction(async tx => {
             const user = await this.storage.userStorage.GetUser(userId, tx)
             if (user.locked) {
@@ -308,7 +308,6 @@ export default class {
             const fromUser = await this.storage.userStorage.GetUser(fromUserId, tx)
             const toUser = await this.storage.userStorage.GetUser(toUserId, tx)
             if (fromUser.balance_sats < amount) {
-                console.log({balance:fromUser.balance_sats, amount})
                 throw new Error("not enough balance to send user to user payment")
             }
             if (!linkedApplication) {

src/services/storage/index.ts

@@ -69,17 +69,14 @@ export default class {
 
     doTransaction(exec: TX) {
         if (this.pendingTx) {
-            throw new Error("cannot start transaction")
+            throw new Error("cannot start DB transaction")
         }
         this.pendingTx = true
-        console.log("starting tx")
         return this.DB.transaction(async tx => {
             try {
                 await exec(tx)
-                console.log("tx done")
                 this.ExecNextInQueue()
             } catch (err) {
-                console.log("tx err")
                 this.ExecNextInQueue()
                 throw err
             }

src/services/storage/userStorage.ts

@@ -13,7 +13,7 @@ export default class {
         if (balance && process.env.ALLOW_BALANCE_MIGRATION !== 'true') {
             throw new Error("balance migration is not allowed")
         }
-        console.log("Adding user with balance", balance)
+        getLogger({})("Adding user with balance", balance)
         const newUser = entityManager.getRepository(User).create({
             user_id: crypto.randomBytes(32).toString('hex'),
             balance_sats: balance
@@ -76,7 +76,6 @@ export default class {
         }
     }
     async UnlockUser(userId: string, entityManager = this.DB) {
-        console.log("unlocking", userId)
         const res = await entityManager.getRepository(User).update({
             user_id: userId
         }, { locked: false })