Merge pull request #694 from shocknet/lsp

lsp

env.example

@@ -30,6 +30,13 @@ LIQUIDITY_PROVIDER_PUB=
 # Will execute when it costs less than 1% of balance and uses a trusted peer
 #BOOTSTRAP=1
 
+#LSP
+OLYMPUS_LSP_URL=https://lsps1.lnolymp.us/api/v1
+VOLTAGE_LSP_URL=https://lsp.voltageapi.com/api/v1
+FLASHSATS_LSP_URL=https://lsp.flashsats.xyz/lsp/channel
+LSP_CHANNEL_THRESHOLD=1000000
+LSP_MAX_FEE_BPS=100
+
 #ROOT_FEES
 # Applied to either debits or credits and sent to an admin account
 # BPS are basis points, 100 BPS = 1%

metricsDatasource.js

@@ -1,10 +1,10 @@
 import { DataSource } from "typeorm"
-import { ChannelRouting } from "./build/src/services/storage/entity/ChannelRouting.js"
+import { LspOrder } from "./build/src/services/storage/entity/LspOrder.js"
 
 
 
 export default new DataSource({
     type: "sqlite",
-    database: "metrics.sqlite",
+    database: "db.sqlite",
-    entities: [ChannelRouting],
+    entities: [LspOrder],
 });

src/services/lnd/index.ts

@@ -7,6 +7,5 @@ export const LoadLndSettingsFromEnv = (): LndSettings => {
     const feeRateLimit = EnvCanBeInteger("OUTBOUND_MAX_FEE_BPS", 60) / 10000
     const feeFixedLimit = EnvCanBeInteger("OUTBOUND_MAX_FEE_EXTRA_SATS", 100)
     const mockLnd = EnvCanBeBoolean("MOCK_LND")
-    const liquidityProviderPub = process.env.LIQUIDITY_PROVIDER_PUB || ""
+    return { mainNode: { lndAddr, lndCertPath, lndMacaroonPath }, feeRateLimit, feeFixedLimit, mockLnd }
-    return { mainNode: { lndAddr, lndCertPath, lndMacaroonPath }, feeRateLimit, feeFixedLimit, mockLnd, liquidityProviderPub, useOnlyLiquidityProvider: false }
 }

src/services/lnd/lnd.ts

@@ -36,7 +36,8 @@ export default class {
     log = getLogger({ component: 'lndManager' })
     outgoingOpsLocked = false
     liquidProvider: LiquidityProvider
-    constructor(settings: LndSettings, liquidProvider: LiquidityProvider, addressPaidCb: AddressPaidCb, invoicePaidCb: InvoicePaidCb, newBlockCb: NewBlockCb, htlcCb: HtlcCb) {
+    useOnlyLiquidityProvider = false
+    constructor(settings: LndSettings, provider: { liquidProvider: LiquidityProvider, useOnly?: boolean }, addressPaidCb: AddressPaidCb, invoicePaidCb: InvoicePaidCb, newBlockCb: NewBlockCb, htlcCb: HtlcCb) {
         this.settings = settings
         this.addressPaidCb = addressPaidCb
         this.invoicePaidCb = invoicePaidCb
@@ -62,7 +63,8 @@ export default class {
         this.invoices = new InvoicesClient(transport)
         this.router = new RouterClient(transport)
         this.chainNotifier = new ChainNotifierClient(transport)
-        this.liquidProvider = liquidProvider
+        this.liquidProvider = provider.liquidProvider
+        this.useOnlyLiquidityProvider = !!provider.useOnly
     }
 
     LockOutgoingOperations(): void {
@@ -81,7 +83,7 @@ export default class {
     }
 
     async ShouldUseLiquidityProvider(req: LiquidityRequest): Promise<boolean> {
-        if (this.settings.useOnlyLiquidityProvider) {
+        if (this.useOnlyLiquidityProvider) {
             return true
         }
         if (!this.liquidProvider.CanProviderHandle(req)) {

src/services/lnd/lsp.ts

@@ -1,29 +1,329 @@
 import fetch from "node-fetch"
+import { LiquidityProvider } from "./liquidityProvider.js"
+import { getLogger, PubLogger } from '../helpers/logger.js'
+import LND from "./lnd.js"
+import { AddressType } from "../../../proto/autogenerated/ts/types.js"
+import { EnvCanBeInteger } from "../helpers/envParser.js"
+export type LSPSettings = {
+    olympusServiceUrl: string
+    voltageServiceUrl: string
+    flashsatsServiceUrl: string
+    channelThreshold: number
+    maxRelativeFee: number
+}
 
-export class LSP {
+export const LoadLSPSettingsFromEnv = (): LSPSettings => {
-    serviceUrl: string
+    const olympusServiceUrl = process.env.OLYMPUS_LSP_URL || "https://lsps1.lnolymp.us/api/v1"
-    constructor(serviceUrl: string) {
+    const voltageServiceUrl = process.env.VOLTAGE_LSP_URL || "https://lsp.voltageapi.com/api/v1"
-        this.serviceUrl = serviceUrl
+    const flashsatsServiceUrl = process.env.FLASHSATS_LSP_URL || "https://lsp.flashsats.xyz/lsp/channel"
+    const channelThreshold = EnvCanBeInteger("LSP_CHANNEL_THRESHOLD", 1000000)
+    const maxRelativeFee = EnvCanBeInteger("LSP_MAX_FEE_BPS", 100) / 10000
+    return { olympusServiceUrl, voltageServiceUrl, channelThreshold, maxRelativeFee, flashsatsServiceUrl }
+
+}
+type OlympusOrder = {
+    "lsp_balance_sat": string,
+    "client_balance_sat": string,
+    "required_channel_confirmations": number,
+    "funding_confirms_within_blocks": number,
+    "channel_expiry_blocks": number,
+    "refund_onchain_address": string,
+    "announce_channel": boolean,
+    "public_key": string
+
+}
+type FlashsatsOrder = {
+    "node_connection_info": string,
+    "lsp_balance_sat": number,
+    "client_balance_sat": number,
+    "confirms_within_blocks": number,
+    "channel_expiry_blocks": number,
+    "announce_channel": boolean,
+    "token": string
+}
+
+type OrderResponse = {
+    orderId: string
+    invoice: string
+    totalSats: number
+    fees: number
+}
+
+class LSP {
+    settings: LSPSettings
+    liquidityProvider: LiquidityProvider
+    lnd: LND
+    log: PubLogger
+    constructor(serviceName: string, settings: LSPSettings, lnd: LND, liquidityProvider: LiquidityProvider) {
+        this.settings = settings
+        this.lnd = lnd
+        this.liquidityProvider = liquidityProvider
+        this.log = getLogger({ component: serviceName })
     }
 
+    shouldOpenChannel = async (): Promise<{ shouldOpen: false } | { shouldOpen: true, maxSpendable: number }> => {
+        if (this.settings.channelThreshold === 0) {
+            this.log("channel threshold is 0")
+            return { shouldOpen: false }
+        }
+        const channels = await this.lnd.ListChannels()
+        if (channels.channels.length > 0) {
+            this.log("this node already has open channels")
+            return { shouldOpen: false }
+        }
+        const pendingChannels = await this.lnd.ListPendingChannels()
+        if (pendingChannels.pendingOpenChannels.length > 0) {
+            this.log("this node already has pending channels")
+            return { shouldOpen: false }
+        }
+        const userState = await this.liquidityProvider.CheckUserState()
+        if (!userState || userState.max_withdrawable < this.settings.channelThreshold) {
+            this.log("balance of", userState?.max_withdrawable || 0, "is lower than channel threshold of", this.settings.channelThreshold)
+            return { shouldOpen: false }
+        }
+        return { shouldOpen: true, maxSpendable: userState.max_withdrawable }
+    }
+
+    addPeer = async (pubKey: string, host: string) => {
+        const { peers } = await this.lnd.ListPeers()
+        if (!peers.find(p => p.pubKey === pubKey)) {
+            await this.lnd.ConnectPeer({ host, pubkey: pubKey })
+        }
+
+    }
+}
+
+export class FlashsatsLSP extends LSP {
+    constructor(settings: LSPSettings, lnd: LND, liquidityProvider: LiquidityProvider) {
+        super("FlashsatsLSP", settings, lnd, liquidityProvider)
+    }
+
+    openChannelIfReady = async (): Promise<OrderResponse | null> => {
+        const shouldOpen = await this.shouldOpenChannel()
+        if (!shouldOpen.shouldOpen) {
+            return null
+        }
+        if (!this.settings.flashsatsServiceUrl) {
+            this.log("no flashsats service url provided")
+            return null
+        }
+        const serviceInfo = await this.getInfo()
+        if (+serviceInfo.options.min_initial_client_balance_sat > shouldOpen.maxSpendable) {
+            this.log("balance of", shouldOpen.maxSpendable, "is lower than service minimum of", serviceInfo.options.min_initial_client_balance_sat)
+            return null
+        }
+        const lndInfo = await this.lnd.GetInfo()
+        const myUri = lndInfo.uris.length > 0 ? lndInfo.uris[0] : ""
+        if (!myUri) {
+            this.log("no uri found for this node,uri is required to use flashsats")
+            return null
+        }
+        const lspBalance = (this.settings.channelThreshold * 2).toString()
+        const chanExpiryBlocks = serviceInfo.options.max_channel_expiry_blocks
+        const order = await this.createOrder({ nodeUri: myUri, lspBalance, clientBalance: "0", chanExpiryBlocks })
+        if (order.payment.state !== 'EXPECT_PAYMENT') {
+            this.log("order not in expect payment state")
+            return null
+        }
+        const decoded = await this.lnd.DecodeInvoice(order.payment.bolt11_invoice)
+        if (decoded.numSatoshis !== +order.payment.order_total_sat) {
+            this.log("invoice of amount", decoded.numSatoshis, "does not match order total of", order.payment.order_total_sat)
+            return null
+        }
+        if (decoded.numSatoshis > shouldOpen.maxSpendable) {
+            this.log("invoice of amount", decoded.numSatoshis, "exceeds user balance of", shouldOpen.maxSpendable)
+            return null
+        }
+        const relativeFee = +order.payment.fee_total_sat / this.settings.channelThreshold
+        if (relativeFee > this.settings.maxRelativeFee) {
+            this.log("invoice relative fee of", relativeFee, "exceeds max relative fee of", this.settings.maxRelativeFee)
+            return null
+        }
+        const res = await this.liquidityProvider.PayInvoice(order.payment.bolt11_invoice)
+        this.log("paid", res.amount_paid, "to open channel")
+        return { orderId: order.order_id, invoice: order.payment.bolt11_invoice, totalSats: +order.payment.order_total_sat, fees: +order.payment.fee_total_sat }
+
+    }
     getInfo = async () => {
-        const res = await fetch(`${this.serviceUrl}/getinfo`)
+        const res = await fetch(`${this.settings.flashsatsServiceUrl}/info`)
-        const json = await res.json() as { options: {}, uris: string[] }
+        const json = await res.json() as { options: { min_initial_client_balance_sat: string, max_channel_expiry_blocks: number } }
+        return json
     }
-
+    createOrder = async (orderInfo: { nodeUri: string, lspBalance: string, clientBalance: string, chanExpiryBlocks: number }) => {
-    createOrder = async (req: { public_key: string }) => {
+        const req: FlashsatsOrder = {
-        const res = await fetch(`${this.serviceUrl}/create_order`, {
+            node_connection_info: orderInfo.nodeUri,
+            announce_channel: true,
+            channel_expiry_blocks: orderInfo.chanExpiryBlocks,
+            client_balance_sat: +orderInfo.clientBalance,
+            lsp_balance_sat: +orderInfo.lspBalance,
+            confirms_within_blocks: 6,
+            token: "flashsats"
+        }
+        const res = await fetch(`${this.settings.flashsatsServiceUrl}/channel`, {
             method: "POST",
             body: JSON.stringify(req),
             headers: { "Content-Type": "application/json" }
         })
-        const json = await res.json() as {}
+        const json = await res.json() as { order_id: string, payment: { state: 'EXPECT_PAYMENT', bolt11_invoice: string, fee_total_sat: string, order_total_sat: string } }
+        return json
+    }
+}
+
+export class OlympusLSP extends LSP {
+    constructor(settings: LSPSettings, lnd: LND, liquidityProvider: LiquidityProvider) {
+        super("OlympusLSP", settings, lnd, liquidityProvider)
+    }
+
+    openChannelIfReady = async (): Promise<OrderResponse | null> => {
+        const shouldOpen = await this.shouldOpenChannel()
+        if (!shouldOpen.shouldOpen) {
+            return null
+        }
+        if (!this.settings.olympusServiceUrl) {
+            this.log("no olympus service url provided")
+            return null
+        }
+        const serviceInfo = await this.getInfo()
+        if (+serviceInfo.options.min_initial_client_balance_sat > shouldOpen.maxSpendable) {
+            this.log("balance of", shouldOpen.maxSpendable, "is lower than service minimum of", serviceInfo.options.min_initial_client_balance_sat)
+            return null
+        }
+        const [servicePub, host] = serviceInfo.uris[0].split('@')
+        await this.addPeer(servicePub, host)
+        const lndInfo = await this.lnd.GetInfo()
+        const myPub = lndInfo.identityPubkey
+        const refundAddr = await this.lnd.NewAddress(AddressType.WITNESS_PUBKEY_HASH)
+        const lspBalance = (this.settings.channelThreshold * 2).toString()
+        const chanExpiryBlocks = serviceInfo.options.max_channel_expiry_blocks
+        const order = await this.createOrder({ pubKey: myPub, refundAddr: refundAddr.address, lspBalance, clientBalance: "0", chanExpiryBlocks })
+        if (order.payment.state !== 'EXPECT_PAYMENT') {
+            this.log("order not in expect payment state")
+            return null
+        }
+        const decoded = await this.lnd.DecodeInvoice(order.payment.bolt11_invoice)
+        if (decoded.numSatoshis !== +order.payment.order_total_sat) {
+            this.log("invoice of amount", decoded.numSatoshis, "does not match order total of", order.payment.order_total_sat)
+            return null
+        }
+        if (decoded.numSatoshis > shouldOpen.maxSpendable) {
+            this.log("invoice of amount", decoded.numSatoshis, "exceeds user balance of", shouldOpen.maxSpendable)
+            return null
+        }
+        const relativeFee = +order.payment.fee_total_sat / this.settings.channelThreshold
+        if (relativeFee > this.settings.maxRelativeFee) {
+            this.log("invoice relative fee of", relativeFee, "exceeds max relative fee of", this.settings.maxRelativeFee)
+            return null
+        }
+        const res = await this.liquidityProvider.PayInvoice(order.payment.bolt11_invoice)
+        this.log("paid", res.amount_paid, "to open channel")
+        return { orderId: order.order_id, invoice: order.payment.bolt11_invoice, totalSats: +order.payment.order_total_sat, fees: +order.payment.fee_total_sat }
+    }
+
+    getInfo = async () => {
+        const res = await fetch(`${this.settings.olympusServiceUrl}/getinfo`)
+        const json = await res.json() as { options: { min_initial_client_balance_sat: string, max_channel_expiry_blocks: number }, uris: string[] }
+        return json
+    }
+
+    createOrder = async (orderInfo: { pubKey: string, refundAddr: string, lspBalance: string, clientBalance: string, chanExpiryBlocks: number }) => {
+        const req: OlympusOrder = {
+            public_key: orderInfo.pubKey,
+            announce_channel: true,
+            refund_onchain_address: orderInfo.refundAddr,
+            lsp_balance_sat: orderInfo.lspBalance,
+            client_balance_sat: orderInfo.clientBalance,
+            channel_expiry_blocks: orderInfo.chanExpiryBlocks,
+            funding_confirms_within_blocks: 6,
+            required_channel_confirmations: 0
+        }
+        const res = await fetch(`${this.settings.olympusServiceUrl}/create_order`, {
+            method: "POST",
+            body: JSON.stringify(req),
+            headers: { "Content-Type": "application/json" }
+        })
+        const json = await res.json() as { order_id: string, payment: { state: 'EXPECT_PAYMENT', bolt11_invoice: string, fee_total_sat: string, order_total_sat: string } }
         return json
     }
 
     getOrder = async (orderId: string) => {
-        const res = await fetch(`${this.serviceUrl}/get_order&order_id=${orderId}`)
+        const res = await fetch(`${this.settings.olympusServiceUrl}/get_order&order_id=${orderId}`)
         const json = await res.json() as {}
         return json
     }
+}
+
+export class VoltageLSP extends LSP {
+    constructor(settings: LSPSettings, lnd: LND, liquidityProvider: LiquidityProvider) {
+        super("VoltageLSP", settings, lnd, liquidityProvider)
+    }
+
+    getInfo = async () => {
+        const res = await fetch(`${this.settings.voltageServiceUrl}/info`)
+        const json = await res.json() as { connection_methods: { address: string, port: string, type: string }[], pubkey: string }
+        return json
+    }
+
+    getFees = async (amtMsat: string, pubkey: string) => {
+        const res = await fetch(`${this.settings.voltageServiceUrl}/fee`, {
+            method: "POST",
+            body: JSON.stringify({ amount_msat: amtMsat, pubkey }),
+            headers: { "Content-Type": "application/json" }
+        })
+        const json = await res.json() as { fee_amount_msat: number, id: string }
+        return json
+    }
+
+    openChannelIfReady = async (): Promise<OrderResponse | null> => {
+        const shouldOpen = await this.shouldOpenChannel()
+        if (!shouldOpen.shouldOpen) {
+            return null
+        }
+
+        if (!this.settings.voltageServiceUrl) {
+            this.log("no voltage service url provided")
+            return null
+        }
+
+        const lndInfo = await this.lnd.GetInfo()
+        const myPub = lndInfo.identityPubkey
+        const amtMsats = this.settings.channelThreshold.toString() + "000"
+        const fee = await this.getFees(amtMsats, myPub)
+        const feeSats = fee.fee_amount_msat / 1000
+        const relativeFee = feeSats / this.settings.channelThreshold
+
+        if (relativeFee > this.settings.maxRelativeFee) {
+            this.log("relative fee of", relativeFee, "exceeds max relative fee of", this.settings.maxRelativeFee)
+            return null
+        }
+
+        const info = await this.getInfo()
+        const ipv4 = info.connection_methods.find(c => c.type === 'ipv4')
+        if (!ipv4) {
+            this.log("no ipv4 address found")
+            return null
+        }
+        await this.addPeer(info.pubkey, `${ipv4.address}:${ipv4.port}`)
+
+        const invoice = await this.lnd.NewInvoice(this.settings.channelThreshold, "open channel", 60 * 60)
+        const res = await this.proposal(invoice.payRequest, fee.id)
+        const decoded = await this.lnd.DecodeInvoice(res.jit_bolt11)
+        if (decoded.numSatoshis !== this.settings.channelThreshold + feeSats) {
+            this.log("invoice of amount", decoded.numSatoshis, "does not match expected amount of", this.settings.channelThreshold + feeSats)
+            return null
+        }
+
+        const invoiceRes = await this.liquidityProvider.PayInvoice(res.jit_bolt11)
+        this.log("paid", invoiceRes.amount_paid, "to open channel")
+        return { orderId: fee.id, invoice: res.jit_bolt11, totalSats: decoded.numSatoshis, fees: feeSats }
+    }
+
+    proposal = async (bolt11: string, feeId: string) => {
+        const res = await fetch(`${this.settings.voltageServiceUrl}/proposal`, {
+            method: "POST",
+            body: JSON.stringify({ bolt11, fee_id: feeId }),
+            headers: { "Content-Type": "application/json" }
+        })
+        const json = await res.json() as { jit_bolt11: string }
+        return json
+    }
 }

src/services/lnd/settings.ts

@@ -9,8 +9,6 @@ export type LndSettings = {
     feeRateLimit: number
     feeFixedLimit: number
     mockLnd: boolean
-    liquidityProviderPub: string
-    useOnlyLiquidityProvider: boolean
 
     otherNode?: NodeSettings
     thirdNode?: NodeSettings

src/services/main/index.ts

@@ -16,6 +16,7 @@ import { NostrSend } from '../nostr/handler.js'
 import MetricsManager from '../metrics/index.js'
 import { LoggedEvent } from '../storage/eventsLog.js'
 import { LiquidityProvider } from "../lnd/liquidityProvider.js"
+import { LiquidityManager } from "./liquidityManager.js"
 
 type UserOperationsSub = {
     id: string
@@ -37,20 +38,23 @@ export default class {
     paymentSubs: Record<string, ((op: Types.UserOperation) => void) | null> = {}
     metricsManager: MetricsManager
     liquidProvider: LiquidityProvider
+    liquidityManager: LiquidityManager
     nostrSend: NostrSend = () => { getLogger({})("nostr send not initialized yet") }
     constructor(settings: MainSettings, storage: Storage) {
         this.settings = settings
         this.storage = storage
-        this.liquidProvider = new LiquidityProvider(settings.lndSettings.liquidityProviderPub, this.invoicePaidCb)
+        this.liquidProvider = new LiquidityProvider(settings.liquiditySettings.liquidityProviderPub, this.invoicePaidCb)
-        this.lnd = new LND(settings.lndSettings, this.liquidProvider, this.addressPaidCb, this.invoicePaidCb, this.newBlockCb, this.htlcCb)
+        const provider = { liquidProvider: this.liquidProvider, useOnly: settings.liquiditySettings.useOnlyLiquidityProvider }
+        this.lnd = new LND(settings.lndSettings, provider, this.addressPaidCb, this.invoicePaidCb, this.newBlockCb, this.htlcCb)
+        this.liquidityManager = new LiquidityManager(this.settings.liquiditySettings, this.storage, this.liquidProvider, this.lnd)
         this.metricsManager = new MetricsManager(this.storage, this.lnd)
 
         this.paymentManager = new PaymentManager(this.storage, this.lnd, this.settings, this.addressPaidCb, this.invoicePaidCb)
         this.productManager = new ProductManager(this.storage, this.paymentManager, this.settings)
         this.applicationManager = new ApplicationManager(this.storage, this.settings, this.paymentManager)
         this.appUserManager = new AppUserManager(this.storage, this.settings, this.applicationManager)
-
     }
+
     Stop() {
         this.lnd.Stop()
         this.applicationManager.Stop()
@@ -187,6 +191,7 @@ export default class {
                 this.sendOperationToNostr(userInvoice.linkedApplication, userInvoice.user.user_id, op)
                 this.createZapReceipt(log, userInvoice)
                 log("paid invoice processed successfully")
+                this.liquidityManager.afterInInvoicePaid()
             } catch (err: any) {
                 log(ERROR, "cannot process paid invoice", err.message || "")
             }

src/services/main/liquidityManager.ts

@@ -0,0 +1,72 @@
+import { getLogger } from "../helpers/logger.js"
+import { LiquidityProvider } from "../lnd/liquidityProvider.js"
+import LND from "../lnd/lnd.js"
+import { FlashsatsLSP, LoadLSPSettingsFromEnv, LSPSettings, OlympusLSP, VoltageLSP } from "../lnd/lsp.js"
+import Storage from '../storage/index.js'
+export type LiquiditySettings = {
+    lspSettings: LSPSettings
+    liquidityProviderPub: string
+    useOnlyLiquidityProvider: boolean
+}
+export const LoadLiquiditySettingsFromEnv = (): LiquiditySettings => {
+    const lspSettings = LoadLSPSettingsFromEnv()
+    const liquidityProviderPub = process.env.LIQUIDITY_PROVIDER_PUB || ""
+    return { lspSettings, liquidityProviderPub, useOnlyLiquidityProvider: false }
+}
+export class LiquidityManager {
+    settings: LiquiditySettings
+    storage: Storage
+    liquidityProvider: LiquidityProvider
+    lnd: LND
+    olympusLSP: OlympusLSP
+    voltageLSP: VoltageLSP
+    flashsatsLSP: FlashsatsLSP
+    log = getLogger({ component: "liquidityManager" })
+    channelRequested = false
+    constructor(settings: LiquiditySettings, storage: Storage, liquidityProvider: LiquidityProvider, lnd: LND) {
+        this.settings = settings
+        this.storage = storage
+        this.liquidityProvider = liquidityProvider
+        this.lnd = lnd
+        this.olympusLSP = new OlympusLSP(settings.lspSettings, lnd, liquidityProvider)
+        this.voltageLSP = new VoltageLSP(settings.lspSettings, lnd, liquidityProvider)
+        this.flashsatsLSP = new FlashsatsLSP(settings.lspSettings, lnd, liquidityProvider)
+    }
+    beforeInvoiceCreation = async () => { }
+    afterInInvoicePaid = async () => {
+        const existingOrder = await this.storage.liquidityStorage.GetLatestLspOrder()
+        if (existingOrder) {
+            return
+        }
+        if (this.channelRequested) {
+            return
+        }
+        this.log("checking if channel should be requested")
+        const olympusOk = await this.olympusLSP.openChannelIfReady()
+        if (olympusOk) {
+            this.log("requested channel from olympus")
+            this.channelRequested = true
+            await this.storage.liquidityStorage.SaveLspOrder({ service_name: 'olympus', invoice: olympusOk.invoice, total_paid: olympusOk.totalSats, order_id: olympusOk.orderId, fees: olympusOk.fees })
+            return
+        }
+        const voltageOk = await this.voltageLSP.openChannelIfReady()
+        if (voltageOk) {
+            this.log("requested channel from voltage")
+            this.channelRequested = true
+            await this.storage.liquidityStorage.SaveLspOrder({ service_name: 'voltage', invoice: voltageOk.invoice, total_paid: voltageOk.totalSats, order_id: voltageOk.orderId, fees: voltageOk.fees })
+            return
+        }
+
+        const flashsatsOk = await this.flashsatsLSP.openChannelIfReady()
+        if (flashsatsOk) {
+            this.log("requested channel from flashsats")
+            this.channelRequested = true
+            await this.storage.liquidityStorage.SaveLspOrder({ service_name: 'flashsats', invoice: flashsatsOk.invoice, total_paid: flashsatsOk.totalSats, order_id: flashsatsOk.orderId, fees: flashsatsOk.fees })
+            return
+        }
+        this.log("no channel requested")
+    }
+
+    beforeOutInvoicePayment = async () => { }
+    afterOutInvoicePaid = async () => { }
+}

src/services/main/settings.ts

@@ -6,10 +6,12 @@ import { EnvCanBeInteger, EnvMustBeInteger, EnvMustBeNonEmptyString } from '../h
 import { getLogger } from '../helpers/logger.js'
 import fs from 'fs'
 import crypto from 'crypto';
+import { LiquiditySettings, LoadLiquiditySettingsFromEnv } from './liquidityManager.js'
 export type MainSettings = {
     storageSettings: StorageSettings,
     lndSettings: LndSettings,
     watchDogSettings: WatchdogSettings,
+    liquiditySettings: LiquiditySettings,
     jwtSecret: string
     incomingTxFee: number
     outgoingTxFee: number
@@ -32,11 +34,13 @@ export type BitcoinCoreSettings = {
 }
 export type TestSettings = MainSettings & { lndSettings: { otherNode: NodeSettings, thirdNode: NodeSettings, fourthNode: NodeSettings }, bitcoinCoreSettings: BitcoinCoreSettings }
 export const LoadMainSettingsFromEnv = (): MainSettings => {
+    const storageSettings = LoadStorageSettingsFromEnv()
     return {
         watchDogSettings: LoadWatchdogSettingsFromEnv(),
         lndSettings: LoadLndSettingsFromEnv(),
-        storageSettings: LoadStorageSettingsFromEnv(),
+        storageSettings: storageSettings,
-        jwtSecret: loadJwtSecret(),
+        liquiditySettings: LoadLiquiditySettingsFromEnv(),
+        jwtSecret: loadJwtSecret(storageSettings.dataDir),
         incomingTxFee: EnvCanBeInteger("INCOMING_CHAIN_FEE_ROOT_BPS", 0) / 10000,
         outgoingTxFee: EnvCanBeInteger("OUTGOING_CHAIN_FEE_ROOT_BPS", 60) / 10000,
         incomingAppInvoiceFee: EnvCanBeInteger("INCOMING_INVOICE_FEE_ROOT_BPS", 0) / 10000,
@@ -58,7 +62,7 @@ export const LoadTestSettingsFromEnv = (): TestSettings => {
     const settings = LoadMainSettingsFromEnv()
     return {
         ...settings,
-        storageSettings: { dbSettings: { ...settings.storageSettings.dbSettings, databaseFile: ":memory:", metricsDatabaseFile: ":memory:" }, eventLogPath },
+        storageSettings: { dbSettings: { ...settings.storageSettings.dbSettings, databaseFile: ":memory:", metricsDatabaseFile: ":memory:" }, eventLogPath, dataDir: "data" },
         lndSettings: {
             ...settings.lndSettings,
             otherNode: {
@@ -76,7 +80,10 @@ export const LoadTestSettingsFromEnv = (): TestSettings => {
                 lndCertPath: EnvMustBeNonEmptyString("LND_FOURTH_CERT_PATH"),
                 lndMacaroonPath: EnvMustBeNonEmptyString("LND_FOURTH_MACAROON_PATH")
             },
-            liquidityProviderPub: ""
+        },
+        liquiditySettings: {
+            ...settings.liquiditySettings,
+            liquidityProviderPub: "",
         },
         skipSanityCheck: true,
         bitcoinCoreSettings: {
@@ -87,20 +94,21 @@ export const LoadTestSettingsFromEnv = (): TestSettings => {
     }
 }
 
-export const loadJwtSecret = (): string => {
+export const loadJwtSecret = (dataDir: string): string => {
     const secret = process.env["JWT_SECRET"]
     const log = getLogger({})
     if (secret) {
         return secret
     }
     log("JWT_SECRET not set in env, checking .jwt_secret file")
+    const secretPath = dataDir !== "" ? `${dataDir}/.jwt_secret` : ".jwt_secret"
     try {
-        const fileContent = fs.readFileSync(".jwt_secret", "utf-8")
+        const fileContent = fs.readFileSync(secretPath, "utf-8")
         return fileContent.trim()
     } catch (e) {
         log(".jwt_secret file not found, generating random secret")
         const secret = crypto.randomBytes(32).toString('hex')
-        fs.writeFileSync(".jwt_secret", secret)
+        fs.writeFileSync(secretPath, secret)
         return secret
     }
 }

src/services/storage/db.ts

@@ -17,6 +17,7 @@ import { BalanceEvent } from "./entity/BalanceEvent.js"
 import { ChannelBalanceEvent } from "./entity/ChannelsBalanceEvent.js"
 import { getLogger } from "../helpers/logger.js"
 import { ChannelRouting } from "./entity/ChannelRouting.js"
+import { LspOrder } from "./entity/LspOrder.js"
 
 
 export type DbSettings = {
@@ -56,7 +57,7 @@ export default async (settings: DbSettings, migrations: Function[]): Promise<{ s
         database: settings.databaseFile,
         // logging: true,
         entities: [User, UserReceivingInvoice, UserReceivingAddress, AddressReceivingTransaction, UserInvoicePayment, UserTransactionPayment,
-            UserBasicAuth, UserEphemeralKey, Product, UserToUserPayment, Application, ApplicationUser, UserToUserPayment],
+            UserBasicAuth, UserEphemeralKey, Product, UserToUserPayment, Application, ApplicationUser, UserToUserPayment, LspOrder],
         //synchronize: true,
         migrations
     }).initialize()

src/services/storage/entity/LspOrder.ts

@@ -0,0 +1,28 @@
+import { Entity, PrimaryGeneratedColumn, Column, CreateDateColumn, UpdateDateColumn } from "typeorm"
+
+@Entity()
+export class LspOrder {
+    @PrimaryGeneratedColumn()
+    serial_id: number
+
+    @Column()
+    service_name: string
+
+    @Column()
+    invoice: string
+
+    @Column()
+    order_id: string
+
+    @Column()
+    total_paid: number
+
+    @Column()
+    fees: number
+
+    @CreateDateColumn()
+    created_at: Date
+
+    @UpdateDateColumn()
+    updated_at: Date
+}

src/services/storage/index.ts

@@ -7,12 +7,14 @@ import PaymentStorage from "./paymentStorage.js";
 import MetricsStorage from "./metricsStorage.js";
 import TransactionsQueue, { TX } from "./transactionsQueue.js";
 import EventsLogManager from "./eventsLog.js";
+import { LiquidityStorage } from "./liquidityStorage.js";
 export type StorageSettings = {
     dbSettings: DbSettings
     eventLogPath: string
+    dataDir: string
 }
 export const LoadStorageSettingsFromEnv = (): StorageSettings => {
-    return { dbSettings: LoadDbSettingsFromEnv(), eventLogPath: "logs/eventLogV2.csv" }
+    return { dbSettings: LoadDbSettingsFromEnv(), eventLogPath: "logs/eventLogV2.csv", dataDir: process.env.DATA_DIR || "" }
 }
 export default class {
     DB: DataSource | EntityManager
@@ -23,6 +25,7 @@ export default class {
     userStorage: UserStorage
     paymentStorage: PaymentStorage
     metricsStorage: MetricsStorage
+    liquidityStorage: LiquidityStorage
     eventsLog: EventsLogManager
     constructor(settings: StorageSettings) {
         this.settings = settings
@@ -37,6 +40,7 @@ export default class {
         this.applicationStorage = new ApplicationStorage(this.DB, this.userStorage, this.txQueue)
         this.paymentStorage = new PaymentStorage(this.DB, this.userStorage, this.txQueue)
         this.metricsStorage = new MetricsStorage(this.settings)
+        this.liquidityStorage = new LiquidityStorage(this.DB, this.txQueue)
         const executedMetricsMigrations = await this.metricsStorage.Connect(metricsMigrations)
         return { executedMigrations, executedMetricsMigrations };
     }

src/services/storage/liquidityStorage.ts

@@ -0,0 +1,20 @@
+import { DataSource, EntityManager, MoreThan } from "typeorm"
+import { LspOrder } from "./entity/LspOrder.js";
+import TransactionsQueue, { TX } from "./transactionsQueue.js";
+export class LiquidityStorage {
+    DB: DataSource | EntityManager
+    txQueue: TransactionsQueue
+    constructor(DB: DataSource | EntityManager, txQueue: TransactionsQueue) {
+        this.DB = DB
+        this.txQueue = txQueue
+    }
+
+    GetLatestLspOrder() {
+        return this.DB.getRepository(LspOrder).findOne({ where: { serial_id: MoreThan(0) }, order: { serial_id: "DESC" } })
+    }
+
+    SaveLspOrder(order: Partial<LspOrder>) {
+        const entry = this.DB.getRepository(LspOrder).create(order)
+        return this.txQueue.PushToQueue<LspOrder>({ exec: async db => db.getRepository(LspOrder).save(entry), dbTx: false })
+    }
+}

src/services/storage/migrations/1718387847693-lsp_order.ts

@@ -0,0 +1,14 @@
+import { MigrationInterface, QueryRunner } from "typeorm";
+
+export class LspOrder1718387847693 implements MigrationInterface {
+    name = 'LspOrder1718387847693'
+
+    public async up(queryRunner: QueryRunner): Promise<void> {
+        await queryRunner.query(`CREATE TABLE "lsp_order" ("serial_id" integer PRIMARY KEY AUTOINCREMENT NOT NULL, "service_name" varchar NOT NULL, "invoice" varchar NOT NULL, "order_id" varchar NOT NULL, "total_paid" integer NOT NULL, "fees" integer NOT NULL, "created_at" datetime NOT NULL DEFAULT (datetime('now')), "updated_at" datetime NOT NULL DEFAULT (datetime('now')))`);
+    }
+
+    public async down(queryRunner: QueryRunner): Promise<void> {
+        await queryRunner.query(`DROP TABLE "lsp_order"`);
+    }
+
+}

src/services/storage/migrations/runner.ts

@@ -4,7 +4,8 @@ import Storage, { StorageSettings } from '../index.js'
 import { Initial1703170309875 } from './1703170309875-initial.js'
 import { LndMetrics1703170330183 } from './1703170330183-lnd_metrics.js'
 import { ChannelRouting1709316653538 } from './1709316653538-channel_routing.js'
-const allMigrations = [Initial1703170309875]
+import { LspOrder1718387847693 } from './1718387847693-lsp_order.js'
+const allMigrations = [Initial1703170309875, LspOrder1718387847693]
 const allMetricsMigrations = [LndMetrics1703170330183, ChannelRouting1709316653538]
 export const TypeOrmMigrationRunner = async (log: PubLogger, storageManager: Storage, settings: DbSettings, arg: string | undefined): Promise<boolean> => {
     if (arg === 'fake_initial_migration') {

src/tests/liquidityProvider.spec.ts

@@ -24,14 +24,16 @@ const testInboundPaymentFromProvider = async (T: TestBase, bootstrapped: Main, b
     const invoiceRes = await bootstrapped.appUserManager.NewInvoice({ app_id: bUser.appId, user_id: bUser.userId, app_user_id: bUser.appUserIdentifier }, { amountSats: 2000, memo: "liquidityTest" })
 
     await T.externalAccessToOtherLnd.PayInvoice(invoiceRes.invoice, 0, 100)
+    await new Promise((resolve) => setTimeout(resolve, 200))
     const userBalance = await bootstrapped.appUserManager.GetUserInfo({ app_id: bUser.appId, user_id: bUser.userId, app_user_id: bUser.appUserIdentifier })
     T.expect(userBalance.balance).to.equal(2000)
-
+    T.d("user balance is 2000")
     const providerBalance = await bootstrapped.liquidProvider.CheckUserState()
     if (!providerBalance) {
         throw new Error("provider balance not found")
     }
     T.expect(providerBalance.balance).to.equal(2000)
+    T.d("provider balance is 2000")
     T.d("testInboundPaymentFromProvider done")
 }
 

src/tests/networkSetup.ts

@@ -8,8 +8,8 @@ export const setupNetwork = async () => {
     const core = new BitcoinCoreWrapper(settings)
     await core.InitAddress()
     await core.Mine(1)
-    const alice = new LND(settings.lndSettings, new LiquidityProvider("", () => { }), () => { }, () => { }, () => { }, () => { })
+    const alice = new LND(settings.lndSettings, { liquidProvider: new LiquidityProvider("", () => { }) }, () => { }, () => { }, () => { }, () => { })
-    const bob = new LND({ ...settings.lndSettings, mainNode: settings.lndSettings.otherNode }, new LiquidityProvider("", () => { }), () => { }, () => { }, () => { }, () => { })
+    const bob = new LND({ ...settings.lndSettings, mainNode: settings.lndSettings.otherNode }, { liquidProvider: new LiquidityProvider("", () => { }) }, () => { }, () => { }, () => { }, () => { })
     await tryUntil<void>(async i => {
         const peers = await alice.ListPeers()
         if (peers.peers.length > 0) {

src/tests/setupBootstrapped.ts

@@ -7,8 +7,8 @@ import * as Types from '../../proto/autogenerated/ts/types.js'
 
 export const initBootstrappedInstance = async (T: TestBase) => {
     const settings = LoadTestSettingsFromEnv()
-    settings.lndSettings.useOnlyLiquidityProvider = true
+    settings.liquiditySettings.useOnlyLiquidityProvider = true
-    settings.lndSettings.liquidityProviderPub = T.app.publicKey
+    settings.liquiditySettings.liquidityProviderPub = T.app.publicKey
     settings.lndSettings.mainNode = settings.lndSettings.thirdNode
     const initialized = await initMainHandler(getLogger({ component: "bootstrapped" }), settings)
     if (!initialized) {

src/tests/testBase.ts

@@ -46,15 +46,15 @@ export const SetupTest = async (d: Describe): Promise<TestBase> => {
     const user2 = { userId: u2.info.userId, appUserIdentifier: u2.identifier, appId: app.appId }
 
 
-    const externalAccessToMainLnd = new LND(settings.lndSettings, new LiquidityProvider("", () => { }), console.log, console.log, () => { }, () => { })
+    const externalAccessToMainLnd = new LND(settings.lndSettings, { liquidProvider: new LiquidityProvider("", () => { }) }, console.log, console.log, () => { }, () => { })
     await externalAccessToMainLnd.Warmup()
 
     const otherLndSetting = { ...settings.lndSettings, mainNode: settings.lndSettings.otherNode }
-    const externalAccessToOtherLnd = new LND(otherLndSetting, new LiquidityProvider("", () => { }), console.log, console.log, () => { }, () => { })
+    const externalAccessToOtherLnd = new LND(otherLndSetting, { liquidProvider: new LiquidityProvider("", () => { }) }, console.log, console.log, () => { }, () => { })
     await externalAccessToOtherLnd.Warmup()
 
     const thirdLndSetting = { ...settings.lndSettings, mainNode: settings.lndSettings.thirdNode }
-    const externalAccessToThirdLnd = new LND(thirdLndSetting, new LiquidityProvider("", () => { }), console.log, console.log, () => { }, () => { })
+    const externalAccessToThirdLnd = new LND(thirdLndSetting, { liquidProvider: new LiquidityProvider("", () => { }) }, console.log, console.log, () => { }, () => { })
     await externalAccessToThirdLnd.Warmup()