aiolabs/lightning-pub
2
0
Fork 0
Code Issues 9 Pull requests 4 Projects Releases Packages Wiki Activity Actions

Merge pull request #694 from shocknet/lsp

Browse source 
lsp
This commit is contained in:
Justin (shocknet) 2024-06-17 11:54:45 -04:00 • committed by GitHub
commit be98507ff8
19 changed files with 503 additions and 42 deletions
Download patch file Download diff file Expand all files Collapse all files

7
env.example
View file

@ -30,6 +30,13 @@ LIQUIDITY_PROVIDER_PUB=
# Will execute when it costs less than 1% of balance and uses a trusted peer
#BOOTSTRAP=1
#LSP
OLYMPUS_LSP_URL=https://lsps1.lnolymp.us/api/v1
VOLTAGE_LSP_URL=https://lsp.voltageapi.com/api/v1
FLASHSATS_LSP_URL=https://lsp.flashsats.xyz/lsp/channel
LSP_CHANNEL_THRESHOLD=1000000
LSP_MAX_FEE_BPS=100
#ROOT_FEES
# Applied to either debits or credits and sent to an admin account
# BPS are basis points, 100 BPS = 1%

6
metricsDatasource.js
View file

@ -1,10 +1,10 @@
import { DataSource } from "typeorm"
import { ChannelRouting } from "./build/src/services/storage/entity/ChannelRouting.js"
import { LspOrder } from "./build/src/services/storage/entity/LspOrder.js"
export default new DataSource({
type: "sqlite",
database: "metrics.sqlite",
entities: [ChannelRouting],
database: "db.sqlite",
entities: [LspOrder],
});

3
src/services/lnd/index.ts
View file

@ -7,6 +7,5 @@ export const LoadLndSettingsFromEnv = (): LndSettings => {
const feeRateLimit = EnvCanBeInteger("OUTBOUND_MAX_FEE_BPS", 60) / 10000
const feeFixedLimit = EnvCanBeInteger("OUTBOUND_MAX_FEE_EXTRA_SATS", 100)
const mockLnd = EnvCanBeBoolean("MOCK_LND")
const liquidityProviderPub = process.env.LIQUIDITY_PROVIDER_PUB || ""
return { mainNode: { lndAddr, lndCertPath, lndMacaroonPath }, feeRateLimit, feeFixedLimit, mockLnd, liquidityProviderPub, useOnlyLiquidityProvider: false }
return { mainNode: { lndAddr, lndCertPath, lndMacaroonPath }, feeRateLimit, feeFixedLimit, mockLnd }
}

8
src/services/lnd/lnd.ts
View file

@ -36,7 +36,8 @@ export default class {
log = getLogger({ component: 'lndManager' })
outgoingOpsLocked = false
liquidProvider: LiquidityProvider
constructor(settings: LndSettings, liquidProvider: LiquidityProvider, addressPaidCb: AddressPaidCb, invoicePaidCb: InvoicePaidCb, newBlockCb: NewBlockCb, htlcCb: HtlcCb) {
useOnlyLiquidityProvider = false
constructor(settings: LndSettings, provider: { liquidProvider: LiquidityProvider, useOnly?: boolean }, addressPaidCb: AddressPaidCb, invoicePaidCb: InvoicePaidCb, newBlockCb: NewBlockCb, htlcCb: HtlcCb) {
this.settings = settings
this.addressPaidCb = addressPaidCb
this.invoicePaidCb = invoicePaidCb
@ -62,7 +63,8 @@ export default class {
this.invoices = new InvoicesClient(transport)
this.router = new RouterClient(transport)
this.chainNotifier = new ChainNotifierClient(transport)
this.liquidProvider = liquidProvider
this.liquidProvider = provider.liquidProvider
this.useOnlyLiquidityProvider = !!provider.useOnly
}
LockOutgoingOperations(): void {
@ -81,7 +83,7 @@ export default class {
}
async ShouldUseLiquidityProvider(req: LiquidityRequest): Promise<boolean> {
if (this.settings.useOnlyLiquidityProvider) {
if (this.useOnlyLiquidityProvider) {
return true
}
if (!this.liquidProvider.CanProviderHandle(req)) {

322
src/services/lnd/lsp.ts
View file

@ -1,29 +1,329 @@
import fetch from "node-fetch"
import { LiquidityProvider } from "./liquidityProvider.js"
import { getLogger, PubLogger } from '../helpers/logger.js'
import LND from "./lnd.js"
import { AddressType } from "../../../proto/autogenerated/ts/types.js"
import { EnvCanBeInteger } from "../helpers/envParser.js"
export type LSPSettings = {
olympusServiceUrl: string
voltageServiceUrl: string
flashsatsServiceUrl: string
channelThreshold: number
maxRelativeFee: number
}
export class LSP {
serviceUrl: string
constructor(serviceUrl: string) {
this.serviceUrl = serviceUrl
export const LoadLSPSettingsFromEnv = (): LSPSettings => {
const olympusServiceUrl = process.env.OLYMPUS_LSP_URL || "https://lsps1.lnolymp.us/api/v1"
const voltageServiceUrl = process.env.VOLTAGE_LSP_URL || "https://lsp.voltageapi.com/api/v1"
const flashsatsServiceUrl = process.env.FLASHSATS_LSP_URL || "https://lsp.flashsats.xyz/lsp/channel"
const channelThreshold = EnvCanBeInteger("LSP_CHANNEL_THRESHOLD", 1000000)
const maxRelativeFee = EnvCanBeInteger("LSP_MAX_FEE_BPS", 100) / 10000
return { olympusServiceUrl, voltageServiceUrl, channelThreshold, maxRelativeFee, flashsatsServiceUrl }
}
type OlympusOrder = {
"lsp_balance_sat": string,
"client_balance_sat": string,
"required_channel_confirmations": number,
"funding_confirms_within_blocks": number,
"channel_expiry_blocks": number,
"refund_onchain_address": string,
"announce_channel": boolean,
"public_key": string
}
type FlashsatsOrder = {
"node_connection_info": string,
"lsp_balance_sat": number,
"client_balance_sat": number,
"confirms_within_blocks": number,
"channel_expiry_blocks": number,
"announce_channel": boolean,
"token": string
}
type OrderResponse = {
orderId: string
invoice: string
totalSats: number
fees: number
}
class LSP {
settings: LSPSettings
liquidityProvider: LiquidityProvider
lnd: LND
log: PubLogger
constructor(serviceName: string, settings: LSPSettings, lnd: LND, liquidityProvider: LiquidityProvider) {
this.settings = settings
this.lnd = lnd
this.liquidityProvider = liquidityProvider
this.log = getLogger({ component: serviceName })
}
shouldOpenChannel = async (): Promise<{ shouldOpen: false } | { shouldOpen: true, maxSpendable: number }> => {
if (this.settings.channelThreshold === 0) {
this.log("channel threshold is 0")
return { shouldOpen: false }
}
const channels = await this.lnd.ListChannels()
if (channels.channels.length > 0) {
this.log("this node already has open channels")
return { shouldOpen: false }
}
const pendingChannels = await this.lnd.ListPendingChannels()
if (pendingChannels.pendingOpenChannels.length > 0) {
this.log("this node already has pending channels")
return { shouldOpen: false }
}
const userState = await this.liquidityProvider.CheckUserState()
if (!userState || userState.max_withdrawable < this.settings.channelThreshold) {
this.log("balance of", userState?.max_withdrawable || 0, "is lower than channel threshold of", this.settings.channelThreshold)
return { shouldOpen: false }
}
return { shouldOpen: true, maxSpendable: userState.max_withdrawable }
}
addPeer = async (pubKey: string, host: string) => {
const { peers } = await this.lnd.ListPeers()
if (!peers.find(p => p.pubKey === pubKey)) {
await this.lnd.ConnectPeer({ host, pubkey: pubKey })
}
}
}
export class FlashsatsLSP extends LSP {
constructor(settings: LSPSettings, lnd: LND, liquidityProvider: LiquidityProvider) {
super("FlashsatsLSP", settings, lnd, liquidityProvider)
}
openChannelIfReady = async (): Promise<OrderResponse | null> => {
const shouldOpen = await this.shouldOpenChannel()
if (!shouldOpen.shouldOpen) {
return null
}
if (!this.settings.flashsatsServiceUrl) {
this.log("no flashsats service url provided")
return null
}
const serviceInfo = await this.getInfo()
if (+serviceInfo.options.min_initial_client_balance_sat > shouldOpen.maxSpendable) {
this.log("balance of", shouldOpen.maxSpendable, "is lower than service minimum of", serviceInfo.options.min_initial_client_balance_sat)
return null
}
const lndInfo = await this.lnd.GetInfo()
const myUri = lndInfo.uris.length > 0 ? lndInfo.uris[0] : ""
if (!myUri) {
this.log("no uri found for this node,uri is required to use flashsats")
return null
}
const lspBalance = (this.settings.channelThreshold * 2).toString()
const chanExpiryBlocks = serviceInfo.options.max_channel_expiry_blocks
const order = await this.createOrder({ nodeUri: myUri, lspBalance, clientBalance: "0", chanExpiryBlocks })
if (order.payment.state !== 'EXPECT_PAYMENT') {
this.log("order not in expect payment state")
return null
}
const decoded = await this.lnd.DecodeInvoice(order.payment.bolt11_invoice)
if (decoded.numSatoshis !== +order.payment.order_total_sat) {
this.log("invoice of amount", decoded.numSatoshis, "does not match order total of", order.payment.order_total_sat)
return null
}
if (decoded.numSatoshis > shouldOpen.maxSpendable) {
this.log("invoice of amount", decoded.numSatoshis, "exceeds user balance of", shouldOpen.maxSpendable)
return null
}
const relativeFee = +order.payment.fee_total_sat / this.settings.channelThreshold
if (relativeFee > this.settings.maxRelativeFee) {
this.log("invoice relative fee of", relativeFee, "exceeds max relative fee of", this.settings.maxRelativeFee)
return null
}
const res = await this.liquidityProvider.PayInvoice(order.payment.bolt11_invoice)
this.log("paid", res.amount_paid, "to open channel")
return { orderId: order.order_id, invoice: order.payment.bolt11_invoice, totalSats: +order.payment.order_total_sat, fees: +order.payment.fee_total_sat }
}
getInfo = async () => {
const res = await fetch(`${this.serviceUrl}/getinfo`)
const json = await res.json() as { options: {}, uris: string[] }
const res = await fetch(`${this.settings.flashsatsServiceUrl}/info`)
const json = await res.json() as { options: { min_initial_client_balance_sat: string, max_channel_expiry_blocks: number } }
return json
}
createOrder = async (req: { public_key: string }) => {
const res = await fetch(`${this.serviceUrl}/create_order`, {
createOrder = async (orderInfo: { nodeUri: string, lspBalance: string, clientBalance: string, chanExpiryBlocks: number }) => {
const req: FlashsatsOrder = {
node_connection_info: orderInfo.nodeUri,
announce_channel: true,
channel_expiry_blocks: orderInfo.chanExpiryBlocks,
client_balance_sat: +orderInfo.clientBalance,
lsp_balance_sat: +orderInfo.lspBalance,
confirms_within_blocks: 6,
token: "flashsats"
}
const res = await fetch(`${this.settings.flashsatsServiceUrl}/channel`, {
method: "POST",
body: JSON.stringify(req),
headers: { "Content-Type": "application/json" }
})
const json = await res.json() as {}
const json = await res.json() as { order_id: string, payment: { state: 'EXPECT_PAYMENT', bolt11_invoice: string, fee_total_sat: string, order_total_sat: string } }
return json
}
}
export class OlympusLSP extends LSP {
constructor(settings: LSPSettings, lnd: LND, liquidityProvider: LiquidityProvider) {
super("OlympusLSP", settings, lnd, liquidityProvider)
}
openChannelIfReady = async (): Promise<OrderResponse | null> => {
const shouldOpen = await this.shouldOpenChannel()
if (!shouldOpen.shouldOpen) {
return null
}
if (!this.settings.olympusServiceUrl) {
this.log("no olympus service url provided")
return null
}
const serviceInfo = await this.getInfo()
if (+serviceInfo.options.min_initial_client_balance_sat > shouldOpen.maxSpendable) {
this.log("balance of", shouldOpen.maxSpendable, "is lower than service minimum of", serviceInfo.options.min_initial_client_balance_sat)
return null
}
const [servicePub, host] = serviceInfo.uris[0].split('@')
await this.addPeer(servicePub, host)
const lndInfo = await this.lnd.GetInfo()
const myPub = lndInfo.identityPubkey
const refundAddr = await this.lnd.NewAddress(AddressType.WITNESS_PUBKEY_HASH)
const lspBalance = (this.settings.channelThreshold * 2).toString()
const chanExpiryBlocks = serviceInfo.options.max_channel_expiry_blocks
const order = await this.createOrder({ pubKey: myPub, refundAddr: refundAddr.address, lspBalance, clientBalance: "0", chanExpiryBlocks })
if (order.payment.state !== 'EXPECT_PAYMENT') {
this.log("order not in expect payment state")
return null
}
const decoded = await this.lnd.DecodeInvoice(order.payment.bolt11_invoice)
if (decoded.numSatoshis !== +order.payment.order_total_sat) {
this.log("invoice of amount", decoded.numSatoshis, "does not match order total of", order.payment.order_total_sat)
return null
}
if (decoded.numSatoshis > shouldOpen.maxSpendable) {
this.log("invoice of amount", decoded.numSatoshis, "exceeds user balance of", shouldOpen.maxSpendable)
return null
}
const relativeFee = +order.payment.fee_total_sat / this.settings.channelThreshold
if (relativeFee > this.settings.maxRelativeFee) {
this.log("invoice relative fee of", relativeFee, "exceeds max relative fee of", this.settings.maxRelativeFee)
return null
}
const res = await this.liquidityProvider.PayInvoice(order.payment.bolt11_invoice)
this.log("paid", res.amount_paid, "to open channel")
return { orderId: order.order_id, invoice: order.payment.bolt11_invoice, totalSats: +order.payment.order_total_sat, fees: +order.payment.fee_total_sat }
}
getInfo = async () => {
const res = await fetch(`${this.settings.olympusServiceUrl}/getinfo`)
const json = await res.json() as { options: { min_initial_client_balance_sat: string, max_channel_expiry_blocks: number }, uris: string[] }
return json
}
createOrder = async (orderInfo: { pubKey: string, refundAddr: string, lspBalance: string, clientBalance: string, chanExpiryBlocks: number }) => {
const req: OlympusOrder = {
public_key: orderInfo.pubKey,
announce_channel: true,
refund_onchain_address: orderInfo.refundAddr,
lsp_balance_sat: orderInfo.lspBalance,
client_balance_sat: orderInfo.clientBalance,
channel_expiry_blocks: orderInfo.chanExpiryBlocks,
funding_confirms_within_blocks: 6,
required_channel_confirmations: 0
}
const res = await fetch(`${this.settings.olympusServiceUrl}/create_order`, {
method: "POST",
body: JSON.stringify(req),
headers: { "Content-Type": "application/json" }
})
const json = await res.json() as { order_id: string, payment: { state: 'EXPECT_PAYMENT', bolt11_invoice: string, fee_total_sat: string, order_total_sat: string } }
return json
}
getOrder = async (orderId: string) => {
const res = await fetch(`${this.serviceUrl}/get_order&order_id=${orderId}`)
const res = await fetch(`${this.settings.olympusServiceUrl}/get_order&order_id=${orderId}`)
const json = await res.json() as {}
return json
}
}
export class VoltageLSP extends LSP {
constructor(settings: LSPSettings, lnd: LND, liquidityProvider: LiquidityProvider) {
super("VoltageLSP", settings, lnd, liquidityProvider)
}
getInfo = async () => {
const res = await fetch(`${this.settings.voltageServiceUrl}/info`)
const json = await res.json() as { connection_methods: { address: string, port: string, type: string }[], pubkey: string }
return json
}
getFees = async (amtMsat: string, pubkey: string) => {
const res = await fetch(`${this.settings.voltageServiceUrl}/fee`, {
method: "POST",
body: JSON.stringify({ amount_msat: amtMsat, pubkey }),
headers: { "Content-Type": "application/json" }
})
const json = await res.json() as { fee_amount_msat: number, id: string }
return json
}
openChannelIfReady = async (): Promise<OrderResponse | null> => {
const shouldOpen = await this.shouldOpenChannel()
if (!shouldOpen.shouldOpen) {
return null
}
if (!this.settings.voltageServiceUrl) {
this.log("no voltage service url provided")
return null
}
const lndInfo = await this.lnd.GetInfo()
const myPub = lndInfo.identityPubkey
const amtMsats = this.settings.channelThreshold.toString() + "000"
const fee = await this.getFees(amtMsats, myPub)
const feeSats = fee.fee_amount_msat / 1000
const relativeFee = feeSats / this.settings.channelThreshold
if (relativeFee > this.settings.maxRelativeFee) {
this.log("relative fee of", relativeFee, "exceeds max relative fee of", this.settings.maxRelativeFee)
return null
}
const info = await this.getInfo()
const ipv4 = info.connection_methods.find(c => c.type === 'ipv4')
if (!ipv4) {
this.log("no ipv4 address found")
return null
}
await this.addPeer(info.pubkey, `${ipv4.address}:${ipv4.port}`)
const invoice = await this.lnd.NewInvoice(this.settings.channelThreshold, "open channel", 60 * 60)
const res = await this.proposal(invoice.payRequest, fee.id)
const decoded = await this.lnd.DecodeInvoice(res.jit_bolt11)
if (decoded.numSatoshis !== this.settings.channelThreshold + feeSats) {
this.log("invoice of amount", decoded.numSatoshis, "does not match expected amount of", this.settings.channelThreshold + feeSats)
return null
}
const invoiceRes = await this.liquidityProvider.PayInvoice(res.jit_bolt11)
this.log("paid", invoiceRes.amount_paid, "to open channel")
return { orderId: fee.id, invoice: res.jit_bolt11, totalSats: decoded.numSatoshis, fees: feeSats }
}
proposal = async (bolt11: string, feeId: string) => {
const res = await fetch(`${this.settings.voltageServiceUrl}/proposal`, {
method: "POST",
body: JSON.stringify({ bolt11, fee_id: feeId }),
headers: { "Content-Type": "application/json" }
})
const json = await res.json() as { jit_bolt11: string }
return json
}
}

2
src/services/lnd/settings.ts
View file

@ -9,8 +9,6 @@ export type LndSettings = {
feeRateLimit: number
feeFixedLimit: number
mockLnd: boolean
liquidityProviderPub: string
useOnlyLiquidityProvider: boolean
otherNode?: NodeSettings
thirdNode?: NodeSettings

11
src/services/main/index.ts
View file

@ -16,6 +16,7 @@ import { NostrSend } from '../nostr/handler.js'
import MetricsManager from '../metrics/index.js'
import { LoggedEvent } from '../storage/eventsLog.js'
import { LiquidityProvider } from "../lnd/liquidityProvider.js"
import { LiquidityManager } from "./liquidityManager.js"
type UserOperationsSub = {
id: string
@ -37,20 +38,23 @@ export default class {
paymentSubs: Record<string, ((op: Types.UserOperation) => void) | null> = {}
metricsManager: MetricsManager
liquidProvider: LiquidityProvider
liquidityManager: LiquidityManager
nostrSend: NostrSend = () => { getLogger({})("nostr send not initialized yet") }
constructor(settings: MainSettings, storage: Storage) {
this.settings = settings
this.storage = storage
this.liquidProvider = new LiquidityProvider(settings.lndSettings.liquidityProviderPub, this.invoicePaidCb)
this.lnd = new LND(settings.lndSettings, this.liquidProvider, this.addressPaidCb, this.invoicePaidCb, this.newBlockCb, this.htlcCb)
this.liquidProvider = new LiquidityProvider(settings.liquiditySettings.liquidityProviderPub, this.invoicePaidCb)
const provider = { liquidProvider: this.liquidProvider, useOnly: settings.liquiditySettings.useOnlyLiquidityProvider }
this.lnd = new LND(settings.lndSettings, provider, this.addressPaidCb, this.invoicePaidCb, this.newBlockCb, this.htlcCb)
this.liquidityManager = new LiquidityManager(this.settings.liquiditySettings, this.storage, this.liquidProvider, this.lnd)
this.metricsManager = new MetricsManager(this.storage, this.lnd)
this.paymentManager = new PaymentManager(this.storage, this.lnd, this.settings, this.addressPaidCb, this.invoicePaidCb)
this.productManager = new ProductManager(this.storage, this.paymentManager, this.settings)
this.applicationManager = new ApplicationManager(this.storage, this.settings, this.paymentManager)
this.appUserManager = new AppUserManager(this.storage, this.settings, this.applicationManager)
}
Stop() {
this.lnd.Stop()
this.applicationManager.Stop()
@ -187,6 +191,7 @@ export default class {
this.sendOperationToNostr(userInvoice.linkedApplication, userInvoice.user.user_id, op)
this.createZapReceipt(log, userInvoice)
log("paid invoice processed successfully")
this.liquidityManager.afterInInvoicePaid()
} catch (err: any) {
log(ERROR, "cannot process paid invoice", err.message || "")
}

72
src/services/main/liquidityManager.ts Normal file
View file

@ -0,0 +1,72 @@
import { getLogger } from "../helpers/logger.js"
import { LiquidityProvider } from "../lnd/liquidityProvider.js"
import LND from "../lnd/lnd.js"
import { FlashsatsLSP, LoadLSPSettingsFromEnv, LSPSettings, OlympusLSP, VoltageLSP } from "../lnd/lsp.js"
import Storage from '../storage/index.js'
export type LiquiditySettings = {
lspSettings: LSPSettings
liquidityProviderPub: string
useOnlyLiquidityProvider: boolean
}
export const LoadLiquiditySettingsFromEnv = (): LiquiditySettings => {
const lspSettings = LoadLSPSettingsFromEnv()
const liquidityProviderPub = process.env.LIQUIDITY_PROVIDER_PUB || ""
return { lspSettings, liquidityProviderPub, useOnlyLiquidityProvider: false }
}
export class LiquidityManager {
settings: LiquiditySettings
storage: Storage
liquidityProvider: LiquidityProvider
lnd: LND
olympusLSP: OlympusLSP
voltageLSP: VoltageLSP
flashsatsLSP: FlashsatsLSP
log = getLogger({ component: "liquidityManager" })
channelRequested = false
constructor(settings: LiquiditySettings, storage: Storage, liquidityProvider: LiquidityProvider, lnd: LND) {
this.settings = settings
this.storage = storage
this.liquidityProvider = liquidityProvider
this.lnd = lnd
this.olympusLSP = new OlympusLSP(settings.lspSettings, lnd, liquidityProvider)
this.voltageLSP = new VoltageLSP(settings.lspSettings, lnd, liquidityProvider)
this.flashsatsLSP = new FlashsatsLSP(settings.lspSettings, lnd, liquidityProvider)
}
beforeInvoiceCreation = async () => { }
afterInInvoicePaid = async () => {
const existingOrder = await this.storage.liquidityStorage.GetLatestLspOrder()
if (existingOrder) {
return
}
if (this.channelRequested) {
return
}
this.log("checking if channel should be requested")
const olympusOk = await this.olympusLSP.openChannelIfReady()
if (olympusOk) {
this.log("requested channel from olympus")
this.channelRequested = true
await this.storage.liquidityStorage.SaveLspOrder({ service_name: 'olympus', invoice: olympusOk.invoice, total_paid: olympusOk.totalSats, order_id: olympusOk.orderId, fees: olympusOk.fees })
return
}
const voltageOk = await this.voltageLSP.openChannelIfReady()
if (voltageOk) {
this.log("requested channel from voltage")
this.channelRequested = true
await this.storage.liquidityStorage.SaveLspOrder({ service_name: 'voltage', invoice: voltageOk.invoice, total_paid: voltageOk.totalSats, order_id: voltageOk.orderId, fees: voltageOk.fees })
return
}
const flashsatsOk = await this.flashsatsLSP.openChannelIfReady()
if (flashsatsOk) {
this.log("requested channel from flashsats")
this.channelRequested = true
await this.storage.liquidityStorage.SaveLspOrder({ service_name: 'flashsats', invoice: flashsatsOk.invoice, total_paid: flashsatsOk.totalSats, order_id: flashsatsOk.orderId, fees: flashsatsOk.fees })
return
}
this.log("no channel requested")
}
beforeOutInvoicePayment = async () => { }
afterOutInvoicePaid = async () => { }
}

22
src/services/main/settings.ts
View file

@ -6,10 +6,12 @@ import { EnvCanBeInteger, EnvMustBeInteger, EnvMustBeNonEmptyString } from '../h
import { getLogger } from '../helpers/logger.js'
import fs from 'fs'
import crypto from 'crypto';
import { LiquiditySettings, LoadLiquiditySettingsFromEnv } from './liquidityManager.js'
export type MainSettings = {
storageSettings: StorageSettings,
lndSettings: LndSettings,
watchDogSettings: WatchdogSettings,
liquiditySettings: LiquiditySettings,
jwtSecret: string
incomingTxFee: number
outgoingTxFee: number
@ -32,11 +34,13 @@ export type BitcoinCoreSettings = {
}
export type TestSettings = MainSettings & { lndSettings: { otherNode: NodeSettings, thirdNode: NodeSettings, fourthNode: NodeSettings }, bitcoinCoreSettings: BitcoinCoreSettings }
export const LoadMainSettingsFromEnv = (): MainSettings => {
const storageSettings = LoadStorageSettingsFromEnv()
return {
watchDogSettings: LoadWatchdogSettingsFromEnv(),
lndSettings: LoadLndSettingsFromEnv(),
storageSettings: LoadStorageSettingsFromEnv(),
jwtSecret: loadJwtSecret(),
storageSettings: storageSettings,
liquiditySettings: LoadLiquiditySettingsFromEnv(),
jwtSecret: loadJwtSecret(storageSettings.dataDir),
incomingTxFee: EnvCanBeInteger("INCOMING_CHAIN_FEE_ROOT_BPS", 0) / 10000,
outgoingTxFee: EnvCanBeInteger("OUTGOING_CHAIN_FEE_ROOT_BPS", 60) / 10000,
incomingAppInvoiceFee: EnvCanBeInteger("INCOMING_INVOICE_FEE_ROOT_BPS", 0) / 10000,
@ -58,7 +62,7 @@ export const LoadTestSettingsFromEnv = (): TestSettings => {
const settings = LoadMainSettingsFromEnv()
return {
...settings,
storageSettings: { dbSettings: { ...settings.storageSettings.dbSettings, databaseFile: ":memory:", metricsDatabaseFile: ":memory:" }, eventLogPath },
storageSettings: { dbSettings: { ...settings.storageSettings.dbSettings, databaseFile: ":memory:", metricsDatabaseFile: ":memory:" }, eventLogPath, dataDir: "data" },
lndSettings: {
...settings.lndSettings,
otherNode: {
@ -76,7 +80,10 @@ export const LoadTestSettingsFromEnv = (): TestSettings => {
lndCertPath: EnvMustBeNonEmptyString("LND_FOURTH_CERT_PATH"),
lndMacaroonPath: EnvMustBeNonEmptyString("LND_FOURTH_MACAROON_PATH")
},
liquidityProviderPub: ""
},
liquiditySettings: {
...settings.liquiditySettings,
liquidityProviderPub: "",
},
skipSanityCheck: true,
bitcoinCoreSettings: {
@ -87,20 +94,21 @@ export const LoadTestSettingsFromEnv = (): TestSettings => {
}
}
export const loadJwtSecret = (): string => {
export const loadJwtSecret = (dataDir: string): string => {
const secret = process.env["JWT_SECRET"]
const log = getLogger({})
if (secret) {
return secret
}
log("JWT_SECRET not set in env, checking .jwt_secret file")
const secretPath = dataDir !== "" ? `${dataDir}/.jwt_secret` : ".jwt_secret"
try {
const fileContent = fs.readFileSync(".jwt_secret", "utf-8")
const fileContent = fs.readFileSync(secretPath, "utf-8")
return fileContent.trim()
} catch (e) {
log(".jwt_secret file not found, generating random secret")
const secret = crypto.randomBytes(32).toString('hex')
fs.writeFileSync(".jwt_secret", secret)
fs.writeFileSync(secretPath, secret)
return secret
}
}

3
src/services/storage/db.ts
View file

@ -17,6 +17,7 @@ import { BalanceEvent } from "./entity/BalanceEvent.js"
import { ChannelBalanceEvent } from "./entity/ChannelsBalanceEvent.js"
import { getLogger } from "../helpers/logger.js"
import { ChannelRouting } from "./entity/ChannelRouting.js"
import { LspOrder } from "./entity/LspOrder.js"
export type DbSettings = {
@ -56,7 +57,7 @@ export default async (settings: DbSettings, migrations: Function[]): Promise<{ s
database: settings.databaseFile,
// logging: true,
entities: [User, UserReceivingInvoice, UserReceivingAddress, AddressReceivingTransaction, UserInvoicePayment, UserTransactionPayment,
UserBasicAuth, UserEphemeralKey, Product, UserToUserPayment, Application, ApplicationUser, UserToUserPayment],
UserBasicAuth, UserEphemeralKey, Product, UserToUserPayment, Application, ApplicationUser, UserToUserPayment, LspOrder],
//synchronize: true,
migrations
}).initialize()

28
src/services/storage/entity/LspOrder.ts Normal file
View file

@ -0,0 +1,28 @@
import { Entity, PrimaryGeneratedColumn, Column, CreateDateColumn, UpdateDateColumn } from "typeorm"
@Entity()
export class LspOrder {
@PrimaryGeneratedColumn()
serial_id: number
@Column()
service_name: string
@Column()
invoice: string
@Column()
order_id: string
@Column()
total_paid: number
@Column()
fees: number
@CreateDateColumn()
created_at: Date
@UpdateDateColumn()
updated_at: Date
}

6
src/services/storage/index.ts
View file

@ -7,12 +7,14 @@ import PaymentStorage from "./paymentStorage.js";
import MetricsStorage from "./metricsStorage.js";
import TransactionsQueue, { TX } from "./transactionsQueue.js";
import EventsLogManager from "./eventsLog.js";
import { LiquidityStorage } from "./liquidityStorage.js";
export type StorageSettings = {
dbSettings: DbSettings
eventLogPath: string
dataDir: string
}
export const LoadStorageSettingsFromEnv = (): StorageSettings => {
return { dbSettings: LoadDbSettingsFromEnv(), eventLogPath: "logs/eventLogV2.csv" }
return { dbSettings: LoadDbSettingsFromEnv(), eventLogPath: "logs/eventLogV2.csv", dataDir: process.env.DATA_DIR || "" }
}
export default class {
DB: DataSource | EntityManager
@ -23,6 +25,7 @@ export default class {
userStorage: UserStorage
paymentStorage: PaymentStorage
metricsStorage: MetricsStorage
liquidityStorage: LiquidityStorage
eventsLog: EventsLogManager
constructor(settings: StorageSettings) {
this.settings = settings
@ -37,6 +40,7 @@ export default class {
this.applicationStorage = new ApplicationStorage(this.DB, this.userStorage, this.txQueue)
this.paymentStorage = new PaymentStorage(this.DB, this.userStorage, this.txQueue)
this.metricsStorage = new MetricsStorage(this.settings)
this.liquidityStorage = new LiquidityStorage(this.DB, this.txQueue)
const executedMetricsMigrations = await this.metricsStorage.Connect(metricsMigrations)
return { executedMigrations, executedMetricsMigrations };
}

20
src/services/storage/liquidityStorage.ts Normal file
View file

@ -0,0 +1,20 @@
import { DataSource, EntityManager, MoreThan } from "typeorm"
import { LspOrder } from "./entity/LspOrder.js";
import TransactionsQueue, { TX } from "./transactionsQueue.js";
export class LiquidityStorage {
DB: DataSource | EntityManager
txQueue: TransactionsQueue
constructor(DB: DataSource | EntityManager, txQueue: TransactionsQueue) {
this.DB = DB
this.txQueue = txQueue
}
GetLatestLspOrder() {
return this.DB.getRepository(LspOrder).findOne({ where: { serial_id: MoreThan(0) }, order: { serial_id: "DESC" } })
}
SaveLspOrder(order: Partial<LspOrder>) {
const entry = this.DB.getRepository(LspOrder).create(order)
return this.txQueue.PushToQueue<LspOrder>({ exec: async db => db.getRepository(LspOrder).save(entry), dbTx: false })
}
}

14
src/services/storage/migrations/1718387847693-lsp_order.ts Normal file
View file

@ -0,0 +1,14 @@
import { MigrationInterface, QueryRunner } from "typeorm";
export class LspOrder1718387847693 implements MigrationInterface {
name = 'LspOrder1718387847693'
public async up(queryRunner: QueryRunner): Promise<void> {
await queryRunner.query(`CREATE TABLE "lsp_order" ("serial_id" integer PRIMARY KEY AUTOINCREMENT NOT NULL, "service_name" varchar NOT NULL, "invoice" varchar NOT NULL, "order_id" varchar NOT NULL, "total_paid" integer NOT NULL, "fees" integer NOT NULL, "created_at" datetime NOT NULL DEFAULT (datetime('now')), "updated_at" datetime NOT NULL DEFAULT (datetime('now')))`);
}
public async down(queryRunner: QueryRunner): Promise<void> {
await queryRunner.query(`DROP TABLE "lsp_order"`);
}
}

3
src/services/storage/migrations/runner.ts
View file

@ -4,7 +4,8 @@ import Storage, { StorageSettings } from '../index.js'
import { Initial1703170309875 } from './1703170309875-initial.js'
import { LndMetrics1703170330183 } from './1703170330183-lnd_metrics.js'
import { ChannelRouting1709316653538 } from './1709316653538-channel_routing.js'
const allMigrations = [Initial1703170309875]
import { LspOrder1718387847693 } from './1718387847693-lsp_order.js'
const allMigrations = [Initial1703170309875, LspOrder1718387847693]
const allMetricsMigrations = [LndMetrics1703170330183, ChannelRouting1709316653538]
export const TypeOrmMigrationRunner = async (log: PubLogger, storageManager: Storage, settings: DbSettings, arg: string | undefined): Promise<boolean> => {
if (arg === 'fake_initial_migration') {

4
src/tests/liquidityProvider.spec.ts
View file

@ -24,14 +24,16 @@ const testInboundPaymentFromProvider = async (T: TestBase, bootstrapped: Main, b
const invoiceRes = await bootstrapped.appUserManager.NewInvoice({ app_id: bUser.appId, user_id: bUser.userId, app_user_id: bUser.appUserIdentifier }, { amountSats: 2000, memo: "liquidityTest" })
await T.externalAccessToOtherLnd.PayInvoice(invoiceRes.invoice, 0, 100)
await new Promise((resolve) => setTimeout(resolve, 200))
const userBalance = await bootstrapped.appUserManager.GetUserInfo({ app_id: bUser.appId, user_id: bUser.userId, app_user_id: bUser.appUserIdentifier })
T.expect(userBalance.balance).to.equal(2000)
T.d("user balance is 2000")
const providerBalance = await bootstrapped.liquidProvider.CheckUserState()
if (!providerBalance) {
throw new Error("provider balance not found")
}
T.expect(providerBalance.balance).to.equal(2000)
T.d("provider balance is 2000")
T.d("testInboundPaymentFromProvider done")
}

4
src/tests/networkSetup.ts
View file

@ -8,8 +8,8 @@ export const setupNetwork = async () => {
const core = new BitcoinCoreWrapper(settings)
await core.InitAddress()
await core.Mine(1)
const alice = new LND(settings.lndSettings, new LiquidityProvider("", () => { }), () => { }, () => { }, () => { }, () => { })
const bob = new LND({ ...settings.lndSettings, mainNode: settings.lndSettings.otherNode }, new LiquidityProvider("", () => { }), () => { }, () => { }, () => { }, () => { })
const alice = new LND(settings.lndSettings, { liquidProvider: new LiquidityProvider("", () => { }) }, () => { }, () => { }, () => { }, () => { })
const bob = new LND({ ...settings.lndSettings, mainNode: settings.lndSettings.otherNode }, { liquidProvider: new LiquidityProvider("", () => { }) }, () => { }, () => { }, () => { }, () => { })
await tryUntil<void>(async i => {
const peers = await alice.ListPeers()
if (peers.peers.length > 0) {

4
src/tests/setupBootstrapped.ts
View file

@ -7,8 +7,8 @@ import * as Types from '../../proto/autogenerated/ts/types.js'
export const initBootstrappedInstance = async (T: TestBase) => {
const settings = LoadTestSettingsFromEnv()
settings.lndSettings.useOnlyLiquidityProvider = true
settings.lndSettings.liquidityProviderPub = T.app.publicKey
settings.liquiditySettings.useOnlyLiquidityProvider = true
settings.liquiditySettings.liquidityProviderPub = T.app.publicKey
settings.lndSettings.mainNode = settings.lndSettings.thirdNode
const initialized = await initMainHandler(getLogger({ component: "bootstrapped" }), settings)
if (!initialized) {

6
src/tests/testBase.ts
View file

@ -46,15 +46,15 @@ export const SetupTest = async (d: Describe): Promise<TestBase> => {
const user2 = { userId: u2.info.userId, appUserIdentifier: u2.identifier, appId: app.appId }
const externalAccessToMainLnd = new LND(settings.lndSettings, new LiquidityProvider("", () => { }), console.log, console.log, () => { }, () => { })
const externalAccessToMainLnd = new LND(settings.lndSettings, { liquidProvider: new LiquidityProvider("", () => { }) }, console.log, console.log, () => { }, () => { })
await externalAccessToMainLnd.Warmup()
const otherLndSetting = { ...settings.lndSettings, mainNode: settings.lndSettings.otherNode }
const externalAccessToOtherLnd = new LND(otherLndSetting, new LiquidityProvider("", () => { }), console.log, console.log, () => { }, () => { })
const externalAccessToOtherLnd = new LND(otherLndSetting, { liquidProvider: new LiquidityProvider("", () => { }) }, console.log, console.log, () => { }, () => { })
await externalAccessToOtherLnd.Warmup()
const thirdLndSetting = { ...settings.lndSettings, mainNode: settings.lndSettings.thirdNode }
const externalAccessToThirdLnd = new LND(thirdLndSetting, new LiquidityProvider("", () => { }), console.log, console.log, () => { }, () => { })
const externalAccessToThirdLnd = new LND(thirdLndSetting, { liquidProvider: new LiquidityProvider("", () => { }) }, console.log, console.log, () => { }, () => { })
await externalAccessToThirdLnd.Warmup()