aiolabs/lightning-pub
2
0
Fork 0
Code Issues 9 Pull requests 4 Projects Releases Packages Wiki Activity Actions

Merge pull request #661 from shocknet/sanity_check_fix

Browse source 
move watchdog, write failed payments, fix sanity check
This commit is contained in:
Justin (shocknet) 2024-03-30 12:30:56 -04:00 • committed by GitHub
commit 532e3c6e02
24 changed files with 776 additions and 278 deletions
Download patch file Download diff file Expand all files Collapse all files

2
package.json
View file

@ -4,7 +4,7 @@
"description": "",
"main": "index.js",
"scripts": {
"test": " tsc && node build/src/testRunner.js",
"test": " tsc && node build/src/tests/testRunner.js",
"start": "tsc && node build/src/index.js",
"start:ci": "git reset --hard && git pull && npm run start",
"build_autogenerated": "cd proto && rimraf autogenerated && protoc -I ./service --pub_out=. service/*",

8
src/services/helpers/logger.ts
View file

@ -7,7 +7,6 @@ try {
} catch { }
const z = (n: number) => n < 10 ? `0${n}` : `${n}`
const openWriter = (fileName: string): Writer => {
const logStream = fs.createWriteStream(`logs/${fileName}`, { flags: 'a' });
return (message) => {
logStream.write(message + "\n")
@ -37,6 +36,9 @@ export const getLogger = (params: LoggerParams): PubLogger => {
const timestamp = `${now.getFullYear()}-${z(now.getMonth() + 1)}-${z(now.getDate())} ${z(now.getHours())}:${z(now.getMinutes())}:${z(now.getSeconds())}`
const toLog = [timestamp]
if (params.appName) {
if (disabledApps.includes(params.appName)) {
return
}
toLog.push(params.appName)
}
if (params.userId) {
@ -48,3 +50,7 @@ export const getLogger = (params: LoggerParams): PubLogger => {
writers.forEach(w => w(final))
}
}
const disabledApps: string[] = []
export const disableLoggers = (appNamesToDisable: string[]) => {
disabledApps.push(...appNamesToDisable)
}

2
src/services/lnd/index.ts
View file

@ -12,7 +12,7 @@ export const LoadLndSettingsFromEnv = (): LndSettings => {
const feeRateLimit = EnvMustBeInteger("OUTBOUND_MAX_FEE_BPS") / 10000
const feeFixedLimit = EnvMustBeInteger("OUTBOUND_MAX_FEE_EXTRA_SATS")
const mockLnd = EnvCanBeBoolean("MOCK_LND")
return { lndAddr, lndCertPath, lndMacaroonPath, feeRateLimit, feeFixedLimit, mockLnd }
return { lndAddr, lndCertPath, lndMacaroonPath, feeRateLimit, feeFixedLimit, mockLnd, otherLndAddr: "", otherLndCertPath: "", otherLndMacaroonPath: "" }
}
export interface LightningHandler {
Stop(): void

32
src/services/lnd/lnd.spec.ts
View file

@ -1,32 +0,0 @@
import 'dotenv/config' // TODO - test env
import { expect } from 'chai'
import * as Types from '../../../proto/autogenerated/ts/types.js';
import NewLightningHandler, { LightningHandler, LoadLndSettingsFromEnv } from '../lnd/index.js'
let lnd: LightningHandler
export const ignore = true
export const setup = async () => {
lnd = NewLightningHandler(LoadLndSettingsFromEnv(), console.log, console.log, console.log, console.log)
await lnd.Warmup()
}
export const teardown = () => {
lnd.Stop()
}
export default async (d: (message: string, failure?: boolean) => void) => {
const info = await lnd.GetInfo()
expect(info.alias).to.equal("alice")
d("get alias ok")
const addr = await lnd.NewAddress(Types.AddressType.WITNESS_PUBKEY_HASH)
console.log(addr)
d("new address ok")
const invoice = await lnd.NewInvoice(1000, "", 60 * 60)
console.log(invoice)
d("new invoice ok")
const res = await lnd.EstimateChainFees("bcrt1qajzzx453x9fx5gtlyax8zrsennckrw3syd2llt", 1000, 100)
console.log(res)
d("estimate fee ok")
//const res = await this.lnd.OpenChannel("025ed7fc85fc05a07fc5acc13a6e3836cd11c5587c1d400afcd22630a9e230eb7a", "", 20000, 0)
}

4
src/services/lnd/settings.ts
View file

@ -7,6 +7,10 @@ export type LndSettings = {
feeRateLimit: number
feeFixedLimit: number
mockLnd: boolean
otherLndAddr: string
otherLndCertPath: string
otherLndMacaroonPath: string
}
type TxOutput = {
hash: string

82
src/services/main/index.ts
View file

@ -1,4 +1,3 @@
import crypto from 'crypto'
import fetch from "node-fetch"
import Storage, { LoadStorageSettingsFromEnv } from '../storage/index.js'
import * as Types from '../../../proto/autogenerated/ts/types.js'
@ -223,85 +222,4 @@ export default class {
log({ unsigned: event })
this.nostrSend(invoice.linkedApplication.app_id, { type: 'event', event })
}
async VerifyEventsLog() {
const events = await this.storage.eventsLog.GetAllLogs()
const invoices = await this.lnd.GetAllPaidInvoices(1000)
const payments = await this.lnd.GetAllPayments(1000)
const incrementSources: Record<string, boolean> = {}
const decrementSources: Record<string, boolean> = {}
const users: Record<string, { ts: number, updatedBalance: number }> = {}
for (let i = 0; i < events.length; i++) {
const e = events[i]
if (e.type === 'balance_decrement') {
users[e.userId] = this.checkUserEntry(e, users[e.userId])
if (LN_INVOICE_REGEX.test(e.data)) {
if (decrementSources[e.data]) {
throw new Error("payment decremented more that once " + e.data)
}
decrementSources[e.data] = true
const paymentEntry = await this.storage.paymentStorage.GetPaymentOwner(e.data)
if (!paymentEntry) {
throw new Error("payment entry not found for " + e.data)
}
if (paymentEntry.paid_at_unix === 0) {
throw new Error("payment was never paid " + e.data)
}
if (!paymentEntry.internal) {
const entry = payments.payments.find(i => i.paymentRequest === e.data)
if (!entry) {
throw new Error("payment not found in lnd " + e.data)
}
}
}
} else if (e.type === 'balance_increment') {
users[e.userId] = this.checkUserEntry(e, users[e.userId])
if (LN_INVOICE_REGEX.test(e.data)) {
if (incrementSources[e.data]) {
throw new Error("invoice incremented more that once " + e.data)
}
incrementSources[e.data] = true
const invoiceEntry = await this.storage.paymentStorage.GetInvoiceOwner(e.data)
if (!invoiceEntry) {
throw new Error("invoice entry not found for " + e.data)
}
if (invoiceEntry.paid_at_unix === 0) {
throw new Error("invoice was never paid " + e.data)
}
if (!invoiceEntry.internal) {
const entry = invoices.invoices.find(i => i.paymentRequest === e.data)
if (!entry) {
throw new Error("invoice not found in lnd " + e.data)
}
}
}
} else {
await this.storage.paymentStorage.VerifyDbEvent(e)
}
}
await Promise.all(Object.entries(users).map(async ([userId, u]) => {
const user = await this.storage.userStorage.GetUser(userId)
if (user.balance_sats !== u.updatedBalance) {
throw new Error("sanity check on balance failed, expected: " + u.updatedBalance + " found: " + user.balance_sats)
}
}))
}
checkUserEntry(e: LoggedEvent, u: { ts: number, updatedBalance: number } | undefined) {
const newEntry = { ts: e.timestampMs, updatedBalance: e.balance + e.amount * (e.type === 'balance_decrement' ? -1 : 1) }
if (!u) {
return newEntry
}
if (e.timestampMs < u.ts) {
throw new Error("entry out of order " + e.timestampMs + " " + u.ts)
}
if (e.balance !== u.updatedBalance) {
throw new Error("inconsistent balance update got: " + e.balance + " expected " + u.updatedBalance)
}
return newEntry
}
}
const LN_INVOICE_REGEX = /^(lightning:)?(lnbc|lntb)[0-9a-zA-Z]+$/;

7
src/services/main/init.ts
View file

@ -2,6 +2,7 @@ import { PubLogger, getLogger } from "../helpers/logger.js"
import Storage from "../storage/index.js"
import { TypeOrmMigrationRunner } from "../storage/migrations/runner.js"
import Main from "./index.js"
import SanityChecker from "./sanityChecker.js"
import { MainSettings } from "./settings.js"
export type AppData = {
privateKey: string;
@ -18,10 +19,10 @@ export const initMainHandler = async (log: PubLogger, mainSettings: MainSettings
const mainHandler = new Main(mainSettings, storageManager)
await mainHandler.lnd.Warmup()
if (!mainSettings.skipSanityCheck) {
await mainHandler.VerifyEventsLog()
const sanityChecker = new SanityChecker(storageManager, mainHandler.lnd)
await sanityChecker.VerifyEventsLog()
}
const totalUsersBalance = await mainHandler.storage.paymentStorage.GetTotalUsersBalance()
await mainHandler.paymentManager.watchDog.SeedLndBalance(totalUsersBalance || 0)
await mainHandler.paymentManager.watchDog.Start()
const appsData = await mainHandler.storage.applicationStorage.GetApplications()
const existingWalletApp = await appsData.find(app => app.name === 'wallet' || app.name === 'wallet-test')
if (!existingWalletApp) {

47
src/services/main/main.spec.ts
View file

@ -1,47 +0,0 @@
import 'dotenv/config' // TODO - test env
import chai from 'chai'
import { AppData, initMainHandler } from './init.js'
import Main from './index.js'
import { User } from '../storage/entity/User.js'
import { LoadMainSettingsFromEnv, LoadTestSettingsFromEnv, MainSettings } from './settings.js'
import chaiString from 'chai-string'
import { defaultInvoiceExpiry } from '../storage/paymentStorage.js'
chai.use(chaiString)
const expect = chai.expect
export const ignore = false
let main: Main
let app: AppData
let user1: { userId: string, appUserIdentifier: string, appId: string }
let user2: { userId: string, appUserIdentifier: string, appId: string }
export const setup = async () => {
const settings = LoadTestSettingsFromEnv()
const initialized = await initMainHandler(console.log, settings)
if (!initialized) {
throw new Error("failed to initialize main handler")
}
main = initialized.mainHandler
app = initialized.apps[0]
const u1 = await main.applicationManager.AddAppUser(app.appId, { identifier: "user1", balance: 0, fail_if_exists: true })
const u2 = await main.applicationManager.AddAppUser(app.appId, { identifier: "user2", balance: 2000, fail_if_exists: true })
user1 = { userId: u1.info.userId, appUserIdentifier: u1.identifier, appId: app.appId }
user2 = { userId: u2.info.userId, appUserIdentifier: u2.identifier, appId: app.appId }
}
export const teardown = () => {
console.log("teardown")
}
export default async (d: (message: string, failure?: boolean) => void) => {
const application = await main.storage.applicationStorage.GetApplication(app.appId)
const invoice = await main.paymentManager.NewInvoice(user1.userId, { amountSats: 1000, memo: "test" }, { linkedApplication: application, expiry: defaultInvoiceExpiry })
expect(invoice.invoice).to.startWith("lnbcrtmockin")
d("got the invoice")
const pay = await main.paymentManager.PayInvoice(user2.userId, { invoice: invoice.invoice, amount: 0 }, application)
expect(pay.amount_paid).to.be.equal(1000)
const u1 = await main.storage.userStorage.GetUser(user1.userId)
const u2 = await main.storage.userStorage.GetUser(user2.userId)
const owner = await main.storage.userStorage.GetUser(application.owner.user_id)
expect(u1.balance_sats).to.be.equal(1000)
expect(u2.balance_sats).to.be.equal(994)
expect(owner.balance_sats).to.be.equal(6)
}

106
src/services/main/paymentManager.ts
View file

@ -14,7 +14,7 @@ import { SendCoinsResponse } from '../../../proto/lnd/lightning.js'
import { Event, verifiedSymbol, verifySignature } from '../nostr/tools/event.js'
import { AddressReceivingTransaction } from '../storage/entity/AddressReceivingTransaction.js'
import { UserTransactionPayment } from '../storage/entity/UserTransactionPayment.js'
import { Watchdog } from '../lnd/watchdog.js'
import { Watchdog } from './watchdog.js'
interface UserOperationInfo {
serial_id: number
paid_amount: number
@ -39,7 +39,6 @@ const defaultLnurlPayMetadata = `[["text/plain", "lnurl pay to Lightning.pub"]]`
const confInOne = 1000 * 1000
const confInTwo = 100 * 1000 * 1000
export default class {
storage: Storage
settings: MainSettings
lnd: LightningHandler
@ -51,7 +50,7 @@ export default class {
this.storage = storage
this.settings = settings
this.lnd = lnd
this.watchDog = new Watchdog(settings.watchDogSettings, lnd)
this.watchDog = new Watchdog(settings.watchDogSettings, lnd, storage)
this.addressPaidCb = addressPaidCb
this.invoicePaidCb = invoicePaidCb
}
@ -143,14 +142,9 @@ export default class {
}
}
async WatchdogCheck() {
const total = await this.storage.paymentStorage.GetTotalUsersBalance()
await this.watchDog.PaymentRequested(total || 0)
}
async PayInvoice(userId: string, req: Types.PayInvoiceRequest, linkedApplication: Application): Promise<Types.PayInvoiceResponse> {
this.log("paying invoice", req.invoice, "for user", userId, "with amount", req.amount)
await this.WatchdogCheck()
await this.watchDog.PaymentRequested()
const maybeBanned = await this.storage.userStorage.GetUser(userId)
if (maybeBanned.locked) {
throw new Error("user is banned, cannot send payment")
@ -165,53 +159,73 @@ export default class {
const payAmount = req.amount !== 0 ? req.amount : Number(decoded.numSatoshis)
const isAppUserPayment = userId !== linkedApplication.owner.user_id
const serviceFee = this.getServiceFee(Types.UserOperationType.OUTGOING_INVOICE, payAmount, isAppUserPayment)
const totalAmountToDecrement = payAmount + serviceFee
const internalInvoice = await this.storage.paymentStorage.GetInvoiceOwner(req.invoice)
let payment: PaidInvoice | null = null
if (!internalInvoice) {
if (this.settings.disableExternalPayments) {
throw new Error("something went wrong sending payment, please try again later")
}
this.log("paying external invoice", req.invoice)
const routingFeeLimit = this.lnd.GetFeeLimitAmount(payAmount)
await this.storage.userStorage.DecrementUserBalance(userId, totalAmountToDecrement + routingFeeLimit, req.invoice)
try {
payment = await this.lnd.PayInvoice(req.invoice, req.amount, routingFeeLimit)
if (routingFeeLimit - payment.feeSat > 0) {
await this.storage.userStorage.IncrementUserBalance(userId, routingFeeLimit - payment.feeSat, "routing_fee_refund")
}
} catch (err) {
await this.storage.userStorage.IncrementUserBalance(userId, totalAmountToDecrement + routingFeeLimit, "payment_refund")
throw err
}
let paymentInfo = { preimage: "", amtPaid: 0, networkFee: 0, serialId: 0 }
if (internalInvoice) {
paymentInfo = await this.PayInternalInvoice(userId, internalInvoice, { payAmount, serviceFee }, linkedApplication)
} else {
this.log("paying internal invoice", req.invoice)
if (internalInvoice.paid_at_unix > 0) {
throw new Error("this invoice was already paid")
}
await this.storage.userStorage.DecrementUserBalance(userId, totalAmountToDecrement, req.invoice)
this.invoicePaidCb(req.invoice, payAmount, true)
paymentInfo = await this.PayExternalInvoice(userId, req.invoice, { payAmount, serviceFee, amountForLnd: req.amount }, linkedApplication)
}
if (isAppUserPayment && serviceFee > 0) {
await this.storage.userStorage.IncrementUserBalance(linkedApplication.owner.user_id, serviceFee, "fees")
}
const routingFees = payment ? payment.feeSat : 0
const newPayment = await this.storage.paymentStorage.AddUserInvoicePayment(userId, req.invoice, payAmount, routingFees, serviceFee, !!internalInvoice, linkedApplication)
const user = await this.storage.userStorage.GetUser(userId)
this.storage.eventsLog.LogEvent({ type: 'invoice_payment', userId, appId: linkedApplication.app_id, appUserId: "", balance: user.balance_sats, data: req.invoice, amount: payAmount })
return {
preimage: payment ? payment.paymentPreimage : "",
amount_paid: payment ? Number(payment.valueSat) : payAmount,
operation_id: `${Types.UserOperationType.OUTGOING_INVOICE}-${newPayment.serial_id}`,
network_fee: routingFees,
preimage: paymentInfo.preimage,
amount_paid: paymentInfo.amtPaid,
operation_id: `${Types.UserOperationType.OUTGOING_INVOICE}-${paymentInfo.serialId}`,
network_fee: paymentInfo.networkFee,
service_fee: serviceFee
}
}
async PayExternalInvoice(userId: string, invoice: string, amounts: { payAmount: number, serviceFee: number, amountForLnd: number }, linkedApplication: Application) {
if (this.settings.disableExternalPayments) {
throw new Error("something went wrong sending payment, please try again later")
}
const { amountForLnd, payAmount, serviceFee } = amounts
const totalAmountToDecrement = payAmount + serviceFee
this.log("paying external invoice", invoice)
const routingFeeLimit = this.lnd.GetFeeLimitAmount(payAmount)
await this.storage.userStorage.DecrementUserBalance(userId, totalAmountToDecrement + routingFeeLimit, invoice)
console.log("decremented")
const pendingPayment = await this.storage.paymentStorage.AddPendingExternalPayment(userId, invoice, payAmount, linkedApplication)
try {
const payment = await this.lnd.PayInvoice(invoice, amountForLnd, routingFeeLimit)
if (routingFeeLimit - payment.feeSat > 0) {
this.log("refund routing fee", routingFeeLimit, payment.feeSat, "sats")
await this.storage.userStorage.IncrementUserBalance(userId, routingFeeLimit - payment.feeSat, "routing_fee_refund:" + invoice)
}
await this.storage.paymentStorage.UpdateExternalPayment(pendingPayment.serial_id, payment.feeSat, serviceFee, true)
return { preimage: payment.paymentPreimage, amtPaid: payment.valueSat, networkFee: payment.feeSat, serialId: pendingPayment.serial_id }
} catch (err) {
await this.storage.userStorage.IncrementUserBalance(userId, totalAmountToDecrement + routingFeeLimit, "payment_refund:" + invoice)
await this.storage.paymentStorage.UpdateExternalPayment(pendingPayment.serial_id, 0, 0, false)
throw err
}
}
async PayInternalInvoice(userId: string, internalInvoice: UserReceivingInvoice, amounts: { payAmount: number, serviceFee: number }, linkedApplication: Application) {
this.log("paying internal invoice", internalInvoice.invoice)
if (internalInvoice.paid_at_unix > 0) {
throw new Error("this invoice was already paid")
}
const { payAmount, serviceFee } = amounts
const totalAmountToDecrement = payAmount + serviceFee
await this.storage.userStorage.DecrementUserBalance(userId, totalAmountToDecrement, internalInvoice.invoice)
this.invoicePaidCb(internalInvoice.invoice, payAmount, true)
const newPayment = await this.storage.paymentStorage.AddInternalPayment(userId, internalInvoice.invoice, payAmount, serviceFee, linkedApplication)
return { preimage: "", amtPaid: payAmount, networkFee: 0, serialId: newPayment.serial_id }
}
async PayAddress(ctx: Types.UserContext, req: Types.PayAddressRequest): Promise<Types.PayAddressResponse> {
throw new Error("address payment currently disabled, use Lightning instead")
await this.WatchdogCheck()
await this.watchDog.PaymentRequested()
this.log("paying address", req.address, "for user", ctx.user_id, "with amount", req.amoutSats)
const maybeBanned = await this.storage.userStorage.GetUser(ctx.user_id)
if (maybeBanned.locked) {
@ -230,18 +244,21 @@ export default class {
const vBytes = Math.ceil(Number(estimate.feeSat / estimate.satPerVbyte))
chainFees = vBytes * req.satsPerVByte
const total = req.amoutSats + chainFees
// WARNING, before re-enabling this, make sure to add the tx_hash to the DecrementUserBalance "reason"!!
this.storage.userStorage.DecrementUserBalance(ctx.user_id, total + serviceFee, req.address)
try {
const payment = await this.lnd.PayAddress(req.address, req.amoutSats, req.satsPerVByte)
txId = payment.txid
} catch (err) {
// WARNING, before re-enabling this, make sure to add the tx_hash to the IncrementUserBalance "reason"!!
await this.storage.userStorage.IncrementUserBalance(ctx.user_id, total + serviceFee, req.address)
throw err
}
} else {
this.log("paying internal address")
await this.storage.userStorage.DecrementUserBalance(ctx.user_id, req.amoutSats + serviceFee, req.address)
txId = crypto.randomBytes(32).toString("hex")
const addressData = `${req.address}:${txId}`
await this.storage.userStorage.DecrementUserBalance(ctx.user_id, req.amoutSats + serviceFee, addressData)
this.addressPaidCb({ hash: txId, index: 0 }, req.address, req.amoutSats, true)
}
@ -520,9 +537,10 @@ export default class {
const isAppUserPayment = fromUser.user_id !== linkedApplication.owner.user_id
let fee = this.getServiceFee(Types.UserOperationType.OUTGOING_USER_TO_USER, amount, isAppUserPayment)
const toIncrement = amount - fee
await this.storage.userStorage.DecrementUserBalance(fromUser.user_id, amount, toUserId, tx)
await this.storage.userStorage.IncrementUserBalance(toUser.user_id, toIncrement, fromUserId, tx)
await this.storage.paymentStorage.AddUserToUserPayment(fromUserId, toUserId, amount, fee, linkedApplication, tx)
const paymentEntry = await this.storage.paymentStorage.CreateUserToUserPayment(fromUserId, toUserId, amount, fee, linkedApplication, tx)
await this.storage.userStorage.DecrementUserBalance(fromUser.user_id, amount, `${toUserId}:${paymentEntry.serial_id}`, tx)
await this.storage.userStorage.IncrementUserBalance(toUser.user_id, toIncrement, `${fromUserId}:${paymentEntry.serial_id}`, tx)
await this.storage.paymentStorage.SaveUserToUserPayment(paymentEntry, tx)
if (isAppUserPayment && fee > 0) {
await this.storage.userStorage.IncrementUserBalance(linkedApplication.owner.user_id, fee, 'fees', tx)
}

268
src/services/main/sanityChecker.ts Normal file
View file

@ -0,0 +1,268 @@
import Storage from '../storage/index.js'
import { LightningHandler } from "../lnd/index.js"
import { LoggedEvent } from '../storage/eventsLog.js'
import { Invoice, Payment } from '../../../proto/lnd/lightning';
const LN_INVOICE_REGEX = /^(lightning:)?(lnbc|lntb)[0-9a-zA-Z]+$/;
const BITCOIN_ADDRESS_REGEX = /^(bitcoin:)?([13][a-km-zA-HJ-NP-Z1-9]{25,34}|bc1[a-zA-HJ-NP-Z0-9]{39,59})$/;
type UniqueDecrementReasons = 'ban'
type UniqueIncrementReasons = 'fees' | 'routing_fee_refund' | 'payment_refund'
type CommonReasons = 'invoice' | 'address' | 'u2u'
type Reason = UniqueDecrementReasons | UniqueIncrementReasons | CommonReasons
const incrementTwiceAllowed = ['fees', 'ban']
export default class SanityChecker {
storage: Storage
lnd: LightningHandler
events: LoggedEvent[] = []
invoices: Invoice[] = []
payments: Payment[] = []
incrementSources: Record<string, boolean> = {}
decrementSources: Record<string, boolean> = {}
decrementEvents: Record<string, { userId: string, refund: number, failure: boolean }> = {}
users: Record<string, { ts: number, updatedBalance: number }> = {}
constructor(storage: Storage, lnd: LightningHandler) {
this.storage = storage
this.lnd = lnd
}
parseDataField(data: string): { type: Reason, data: string, txHash?: string, serialId?: number } {
const parts = data.split(":")
if (parts.length === 1) {
const [fullData] = parts
if (fullData === 'fees' || fullData === 'ban') {
return { type: fullData, data: fullData }
} else if (LN_INVOICE_REGEX.test(fullData)) {
return { type: 'invoice', data: fullData }
} else if (BITCOIN_ADDRESS_REGEX.test(fullData)) {
return { type: 'address', data: fullData }
} else {
return { type: 'u2u', data: fullData }
}
} else if (parts.length === 2) {
const [prefix, data] = parts
if (prefix === 'routing_fee_refund' || prefix === 'payment_refund') {
return { type: prefix, data }
} else if (BITCOIN_ADDRESS_REGEX.test(prefix)) {
return { type: 'address', data: prefix, txHash: data }
} else {
return { type: 'u2u', data: prefix, serialId: +data }
}
}
throw new Error("unknown data format")
}
async verifyDecrementEvent(e: LoggedEvent) {
if (this.decrementSources[e.data]) {
throw new Error("entry decremented more that once " + e.data)
}
this.decrementSources[e.data] = !incrementTwiceAllowed.includes(e.data)
this.users[e.userId] = this.checkUserEntry(e, this.users[e.userId])
const parsed = this.parseDataField(e.data)
switch (parsed.type) {
case 'ban':
return
case 'address':
return this.validateUserTransactionPayment({ address: parsed.data, txHash: parsed.txHash, userId: e.userId })
case 'invoice':
return this.validateUserInvoicePayment({ invoice: parsed.data, userId: e.userId, amt: e.amount })
case 'u2u':
return this.validateUser2UserPayment({ fromUser: e.userId, toUser: parsed.data, serialId: parsed.serialId })
default:
throw new Error("unknown decrement type " + parsed.type)
}
}
async validateUserTransactionPayment({ address, txHash, userId }: { userId: string, address: string, txHash?: string }) {
if (!txHash) {
throw new Error("no tx hash provided to payment for address " + address)
}
const entry = await this.storage.paymentStorage.GetUserTransactionPaymentOwner(address, txHash)
if (!entry) {
throw new Error("no payment found for tx hash " + txHash)
}
if (entry.user.user_id !== userId) {
throw new Error("payment user id mismatch for tx hash " + txHash)
}
if (entry.paid_at_unix <= 0) {
throw new Error("payment not paid for tx hash " + txHash)
}
}
async validateUserInvoicePayment({ invoice, userId, amt }: { userId: string, invoice: string, amt: number }) {
const entry = await this.storage.paymentStorage.GetPaymentOwner(invoice)
if (!entry) {
throw new Error("no payment found for invoice " + invoice)
}
if (entry.user.user_id !== userId) {
throw new Error("payment user id mismatch for invoice " + invoice)
}
if (entry.paid_at_unix === 0) {
throw new Error("payment never settled for invoice " + invoice) // TODO: check if this is correct
}
if (entry.paid_at_unix === -1) {
this.decrementEvents[invoice] = { userId, refund: amt, failure: true }
} else {
const refund = amt - (entry.paid_amount + entry.routing_fees + entry.service_fees)
this.decrementEvents[invoice] = { userId, refund, failure: false }
}
if (!entry.internal) {
const lndEntry = this.payments.find(i => i.paymentRequest === invoice)
if (!lndEntry) {
throw new Error("payment not found in lnd for invoice " + invoice)
}
}
}
async validateUser2UserPayment({ fromUser, toUser, serialId }: { fromUser: string, toUser: string, serialId?: number }) {
if (!serialId) {
throw new Error("no serial id provided to u2u payment")
}
const entry = await this.storage.paymentStorage.GetUser2UserPayment(serialId)
if (!entry) {
throw new Error("no payment u2u found for serial id " + serialId)
}
if (entry.from_user.user_id !== fromUser || entry.to_user.user_id !== toUser) {
throw new Error("u2u payment user id mismatch for serial id " + serialId)
}
if (entry.paid_at_unix <= 0) {
throw new Error("payment not paid for serial id " + serialId)
}
}
async verifyIncrementEvent(e: LoggedEvent) {
if (this.incrementSources[e.data]) {
throw new Error("entry incremented more that once " + e.data)
}
this.incrementSources[e.data] = !incrementTwiceAllowed.includes(e.data)
this.users[e.userId] = this.checkUserEntry(e, this.users[e.userId])
const parsed = this.parseDataField(e.data)
switch (parsed.type) {
case 'fees':
return
case 'address':
return this.validateAddressReceivingTransaction({ address: parsed.data, txHash: parsed.txHash, userId: e.userId })
case 'invoice':
return this.validateReceivingInvoice({ invoice: parsed.data, userId: e.userId })
case 'u2u':
return this.validateUser2UserPayment({ fromUser: parsed.data, toUser: e.userId, serialId: parsed.serialId })
case 'routing_fee_refund':
return this.validateRoutingFeeRefund({ amt: e.amount, invoice: parsed.data, userId: e.userId })
case 'payment_refund':
return this.validatePaymentRefund({ amt: e.amount, invoice: parsed.data, userId: e.userId })
default:
throw new Error("unknown increment type " + parsed.type)
}
}
async validateAddressReceivingTransaction({ userId, address, txHash }: { userId: string, address: string, txHash?: string }) {
if (!txHash) {
throw new Error("no tx hash provided to address " + address)
}
const entry = await this.storage.paymentStorage.GetAddressReceivingTransactionOwner(address, txHash)
if (!entry) {
throw new Error("no tx found for tx hash " + txHash)
}
if (entry.user_address.user.user_id !== userId) {
throw new Error("tx user id mismatch for tx hash " + txHash)
}
if (entry.paid_at_unix <= 0) {
throw new Error("tx not paid for tx hash " + txHash)
}
}
async validateReceivingInvoice({ userId, invoice }: { userId: string, invoice: string }) {
const entry = await this.storage.paymentStorage.GetInvoiceOwner(invoice)
if (!entry) {
throw new Error("no invoice found for invoice " + invoice)
}
if (entry.user.user_id !== userId) {
throw new Error("invoice user id mismatch for invoice " + invoice)
}
if (entry.paid_at_unix <= 0) {
throw new Error("invoice not paid for invoice " + invoice)
}
if (!entry.internal) {
const entry = this.invoices.find(i => i.paymentRequest === invoice)
if (!entry) {
throw new Error("invoice not found in lnd " + invoice)
}
}
}
async validateRoutingFeeRefund({ amt, invoice, userId }: { userId: string, invoice: string, amt: number }) {
const entry = this.decrementEvents[invoice]
if (!entry) {
throw new Error("no decrement event found for invoice routing fee refound " + invoice)
}
if (entry.userId !== userId) {
throw new Error("user id mismatch for routing fee refund " + invoice)
}
if (entry.failure) {
throw new Error("payment failled, should not refund routing fees " + invoice)
}
if (entry.refund !== amt) {
console.log(entry.refund, amt)
throw new Error("refund amount mismatch for routing fee refund " + invoice)
}
}
async validatePaymentRefund({ amt, invoice, userId }: { userId: string, invoice: string, amt: number }) {
const entry = this.decrementEvents[invoice]
if (!entry) {
throw new Error("no decrement event found for invoice payment refund " + invoice)
}
if (entry.userId !== userId) {
throw new Error("user id mismatch for payment refund " + invoice)
}
if (!entry.failure) {
throw new Error("payment did not fail, should not refund payment " + invoice)
}
if (entry.refund !== amt) {
throw new Error("refund amount mismatch for payment refund " + invoice)
}
}
async VerifyEventsLog() {
this.events = await this.storage.eventsLog.GetAllLogs()
this.invoices = (await this.lnd.GetAllPaidInvoices(1000)).invoices
this.payments = (await this.lnd.GetAllPayments(1000)).payments
this.incrementSources = {}
this.decrementSources = {}
this.users = {}
this.users = {}
this.decrementEvents = {}
for (let i = 0; i < this.events.length; i++) {
const e = this.events[i]
if (e.type === 'balance_decrement') {
await this.verifyDecrementEvent(e)
} else if (e.type === 'balance_increment') {
await this.verifyIncrementEvent(e)
} else {
await this.storage.paymentStorage.VerifyDbEvent(e)
}
}
await Promise.all(Object.entries(this.users).map(async ([userId, u]) => {
const user = await this.storage.userStorage.GetUser(userId)
if (user.balance_sats !== u.updatedBalance) {
throw new Error("sanity check on balance failed, expected: " + u.updatedBalance + " found: " + user.balance_sats)
}
}))
}
checkUserEntry(e: LoggedEvent, u: { ts: number, updatedBalance: number } | undefined) {
const newEntry = { ts: e.timestampMs, updatedBalance: e.balance + e.amount * (e.type === 'balance_decrement' ? -1 : 1) }
console.log(e)
if (!u) {
console.log(e.userId, "balance starts at", e.balance, "sats and moves by", e.amount * (e.type === 'balance_decrement' ? -1 : 1), "sats, resulting in", newEntry.updatedBalance, "sats")
return newEntry
}
if (e.timestampMs < u.ts) {
throw new Error("entry out of order " + e.timestampMs + " " + u.ts)
}
if (e.balance !== u.updatedBalance) {
throw new Error("inconsistent balance update got: " + e.balance + " expected " + u.updatedBalance)
}
console.log(e.userId, "balance updates from", e.balance, "sats and moves by", e.amount * (e.type === 'balance_decrement' ? -1 : 1), "sats, resulting in", newEntry.updatedBalance, "sats")
return newEntry
}
}

14
src/services/main/settings.ts
View file

@ -1,6 +1,6 @@
import { LoadStorageSettingsFromEnv, StorageSettings } from '../storage/index.js'
import { LndSettings } from '../lnd/settings.js'
import { LoadWatchdogSettingsFromEnv, WatchdogSettings } from '../lnd/watchdog.js'
import { LoadWatchdogSettingsFromEnv, WatchdogSettings } from './watchdog.js'
import { LoadLndSettingsFromEnv } from '../lnd/index.js'
import { EnvMustBeInteger, EnvMustBeNonEmptyString } from '../helpers/envParser.js'
export type MainSettings = {
@ -40,16 +40,22 @@ export const LoadMainSettingsFromEnv = (): MainSettings => {
servicePort: EnvMustBeInteger("PORT"),
recordPerformance: process.env.RECORD_PERFORMANCE === 'true' || false,
skipSanityCheck: process.env.SKIP_SANITY_CHECK === 'true' || false,
disableExternalPayments: process.env.DISABLE_EXTERNAL_PAYMENTS === 'true' || false,
disableExternalPayments: process.env.DISABLE_EXTERNAL_PAYMENTS === 'true' || false
}
}
export const LoadTestSettingsFromEnv = (): MainSettings => {
const eventLogPath = `logs/eventLogV2Test${Date.now()}.csv`
const settings = LoadMainSettingsFromEnv()
return {
...settings,
storageSettings: { dbSettings: { ...settings.storageSettings.dbSettings, databaseFile: ":memory:", metricsDatabaseFile: ":memory:" } },
storageSettings: { dbSettings: { ...settings.storageSettings.dbSettings, databaseFile: ":memory:", metricsDatabaseFile: ":memory:" }, eventLogPath },
lndSettings: {
...settings.lndSettings,
otherLndAddr: EnvMustBeNonEmptyString("LND_OTHER_ADDR"),
otherLndCertPath: EnvMustBeNonEmptyString("LND_OTHER_CERT_PATH"),
otherLndMacaroonPath: EnvMustBeNonEmptyString("LND_OTHER_MACAROON_PATH")
},
skipSanityCheck: true
}
}

30
src/services/lnd/watchdog.ts → src/services/main/watchdog.ts
View file

@ -1,6 +1,7 @@
import { EnvCanBeInteger } from "../helpers/envParser.js";
import { getLogger } from "../helpers/logger.js";
import { LightningHandler } from "./index.js";
import { LightningHandler } from "../lnd/index.js";
import Storage from '../storage/index.js'
export type WatchdogSettings = {
maxDiffSats: number
}
@ -10,21 +11,40 @@ export const LoadWatchdogSettingsFromEnv = (test = false): WatchdogSettings => {
}
}
export class Watchdog {
initialLndBalance: number;
initialUsersBalance: number;
lnd: LightningHandler;
settings: WatchdogSettings;
storage: Storage;
latestCheckStart = 0
log = getLogger({ appName: "watchdog" })
enabled = false
constructor(settings: WatchdogSettings, lnd: LightningHandler) {
interval: NodeJS.Timer;
constructor(settings: WatchdogSettings, lnd: LightningHandler, storage: Storage) {
this.lnd = lnd;
this.settings = settings;
this.storage = storage;
}
SeedLndBalance = async (totalUsersBalance: number) => {
Stop() {
if (this.interval) {
clearInterval(this.interval)
}
}
Start = async () => {
const totalUsersBalance = await this.storage.paymentStorage.GetTotalUsersBalance()
this.initialLndBalance = await this.getTotalLndBalance()
this.initialUsersBalance = totalUsersBalance
this.enabled = true
this.interval = setInterval(() => {
if (this.latestCheckStart + (1000 * 60) < Date.now()) {
this.log("No balance check was made in the last minute, checking now")
this.PaymentRequested()
}
}, 1000 * 60)
}
getTotalLndBalance = async () => {
@ -77,12 +97,14 @@ export class Watchdog {
return false
}
PaymentRequested = async (totalUsersBalance: number) => {
PaymentRequested = async () => {
this.log("Payment requested, checking balance")
if (!this.enabled) {
this.log("WARNING! Watchdog not enabled, skipping balance check")
return
}
this.latestCheckStart = Date.now()
const totalUsersBalance = await this.storage.paymentStorage.GetTotalUsersBalance()
const totalLndBalance = await this.getTotalLndBalance()
const deltaLnd = totalLndBalance - this.initialLndBalance
const deltaUsers = totalUsersBalance - this.initialUsersBalance

8
src/services/storage/eventsLog.ts
View file

@ -1,7 +1,7 @@
import fs from 'fs'
import { parse, stringify } from 'csv'
import { getLogger } from '../helpers/logger.js'
const eventLogPath = "logs/eventLog.csv"
//const eventLogPath = "logs/eventLogV2.csv"
type LoggedEventType = 'new_invoice' | 'new_address' | 'address_paid' | 'invoice_paid' | 'invoice_payment' | 'address_payment' | 'u2u_receiver' | 'u2u_sender' | 'balance_increment' | 'balance_decrement'
export type LoggedEvent = {
timestampMs: number
@ -22,9 +22,11 @@ type TimeEntry = {
const columns = ["timestampMs", "userId", "appUserId", "appId", "balance", "type", "data", "amount"]
type StringerWrite = (chunk: any, cb: (error: Error | null | undefined) => void) => boolean
export default class EventsLogManager {
eventLogPath: string
log = getLogger({ appName: "EventsLogManager" })
stringerWrite: StringerWrite
constructor() {
constructor(eventLogPath: string) {
this.eventLogPath = eventLogPath
const exists = fs.existsSync(eventLogPath)
if (!exists) {
const stringer = stringify({ header: true, columns })
@ -51,7 +53,7 @@ export default class EventsLogManager {
}
Read = async (path?: string): Promise<LoggedEvent[]> => {
const filePath = path ? path : eventLogPath
const filePath = path ? path : this.eventLogPath
const exists = fs.existsSync(filePath)
if (!exists) {
return []

5
src/services/storage/index.ts
View file

@ -9,9 +9,10 @@ import TransactionsQueue, { TX } from "./transactionsQueue.js";
import EventsLogManager from "./eventsLog.js";
export type StorageSettings = {
dbSettings: DbSettings
eventLogPath: string
}
export const LoadStorageSettingsFromEnv = (): StorageSettings => {
return { dbSettings: LoadDbSettingsFromEnv() }
return { dbSettings: LoadDbSettingsFromEnv(), eventLogPath: "logs/eventLogV2.csv" }
}
export default class {
DB: DataSource | EntityManager
@ -25,7 +26,7 @@ export default class {
eventsLog: EventsLogManager
constructor(settings: StorageSettings) {
this.settings = settings
this.eventsLog = new EventsLogManager()
this.eventsLog = new EventsLogManager(settings.eventLogPath)
}
async Connect(migrations: Function[], metricsMigrations: Function[]) {
const { source, executedMigrations } = await NewDB(this.settings.dbSettings, migrations)

73
src/services/storage/paymentStorage.ts
View file

@ -114,6 +114,23 @@ export default class {
})
}
async GetAddressReceivingTransactionOwner(address: string, txHash: string, entityManager = this.DB): Promise<AddressReceivingTransaction | null> {
return entityManager.getRepository(AddressReceivingTransaction).findOne({
where: {
user_address: { address },
tx_hash: txHash
}
})
}
async GetUserTransactionPaymentOwner(address: string, txHash: string, entityManager = this.DB): Promise<UserTransactionPayment | null> {
return entityManager.getRepository(UserTransactionPayment).findOne({
where: {
address,
tx_hash: txHash
}
})
}
async GetInvoiceOwner(paymentRequest: string, entityManager = this.DB): Promise<UserReceivingInvoice | null> {
return entityManager.getRepository(UserReceivingInvoice).findOne({
where: {
@ -128,19 +145,48 @@ export default class {
}
})
}
async GetUser2UserPayment(serialId: number, entityManager = this.DB): Promise<UserToUserPayment | null> {
return entityManager.getRepository(UserToUserPayment).findOne({
where: {
serial_id: serialId
}
})
}
async AddUserInvoicePayment(userId: string, invoice: string, amount: number, routingFees: number, serviceFees: number, internal: boolean, linkedApplication: Application): Promise<UserInvoicePayment> {
async AddPendingExternalPayment(userId: string, invoice: string, amount: number, linkedApplication: Application): Promise<UserInvoicePayment> {
const newPayment = this.DB.getRepository(UserInvoicePayment).create({
user: await this.userStorage.GetUser(userId),
paid_amount: amount,
invoice,
routing_fees: routingFees,
service_fees: serviceFees,
paid_at_unix: Math.floor(Date.now() / 1000),
internal,
routing_fees: 0,
service_fees: 0,
paid_at_unix: 0,
internal: false,
linkedApplication
})
return this.txQueue.PushToQueue<UserInvoicePayment>({ exec: async db => db.getRepository(UserInvoicePayment).save(newPayment), dbTx: false, description: `add invoice payment for ${userId} linked to ${linkedApplication.app_id}: ${invoice}, amt: ${amount} ` })
return this.txQueue.PushToQueue<UserInvoicePayment>({ exec: async db => db.getRepository(UserInvoicePayment).save(newPayment), dbTx: false, description: `add pending invoice payment for ${userId} linked to ${linkedApplication.app_id}: ${invoice}, amt: ${amount} ` })
}
async UpdateExternalPayment(invoicePaymentSerialId: number, routingFees: number, serviceFees: number, success: boolean) {
return this.DB.getRepository(UserInvoicePayment).update(invoicePaymentSerialId, {
routing_fees: routingFees,
service_fees: serviceFees,
paid_at_unix: success ? Math.floor(Date.now() / 1000) : -1
})
}
async AddInternalPayment(userId: string, invoice: string, amount: number, serviceFees: number, linkedApplication: Application): Promise<UserInvoicePayment> {
const newPayment = this.DB.getRepository(UserInvoicePayment).create({
user: await this.userStorage.GetUser(userId),
paid_amount: amount,
invoice,
routing_fees: 0,
service_fees: serviceFees,
paid_at_unix: Math.floor(Date.now() / 1000),
internal: true,
linkedApplication
})
return this.txQueue.PushToQueue<UserInvoicePayment>({ exec: async db => db.getRepository(UserInvoicePayment).save(newPayment), dbTx: false, description: `add internal invoice payment for ${userId} linked to ${linkedApplication.app_id}: ${invoice}, amt: ${amount} ` })
}
GetUserInvoicePayments(userId: string, fromIndex: number, take = 50, entityManager = this.DB): Promise<UserInvoicePayment[]> {
@ -237,16 +283,18 @@ export default class {
return found
}
async AddUserToUserPayment(fromUserId: string, toUserId: string, amount: number, fee: number, linkedApplication: Application, dbTx: DataSource | EntityManager) {
const newKey = dbTx.getRepository(UserToUserPayment).create({
from_user: await this.userStorage.GetUser(fromUserId),
to_user: await this.userStorage.GetUser(toUserId),
async CreateUserToUserPayment(fromUserId: string, toUserId: string, amount: number, fee: number, linkedApplication: Application, dbTx: DataSource | EntityManager) {
return dbTx.getRepository(UserToUserPayment).create({
from_user: await this.userStorage.GetUser(fromUserId, dbTx),
to_user: await this.userStorage.GetUser(toUserId, dbTx),
paid_at_unix: Math.floor(Date.now() / 1000),
paid_amount: amount,
service_fees: fee,
linkedApplication
})
return dbTx.getRepository(UserToUserPayment).save(newKey)
}
async SaveUserToUserPayment(payment: UserToUserPayment, dbTx: DataSource | EntityManager) {
return dbTx.getRepository(UserToUserPayment).save(payment)
}
GetUserToUserReceivedPayments(userId: string, fromIndex: number, take = 50, entityManager = this.DB) {
@ -362,6 +410,7 @@ export default class {
}
async GetTotalUsersBalance(entityManager = this.DB) {
return entityManager.getRepository(User).sum("balance_sats")
const total = await entityManager.getRepository(User).sum("balance_sats")
return total || 0
}
}

14
src/services/storage/transactionsQueue.ts
View file

@ -20,6 +20,7 @@ export default class {
PushToQueue<T>(op: TxOperation<T>) {
if (!this.pendingTx) {
this.log("queue empty, starting transaction", this.transactionsQueue.length)
return this.execQueueItem(op)
}
this.log("queue not empty, possibly stuck")
@ -29,6 +30,7 @@ export default class {
}
async execNextInQueue() {
this.log("executing next in queue")
this.pendingTx = false
const next = this.transactionsQueue.pop()
if (!next) {
@ -49,6 +51,7 @@ export default class {
throw new Error("cannot start DB transaction")
}
this.pendingTx = true
this.log("starting", op.dbTx ? "db transaction" : "operation", op.description || "")
if (op.dbTx) {
return this.doTransaction(op.exec)
}
@ -67,16 +70,17 @@ export default class {
}
doTransaction<T>(exec: TX<T>) {
return this.DB.transaction(async tx => {
async doTransaction<T>(exec: TX<T>) {
try {
const res = await exec(tx)
const res = await this.DB.transaction(async tx => {
return exec(tx)
})
this.execNextInQueue()
return res
} catch (err) {
} catch (err: any) {
this.execNextInQueue()
this.log("transaction failed", err.message)
throw err
}
})
}
}

39
src/services/storage/userStorage.ts
View file

@ -76,9 +76,21 @@ export default class {
}
return user
}
async IncrementUserBalance(userId: string, increment: number, reason: string, entityManager = this.DB) {
const user = await this.GetUser(userId, entityManager)
const res = await entityManager.getRepository(User).increment({
async IncrementUserBalance(userId: string, increment: number, reason: string, entityManager?: DataSource | EntityManager) {
if (entityManager) {
return this.IncrementUserBalanceInTx(userId, increment, reason, entityManager)
}
await this.txQueue.PushToQueue({
dbTx: true,
description: `incrementing user ${userId} balance by ${increment}`,
exec: async tx => {
await this.IncrementUserBalanceInTx(userId, increment, reason, tx)
}
})
}
async IncrementUserBalanceInTx(userId: string, increment: number, reason: string, dbTx: DataSource | EntityManager) {
const user = await this.GetUser(userId, dbTx)
const res = await dbTx.getRepository(User).increment({
user_id: userId,
}, "balance_sats", increment)
if (!res.affected) {
@ -88,13 +100,26 @@ export default class {
getLogger({ userId: userId, appName: "balanceUpdates" })("incremented balance from", user.balance_sats, "sats, by", increment, "sats")
this.eventsLog.LogEvent({ type: 'balance_increment', userId, appId: "", appUserId: "", balance: user.balance_sats, data: reason, amount: increment })
}
async DecrementUserBalance(userId: string, decrement: number, reason: string, entityManager = this.DB) {
const user = await this.GetUser(userId, entityManager)
async DecrementUserBalance(userId: string, decrement: number, reason: string, entityManager?: DataSource | EntityManager) {
if (entityManager) {
return this.DecrementUserBalanceInTx(userId, decrement, reason, entityManager)
}
await this.txQueue.PushToQueue({
dbTx: true,
description: `decrementing user ${userId} balance by ${decrement}`,
exec: async tx => {
await this.DecrementUserBalanceInTx(userId, decrement, reason, tx)
}
})
}
async DecrementUserBalanceInTx(userId: string, decrement: number, reason: string, dbTx: DataSource | EntityManager) {
const user = await this.GetUser(userId, dbTx)
if (!user || user.balance_sats < decrement) {
getLogger({ userId: userId, appName: "balanceUpdates" })("user to decrement not found")
getLogger({ userId: userId, appName: "balanceUpdates" })("not enough balance to decrement")
throw new Error("not enough balance to decrement")
}
const res = await entityManager.getRepository(User).decrement({
const res = await dbTx.getRepository(User).decrement({
user_id: userId,
}, "balance_sats", decrement)
if (!res.affected) {

5
src/test.ts
View file

@ -1,5 +0,0 @@
const failure = true
export default async (describe: (message: string, failure?: boolean) => void) => {
describe("all good")
describe("oh no", failure)
}

29
src/tests/externalPayment.spec.ts Normal file
View file

@ -0,0 +1,29 @@
import { defaultInvoiceExpiry } from '../services/storage/paymentStorage.js'
import { Describe, expect, expectThrowsAsync, runSanityCheck, safelySetUserBalance, SetupTest, TestBase } from './testBase.js'
export const ignore = false
export default async (T: TestBase) => {
await safelySetUserBalance(T, T.user1, 2000)
await testSuccessfulExternalPayment(T)
await runSanityCheck(T)
}
const testSuccessfulExternalPayment = async (T: TestBase) => {
const application = await T.main.storage.applicationStorage.GetApplication(T.app.appId)
const invoice = await T.externalAccessToOtherLnd.NewInvoice(500, "test", defaultInvoiceExpiry)
expect(invoice.payRequest).to.startWith("lnbcrt5u")
T.d("generated 500 sats invoice for external node")
const pay = await T.main.paymentManager.PayInvoice(T.user1.userId, { invoice: invoice.payRequest, amount: 0 }, application)
expect(pay.amount_paid).to.be.equal(500)
T.d("paid 500 sats invoice from user1")
const u1 = await T.main.storage.userStorage.GetUser(T.user1.userId)
const owner = await T.main.storage.userStorage.GetUser(application.owner.user_id)
expect(u1.balance_sats).to.be.equal(1496)
T.d("user1 balance is now 1496 (2000 - (500 + 3 fee + 1 routing))")
expect(owner.balance_sats).to.be.equal(3)
T.d("app balance is 3 sats")
}

39
src/tests/internalPayment.spec.ts Normal file
View file

@ -0,0 +1,39 @@
import { defaultInvoiceExpiry } from '../services/storage/paymentStorage.js'
import { Describe, expect, expectThrowsAsync, runSanityCheck, safelySetUserBalance, SetupTest, TestBase } from './testBase.js'
export const ignore = false
export default async (T: TestBase) => {
await safelySetUserBalance(T, T.user1, 2000)
await testSuccessfulInternalPayment(T)
await testFailedInternalPayment(T)
await runSanityCheck(T)
}
const testSuccessfulInternalPayment = async (T: TestBase) => {
const application = await T.main.storage.applicationStorage.GetApplication(T.app.appId)
const invoice = await T.main.paymentManager.NewInvoice(T.user2.userId, { amountSats: 1000, memo: "test" }, { linkedApplication: application, expiry: defaultInvoiceExpiry })
expect(invoice.invoice).to.startWith("lnbcrt10u")
T.d("generated 1000 sats invoice for user2")
const pay = await T.main.paymentManager.PayInvoice(T.user1.userId, { invoice: invoice.invoice, amount: 0 }, application)
expect(pay.amount_paid).to.be.equal(1000)
T.d("paid 1000 sats invoice from user1")
const u1 = await T.main.storage.userStorage.GetUser(T.user1.userId)
const u2 = await T.main.storage.userStorage.GetUser(T.user2.userId)
const owner = await T.main.storage.userStorage.GetUser(application.owner.user_id)
expect(u2.balance_sats).to.be.equal(1000)
T.d("user2 balance is 1000")
expect(u1.balance_sats).to.be.equal(994)
T.d("user1 balance is 994 cuz he paid 6 sats fee")
expect(owner.balance_sats).to.be.equal(6)
T.d("app balance is 6 sats")
}
const testFailedInternalPayment = async (T: TestBase) => {
const application = await T.main.storage.applicationStorage.GetApplication(T.app.appId)
const invoice = await T.main.paymentManager.NewInvoice(T.user2.userId, { amountSats: 1000, memo: "test" }, { linkedApplication: application, expiry: defaultInvoiceExpiry })
expect(invoice.invoice).to.startWith("lnbcrt10u")
T.d("generated 1000 sats invoice for user2")
await expectThrowsAsync(T.main.paymentManager.PayInvoice(T.user1.userId, { invoice: invoice.invoice, amount: 0 }, application), "not enough balance to decrement")
T.d("payment failed as expected, with the expected error message")
}

46
src/tests/spamExternalPayments.spec.ts Normal file
View file

@ -0,0 +1,46 @@
import { disableLoggers } from '../services/helpers/logger.js'
import { defaultInvoiceExpiry } from '../services/storage/paymentStorage.js'
import { Describe, expect, expectThrowsAsync, runSanityCheck, safelySetUserBalance, SetupTest, TestBase } from './testBase.js'
export const ignore = false
export default async (T: TestBase) => {
disableLoggers(["EventsLogManager", "htlcTracker", "watchdog"])
await safelySetUserBalance(T, T.user1, 2000)
await testSpamExternalPayment(T)
await runSanityCheck(T)
}
const testSpamExternalPayment = async (T: TestBase) => {
const application = await T.main.storage.applicationStorage.GetApplication(T.app.appId)
const invoices = await Promise.all(new Array(10).fill(0).map(() => T.externalAccessToOtherLnd.NewInvoice(500, "test", defaultInvoiceExpiry)))
T.d("generated 10 500 sats invoices for external node")
const res = await Promise.all(invoices.map(async (invoice, i) => {
try {
T.d("trying to pay invoice " + i)
const result = await T.main.paymentManager.PayInvoice(T.user1.userId, { invoice: invoice.payRequest, amount: 0 }, application)
T.d("payment succeeded " + i)
return { success: true, result }
} catch (e: any) {
T.d("payment failed " + i)
console.log(e, i)
return { success: false, err: e }
}
}))
const successfulPayments = res.filter(r => r.success)
const failedPayments = res.filter(r => !r.success)
failedPayments.forEach(f => expect(f.err).to.be.equal("not enough balance to decrement"))
successfulPayments.forEach(s => expect(s.result).to.contain({ amount_paid: 500, network_fee: 1, service_fee: 3 }))
expect(successfulPayments.length).to.be.equal(3)
expect(failedPayments.length).to.be.equal(7)
T.d("3 payments succeeded, 7 failed as expected")
const u = await T.main.storage.userStorage.GetUser(T.user1.userId)
const owner = await T.main.storage.userStorage.GetUser(application.owner.user_id)
expect(u.balance_sats).to.be.equal(488)
T.d("user1 balance is now 488 (2000 - (500 + 3 fee + 1 routing) * 3)")
expect(owner.balance_sats).to.be.equal(9)
T.d("app balance is 9 sats")
}

54
src/tests/spamMixedPayments.spec.ts Normal file
View file

@ -0,0 +1,54 @@
import { disableLoggers } from '../services/helpers/logger.js'
import { defaultInvoiceExpiry } from '../services/storage/paymentStorage.js'
import { Describe, expect, expectThrowsAsync, runSanityCheck, safelySetUserBalance, SetupTest, TestBase } from './testBase.js'
import * as Types from '../../proto/autogenerated/ts/types.js'
export const ignore = false
export default async (T: TestBase) => {
disableLoggers(["EventsLogManager", "htlcTracker", "watchdog"])
await safelySetUserBalance(T, T.user1, 2000)
await testSpamExternalPayment(T)
await runSanityCheck(T)
}
const testSpamExternalPayment = async (T: TestBase) => {
const application = await T.main.storage.applicationStorage.GetApplication(T.app.appId)
const invoicesForExternal = await Promise.all(new Array(5).fill(0).map(() => T.externalAccessToOtherLnd.NewInvoice(500, "test", defaultInvoiceExpiry)))
const invoicesForUser2 = await Promise.all(new Array(5).fill(0).map(() => T.main.paymentManager.NewInvoice(T.user2.userId, { amountSats: 500, memo: "test" }, { linkedApplication: application, expiry: defaultInvoiceExpiry })))
const invoices = invoicesForExternal.map(i => i.payRequest).concat(invoicesForUser2.map(i => i.invoice))
T.d("generated 10 500 sats mixed invoices between external node and user 2")
const res = await Promise.all(invoices.map(async (invoice, i) => {
try {
T.d("trying to pay invoice " + i)
const result = await T.main.paymentManager.PayInvoice(T.user1.userId, { invoice: invoice, amount: 0 }, application)
T.d("payment succeeded " + i)
return { success: true, result }
} catch (e: any) {
T.d("payment failed " + i)
console.log(e, i)
return { success: false, err: e }
}
}))
const successfulPayments = res.filter(r => r.success) as { success: true, result: Types.PayInvoiceResponse }[]
const failedPayments = res.filter(r => !r.success)
failedPayments.forEach(f => expect(f.err).to.be.equal("not enough balance to decrement"))
expect(successfulPayments.length).to.be.equal(3)
expect(failedPayments.length).to.be.equal(7)
T.d("3 payments succeeded, 7 failed as expected")
const networkPayments = successfulPayments.filter(s => s.result.network_fee > 0)
const internalPayments = successfulPayments.filter(s => s.result.network_fee === 0)
expect(networkPayments.length).to.be.equal(1)
expect(internalPayments.length).to.be.equal(2)
networkPayments.forEach(s => expect(s.result).to.contain({ amount_paid: 500, service_fee: 3, network_fee: 1 }))
internalPayments.forEach(s => expect(s.result).to.contain({ amount_paid: 500, service_fee: 3 }))
const u = await T.main.storage.userStorage.GetUser(T.user1.userId)
const owner = await T.main.storage.userStorage.GetUser(application.owner.user_id)
expect(u.balance_sats).to.be.equal(490)
T.d("user1 balance is now 490 (2000 - (500 + 3 fee + 1 routing + (500 + 3fee) * 2))")
expect(owner.balance_sats).to.be.equal(9)
T.d("app balance is 9 sats")
}

97
src/tests/testBase.ts Normal file
View file

@ -0,0 +1,97 @@
import 'dotenv/config' // TODO - test env
import chai from 'chai'
import { AppData, initMainHandler } from '../services/main/init.js'
import Main from '../services/main/index.js'
import Storage from '../services/storage/index.js'
import { User } from '../services/storage/entity/User.js'
import { LoadMainSettingsFromEnv, LoadTestSettingsFromEnv, MainSettings } from '../services/main/settings.js'
import chaiString from 'chai-string'
import { defaultInvoiceExpiry } from '../services/storage/paymentStorage.js'
import SanityChecker from '../services/main/sanityChecker.js'
import LND from '../services/lnd/lnd.js'
import { LightningHandler } from '../services/lnd/index.js'
chai.use(chaiString)
export const expect = chai.expect
export type Describe = (message: string, failure?: boolean) => void
export type TestUserData = {
userId: string;
appUserIdentifier: string;
appId: string;
}
export type TestBase = {
expect: Chai.ExpectStatic;
main: Main
app: AppData
user1: TestUserData
user2: TestUserData
externalAccessToMainLnd: LND
externalAccessToOtherLnd: LND
d: Describe
}
export const SetupTest = async (d: Describe): Promise<TestBase> => {
const settings = LoadTestSettingsFromEnv()
const initialized = await initMainHandler(console.log, settings)
if (!initialized) {
throw new Error("failed to initialize main handler")
}
const main = initialized.mainHandler
const app = initialized.apps[0]
const u1 = await main.applicationManager.AddAppUser(app.appId, { identifier: "user1", balance: 0, fail_if_exists: true })
const u2 = await main.applicationManager.AddAppUser(app.appId, { identifier: "user2", balance: 0, fail_if_exists: true })
const user1 = { userId: u1.info.userId, appUserIdentifier: u1.identifier, appId: app.appId }
const user2 = { userId: u2.info.userId, appUserIdentifier: u2.identifier, appId: app.appId }
const externalAccessToMainLnd = new LND(settings.lndSettings, console.log, console.log, () => { }, () => { })
const otherLndSetting = { ...settings.lndSettings, lndCertPath: settings.lndSettings.otherLndCertPath, lndMacaroonPath: settings.lndSettings.otherLndMacaroonPath, lndAddr: settings.lndSettings.otherLndAddr }
const externalAccessToOtherLnd = new LND(otherLndSetting, console.log, console.log, () => { }, () => { })
await externalAccessToMainLnd.Warmup()
await externalAccessToOtherLnd.Warmup()
return {
expect, main, app,
user1, user2,
externalAccessToMainLnd, externalAccessToOtherLnd,
d
}
}
export const teardown = async (T: TestBase) => {
T.main.paymentManager.watchDog.Stop()
T.main.lnd.Stop()
T.externalAccessToMainLnd.Stop()
T.externalAccessToOtherLnd.Stop()
console.log("teardown")
}
export const safelySetUserBalance = async (T: TestBase, user: TestUserData, amount: number) => {
const app = await T.main.storage.applicationStorage.GetApplication(user.appId)
const invoice = await T.main.paymentManager.NewInvoice(user.userId, { amountSats: amount, memo: "test" }, { linkedApplication: app, expiry: defaultInvoiceExpiry })
await T.externalAccessToOtherLnd.PayInvoice(invoice.invoice, 0, 100)
const u = await T.main.storage.userStorage.GetUser(user.userId)
expect(u.balance_sats).to.be.equal(amount)
T.d(`user ${user.appUserIdentifier} balance is now ${amount}`)
}
export const runSanityCheck = async (T: TestBase) => {
const sanityChecker = new SanityChecker(T.main.storage, T.main.lnd)
await sanityChecker.VerifyEventsLog()
}
export const expectThrowsAsync = async (promise: Promise<any>, errorMessage?: string) => {
let error: Error | null = null
try {
await promise
}
catch (err: any) {
error = err as Error
}
expect(error).to.be.an('Error')
console.log(error!.message)
if (errorMessage) {
expect(error!.message).to.equal(errorMessage)
}
}

29
src/testRunner.ts → src/tests/testRunner.ts
View file

@ -1,11 +1,10 @@
import { globby } from 'globby'
type Describe = (message: string, failure?: boolean) => void
import { Describe, SetupTest, teardown, TestBase } from './testBase.js'
type TestModule = {
ignore?: boolean
setup?: () => Promise<void>
default: (describe: Describe) => Promise<void>
teardown?: () => Promise<void>
default: (T: TestBase) => Promise<void>
}
let failures = 0
const start = async () => {
@ -13,7 +12,7 @@ const start = async () => {
const files = await globby("**/*.spec.js")
for (const file of files) {
console.log(file)
const module = await import(`./${file.slice("build/src/".length)}`) as TestModule
const module = await import(`./${file.slice("build/src/tests/".length)}`) as TestModule
await runTestFile(file, module)
}
if (failures) {
@ -30,21 +29,15 @@ const runTestFile = async (fileName: string, mod: TestModule) => {
d("-----ignoring file-----")
return
}
const T = await SetupTest(d)
try {
if (mod.setup) {
d("setup started")
await mod.setup()
}
d("tests starting")
await mod.default(d)
d("tests finished")
if (mod.teardown) {
await mod.teardown()
d("teardown finished")
}
d("test starting")
await mod.default(T)
d("test finished")
await teardown(T)
} catch (e: any) {
d("FAILURE", true)
d(e, true)
await teardown(T)
}
}
@ -52,7 +45,7 @@ const getDescribe = (fileName: string): Describe => {
return (message, failure) => {
if (failure) {
failures++
console.error(redConsole, fileName, ":", message, resetConsole)
console.error(redConsole, fileName, ": FAILURE ", message, resetConsole)
} else {
console.log(greenConsole, fileName, ":", message, resetConsole)
}