fix and add tests

package.json

@@ -4,7 +4,7 @@
   "description": "",
   "main": "index.js",
   "scripts": {
-    "test": " tsc  && node build/src/testRunner.js",
+    "test": " tsc  && node build/src/tests/testRunner.js",
     "start": "tsc && node build/src/index.js",
     "start:ci": "git reset --hard && git pull &&  npm run start",
     "build_autogenerated": "cd proto && rimraf autogenerated && protoc -I ./service --pub_out=. service/*",
@@ -77,4 +77,4 @@
     "ts-node": "10.7.0",
     "typescript": "4.5.2"
   }
-}
+}

src/services/helpers/logger.ts

@@ -7,7 +7,6 @@ try {
 } catch { }
 const z = (n: number) => n < 10 ? `0${n}` : `${n}`
 const openWriter = (fileName: string): Writer => {
-
     const logStream = fs.createWriteStream(`logs/${fileName}`, { flags: 'a' });
     return (message) => {
         logStream.write(message + "\n")
@@ -37,6 +36,9 @@ export const getLogger = (params: LoggerParams): PubLogger => {
         const timestamp = `${now.getFullYear()}-${z(now.getMonth() + 1)}-${z(now.getDate())} ${z(now.getHours())}:${z(now.getMinutes())}:${z(now.getSeconds())}`
         const toLog = [timestamp]
         if (params.appName) {
+            if (disabledApps.includes(params.appName)) {
+                return
+            }
             toLog.push(params.appName)
         }
         if (params.userId) {
@@ -48,3 +50,7 @@ export const getLogger = (params: LoggerParams): PubLogger => {
         writers.forEach(w => w(final))
     }
 }
+const disabledApps: string[] = []
+export const disableLoggers = (appNamesToDisable: string[]) => {
+    disabledApps.push(...appNamesToDisable)
+}

src/services/lnd/index.ts

@@ -12,7 +12,7 @@ export const LoadLndSettingsFromEnv = (): LndSettings => {
     const feeRateLimit = EnvMustBeInteger("OUTBOUND_MAX_FEE_BPS") / 10000
     const feeFixedLimit = EnvMustBeInteger("OUTBOUND_MAX_FEE_EXTRA_SATS")
     const mockLnd = EnvCanBeBoolean("MOCK_LND")
-    return { lndAddr, lndCertPath, lndMacaroonPath, feeRateLimit, feeFixedLimit, mockLnd }
+    return { lndAddr, lndCertPath, lndMacaroonPath, feeRateLimit, feeFixedLimit, mockLnd, otherLndAddr: "", otherLndCertPath: "", otherLndMacaroonPath: "" }
 }
 export interface LightningHandler {
     Stop(): void

src/services/lnd/lnd.spec.ts

@@ -1,32 +0,0 @@
-import 'dotenv/config' // TODO - test env
-import { expect } from 'chai'
-import * as Types from '../../../proto/autogenerated/ts/types.js';
-import NewLightningHandler, { LightningHandler, LoadLndSettingsFromEnv } from '../lnd/index.js'
-let lnd: LightningHandler
-export const ignore = true
-export const setup = async () => {
-    lnd = NewLightningHandler(LoadLndSettingsFromEnv(), console.log, console.log, console.log, console.log)
-    await lnd.Warmup()
-}
-export const teardown = () => {
-    lnd.Stop()
-}
-
-export default async (d: (message: string, failure?: boolean) => void) => {
-    const info = await lnd.GetInfo()
-    expect(info.alias).to.equal("alice")
-    d("get alias ok")
-
-    const addr = await lnd.NewAddress(Types.AddressType.WITNESS_PUBKEY_HASH)
-    console.log(addr)
-    d("new address ok")
-
-    const invoice = await lnd.NewInvoice(1000, "", 60 * 60)
-    console.log(invoice)
-    d("new invoice ok")
-
-    const res = await lnd.EstimateChainFees("bcrt1qajzzx453x9fx5gtlyax8zrsennckrw3syd2llt", 1000, 100)
-    console.log(res)
-    d("estimate fee ok")
-    //const res = await this.lnd.OpenChannel("025ed7fc85fc05a07fc5acc13a6e3836cd11c5587c1d400afcd22630a9e230eb7a", "", 20000, 0)
-}

src/services/lnd/settings.ts

@@ -7,6 +7,10 @@ export type LndSettings = {
     feeRateLimit: number
     feeFixedLimit: number
     mockLnd: boolean
+
+    otherLndAddr: string
+    otherLndCertPath: string
+    otherLndMacaroonPath: string
 }
 type TxOutput = {
     hash: string

src/services/main/main.spec.ts

@@ -1,47 +0,0 @@
-import 'dotenv/config' // TODO - test env
-import chai from 'chai'
-import { AppData, initMainHandler } from './init.js'
-import Main from './index.js'
-import { User } from '../storage/entity/User.js'
-import { LoadMainSettingsFromEnv, LoadTestSettingsFromEnv, MainSettings } from './settings.js'
-import chaiString from 'chai-string'
-import { defaultInvoiceExpiry } from '../storage/paymentStorage.js'
-chai.use(chaiString)
-const expect = chai.expect
-export const ignore = false
-let main: Main
-let app: AppData
-let user1: { userId: string, appUserIdentifier: string, appId: string }
-let user2: { userId: string, appUserIdentifier: string, appId: string }
-export const setup = async () => {
-    const settings = LoadTestSettingsFromEnv()
-    const initialized = await initMainHandler(console.log, settings)
-    if (!initialized) {
-        throw new Error("failed to initialize main handler")
-    }
-    main = initialized.mainHandler
-    app = initialized.apps[0]
-    const u1 = await main.applicationManager.AddAppUser(app.appId, { identifier: "user1", balance: 0, fail_if_exists: true })
-    const u2 = await main.applicationManager.AddAppUser(app.appId, { identifier: "user2", balance: 2000, fail_if_exists: true })
-    user1 = { userId: u1.info.userId, appUserIdentifier: u1.identifier, appId: app.appId }
-    user2 = { userId: u2.info.userId, appUserIdentifier: u2.identifier, appId: app.appId }
-}
-export const teardown = () => {
-    console.log("teardown")
-}
-
-export default async (d: (message: string, failure?: boolean) => void) => {
-    const application = await main.storage.applicationStorage.GetApplication(app.appId)
-    const invoice = await main.paymentManager.NewInvoice(user1.userId, { amountSats: 1000, memo: "test" }, { linkedApplication: application, expiry: defaultInvoiceExpiry })
-    expect(invoice.invoice).to.startWith("lnbcrtmockin")
-    d("got the invoice")
-
-    const pay = await main.paymentManager.PayInvoice(user2.userId, { invoice: invoice.invoice, amount: 0 }, application)
-    expect(pay.amount_paid).to.be.equal(1000)
-    const u1 = await main.storage.userStorage.GetUser(user1.userId)
-    const u2 = await main.storage.userStorage.GetUser(user2.userId)
-    const owner = await main.storage.userStorage.GetUser(application.owner.user_id)
-    expect(u1.balance_sats).to.be.equal(1000)
-    expect(u2.balance_sats).to.be.equal(994)
-    expect(owner.balance_sats).to.be.equal(6)
-}

src/services/main/paymentManager.ts

@@ -159,7 +159,6 @@ export default class {
         const payAmount = req.amount !== 0 ? req.amount : Number(decoded.numSatoshis)
         const isAppUserPayment = userId !== linkedApplication.owner.user_id
         const serviceFee = this.getServiceFee(Types.UserOperationType.OUTGOING_INVOICE, payAmount, isAppUserPayment)
-        const totalAmountToDecrement = payAmount + serviceFee
         const internalInvoice = await this.storage.paymentStorage.GetInvoiceOwner(req.invoice)
         let paymentInfo = { preimage: "", amtPaid: 0, networkFee: 0, serialId: 0 }
         if (internalInvoice) {
@@ -190,10 +189,13 @@ export default class {
         this.log("paying external invoice", invoice)
         const routingFeeLimit = this.lnd.GetFeeLimitAmount(payAmount)
         await this.storage.userStorage.DecrementUserBalance(userId, totalAmountToDecrement + routingFeeLimit, invoice)
+        console.log("decremented")
         const pendingPayment = await this.storage.paymentStorage.AddPendingExternalPayment(userId, invoice, payAmount, linkedApplication)
         try {
             const payment = await this.lnd.PayInvoice(invoice, amountForLnd, routingFeeLimit)
+
             if (routingFeeLimit - payment.feeSat > 0) {
+                this.log("refund routing fee", routingFeeLimit, payment.feeSat, "sats")
                 await this.storage.userStorage.IncrementUserBalance(userId, routingFeeLimit - payment.feeSat, "routing_fee_refund:" + invoice)
             }
             await this.storage.paymentStorage.UpdateExternalPayment(pendingPayment.serial_id, payment.feeSat, serviceFee, true)

src/services/main/sanityChecker.ts

@@ -8,6 +8,7 @@ type UniqueDecrementReasons = 'ban'
 type UniqueIncrementReasons = 'fees' | 'routing_fee_refund' | 'payment_refund'
 type CommonReasons = 'invoice' | 'address' | 'u2u'
 type Reason = UniqueDecrementReasons | UniqueIncrementReasons | CommonReasons
+const incrementTwiceAllowed = ['fees', 'ban']
 export default class SanityChecker {
     storage: Storage
     lnd: LightningHandler
@@ -17,7 +18,7 @@ export default class SanityChecker {
     payments: Payment[] = []
     incrementSources: Record<string, boolean> = {}
     decrementSources: Record<string, boolean> = {}
-    decrementEvents: Record<string, { userId: string, refund: number, falure: boolean }> = {}
+    decrementEvents: Record<string, { userId: string, refund: number, failure: boolean }> = {}
     users: Record<string, { ts: number, updatedBalance: number }> = {}
     constructor(storage: Storage, lnd: LightningHandler) {
         this.storage = storage
@@ -54,7 +55,7 @@ export default class SanityChecker {
         if (this.decrementSources[e.data]) {
             throw new Error("entry decremented more that once " + e.data)
         }
-        this.decrementSources[e.data] = true
+        this.decrementSources[e.data] = !incrementTwiceAllowed.includes(e.data)
         this.users[e.userId] = this.checkUserEntry(e, this.users[e.userId])
         const parsed = this.parseDataField(e.data)
         switch (parsed.type) {
@@ -99,10 +100,10 @@ export default class SanityChecker {
             throw new Error("payment never settled for invoice " + invoice) // TODO: check if this is correct
         }
         if (entry.paid_at_unix === -1) {
-            const refund = amt - (entry.paid_amount + entry.routing_fees + entry.service_fees)
+            this.decrementEvents[invoice] = { userId, refund: amt, failure: true }
-            this.decrementEvents[invoice] = { userId, refund, falure: true }
         } else {
-            this.decrementEvents[invoice] = { userId, refund: amt, falure: false }
+            const refund = amt - (entry.paid_amount + entry.routing_fees + entry.service_fees)
+            this.decrementEvents[invoice] = { userId, refund, failure: false }
         }
         if (!entry.internal) {
             const lndEntry = this.payments.find(i => i.paymentRequest === invoice)
@@ -132,7 +133,7 @@ export default class SanityChecker {
         if (this.incrementSources[e.data]) {
             throw new Error("entry incremented more that once " + e.data)
         }
-        this.incrementSources[e.data] = true
+        this.incrementSources[e.data] = !incrementTwiceAllowed.includes(e.data)
         this.users[e.userId] = this.checkUserEntry(e, this.users[e.userId])
         const parsed = this.parseDataField(e.data)
         switch (parsed.type) {
@@ -196,10 +197,11 @@ export default class SanityChecker {
         if (entry.userId !== userId) {
             throw new Error("user id mismatch for routing fee refund " + invoice)
         }
-        if (entry.falure) {
+        if (entry.failure) {
             throw new Error("payment failled, should not refund routing fees " + invoice)
         }
         if (entry.refund !== amt) {
+            console.log(entry.refund, amt)
             throw new Error("refund amount mismatch for routing fee refund " + invoice)
         }
     }
@@ -212,7 +214,7 @@ export default class SanityChecker {
         if (entry.userId !== userId) {
             throw new Error("user id mismatch for payment refund " + invoice)
         }
-        if (!entry.falure) {
+        if (!entry.failure) {
             throw new Error("payment did not fail, should not refund payment " + invoice)
         }
         if (entry.refund !== amt) {
@@ -249,7 +251,9 @@ export default class SanityChecker {
 
     checkUserEntry(e: LoggedEvent, u: { ts: number, updatedBalance: number } | undefined) {
         const newEntry = { ts: e.timestampMs, updatedBalance: e.balance + e.amount * (e.type === 'balance_decrement' ? -1 : 1) }
+        console.log(e)
         if (!u) {
+            console.log(e.userId, "balance starts at", e.balance, "sats and moves by", e.amount * (e.type === 'balance_decrement' ? -1 : 1), "sats, resulting in", newEntry.updatedBalance, "sats")
             return newEntry
         }
         if (e.timestampMs < u.ts) {
@@ -258,6 +262,7 @@ export default class SanityChecker {
         if (e.balance !== u.updatedBalance) {
             throw new Error("inconsistent balance update got: " + e.balance + " expected " + u.updatedBalance)
         }
+        console.log(e.userId, "balance updates from", e.balance, "sats and moves by", e.amount * (e.type === 'balance_decrement' ? -1 : 1), "sats, resulting in", newEntry.updatedBalance, "sats")
         return newEntry
     }
 }

src/services/main/settings.ts

@@ -40,16 +40,22 @@ export const LoadMainSettingsFromEnv = (): MainSettings => {
         servicePort: EnvMustBeInteger("PORT"),
         recordPerformance: process.env.RECORD_PERFORMANCE === 'true' || false,
         skipSanityCheck: process.env.SKIP_SANITY_CHECK === 'true' || false,
-        disableExternalPayments: process.env.DISABLE_EXTERNAL_PAYMENTS === 'true' || false,
+        disableExternalPayments: process.env.DISABLE_EXTERNAL_PAYMENTS === 'true' || false
-
     }
 }
 
 export const LoadTestSettingsFromEnv = (): MainSettings => {
+    const eventLogPath = `logs/eventLogV2Test${Date.now()}.csv`
     const settings = LoadMainSettingsFromEnv()
     return {
         ...settings,
-        storageSettings: { dbSettings: { ...settings.storageSettings.dbSettings, databaseFile: ":memory:", metricsDatabaseFile: ":memory:" } },
+        storageSettings: { dbSettings: { ...settings.storageSettings.dbSettings, databaseFile: ":memory:", metricsDatabaseFile: ":memory:" }, eventLogPath },
+        lndSettings: {
+            ...settings.lndSettings,
+            otherLndAddr: EnvMustBeNonEmptyString("LND_OTHER_ADDR"),
+            otherLndCertPath: EnvMustBeNonEmptyString("LND_OTHER_CERT_PATH"),
+            otherLndMacaroonPath: EnvMustBeNonEmptyString("LND_OTHER_MACAROON_PATH")
+        },
         skipSanityCheck: true
     }
 }

src/services/main/watchdog.ts

@@ -11,6 +11,7 @@ export const LoadWatchdogSettingsFromEnv = (test = false): WatchdogSettings => {
     }
 }
 export class Watchdog {
+
     initialLndBalance: number;
     initialUsersBalance: number;
     lnd: LightningHandler;
@@ -19,19 +20,26 @@ export class Watchdog {
     latestCheckStart = 0
     log = getLogger({ appName: "watchdog" })
     enabled = false
+    interval: NodeJS.Timer;
     constructor(settings: WatchdogSettings, lnd: LightningHandler, storage: Storage) {
         this.lnd = lnd;
         this.settings = settings;
         this.storage = storage;
     }
 
+    Stop() {
+        if (this.interval) {
+            clearInterval(this.interval)
+        }
+    }
+
     Start = async () => {
         const totalUsersBalance = await this.storage.paymentStorage.GetTotalUsersBalance()
         this.initialLndBalance = await this.getTotalLndBalance()
         this.initialUsersBalance = totalUsersBalance
         this.enabled = true
 
-        setInterval(() => {
+        this.interval = setInterval(() => {
             if (this.latestCheckStart + (1000 * 60) < Date.now()) {
                 this.log("No balance check was made in the last minute, checking now")
                 this.PaymentRequested()

src/services/storage/eventsLog.ts

@@ -1,7 +1,7 @@
 import fs from 'fs'
 import { parse, stringify } from 'csv'
 import { getLogger } from '../helpers/logger.js'
-const eventLogPath = "logs/eventLogV2.csv"
+//const eventLogPath = "logs/eventLogV2.csv"
 type LoggedEventType = 'new_invoice' | 'new_address' | 'address_paid' | 'invoice_paid' | 'invoice_payment' | 'address_payment' | 'u2u_receiver' | 'u2u_sender' | 'balance_increment' | 'balance_decrement'
 export type LoggedEvent = {
     timestampMs: number
@@ -22,9 +22,11 @@ type TimeEntry = {
 const columns = ["timestampMs", "userId", "appUserId", "appId", "balance", "type", "data", "amount"]
 type StringerWrite = (chunk: any, cb: (error: Error | null | undefined) => void) => boolean
 export default class EventsLogManager {
+    eventLogPath: string
     log = getLogger({ appName: "EventsLogManager" })
     stringerWrite: StringerWrite
-    constructor() {
+    constructor(eventLogPath: string) {
+        this.eventLogPath = eventLogPath
         const exists = fs.existsSync(eventLogPath)
         if (!exists) {
             const stringer = stringify({ header: true, columns })
@@ -51,7 +53,7 @@ export default class EventsLogManager {
     }
 
     Read = async (path?: string): Promise<LoggedEvent[]> => {
-        const filePath = path ? path : eventLogPath
+        const filePath = path ? path : this.eventLogPath
         const exists = fs.existsSync(filePath)
         if (!exists) {
             return []

src/services/storage/index.ts

@@ -9,9 +9,10 @@ import TransactionsQueue, { TX } from "./transactionsQueue.js";
 import EventsLogManager from "./eventsLog.js";
 export type StorageSettings = {
     dbSettings: DbSettings
+    eventLogPath: string
 }
 export const LoadStorageSettingsFromEnv = (): StorageSettings => {
-    return { dbSettings: LoadDbSettingsFromEnv() }
+    return { dbSettings: LoadDbSettingsFromEnv(), eventLogPath: "logs/eventLogV2.csv" }
 }
 export default class {
     DB: DataSource | EntityManager
@@ -25,7 +26,7 @@ export default class {
     eventsLog: EventsLogManager
     constructor(settings: StorageSettings) {
         this.settings = settings
-        this.eventsLog = new EventsLogManager()
+        this.eventsLog = new EventsLogManager(settings.eventLogPath)
     }
     async Connect(migrations: Function[], metricsMigrations: Function[]) {
         const { source, executedMigrations } = await NewDB(this.settings.dbSettings, migrations)

src/services/storage/transactionsQueue.ts

@@ -20,6 +20,7 @@ export default class {
 
     PushToQueue<T>(op: TxOperation<T>) {
         if (!this.pendingTx) {
+            this.log("queue empty, starting transaction", this.transactionsQueue.length)
             return this.execQueueItem(op)
         }
         this.log("queue not empty, possibly stuck")
@@ -29,6 +30,7 @@ export default class {
     }
 
     async execNextInQueue() {
+        this.log("executing next in queue")
         this.pendingTx = false
         const next = this.transactionsQueue.pop()
         if (!next) {
@@ -49,6 +51,7 @@ export default class {
             throw new Error("cannot start DB transaction")
         }
         this.pendingTx = true
+        this.log("starting", op.dbTx ? "db transaction" : "operation", op.description || "")
         if (op.dbTx) {
             return this.doTransaction(op.exec)
         }
@@ -67,16 +70,17 @@ export default class {
     }
 
 
-    doTransaction<T>(exec: TX<T>) {
+    async doTransaction<T>(exec: TX<T>) {
-        return this.DB.transaction(async tx => {
+        try {
-            try {
+            const res = await this.DB.transaction(async tx => {
-                const res = await exec(tx)
+                return exec(tx)
-                this.execNextInQueue()
+            })
-                return res
+            this.execNextInQueue()
-            } catch (err) {
+            return res
-                this.execNextInQueue()
+        } catch (err: any) {
-                throw err
+            this.execNextInQueue()
-            }
+            this.log("transaction failed", err.message)
-        })
+            throw err
+        }
     }
 }

src/services/storage/userStorage.ts

@@ -76,9 +76,21 @@ export default class {
         }
         return user
     }
-    async IncrementUserBalance(userId: string, increment: number, reason: string, entityManager = this.DB) {
+    async IncrementUserBalance(userId: string, increment: number, reason: string, entityManager?: DataSource | EntityManager) {
-        const user = await this.GetUser(userId, entityManager)
+        if (entityManager) {
-        const res = await entityManager.getRepository(User).increment({
+            return this.IncrementUserBalanceInTx(userId, increment, reason, entityManager)
+        }
+        await this.txQueue.PushToQueue({
+            dbTx: true,
+            description: `incrementing user ${userId} balance by ${increment}`,
+            exec: async tx => {
+                await this.IncrementUserBalanceInTx(userId, increment, reason, tx)
+            }
+        })
+    }
+    async IncrementUserBalanceInTx(userId: string, increment: number, reason: string, dbTx: DataSource | EntityManager) {
+        const user = await this.GetUser(userId, dbTx)
+        const res = await dbTx.getRepository(User).increment({
             user_id: userId,
         }, "balance_sats", increment)
         if (!res.affected) {
@@ -88,13 +100,26 @@ export default class {
         getLogger({ userId: userId, appName: "balanceUpdates" })("incremented balance from", user.balance_sats, "sats, by", increment, "sats")
         this.eventsLog.LogEvent({ type: 'balance_increment', userId, appId: "", appUserId: "", balance: user.balance_sats, data: reason, amount: increment })
     }
-    async DecrementUserBalance(userId: string, decrement: number, reason: string, entityManager = this.DB) {
+    async DecrementUserBalance(userId: string, decrement: number, reason: string, entityManager?: DataSource | EntityManager) {
-        const user = await this.GetUser(userId, entityManager)
+        if (entityManager) {
+            return this.DecrementUserBalanceInTx(userId, decrement, reason, entityManager)
+        }
+        await this.txQueue.PushToQueue({
+            dbTx: true,
+            description: `decrementing user ${userId} balance by ${decrement}`,
+            exec: async tx => {
+                await this.DecrementUserBalanceInTx(userId, decrement, reason, tx)
+            }
+        })
+    }
+
+    async DecrementUserBalanceInTx(userId: string, decrement: number, reason: string, dbTx: DataSource | EntityManager) {
+        const user = await this.GetUser(userId, dbTx)
         if (!user || user.balance_sats < decrement) {
-            getLogger({ userId: userId, appName: "balanceUpdates" })("user to decrement not found")
+            getLogger({ userId: userId, appName: "balanceUpdates" })("not enough balance to decrement")
             throw new Error("not enough balance to decrement")
         }
-        const res = await entityManager.getRepository(User).decrement({
+        const res = await dbTx.getRepository(User).decrement({
             user_id: userId,
         }, "balance_sats", decrement)
         if (!res.affected) {

src/test.ts

@@ -1,5 +0,0 @@
-const failure = true
-export default async (describe: (message: string, failure?: boolean) => void) => {
-    describe("all good")
-    describe("oh no", failure)
-}

src/tests/externalPayment.spec.ts

@@ -0,0 +1,29 @@
+import { defaultInvoiceExpiry } from '../services/storage/paymentStorage.js'
+import { Describe, expect, expectThrowsAsync, runSanityCheck, safelySetUserBalance, SetupTest, TestBase } from './testBase.js'
+export const ignore = false
+
+export default async (T: TestBase) => {
+    await safelySetUserBalance(T, T.user1, 2000)
+    await testSuccessfulExternalPayment(T)
+    await runSanityCheck(T)
+}
+
+
+const testSuccessfulExternalPayment = async (T: TestBase) => {
+    const application = await T.main.storage.applicationStorage.GetApplication(T.app.appId)
+    const invoice = await T.externalAccessToOtherLnd.NewInvoice(500, "test", defaultInvoiceExpiry)
+    expect(invoice.payRequest).to.startWith("lnbcrt5u")
+    T.d("generated 500 sats invoice for external node")
+
+    const pay = await T.main.paymentManager.PayInvoice(T.user1.userId, { invoice: invoice.payRequest, amount: 0 }, application)
+    expect(pay.amount_paid).to.be.equal(500)
+    T.d("paid 500 sats invoice from user1")
+    const u1 = await T.main.storage.userStorage.GetUser(T.user1.userId)
+    const owner = await T.main.storage.userStorage.GetUser(application.owner.user_id)
+    expect(u1.balance_sats).to.be.equal(1496)
+    T.d("user1 balance is now 1496 (2000 - (500 + 3 fee + 1 routing))")
+    expect(owner.balance_sats).to.be.equal(3)
+    T.d("app balance is 3 sats")
+
+}
+

src/tests/internalPayment.spec.ts

@@ -0,0 +1,39 @@
+import { defaultInvoiceExpiry } from '../services/storage/paymentStorage.js'
+import { Describe, expect, expectThrowsAsync, runSanityCheck, safelySetUserBalance, SetupTest, TestBase } from './testBase.js'
+export const ignore = false
+
+export default async (T: TestBase) => {
+    await safelySetUserBalance(T, T.user1, 2000)
+    await testSuccessfulInternalPayment(T)
+    await testFailedInternalPayment(T)
+    await runSanityCheck(T)
+}
+
+const testSuccessfulInternalPayment = async (T: TestBase) => {
+    const application = await T.main.storage.applicationStorage.GetApplication(T.app.appId)
+    const invoice = await T.main.paymentManager.NewInvoice(T.user2.userId, { amountSats: 1000, memo: "test" }, { linkedApplication: application, expiry: defaultInvoiceExpiry })
+    expect(invoice.invoice).to.startWith("lnbcrt10u")
+    T.d("generated 1000 sats invoice for user2")
+    const pay = await T.main.paymentManager.PayInvoice(T.user1.userId, { invoice: invoice.invoice, amount: 0 }, application)
+    expect(pay.amount_paid).to.be.equal(1000)
+    T.d("paid 1000 sats invoice from user1")
+    const u1 = await T.main.storage.userStorage.GetUser(T.user1.userId)
+    const u2 = await T.main.storage.userStorage.GetUser(T.user2.userId)
+    const owner = await T.main.storage.userStorage.GetUser(application.owner.user_id)
+    expect(u2.balance_sats).to.be.equal(1000)
+    T.d("user2 balance is 1000")
+    expect(u1.balance_sats).to.be.equal(994)
+    T.d("user1 balance is 994 cuz he paid 6 sats fee")
+    expect(owner.balance_sats).to.be.equal(6)
+    T.d("app balance is 6 sats")
+}
+
+const testFailedInternalPayment = async (T: TestBase) => {
+    const application = await T.main.storage.applicationStorage.GetApplication(T.app.appId)
+    const invoice = await T.main.paymentManager.NewInvoice(T.user2.userId, { amountSats: 1000, memo: "test" }, { linkedApplication: application, expiry: defaultInvoiceExpiry })
+    expect(invoice.invoice).to.startWith("lnbcrt10u")
+    T.d("generated 1000 sats invoice for user2")
+    await expectThrowsAsync(T.main.paymentManager.PayInvoice(T.user1.userId, { invoice: invoice.invoice, amount: 0 }, application), "not enough balance to decrement")
+    T.d("payment failed as expected, with the expected error message")
+}
+

src/tests/spamExternalPayments.spec.ts

@@ -0,0 +1,46 @@
+import { disableLoggers } from '../services/helpers/logger.js'
+import { defaultInvoiceExpiry } from '../services/storage/paymentStorage.js'
+import { Describe, expect, expectThrowsAsync, runSanityCheck, safelySetUserBalance, SetupTest, TestBase } from './testBase.js'
+export const ignore = false
+
+export default async (T: TestBase) => {
+    disableLoggers(["EventsLogManager", "htlcTracker", "watchdog"])
+    await safelySetUserBalance(T, T.user1, 2000)
+    await testSpamExternalPayment(T)
+    await runSanityCheck(T)
+}
+
+
+const testSpamExternalPayment = async (T: TestBase) => {
+    const application = await T.main.storage.applicationStorage.GetApplication(T.app.appId)
+    const invoices = await Promise.all(new Array(10).fill(0).map(() => T.externalAccessToOtherLnd.NewInvoice(500, "test", defaultInvoiceExpiry)))
+    T.d("generated 10 500 sats invoices for external node")
+    const res = await Promise.all(invoices.map(async (invoice, i) => {
+        try {
+            T.d("trying to pay invoice " + i)
+            const result = await T.main.paymentManager.PayInvoice(T.user1.userId, { invoice: invoice.payRequest, amount: 0 }, application)
+            T.d("payment succeeded " + i)
+            return { success: true, result }
+        } catch (e: any) {
+            T.d("payment failed " + i)
+            console.log(e, i)
+            return { success: false, err: e }
+        }
+    }))
+
+    const successfulPayments = res.filter(r => r.success)
+    const failedPayments = res.filter(r => !r.success)
+    failedPayments.forEach(f => expect(f.err).to.be.equal("not enough balance to decrement"))
+    successfulPayments.forEach(s => expect(s.result).to.contain({ amount_paid: 500, network_fee: 1, service_fee: 3 }))
+    expect(successfulPayments.length).to.be.equal(3)
+    expect(failedPayments.length).to.be.equal(7)
+    T.d("3 payments succeeded, 7 failed as expected")
+    const u = await T.main.storage.userStorage.GetUser(T.user1.userId)
+    const owner = await T.main.storage.userStorage.GetUser(application.owner.user_id)
+    expect(u.balance_sats).to.be.equal(488)
+    T.d("user1 balance is now 488 (2000 - (500 + 3 fee + 1 routing) * 3)")
+    expect(owner.balance_sats).to.be.equal(9)
+    T.d("app balance is 9 sats")
+
+}
+

src/tests/spamMixedPayments.spec.ts

@@ -0,0 +1,54 @@
+import { disableLoggers } from '../services/helpers/logger.js'
+import { defaultInvoiceExpiry } from '../services/storage/paymentStorage.js'
+import { Describe, expect, expectThrowsAsync, runSanityCheck, safelySetUserBalance, SetupTest, TestBase } from './testBase.js'
+import * as Types from '../../proto/autogenerated/ts/types.js'
+export const ignore = false
+
+export default async (T: TestBase) => {
+    disableLoggers(["EventsLogManager", "htlcTracker", "watchdog"])
+    await safelySetUserBalance(T, T.user1, 2000)
+    await testSpamExternalPayment(T)
+    await runSanityCheck(T)
+}
+
+
+const testSpamExternalPayment = async (T: TestBase) => {
+    const application = await T.main.storage.applicationStorage.GetApplication(T.app.appId)
+    const invoicesForExternal = await Promise.all(new Array(5).fill(0).map(() => T.externalAccessToOtherLnd.NewInvoice(500, "test", defaultInvoiceExpiry)))
+    const invoicesForUser2 = await Promise.all(new Array(5).fill(0).map(() => T.main.paymentManager.NewInvoice(T.user2.userId, { amountSats: 500, memo: "test" }, { linkedApplication: application, expiry: defaultInvoiceExpiry })))
+    const invoices = invoicesForExternal.map(i => i.payRequest).concat(invoicesForUser2.map(i => i.invoice))
+    T.d("generated 10 500 sats mixed invoices between external node and user 2")
+    const res = await Promise.all(invoices.map(async (invoice, i) => {
+        try {
+            T.d("trying to pay invoice " + i)
+            const result = await T.main.paymentManager.PayInvoice(T.user1.userId, { invoice: invoice, amount: 0 }, application)
+            T.d("payment succeeded " + i)
+            return { success: true, result }
+        } catch (e: any) {
+            T.d("payment failed " + i)
+            console.log(e, i)
+            return { success: false, err: e }
+        }
+    }))
+
+    const successfulPayments = res.filter(r => r.success) as { success: true, result: Types.PayInvoiceResponse }[]
+    const failedPayments = res.filter(r => !r.success)
+    failedPayments.forEach(f => expect(f.err).to.be.equal("not enough balance to decrement"))
+    expect(successfulPayments.length).to.be.equal(3)
+    expect(failedPayments.length).to.be.equal(7)
+    T.d("3 payments succeeded, 7 failed as expected")
+    const networkPayments = successfulPayments.filter(s => s.result.network_fee > 0)
+    const internalPayments = successfulPayments.filter(s => s.result.network_fee === 0)
+    expect(networkPayments.length).to.be.equal(1)
+    expect(internalPayments.length).to.be.equal(2)
+    networkPayments.forEach(s => expect(s.result).to.contain({ amount_paid: 500, service_fee: 3, network_fee: 1 }))
+    internalPayments.forEach(s => expect(s.result).to.contain({ amount_paid: 500, service_fee: 3 }))
+    const u = await T.main.storage.userStorage.GetUser(T.user1.userId)
+    const owner = await T.main.storage.userStorage.GetUser(application.owner.user_id)
+    expect(u.balance_sats).to.be.equal(490)
+    T.d("user1 balance is now 490 (2000 - (500 + 3 fee + 1 routing + (500 + 3fee) * 2))")
+    expect(owner.balance_sats).to.be.equal(9)
+    T.d("app balance is 9 sats")
+
+}
+

src/tests/testBase.ts

@@ -0,0 +1,97 @@
+import 'dotenv/config' // TODO - test env
+import chai from 'chai'
+import { AppData, initMainHandler } from '../services/main/init.js'
+import Main from '../services/main/index.js'
+import Storage from '../services/storage/index.js'
+import { User } from '../services/storage/entity/User.js'
+import { LoadMainSettingsFromEnv, LoadTestSettingsFromEnv, MainSettings } from '../services/main/settings.js'
+import chaiString from 'chai-string'
+import { defaultInvoiceExpiry } from '../services/storage/paymentStorage.js'
+import SanityChecker from '../services/main/sanityChecker.js'
+import LND from '../services/lnd/lnd.js'
+import { LightningHandler } from '../services/lnd/index.js'
+chai.use(chaiString)
+export const expect = chai.expect
+export type Describe = (message: string, failure?: boolean) => void
+export type TestUserData = {
+    userId: string;
+    appUserIdentifier: string;
+    appId: string;
+
+}
+export type TestBase = {
+    expect: Chai.ExpectStatic;
+    main: Main
+    app: AppData
+    user1: TestUserData
+    user2: TestUserData
+    externalAccessToMainLnd: LND
+    externalAccessToOtherLnd: LND
+    d: Describe
+}
+
+export const SetupTest = async (d: Describe): Promise<TestBase> => {
+    const settings = LoadTestSettingsFromEnv()
+    const initialized = await initMainHandler(console.log, settings)
+    if (!initialized) {
+        throw new Error("failed to initialize main handler")
+    }
+    const main = initialized.mainHandler
+    const app = initialized.apps[0]
+    const u1 = await main.applicationManager.AddAppUser(app.appId, { identifier: "user1", balance: 0, fail_if_exists: true })
+    const u2 = await main.applicationManager.AddAppUser(app.appId, { identifier: "user2", balance: 0, fail_if_exists: true })
+    const user1 = { userId: u1.info.userId, appUserIdentifier: u1.identifier, appId: app.appId }
+    const user2 = { userId: u2.info.userId, appUserIdentifier: u2.identifier, appId: app.appId }
+
+
+    const externalAccessToMainLnd = new LND(settings.lndSettings, console.log, console.log, () => { }, () => { })
+    const otherLndSetting = { ...settings.lndSettings, lndCertPath: settings.lndSettings.otherLndCertPath, lndMacaroonPath: settings.lndSettings.otherLndMacaroonPath, lndAddr: settings.lndSettings.otherLndAddr }
+    const externalAccessToOtherLnd = new LND(otherLndSetting, console.log, console.log, () => { }, () => { })
+    await externalAccessToMainLnd.Warmup()
+    await externalAccessToOtherLnd.Warmup()
+
+
+    return {
+        expect, main, app,
+        user1, user2,
+        externalAccessToMainLnd, externalAccessToOtherLnd,
+        d
+    }
+}
+
+export const teardown = async (T: TestBase) => {
+    T.main.paymentManager.watchDog.Stop()
+    T.main.lnd.Stop()
+    T.externalAccessToMainLnd.Stop()
+    T.externalAccessToOtherLnd.Stop()
+    console.log("teardown")
+}
+
+export const safelySetUserBalance = async (T: TestBase, user: TestUserData, amount: number) => {
+    const app = await T.main.storage.applicationStorage.GetApplication(user.appId)
+    const invoice = await T.main.paymentManager.NewInvoice(user.userId, { amountSats: amount, memo: "test" }, { linkedApplication: app, expiry: defaultInvoiceExpiry })
+    await T.externalAccessToOtherLnd.PayInvoice(invoice.invoice, 0, 100)
+    const u = await T.main.storage.userStorage.GetUser(user.userId)
+    expect(u.balance_sats).to.be.equal(amount)
+    T.d(`user ${user.appUserIdentifier} balance is now ${amount}`)
+}
+
+export const runSanityCheck = async (T: TestBase) => {
+    const sanityChecker = new SanityChecker(T.main.storage, T.main.lnd)
+    await sanityChecker.VerifyEventsLog()
+}
+
+export const expectThrowsAsync = async (promise: Promise<any>, errorMessage?: string) => {
+    let error: Error | null = null
+    try {
+        await promise
+    }
+    catch (err: any) {
+        error = err as Error
+    }
+    expect(error).to.be.an('Error')
+    console.log(error!.message)
+    if (errorMessage) {
+        expect(error!.message).to.equal(errorMessage)
+    }
+}

src/tests/testRunner.ts

@@ -1,11 +1,10 @@
 import { globby } from 'globby'
-type Describe = (message: string, failure?: boolean) => void
+import { Describe, SetupTest, teardown, TestBase } from './testBase.js'
+
 
 type TestModule = {
     ignore?: boolean
-    setup?: () => Promise<void>
+    default: (T: TestBase) => Promise<void>
-    default: (describe: Describe) => Promise<void>
-    teardown?: () => Promise<void>
 }
 let failures = 0
 const start = async () => {
@@ -13,7 +12,7 @@ const start = async () => {
     const files = await globby("**/*.spec.js")
     for (const file of files) {
         console.log(file)
-        const module = await import(`./${file.slice("build/src/".length)}`) as TestModule
+        const module = await import(`./${file.slice("build/src/tests/".length)}`) as TestModule
         await runTestFile(file, module)
     }
     if (failures) {
@@ -30,21 +29,15 @@ const runTestFile = async (fileName: string, mod: TestModule) => {
         d("-----ignoring file-----")
         return
     }
+    const T = await SetupTest(d)
     try {
-        if (mod.setup) {
+        d("test starting")
-            d("setup started")
+        await mod.default(T)
-            await mod.setup()
+        d("test finished")
-        }
+        await teardown(T)
-        d("tests starting")
-        await mod.default(d)
-        d("tests finished")
-        if (mod.teardown) {
-            await mod.teardown()
-            d("teardown finished")
-        }
     } catch (e: any) {
-        d("FAILURE", true)
         d(e, true)
+        await teardown(T)
     }
 }
 
@@ -52,7 +45,7 @@ const getDescribe = (fileName: string): Describe => {
     return (message, failure) => {
         if (failure) {
             failures++
-            console.error(redConsole, fileName, ":", message, resetConsole)
+            console.error(redConsole, fileName, ": FAILURE ", message, resetConsole)
         } else {
             console.log(greenConsole, fileName, ":", message, resetConsole)
         }