Merge pull request #638 from snake-eaterr/standalone-metrics-db

Standalone metrics db

.gitignore

@@ -7,5 +7,6 @@ temp/
 .env
 build/
 db.sqlite
+metrics.sqlite
 .key/
 logs

env.example

@@ -5,6 +5,7 @@ LND_MACAROON_PATH=/root/.lnd/data/chain/bitcoin/mainnet/admin.macaroon
 
 #DB
 DATABASE_FILE=db.sqlite
+METRICS_DATABASE_FILE=metrics.sqlite
 
 #LOCAL
 ADMIN_TOKEN=
@@ -38,3 +39,6 @@ SERVICE_URL=https://test.lightning.pub
 MOCK_LND=false
 ALLOW_BALANCE_MIGRATION=false
 MIGRATE_DB=false
+
+#METRICS
+RECORD_PERFORMANCE=true

metricsDatasource.js

@@ -0,0 +1,12 @@
+import { DataSource } from "typeorm"
+import { BalanceEvent } from "./build/src/services/storage/entity/BalanceEvent.js"
+import { ChannelBalanceEvent } from "./build/src/services/storage/entity/ChannelsBalanceEvent.js"
+import { RoutingEvent } from "./build/src/services/storage/entity/RoutingEvent.js"
+
+
+
+export default new DataSource({
+    type: "sqlite",
+    database: "metrics.sqlite",
+    entities: [ RoutingEvent, BalanceEvent, ChannelBalanceEvent],
+});

src/auth.ts

@@ -11,7 +11,7 @@ const serverOptions = (mainHandler: Main): ServerOptions => {
         AppAuthGuard: async (authHeader) => { return { app_id: mainHandler.applicationManager.DecodeAppToken(stripBearer(authHeader)) } },
         UserAuthGuard: async (authHeader) => { return mainHandler.appUserManager.DecodeUserToken(stripBearer(authHeader)) },
         GuestAuthGuard: async (_) => ({}),
-        metricsCallback: metrics => mainHandler.metricsManager.AddMetrics(metrics),
+        metricsCallback: metrics => mainHandler.settings.recordPerformance ? mainHandler.metricsManager.AddMetrics(metrics) : null,
         allowCors: true
         //throwErrors: true
     }

src/custom-nip19.ts

@@ -0,0 +1,88 @@
+/* 
+	This file contains functions that deal with encoding and decoding nprofiles,
+	but with he addition of bridge urls in the nprofile.
+	These functions are basically the same functions from nostr-tools package
+	but with some tweaks to allow for the bridge inclusion.
+*/
+import { bytesToHex, concatBytes, hexToBytes } from '@noble/hashes/utils';
+import { bech32 } from 'bech32';
+
+export const utf8Decoder = new TextDecoder('utf-8')
+export const utf8Encoder = new TextEncoder()
+
+
+export type CustomProfilePointer = {
+  pubkey: string
+  relays?: string[]
+	bridge?: string[] // one bridge
+}
+
+
+
+type TLV = { [t: number]: Uint8Array[] }
+
+
+const encodeTLV = (tlv: TLV): Uint8Array => {
+	const entries: Uint8Array[] = []
+
+  Object.entries(tlv)
+	/* 
+		the original function does a reverse() here,
+		but here it causes the nprofile string to be different,
+		even though it would still decode to the correct original inputs
+	*/
+		//.reverse() 
+    .forEach(([t, vs]) => {
+      vs.forEach(v => {
+        const entry = new Uint8Array(v.length + 2)
+        entry.set([parseInt(t)], 0)
+        entry.set([v.length], 1)
+        entry.set(v, 2)
+        entries.push(entry)
+      })
+    })
+	return concatBytes(...entries);
+}
+
+export const encodeNprofile = (profile: CustomProfilePointer): string => {
+	const data = encodeTLV({
+    0: [hexToBytes(profile.pubkey)],
+    1: (profile.relays || []).map(url => utf8Encoder.encode(url)),
+		2: (profile.bridge || []).map(url => utf8Encoder.encode(url))
+  });
+	const words = bech32.toWords(data)
+  return bech32.encode("nprofile", words, 5000);
+}
+
+const parseTLV = (data: Uint8Array): TLV => {
+  const result: TLV = {}
+  let rest = data
+  while (rest.length > 0) {
+    const t = rest[0]
+    const l = rest[1]
+    const v = rest.slice(2, 2 + l)
+    rest = rest.slice(2 + l)
+    if (v.length < l) throw new Error(`not enough data to read on TLV ${t}`)
+    result[t] = result[t] || []
+    result[t].push(v)
+  }
+  return result
+}
+
+export const decodeNprofile = (nprofile: string): CustomProfilePointer => {
+	const { prefix, words } = bech32.decode(nprofile, 5000)
+	if (prefix !== "nprofile") {
+		throw new Error ("Expected nprofile prefix");
+	}
+  const data = new Uint8Array(bech32.fromWords(words))
+
+	const tlv = parseTLV(data);
+	if (!tlv[0]?.[0]) throw new Error('missing TLV 0 for nprofile')
+	if (tlv[0][0].length !== 32) throw new Error('TLV 0 should be 32 bytes')
+
+	return {
+		pubkey: bytesToHex(tlv[0][0]),
+		relays: tlv[1] ? tlv[1].map(d => utf8Decoder.decode(d)) : [],
+		bridge: tlv[2] ? tlv[2].map(d => utf8Decoder.decode(d)): []
+	}
+}

src/nostrMiddleware.ts

@@ -11,7 +11,7 @@ export default (serverMethods: Types.ServerMethods, mainHandler: Main, nostrSett
             let nostrUser = await mainHandler.storage.applicationStorage.GetOrCreateNostrAppUser(app, pub || "")
             return { user_id: nostrUser.user.user_id, app_user_id: nostrUser.identifier, app_id: appId || "" }
         },
-        metricsCallback: metrics => mainHandler.metricsManager.AddMetrics(metrics)
+        metricsCallback: metrics => mainHandler.settings.recordPerformance ?  mainHandler.metricsManager.AddMetrics(metrics) : null
     })
     const nostr = new Nostr(nostrSettings, event => {
         let j: NostrRequest

src/services/main/index.ts

@@ -30,7 +30,8 @@ export const LoadMainSettingsFromEnv = (test = false): MainSettings => {
         userToUserFee: EnvMustBeInteger("TX_FEE_INTERNAL_USER_BPS") / 10000,
         appToUserFee: EnvMustBeInteger("TX_FEE_INTERNAL_ROOT_BPS") / 10000,
         serviceUrl: EnvMustBeNonEmptyString("SERVICE_URL"),
-        servicePort: EnvMustBeInteger("PORT")
+        servicePort: EnvMustBeInteger("PORT"),
+        recordPerformance: process.env.RECORD_PERFORMANCE === 'true' || false
     }
 }
 

src/services/main/settings.ts

@@ -14,5 +14,6 @@ export type MainSettings = {
     appToUserFee: number
     serviceUrl: string
     servicePort: number
+    recordPerformance: boolean
 
 }

src/services/nostr/handler.ts

@@ -2,6 +2,7 @@
 import { SimplePool, Sub, Event, UnsignedEvent, getEventHash, finishEvent, relayInit } from './tools/index.js'
 import { encryptData, decryptData, getSharedSecret, decodePayload, encodePayload } from './nip44.js'
 import { getLogger } from '../helpers/logger.js'
+import { encodeNprofile } from '../../custom-nip19.js'
 const handledEvents: string[] = [] // TODO: - big memory leak here, add TTL
 type AppInfo = { appId: string, publicKey: string, privateKey: string, name: string }
 export type SendData = { type: "content", content: string, pub: string } | { type: "event", event: UnsignedEvent }
@@ -88,7 +89,18 @@ export default class Handler {
     eventCallback: (event: NostrEvent) => void
     constructor(settings: NostrSettings, eventCallback: (event: NostrEvent) => void) {
         this.settings = settings
-        console.log(settings)
+        console.log(
+            {
+                ...settings,
+                apps: settings.apps.map(app => {
+                    const { privateKey, ...rest } = app;
+                    return {
+                        ...rest,
+                        nprofile: encodeNprofile({ pubkey: rest.publicKey, relays: settings.relays })
+                    }
+                })
+            }
+        )
         this.eventCallback = eventCallback
         this.settings.apps.forEach(app => {
             this.apps[app.publicKey] = app

src/services/storage/db.ts

@@ -24,21 +24,41 @@ import { LndMetrics1703170330183 } from "./migrations/1703170330183-lnd_metrics.
 export type DbSettings = {
     databaseFile: string
     migrate: boolean
+    metricsDatabaseFile: string
 }
 export const LoadDbSettingsFromEnv = (test = false): DbSettings => {
     return {
         databaseFile: test ? ":memory:" : EnvMustBeNonEmptyString("DATABASE_FILE"),
         migrate: process.env.MIGRATE_DB === 'true' || false,
+        metricsDatabaseFile: test ? ":memory" : EnvMustBeNonEmptyString("METRICS_DATABASE_FILE")
     }
 }
 
+export const newMetricsDb = async (settings: DbSettings, metricsMigrations: Function[]): Promise<{ source: DataSource, executedMigrations: Migration[] }> => {
+    const source = await new DataSource({
+        type: "sqlite",
+        database: settings.metricsDatabaseFile,
+        entities: [ RoutingEvent, BalanceEvent, ChannelBalanceEvent],
+        migrations: metricsMigrations
+    }).initialize();
+    const log = getLogger({});
+    const pendingMigrations = await source.showMigrations()
+    if (pendingMigrations) {
+        log("Migrations found, migrating...")
+        const executedMigrations = await source.runMigrations({ transaction: 'all' })
+        return { source, executedMigrations }
+    }
+    return { source, executedMigrations: [] }
+
+}
+
 export default async (settings: DbSettings, migrations: Function[]): Promise<{ source: DataSource, executedMigrations: Migration[] }> => {
     const source = await new DataSource({
         type: "sqlite",
         database: settings.databaseFile,
         // logging: true,
         entities: [User, UserReceivingInvoice, UserReceivingAddress, AddressReceivingTransaction, UserInvoicePayment, UserTransactionPayment,
-            UserBasicAuth, UserEphemeralKey, Product, UserToUserPayment, Application, ApplicationUser, UserToUserPayment, RoutingEvent, BalanceEvent, ChannelBalanceEvent],
+            UserBasicAuth, UserEphemeralKey, Product, UserToUserPayment, Application, ApplicationUser, UserToUserPayment],
         //synchronize: true,
         migrations
     }).initialize()

src/services/storage/index.ts

@@ -25,7 +25,7 @@ export default class {
     constructor(settings: StorageSettings) {
         this.settings = settings
     }
-    async Connect(migrations: Function[]) {
+    async Connect(migrations: Function[], metricsMigrations: Function []) {
         const { source, executedMigrations } = await NewDB(this.settings.dbSettings, migrations)
         this.DB = source
         this.txQueue = new TransactionsQueue(this.DB)
@@ -33,8 +33,9 @@ export default class {
         this.productStorage = new ProductStorage(this.DB, this.txQueue)
         this.applicationStorage = new ApplicationStorage(this.DB, this.userStorage, this.txQueue)
         this.paymentStorage = new PaymentStorage(this.DB, this.userStorage, this.txQueue)
-        this.metricsStorage = new MetricsStorage(this.DB, this.txQueue)
-        return executedMigrations
+        this.metricsStorage = new MetricsStorage(this.settings)
+        const executedMetricsMigrations = await this.metricsStorage.Connect(metricsMigrations)
+        return { executedMigrations, executedMetricsMigrations };
     }
 
     StartTransaction(exec: TX<void>) {

src/services/storage/metricsStorage.ts

@@ -3,12 +3,20 @@ import { RoutingEvent } from "./entity/RoutingEvent.js"
 import { BalanceEvent } from "./entity/BalanceEvent.js"
 import { ChannelBalanceEvent } from "./entity/ChannelsBalanceEvent.js"
 import TransactionsQueue, { TX } from "./transactionsQueue.js";
+import { StorageSettings } from "./index.js";
+import { newMetricsDb } from "./db.js";
 export default class {
     DB: DataSource | EntityManager
+    settings: StorageSettings
     txQueue: TransactionsQueue
-    constructor(DB: DataSource | EntityManager, txQueue: TransactionsQueue) {
-        this.DB = DB
-        this.txQueue = txQueue
+    constructor(settings: StorageSettings) {
+        this.settings = settings;
+    }
+    async Connect(metricsMigrations: Function[]) {
+        const { source, executedMigrations } = await newMetricsDb(this.settings.dbSettings, metricsMigrations)
+        this.DB = source;
+        this.txQueue = new TransactionsQueue(this.DB)
+        return executedMigrations;
     }
     async SaveRoutingEvent(event: Partial<RoutingEvent>) {
         const entry = this.DB.getRepository(RoutingEvent).create(event)

src/services/storage/migrations/runner.ts

@@ -6,33 +6,38 @@ import { LndMetrics1703170330183 } from './1703170330183-lnd_metrics.js'
 const allMigrations = [LndMetrics1703170330183]
 export const TypeOrmMigrationRunner = async (log: PubLogger, storageManager: Storage, settings: DbSettings, arg: string | undefined): Promise<boolean> => {
     if (arg === 'initial_migration') {
-        await connectAndMigrate(log, storageManager, true, settings, [Initial1703170309875])
+        await connectAndMigrate(log, storageManager, true, settings, [Initial1703170309875], [])
         return true
     } else if (arg === 'lnd_metrics_migration') {
-        await connectAndMigrate(log, storageManager, true, settings, [LndMetrics1703170330183])
+        await connectAndMigrate(log, storageManager, true, settings, [], [LndMetrics1703170330183])
         return true
     } else if (arg === 'all_migrations') {
-        await connectAndMigrate(log, storageManager, true, settings, allMigrations)
+        await connectAndMigrate(log, storageManager, true, settings, [], allMigrations)
         return true
     } else if (settings.migrate) {
-        await connectAndMigrate(log, storageManager, false, settings, allMigrations)
+        await connectAndMigrate(log, storageManager, false, settings, [], allMigrations)
         return false
     }
-    await connectAndMigrate(log, storageManager, false, settings, [])
+    await connectAndMigrate(log, storageManager, false, settings, [], [])
     return false
 }
 
-const connectAndMigrate = async (log: PubLogger, storageManager: Storage, manual: boolean, settings: DbSettings, migrations: Function[]) => {
+const connectAndMigrate = async (log: PubLogger, storageManager: Storage, manual: boolean, settings: DbSettings, migrations: Function[], metricsMigrations: Function[]) => {
     if (manual && settings.migrate) {
         throw new Error("auto migration is enabled, no need to run manual migration")
     }
     if (migrations.length > 0) {
         log("will add", migrations.length, "typeorm migrations...")
     }
-    const executedMigrations = await storageManager.Connect(migrations)
+    const { executedMigrations, executedMetricsMigrations } = await storageManager.Connect(migrations, metricsMigrations)
     if (migrations.length > 0) {
         log(executedMigrations.length, "of", migrations.length, "migrations were executed correctly")
         log(executedMigrations)
+        log("-------------------")
+        
+    } if (metricsMigrations.length > 0) {
+        log(executedMetricsMigrations.length, "of", migrations.length, "metrics migrations were executed correctly")
+        log(executedMetricsMigrations)
     }
 
 }