diff --git a/.gitignore b/.gitignore index dbe0467d..eb8ee274 100644 --- a/.gitignore +++ b/.gitignore @@ -7,5 +7,6 @@ temp/ .env build/ db.sqlite +metrics.sqlite .key/ logs \ No newline at end of file diff --git a/env.example b/env.example index dbb52845..b3cad64a 100644 --- a/env.example +++ b/env.example @@ -5,6 +5,7 @@ LND_MACAROON_PATH=/root/.lnd/data/chain/bitcoin/mainnet/admin.macaroon #DB DATABASE_FILE=db.sqlite +METRICS_DATABASE_FILE=metrics.sqlite #LOCAL ADMIN_TOKEN= @@ -38,3 +39,6 @@ SERVICE_URL=https://test.lightning.pub MOCK_LND=false ALLOW_BALANCE_MIGRATION=false MIGRATE_DB=false + +#METRICS +RECORD_PERFORMANCE=true diff --git a/metricsDatasource.js b/metricsDatasource.js new file mode 100644 index 00000000..9801c558 --- /dev/null +++ b/metricsDatasource.js @@ -0,0 +1,12 @@ +import { DataSource } from "typeorm" +import { BalanceEvent } from "./build/src/services/storage/entity/BalanceEvent.js" +import { ChannelBalanceEvent } from "./build/src/services/storage/entity/ChannelsBalanceEvent.js" +import { RoutingEvent } from "./build/src/services/storage/entity/RoutingEvent.js" + + + +export default new DataSource({ + type: "sqlite", + database: "metrics.sqlite", + entities: [ RoutingEvent, BalanceEvent, ChannelBalanceEvent], +}); \ No newline at end of file diff --git a/src/auth.ts b/src/auth.ts index de6ac223..74485cb5 100644 --- a/src/auth.ts +++ b/src/auth.ts @@ -11,7 +11,7 @@ const serverOptions = (mainHandler: Main): ServerOptions => { AppAuthGuard: async (authHeader) => { return { app_id: mainHandler.applicationManager.DecodeAppToken(stripBearer(authHeader)) } }, UserAuthGuard: async (authHeader) => { return mainHandler.appUserManager.DecodeUserToken(stripBearer(authHeader)) }, GuestAuthGuard: async (_) => ({}), - metricsCallback: metrics => mainHandler.metricsManager.AddMetrics(metrics), + metricsCallback: metrics => mainHandler.settings.recordPerformance ? mainHandler.metricsManager.AddMetrics(metrics) : null, allowCors: true //throwErrors: true } diff --git a/src/custom-nip19.ts b/src/custom-nip19.ts new file mode 100644 index 00000000..62c7c56a --- /dev/null +++ b/src/custom-nip19.ts @@ -0,0 +1,88 @@ +/* + This file contains functions that deal with encoding and decoding nprofiles, + but with he addition of bridge urls in the nprofile. + These functions are basically the same functions from nostr-tools package + but with some tweaks to allow for the bridge inclusion. +*/ +import { bytesToHex, concatBytes, hexToBytes } from '@noble/hashes/utils'; +import { bech32 } from 'bech32'; + +export const utf8Decoder = new TextDecoder('utf-8') +export const utf8Encoder = new TextEncoder() + + +export type CustomProfilePointer = { + pubkey: string + relays?: string[] + bridge?: string[] // one bridge +} + + + +type TLV = { [t: number]: Uint8Array[] } + + +const encodeTLV = (tlv: TLV): Uint8Array => { + const entries: Uint8Array[] = [] + + Object.entries(tlv) + /* + the original function does a reverse() here, + but here it causes the nprofile string to be different, + even though it would still decode to the correct original inputs + */ + //.reverse() + .forEach(([t, vs]) => { + vs.forEach(v => { + const entry = new Uint8Array(v.length + 2) + entry.set([parseInt(t)], 0) + entry.set([v.length], 1) + entry.set(v, 2) + entries.push(entry) + }) + }) + return concatBytes(...entries); +} + +export const encodeNprofile = (profile: CustomProfilePointer): string => { + const data = encodeTLV({ + 0: [hexToBytes(profile.pubkey)], + 1: (profile.relays || []).map(url => utf8Encoder.encode(url)), + 2: (profile.bridge || []).map(url => utf8Encoder.encode(url)) + }); + const words = bech32.toWords(data) + return bech32.encode("nprofile", words, 5000); +} + +const parseTLV = (data: Uint8Array): TLV => { + const result: TLV = {} + let rest = data + while (rest.length > 0) { + const t = rest[0] + const l = rest[1] + const v = rest.slice(2, 2 + l) + rest = rest.slice(2 + l) + if (v.length < l) throw new Error(`not enough data to read on TLV ${t}`) + result[t] = result[t] || [] + result[t].push(v) + } + return result +} + +export const decodeNprofile = (nprofile: string): CustomProfilePointer => { + const { prefix, words } = bech32.decode(nprofile, 5000) + if (prefix !== "nprofile") { + throw new Error ("Expected nprofile prefix"); + } + const data = new Uint8Array(bech32.fromWords(words)) + + const tlv = parseTLV(data); + if (!tlv[0]?.[0]) throw new Error('missing TLV 0 for nprofile') + if (tlv[0][0].length !== 32) throw new Error('TLV 0 should be 32 bytes') + + return { + pubkey: bytesToHex(tlv[0][0]), + relays: tlv[1] ? tlv[1].map(d => utf8Decoder.decode(d)) : [], + bridge: tlv[2] ? tlv[2].map(d => utf8Decoder.decode(d)): [] + } +} \ No newline at end of file diff --git a/src/nostrMiddleware.ts b/src/nostrMiddleware.ts index 215fa2c6..3eb877d9 100644 --- a/src/nostrMiddleware.ts +++ b/src/nostrMiddleware.ts @@ -11,7 +11,7 @@ export default (serverMethods: Types.ServerMethods, mainHandler: Main, nostrSett let nostrUser = await mainHandler.storage.applicationStorage.GetOrCreateNostrAppUser(app, pub || "") return { user_id: nostrUser.user.user_id, app_user_id: nostrUser.identifier, app_id: appId || "" } }, - metricsCallback: metrics => mainHandler.metricsManager.AddMetrics(metrics) + metricsCallback: metrics => mainHandler.settings.recordPerformance ? mainHandler.metricsManager.AddMetrics(metrics) : null }) const nostr = new Nostr(nostrSettings, event => { let j: NostrRequest diff --git a/src/services/main/index.ts b/src/services/main/index.ts index 4f5f5adf..43f0dba0 100644 --- a/src/services/main/index.ts +++ b/src/services/main/index.ts @@ -30,7 +30,8 @@ export const LoadMainSettingsFromEnv = (test = false): MainSettings => { userToUserFee: EnvMustBeInteger("TX_FEE_INTERNAL_USER_BPS") / 10000, appToUserFee: EnvMustBeInteger("TX_FEE_INTERNAL_ROOT_BPS") / 10000, serviceUrl: EnvMustBeNonEmptyString("SERVICE_URL"), - servicePort: EnvMustBeInteger("PORT") + servicePort: EnvMustBeInteger("PORT"), + recordPerformance: process.env.RECORD_PERFORMANCE === 'true' || false } } diff --git a/src/services/main/settings.ts b/src/services/main/settings.ts index 1be670f2..a4b705a1 100644 --- a/src/services/main/settings.ts +++ b/src/services/main/settings.ts @@ -14,5 +14,6 @@ export type MainSettings = { appToUserFee: number serviceUrl: string servicePort: number + recordPerformance: boolean } \ No newline at end of file diff --git a/src/services/nostr/handler.ts b/src/services/nostr/handler.ts index ffd94424..6bb1fce2 100644 --- a/src/services/nostr/handler.ts +++ b/src/services/nostr/handler.ts @@ -2,6 +2,7 @@ import { SimplePool, Sub, Event, UnsignedEvent, getEventHash, finishEvent, relayInit } from './tools/index.js' import { encryptData, decryptData, getSharedSecret, decodePayload, encodePayload } from './nip44.js' import { getLogger } from '../helpers/logger.js' +import { encodeNprofile } from '../../custom-nip19.js' const handledEvents: string[] = [] // TODO: - big memory leak here, add TTL type AppInfo = { appId: string, publicKey: string, privateKey: string, name: string } export type SendData = { type: "content", content: string, pub: string } | { type: "event", event: UnsignedEvent } @@ -88,7 +89,18 @@ export default class Handler { eventCallback: (event: NostrEvent) => void constructor(settings: NostrSettings, eventCallback: (event: NostrEvent) => void) { this.settings = settings - console.log(settings) + console.log( + { + ...settings, + apps: settings.apps.map(app => { + const { privateKey, ...rest } = app; + return { + ...rest, + nprofile: encodeNprofile({ pubkey: rest.publicKey, relays: settings.relays }) + } + }) + } + ) this.eventCallback = eventCallback this.settings.apps.forEach(app => { this.apps[app.publicKey] = app diff --git a/src/services/storage/db.ts b/src/services/storage/db.ts index 0754bd13..e3b40e8f 100644 --- a/src/services/storage/db.ts +++ b/src/services/storage/db.ts @@ -24,21 +24,41 @@ import { LndMetrics1703170330183 } from "./migrations/1703170330183-lnd_metrics. export type DbSettings = { databaseFile: string migrate: boolean + metricsDatabaseFile: string } export const LoadDbSettingsFromEnv = (test = false): DbSettings => { return { databaseFile: test ? ":memory:" : EnvMustBeNonEmptyString("DATABASE_FILE"), migrate: process.env.MIGRATE_DB === 'true' || false, + metricsDatabaseFile: test ? ":memory" : EnvMustBeNonEmptyString("METRICS_DATABASE_FILE") } } +export const newMetricsDb = async (settings: DbSettings, metricsMigrations: Function[]): Promise<{ source: DataSource, executedMigrations: Migration[] }> => { + const source = await new DataSource({ + type: "sqlite", + database: settings.metricsDatabaseFile, + entities: [ RoutingEvent, BalanceEvent, ChannelBalanceEvent], + migrations: metricsMigrations + }).initialize(); + const log = getLogger({}); + const pendingMigrations = await source.showMigrations() + if (pendingMigrations) { + log("Migrations found, migrating...") + const executedMigrations = await source.runMigrations({ transaction: 'all' }) + return { source, executedMigrations } + } + return { source, executedMigrations: [] } + +} + export default async (settings: DbSettings, migrations: Function[]): Promise<{ source: DataSource, executedMigrations: Migration[] }> => { const source = await new DataSource({ type: "sqlite", database: settings.databaseFile, // logging: true, entities: [User, UserReceivingInvoice, UserReceivingAddress, AddressReceivingTransaction, UserInvoicePayment, UserTransactionPayment, - UserBasicAuth, UserEphemeralKey, Product, UserToUserPayment, Application, ApplicationUser, UserToUserPayment, RoutingEvent, BalanceEvent, ChannelBalanceEvent], + UserBasicAuth, UserEphemeralKey, Product, UserToUserPayment, Application, ApplicationUser, UserToUserPayment], //synchronize: true, migrations }).initialize() diff --git a/src/services/storage/index.ts b/src/services/storage/index.ts index 61c10645..98f5124d 100644 --- a/src/services/storage/index.ts +++ b/src/services/storage/index.ts @@ -25,7 +25,7 @@ export default class { constructor(settings: StorageSettings) { this.settings = settings } - async Connect(migrations: Function[]) { + async Connect(migrations: Function[], metricsMigrations: Function []) { const { source, executedMigrations } = await NewDB(this.settings.dbSettings, migrations) this.DB = source this.txQueue = new TransactionsQueue(this.DB) @@ -33,8 +33,9 @@ export default class { this.productStorage = new ProductStorage(this.DB, this.txQueue) this.applicationStorage = new ApplicationStorage(this.DB, this.userStorage, this.txQueue) this.paymentStorage = new PaymentStorage(this.DB, this.userStorage, this.txQueue) - this.metricsStorage = new MetricsStorage(this.DB, this.txQueue) - return executedMigrations + this.metricsStorage = new MetricsStorage(this.settings) + const executedMetricsMigrations = await this.metricsStorage.Connect(metricsMigrations) + return { executedMigrations, executedMetricsMigrations }; } StartTransaction(exec: TX) { diff --git a/src/services/storage/metricsStorage.ts b/src/services/storage/metricsStorage.ts index fc8bd487..d6c7c2a4 100644 --- a/src/services/storage/metricsStorage.ts +++ b/src/services/storage/metricsStorage.ts @@ -3,12 +3,20 @@ import { RoutingEvent } from "./entity/RoutingEvent.js" import { BalanceEvent } from "./entity/BalanceEvent.js" import { ChannelBalanceEvent } from "./entity/ChannelsBalanceEvent.js" import TransactionsQueue, { TX } from "./transactionsQueue.js"; +import { StorageSettings } from "./index.js"; +import { newMetricsDb } from "./db.js"; export default class { DB: DataSource | EntityManager + settings: StorageSettings txQueue: TransactionsQueue - constructor(DB: DataSource | EntityManager, txQueue: TransactionsQueue) { - this.DB = DB - this.txQueue = txQueue + constructor(settings: StorageSettings) { + this.settings = settings; + } + async Connect(metricsMigrations: Function[]) { + const { source, executedMigrations } = await newMetricsDb(this.settings.dbSettings, metricsMigrations) + this.DB = source; + this.txQueue = new TransactionsQueue(this.DB) + return executedMigrations; } async SaveRoutingEvent(event: Partial) { const entry = this.DB.getRepository(RoutingEvent).create(event) diff --git a/src/services/storage/migrations/runner.ts b/src/services/storage/migrations/runner.ts index 81b31670..7a7780e5 100644 --- a/src/services/storage/migrations/runner.ts +++ b/src/services/storage/migrations/runner.ts @@ -6,33 +6,38 @@ import { LndMetrics1703170330183 } from './1703170330183-lnd_metrics.js' const allMigrations = [LndMetrics1703170330183] export const TypeOrmMigrationRunner = async (log: PubLogger, storageManager: Storage, settings: DbSettings, arg: string | undefined): Promise => { if (arg === 'initial_migration') { - await connectAndMigrate(log, storageManager, true, settings, [Initial1703170309875]) + await connectAndMigrate(log, storageManager, true, settings, [Initial1703170309875], []) return true } else if (arg === 'lnd_metrics_migration') { - await connectAndMigrate(log, storageManager, true, settings, [LndMetrics1703170330183]) + await connectAndMigrate(log, storageManager, true, settings, [], [LndMetrics1703170330183]) return true } else if (arg === 'all_migrations') { - await connectAndMigrate(log, storageManager, true, settings, allMigrations) + await connectAndMigrate(log, storageManager, true, settings, [], allMigrations) return true } else if (settings.migrate) { - await connectAndMigrate(log, storageManager, false, settings, allMigrations) + await connectAndMigrate(log, storageManager, false, settings, [], allMigrations) return false } - await connectAndMigrate(log, storageManager, false, settings, []) + await connectAndMigrate(log, storageManager, false, settings, [], []) return false } -const connectAndMigrate = async (log: PubLogger, storageManager: Storage, manual: boolean, settings: DbSettings, migrations: Function[]) => { +const connectAndMigrate = async (log: PubLogger, storageManager: Storage, manual: boolean, settings: DbSettings, migrations: Function[], metricsMigrations: Function[]) => { if (manual && settings.migrate) { throw new Error("auto migration is enabled, no need to run manual migration") } if (migrations.length > 0) { log("will add", migrations.length, "typeorm migrations...") } - const executedMigrations = await storageManager.Connect(migrations) + const { executedMigrations, executedMetricsMigrations } = await storageManager.Connect(migrations, metricsMigrations) if (migrations.length > 0) { log(executedMigrations.length, "of", migrations.length, "migrations were executed correctly") log(executedMigrations) + log("-------------------") + + } if (metricsMigrations.length > 0) { + log(executedMetricsMigrations.length, "of", migrations.length, "metrics migrations were executed correctly") + log(executedMetricsMigrations) } } \ No newline at end of file