From 5efdbb3312c1ffa882364db6226b02a3633423be Mon Sep 17 00:00:00 2001 From: "Justin (shocknet)" <34176400+capitalistdog@users.noreply.github.com> Date: Sun, 11 Feb 2024 15:12:29 -0500 Subject: [PATCH 1/4] Update README.md --- README.md | 25 ++++++++++++++----------- 1 file changed, 14 insertions(+), 11 deletions(-) diff --git a/README.md b/README.md index 5dbf6c3d..1bbabcb7 100644 --- a/README.md +++ b/README.md @@ -8,36 +8,38 @@ ### Don't just run a Lightning Node, run a Lightning Pub. -`Pub` enables your Lightning node with public API's and accounts over `nostr`, allowing LN nodes to act like a webserver without the complexity of networking and SSL configurations. +"Pub" is a `nostr` native account system that makes connecting your node to apps and websites super easy. -The Permissionless WebApps that use these API's promote a more decentralized Lightning Network, by removing hurdles for self-custodial home nodes to power connections from Friends, Family and Customers. +Using Nostr relays as transport for encrypted RPCs, Pub eliminates the complexity of WebServer and SSL configurations. + +By solving the networking and programability hurdles, Pub enables node-runners and Uncle Jim's to bring their Friends, Family and Customers into Bitcoin's permissionless circular economy. All while keeping the Lightning Network decentralized, and custodial scaling free of fiat shitcoin rails and large banks. #### Features: - Wrapper for [`LND`](https://github.com/lightningnetwork/lnd/releases) that can serve accounts over LNURL and NOSTR - A growing number of [methods](https://github.com/shocknet/Lightning.Pub/blob/master/proto/autogenerated/client.md) - Accounting SubLayers for Application Pools and Users - - A fee regime allows applications owners to tax users, or node operators to tax applications. + - A fee regime allows applications owners to monetize users, or node operators to host distinctly monetized applications. ![Accounts](https://github.com/shocknet/Lightning.Pub/raw/master/accounting_layers.png) #### Planned - [ ] Management Dashboard is being integrated into [ShockWallet](https://github.com/shocknet/wallet2) +- [ ] Nostr native "offers" - [ ] Channel Automation +- [ ] Bootstarp Peering (Passive "LSP") - [ ] Subscriptions / Notifications - [ ] Submarine Swaps -- [ ] High-Availabilty +- [ ] High-Availabilty / Clustering Dashboard: Pub Dashboard -#### ShockWallet and most of Lightning.Pub were developed as part of the [Bolt.Fun hackathon](https://bolt.fun/project/shocknet). If you would like to see continued development, please show your support there and help us win :) - -#### See the original NostrHack presentation: https://lightning.video/f0f64fa1fc3744fb6a3880e2bd8f6a254ceb3caee112d9708271f2d6a09a2f00 +#### ShockWallet and Lightning.Pub are free software. If you would like to see continued development, please show your [support](https://github.com/sponsors/shocknet) :) -> **WARNING:** This repository is under rapid iteration and security is not guaranteed. Use tagged releases for non-development. +> **WARNING:** While this software has been used in production for many months, it is still bleeding edge and security or reliabilty is not guaranteed. ## Manual Installation @@ -64,16 +66,17 @@ cd Lightning.Pub && npm i 3) `cp env.example .env` 4) Add values to env file - - You can generate a keypair with `node genkey.js` 5) `npm start` -6) Create an Application Pool +6) Create an Application Pool + +A default "wallet" pool will be automatically created and keys generated automatically, if you wish to create something other: ``` curl -XPOST -H 'Authorization: Bearer defined_in_constants.ts' -H "Content-type: application/json" -d '{"name":"ExampleApplicationPoolName"}' 'http://localhost:8080/api/admin/app/add' ``` -7) Connect with [wallet2](https://github.com/shocknet/wallet2) using the npub response in step 6. +7) Connect with [wallet2](https://github.com/shocknet/wallet2) using the npub response in step 6 or the the wallet application nprofile logged at startup. From 6f0700c58c9a4b2fa0365589bdd08bf050661252 Mon Sep 17 00:00:00 2001 From: Mothana Date: Mon, 12 Feb 2024 16:54:37 +0400 Subject: [PATCH 2/4] standalone metrics db --- .gitignore | 1 + env.example | 1 + metricsDatasource.js | 12 ++++++++++++ src/services/storage/db.ts | 22 +++++++++++++++++++++- src/services/storage/index.ts | 7 ++++--- src/services/storage/metricsStorage.ts | 14 +++++++++++--- src/services/storage/migrations/runner.ts | 19 ++++++++++++------- 7 files changed, 62 insertions(+), 14 deletions(-) create mode 100644 metricsDatasource.js diff --git a/.gitignore b/.gitignore index dbe0467d..eb8ee274 100644 --- a/.gitignore +++ b/.gitignore @@ -7,5 +7,6 @@ temp/ .env build/ db.sqlite +metrics.sqlite .key/ logs \ No newline at end of file diff --git a/env.example b/env.example index dbb52845..afb66fff 100644 --- a/env.example +++ b/env.example @@ -5,6 +5,7 @@ LND_MACAROON_PATH=/root/.lnd/data/chain/bitcoin/mainnet/admin.macaroon #DB DATABASE_FILE=db.sqlite +METRICS_DATABASE_FILE=metrics.sqlite #LOCAL ADMIN_TOKEN= diff --git a/metricsDatasource.js b/metricsDatasource.js new file mode 100644 index 00000000..9801c558 --- /dev/null +++ b/metricsDatasource.js @@ -0,0 +1,12 @@ +import { DataSource } from "typeorm" +import { BalanceEvent } from "./build/src/services/storage/entity/BalanceEvent.js" +import { ChannelBalanceEvent } from "./build/src/services/storage/entity/ChannelsBalanceEvent.js" +import { RoutingEvent } from "./build/src/services/storage/entity/RoutingEvent.js" + + + +export default new DataSource({ + type: "sqlite", + database: "metrics.sqlite", + entities: [ RoutingEvent, BalanceEvent, ChannelBalanceEvent], +}); \ No newline at end of file diff --git a/src/services/storage/db.ts b/src/services/storage/db.ts index 0754bd13..e3b40e8f 100644 --- a/src/services/storage/db.ts +++ b/src/services/storage/db.ts @@ -24,21 +24,41 @@ import { LndMetrics1703170330183 } from "./migrations/1703170330183-lnd_metrics. export type DbSettings = { databaseFile: string migrate: boolean + metricsDatabaseFile: string } export const LoadDbSettingsFromEnv = (test = false): DbSettings => { return { databaseFile: test ? ":memory:" : EnvMustBeNonEmptyString("DATABASE_FILE"), migrate: process.env.MIGRATE_DB === 'true' || false, + metricsDatabaseFile: test ? ":memory" : EnvMustBeNonEmptyString("METRICS_DATABASE_FILE") } } +export const newMetricsDb = async (settings: DbSettings, metricsMigrations: Function[]): Promise<{ source: DataSource, executedMigrations: Migration[] }> => { + const source = await new DataSource({ + type: "sqlite", + database: settings.metricsDatabaseFile, + entities: [ RoutingEvent, BalanceEvent, ChannelBalanceEvent], + migrations: metricsMigrations + }).initialize(); + const log = getLogger({}); + const pendingMigrations = await source.showMigrations() + if (pendingMigrations) { + log("Migrations found, migrating...") + const executedMigrations = await source.runMigrations({ transaction: 'all' }) + return { source, executedMigrations } + } + return { source, executedMigrations: [] } + +} + export default async (settings: DbSettings, migrations: Function[]): Promise<{ source: DataSource, executedMigrations: Migration[] }> => { const source = await new DataSource({ type: "sqlite", database: settings.databaseFile, // logging: true, entities: [User, UserReceivingInvoice, UserReceivingAddress, AddressReceivingTransaction, UserInvoicePayment, UserTransactionPayment, - UserBasicAuth, UserEphemeralKey, Product, UserToUserPayment, Application, ApplicationUser, UserToUserPayment, RoutingEvent, BalanceEvent, ChannelBalanceEvent], + UserBasicAuth, UserEphemeralKey, Product, UserToUserPayment, Application, ApplicationUser, UserToUserPayment], //synchronize: true, migrations }).initialize() diff --git a/src/services/storage/index.ts b/src/services/storage/index.ts index 61c10645..98f5124d 100644 --- a/src/services/storage/index.ts +++ b/src/services/storage/index.ts @@ -25,7 +25,7 @@ export default class { constructor(settings: StorageSettings) { this.settings = settings } - async Connect(migrations: Function[]) { + async Connect(migrations: Function[], metricsMigrations: Function []) { const { source, executedMigrations } = await NewDB(this.settings.dbSettings, migrations) this.DB = source this.txQueue = new TransactionsQueue(this.DB) @@ -33,8 +33,9 @@ export default class { this.productStorage = new ProductStorage(this.DB, this.txQueue) this.applicationStorage = new ApplicationStorage(this.DB, this.userStorage, this.txQueue) this.paymentStorage = new PaymentStorage(this.DB, this.userStorage, this.txQueue) - this.metricsStorage = new MetricsStorage(this.DB, this.txQueue) - return executedMigrations + this.metricsStorage = new MetricsStorage(this.settings) + const executedMetricsMigrations = await this.metricsStorage.Connect(metricsMigrations) + return { executedMigrations, executedMetricsMigrations }; } StartTransaction(exec: TX) { diff --git a/src/services/storage/metricsStorage.ts b/src/services/storage/metricsStorage.ts index fc8bd487..d6c7c2a4 100644 --- a/src/services/storage/metricsStorage.ts +++ b/src/services/storage/metricsStorage.ts @@ -3,12 +3,20 @@ import { RoutingEvent } from "./entity/RoutingEvent.js" import { BalanceEvent } from "./entity/BalanceEvent.js" import { ChannelBalanceEvent } from "./entity/ChannelsBalanceEvent.js" import TransactionsQueue, { TX } from "./transactionsQueue.js"; +import { StorageSettings } from "./index.js"; +import { newMetricsDb } from "./db.js"; export default class { DB: DataSource | EntityManager + settings: StorageSettings txQueue: TransactionsQueue - constructor(DB: DataSource | EntityManager, txQueue: TransactionsQueue) { - this.DB = DB - this.txQueue = txQueue + constructor(settings: StorageSettings) { + this.settings = settings; + } + async Connect(metricsMigrations: Function[]) { + const { source, executedMigrations } = await newMetricsDb(this.settings.dbSettings, metricsMigrations) + this.DB = source; + this.txQueue = new TransactionsQueue(this.DB) + return executedMigrations; } async SaveRoutingEvent(event: Partial) { const entry = this.DB.getRepository(RoutingEvent).create(event) diff --git a/src/services/storage/migrations/runner.ts b/src/services/storage/migrations/runner.ts index 81b31670..7a7780e5 100644 --- a/src/services/storage/migrations/runner.ts +++ b/src/services/storage/migrations/runner.ts @@ -6,33 +6,38 @@ import { LndMetrics1703170330183 } from './1703170330183-lnd_metrics.js' const allMigrations = [LndMetrics1703170330183] export const TypeOrmMigrationRunner = async (log: PubLogger, storageManager: Storage, settings: DbSettings, arg: string | undefined): Promise => { if (arg === 'initial_migration') { - await connectAndMigrate(log, storageManager, true, settings, [Initial1703170309875]) + await connectAndMigrate(log, storageManager, true, settings, [Initial1703170309875], []) return true } else if (arg === 'lnd_metrics_migration') { - await connectAndMigrate(log, storageManager, true, settings, [LndMetrics1703170330183]) + await connectAndMigrate(log, storageManager, true, settings, [], [LndMetrics1703170330183]) return true } else if (arg === 'all_migrations') { - await connectAndMigrate(log, storageManager, true, settings, allMigrations) + await connectAndMigrate(log, storageManager, true, settings, [], allMigrations) return true } else if (settings.migrate) { - await connectAndMigrate(log, storageManager, false, settings, allMigrations) + await connectAndMigrate(log, storageManager, false, settings, [], allMigrations) return false } - await connectAndMigrate(log, storageManager, false, settings, []) + await connectAndMigrate(log, storageManager, false, settings, [], []) return false } -const connectAndMigrate = async (log: PubLogger, storageManager: Storage, manual: boolean, settings: DbSettings, migrations: Function[]) => { +const connectAndMigrate = async (log: PubLogger, storageManager: Storage, manual: boolean, settings: DbSettings, migrations: Function[], metricsMigrations: Function[]) => { if (manual && settings.migrate) { throw new Error("auto migration is enabled, no need to run manual migration") } if (migrations.length > 0) { log("will add", migrations.length, "typeorm migrations...") } - const executedMigrations = await storageManager.Connect(migrations) + const { executedMigrations, executedMetricsMigrations } = await storageManager.Connect(migrations, metricsMigrations) if (migrations.length > 0) { log(executedMigrations.length, "of", migrations.length, "migrations were executed correctly") log(executedMigrations) + log("-------------------") + + } if (metricsMigrations.length > 0) { + log(executedMetricsMigrations.length, "of", migrations.length, "metrics migrations were executed correctly") + log(executedMetricsMigrations) } } \ No newline at end of file From 873f9e81d5f5c0dfc1e150d441d15a99c7477889 Mon Sep 17 00:00:00 2001 From: Mothana Date: Mon, 12 Feb 2024 17:22:43 +0400 Subject: [PATCH 3/4] env toggle for perf --- env.example | 3 +++ src/auth.ts | 2 +- src/nostrMiddleware.ts | 2 +- src/services/main/index.ts | 3 ++- src/services/main/settings.ts | 1 + 5 files changed, 8 insertions(+), 3 deletions(-) diff --git a/env.example b/env.example index afb66fff..b3cad64a 100644 --- a/env.example +++ b/env.example @@ -39,3 +39,6 @@ SERVICE_URL=https://test.lightning.pub MOCK_LND=false ALLOW_BALANCE_MIGRATION=false MIGRATE_DB=false + +#METRICS +RECORD_PERFORMANCE=true diff --git a/src/auth.ts b/src/auth.ts index de6ac223..74485cb5 100644 --- a/src/auth.ts +++ b/src/auth.ts @@ -11,7 +11,7 @@ const serverOptions = (mainHandler: Main): ServerOptions => { AppAuthGuard: async (authHeader) => { return { app_id: mainHandler.applicationManager.DecodeAppToken(stripBearer(authHeader)) } }, UserAuthGuard: async (authHeader) => { return mainHandler.appUserManager.DecodeUserToken(stripBearer(authHeader)) }, GuestAuthGuard: async (_) => ({}), - metricsCallback: metrics => mainHandler.metricsManager.AddMetrics(metrics), + metricsCallback: metrics => mainHandler.settings.recordPerformance ? mainHandler.metricsManager.AddMetrics(metrics) : null, allowCors: true //throwErrors: true } diff --git a/src/nostrMiddleware.ts b/src/nostrMiddleware.ts index 215fa2c6..3eb877d9 100644 --- a/src/nostrMiddleware.ts +++ b/src/nostrMiddleware.ts @@ -11,7 +11,7 @@ export default (serverMethods: Types.ServerMethods, mainHandler: Main, nostrSett let nostrUser = await mainHandler.storage.applicationStorage.GetOrCreateNostrAppUser(app, pub || "") return { user_id: nostrUser.user.user_id, app_user_id: nostrUser.identifier, app_id: appId || "" } }, - metricsCallback: metrics => mainHandler.metricsManager.AddMetrics(metrics) + metricsCallback: metrics => mainHandler.settings.recordPerformance ? mainHandler.metricsManager.AddMetrics(metrics) : null }) const nostr = new Nostr(nostrSettings, event => { let j: NostrRequest diff --git a/src/services/main/index.ts b/src/services/main/index.ts index 4f5f5adf..43f0dba0 100644 --- a/src/services/main/index.ts +++ b/src/services/main/index.ts @@ -30,7 +30,8 @@ export const LoadMainSettingsFromEnv = (test = false): MainSettings => { userToUserFee: EnvMustBeInteger("TX_FEE_INTERNAL_USER_BPS") / 10000, appToUserFee: EnvMustBeInteger("TX_FEE_INTERNAL_ROOT_BPS") / 10000, serviceUrl: EnvMustBeNonEmptyString("SERVICE_URL"), - servicePort: EnvMustBeInteger("PORT") + servicePort: EnvMustBeInteger("PORT"), + recordPerformance: process.env.RECORD_PERFORMANCE === 'true' || false } } diff --git a/src/services/main/settings.ts b/src/services/main/settings.ts index 1be670f2..a4b705a1 100644 --- a/src/services/main/settings.ts +++ b/src/services/main/settings.ts @@ -14,5 +14,6 @@ export type MainSettings = { appToUserFee: number serviceUrl: string servicePort: number + recordPerformance: boolean } \ No newline at end of file From b86de671e1740ee713a4787230fb165d25f4bdc6 Mon Sep 17 00:00:00 2001 From: Mothana Date: Mon, 12 Feb 2024 17:45:32 +0400 Subject: [PATCH 4/4] don't log privateKey, log nprofile --- src/custom-nip19.ts | 88 +++++++++++++++++++++++++++++++++++ src/services/nostr/handler.ts | 14 +++++- 2 files changed, 101 insertions(+), 1 deletion(-) create mode 100644 src/custom-nip19.ts diff --git a/src/custom-nip19.ts b/src/custom-nip19.ts new file mode 100644 index 00000000..62c7c56a --- /dev/null +++ b/src/custom-nip19.ts @@ -0,0 +1,88 @@ +/* + This file contains functions that deal with encoding and decoding nprofiles, + but with he addition of bridge urls in the nprofile. + These functions are basically the same functions from nostr-tools package + but with some tweaks to allow for the bridge inclusion. +*/ +import { bytesToHex, concatBytes, hexToBytes } from '@noble/hashes/utils'; +import { bech32 } from 'bech32'; + +export const utf8Decoder = new TextDecoder('utf-8') +export const utf8Encoder = new TextEncoder() + + +export type CustomProfilePointer = { + pubkey: string + relays?: string[] + bridge?: string[] // one bridge +} + + + +type TLV = { [t: number]: Uint8Array[] } + + +const encodeTLV = (tlv: TLV): Uint8Array => { + const entries: Uint8Array[] = [] + + Object.entries(tlv) + /* + the original function does a reverse() here, + but here it causes the nprofile string to be different, + even though it would still decode to the correct original inputs + */ + //.reverse() + .forEach(([t, vs]) => { + vs.forEach(v => { + const entry = new Uint8Array(v.length + 2) + entry.set([parseInt(t)], 0) + entry.set([v.length], 1) + entry.set(v, 2) + entries.push(entry) + }) + }) + return concatBytes(...entries); +} + +export const encodeNprofile = (profile: CustomProfilePointer): string => { + const data = encodeTLV({ + 0: [hexToBytes(profile.pubkey)], + 1: (profile.relays || []).map(url => utf8Encoder.encode(url)), + 2: (profile.bridge || []).map(url => utf8Encoder.encode(url)) + }); + const words = bech32.toWords(data) + return bech32.encode("nprofile", words, 5000); +} + +const parseTLV = (data: Uint8Array): TLV => { + const result: TLV = {} + let rest = data + while (rest.length > 0) { + const t = rest[0] + const l = rest[1] + const v = rest.slice(2, 2 + l) + rest = rest.slice(2 + l) + if (v.length < l) throw new Error(`not enough data to read on TLV ${t}`) + result[t] = result[t] || [] + result[t].push(v) + } + return result +} + +export const decodeNprofile = (nprofile: string): CustomProfilePointer => { + const { prefix, words } = bech32.decode(nprofile, 5000) + if (prefix !== "nprofile") { + throw new Error ("Expected nprofile prefix"); + } + const data = new Uint8Array(bech32.fromWords(words)) + + const tlv = parseTLV(data); + if (!tlv[0]?.[0]) throw new Error('missing TLV 0 for nprofile') + if (tlv[0][0].length !== 32) throw new Error('TLV 0 should be 32 bytes') + + return { + pubkey: bytesToHex(tlv[0][0]), + relays: tlv[1] ? tlv[1].map(d => utf8Decoder.decode(d)) : [], + bridge: tlv[2] ? tlv[2].map(d => utf8Decoder.decode(d)): [] + } +} \ No newline at end of file diff --git a/src/services/nostr/handler.ts b/src/services/nostr/handler.ts index ffd94424..6bb1fce2 100644 --- a/src/services/nostr/handler.ts +++ b/src/services/nostr/handler.ts @@ -2,6 +2,7 @@ import { SimplePool, Sub, Event, UnsignedEvent, getEventHash, finishEvent, relayInit } from './tools/index.js' import { encryptData, decryptData, getSharedSecret, decodePayload, encodePayload } from './nip44.js' import { getLogger } from '../helpers/logger.js' +import { encodeNprofile } from '../../custom-nip19.js' const handledEvents: string[] = [] // TODO: - big memory leak here, add TTL type AppInfo = { appId: string, publicKey: string, privateKey: string, name: string } export type SendData = { type: "content", content: string, pub: string } | { type: "event", event: UnsignedEvent } @@ -88,7 +89,18 @@ export default class Handler { eventCallback: (event: NostrEvent) => void constructor(settings: NostrSettings, eventCallback: (event: NostrEvent) => void) { this.settings = settings - console.log(settings) + console.log( + { + ...settings, + apps: settings.apps.map(app => { + const { privateKey, ...rest } = app; + return { + ...rest, + nprofile: encodeNprofile({ pubkey: rest.publicKey, relays: settings.relays }) + } + }) + } + ) this.eventCallback = eventCallback this.settings.apps.forEach(app => { this.apps[app.publicKey] = app