From 6f0700c58c9a4b2fa0365589bdd08bf050661252 Mon Sep 17 00:00:00 2001 From: Mothana Date: Mon, 12 Feb 2024 16:54:37 +0400 Subject: [PATCH] standalone metrics db --- .gitignore | 1 + env.example | 1 + metricsDatasource.js | 12 ++++++++++++ src/services/storage/db.ts | 22 +++++++++++++++++++++- src/services/storage/index.ts | 7 ++++--- src/services/storage/metricsStorage.ts | 14 +++++++++++--- src/services/storage/migrations/runner.ts | 19 ++++++++++++------- 7 files changed, 62 insertions(+), 14 deletions(-) create mode 100644 metricsDatasource.js diff --git a/.gitignore b/.gitignore index dbe0467d..eb8ee274 100644 --- a/.gitignore +++ b/.gitignore @@ -7,5 +7,6 @@ temp/ .env build/ db.sqlite +metrics.sqlite .key/ logs \ No newline at end of file diff --git a/env.example b/env.example index dbb52845..afb66fff 100644 --- a/env.example +++ b/env.example @@ -5,6 +5,7 @@ LND_MACAROON_PATH=/root/.lnd/data/chain/bitcoin/mainnet/admin.macaroon #DB DATABASE_FILE=db.sqlite +METRICS_DATABASE_FILE=metrics.sqlite #LOCAL ADMIN_TOKEN= diff --git a/metricsDatasource.js b/metricsDatasource.js new file mode 100644 index 00000000..9801c558 --- /dev/null +++ b/metricsDatasource.js @@ -0,0 +1,12 @@ +import { DataSource } from "typeorm" +import { BalanceEvent } from "./build/src/services/storage/entity/BalanceEvent.js" +import { ChannelBalanceEvent } from "./build/src/services/storage/entity/ChannelsBalanceEvent.js" +import { RoutingEvent } from "./build/src/services/storage/entity/RoutingEvent.js" + + + +export default new DataSource({ + type: "sqlite", + database: "metrics.sqlite", + entities: [ RoutingEvent, BalanceEvent, ChannelBalanceEvent], +}); \ No newline at end of file diff --git a/src/services/storage/db.ts b/src/services/storage/db.ts index 0754bd13..e3b40e8f 100644 --- a/src/services/storage/db.ts +++ b/src/services/storage/db.ts @@ -24,21 +24,41 @@ import { LndMetrics1703170330183 } from "./migrations/1703170330183-lnd_metrics. export type DbSettings = { databaseFile: string migrate: boolean + metricsDatabaseFile: string } export const LoadDbSettingsFromEnv = (test = false): DbSettings => { return { databaseFile: test ? ":memory:" : EnvMustBeNonEmptyString("DATABASE_FILE"), migrate: process.env.MIGRATE_DB === 'true' || false, + metricsDatabaseFile: test ? ":memory" : EnvMustBeNonEmptyString("METRICS_DATABASE_FILE") } } +export const newMetricsDb = async (settings: DbSettings, metricsMigrations: Function[]): Promise<{ source: DataSource, executedMigrations: Migration[] }> => { + const source = await new DataSource({ + type: "sqlite", + database: settings.metricsDatabaseFile, + entities: [ RoutingEvent, BalanceEvent, ChannelBalanceEvent], + migrations: metricsMigrations + }).initialize(); + const log = getLogger({}); + const pendingMigrations = await source.showMigrations() + if (pendingMigrations) { + log("Migrations found, migrating...") + const executedMigrations = await source.runMigrations({ transaction: 'all' }) + return { source, executedMigrations } + } + return { source, executedMigrations: [] } + +} + export default async (settings: DbSettings, migrations: Function[]): Promise<{ source: DataSource, executedMigrations: Migration[] }> => { const source = await new DataSource({ type: "sqlite", database: settings.databaseFile, // logging: true, entities: [User, UserReceivingInvoice, UserReceivingAddress, AddressReceivingTransaction, UserInvoicePayment, UserTransactionPayment, - UserBasicAuth, UserEphemeralKey, Product, UserToUserPayment, Application, ApplicationUser, UserToUserPayment, RoutingEvent, BalanceEvent, ChannelBalanceEvent], + UserBasicAuth, UserEphemeralKey, Product, UserToUserPayment, Application, ApplicationUser, UserToUserPayment], //synchronize: true, migrations }).initialize() diff --git a/src/services/storage/index.ts b/src/services/storage/index.ts index 61c10645..98f5124d 100644 --- a/src/services/storage/index.ts +++ b/src/services/storage/index.ts @@ -25,7 +25,7 @@ export default class { constructor(settings: StorageSettings) { this.settings = settings } - async Connect(migrations: Function[]) { + async Connect(migrations: Function[], metricsMigrations: Function []) { const { source, executedMigrations } = await NewDB(this.settings.dbSettings, migrations) this.DB = source this.txQueue = new TransactionsQueue(this.DB) @@ -33,8 +33,9 @@ export default class { this.productStorage = new ProductStorage(this.DB, this.txQueue) this.applicationStorage = new ApplicationStorage(this.DB, this.userStorage, this.txQueue) this.paymentStorage = new PaymentStorage(this.DB, this.userStorage, this.txQueue) - this.metricsStorage = new MetricsStorage(this.DB, this.txQueue) - return executedMigrations + this.metricsStorage = new MetricsStorage(this.settings) + const executedMetricsMigrations = await this.metricsStorage.Connect(metricsMigrations) + return { executedMigrations, executedMetricsMigrations }; } StartTransaction(exec: TX) { diff --git a/src/services/storage/metricsStorage.ts b/src/services/storage/metricsStorage.ts index fc8bd487..d6c7c2a4 100644 --- a/src/services/storage/metricsStorage.ts +++ b/src/services/storage/metricsStorage.ts @@ -3,12 +3,20 @@ import { RoutingEvent } from "./entity/RoutingEvent.js" import { BalanceEvent } from "./entity/BalanceEvent.js" import { ChannelBalanceEvent } from "./entity/ChannelsBalanceEvent.js" import TransactionsQueue, { TX } from "./transactionsQueue.js"; +import { StorageSettings } from "./index.js"; +import { newMetricsDb } from "./db.js"; export default class { DB: DataSource | EntityManager + settings: StorageSettings txQueue: TransactionsQueue - constructor(DB: DataSource | EntityManager, txQueue: TransactionsQueue) { - this.DB = DB - this.txQueue = txQueue + constructor(settings: StorageSettings) { + this.settings = settings; + } + async Connect(metricsMigrations: Function[]) { + const { source, executedMigrations } = await newMetricsDb(this.settings.dbSettings, metricsMigrations) + this.DB = source; + this.txQueue = new TransactionsQueue(this.DB) + return executedMigrations; } async SaveRoutingEvent(event: Partial) { const entry = this.DB.getRepository(RoutingEvent).create(event) diff --git a/src/services/storage/migrations/runner.ts b/src/services/storage/migrations/runner.ts index 81b31670..7a7780e5 100644 --- a/src/services/storage/migrations/runner.ts +++ b/src/services/storage/migrations/runner.ts @@ -6,33 +6,38 @@ import { LndMetrics1703170330183 } from './1703170330183-lnd_metrics.js' const allMigrations = [LndMetrics1703170330183] export const TypeOrmMigrationRunner = async (log: PubLogger, storageManager: Storage, settings: DbSettings, arg: string | undefined): Promise => { if (arg === 'initial_migration') { - await connectAndMigrate(log, storageManager, true, settings, [Initial1703170309875]) + await connectAndMigrate(log, storageManager, true, settings, [Initial1703170309875], []) return true } else if (arg === 'lnd_metrics_migration') { - await connectAndMigrate(log, storageManager, true, settings, [LndMetrics1703170330183]) + await connectAndMigrate(log, storageManager, true, settings, [], [LndMetrics1703170330183]) return true } else if (arg === 'all_migrations') { - await connectAndMigrate(log, storageManager, true, settings, allMigrations) + await connectAndMigrate(log, storageManager, true, settings, [], allMigrations) return true } else if (settings.migrate) { - await connectAndMigrate(log, storageManager, false, settings, allMigrations) + await connectAndMigrate(log, storageManager, false, settings, [], allMigrations) return false } - await connectAndMigrate(log, storageManager, false, settings, []) + await connectAndMigrate(log, storageManager, false, settings, [], []) return false } -const connectAndMigrate = async (log: PubLogger, storageManager: Storage, manual: boolean, settings: DbSettings, migrations: Function[]) => { +const connectAndMigrate = async (log: PubLogger, storageManager: Storage, manual: boolean, settings: DbSettings, migrations: Function[], metricsMigrations: Function[]) => { if (manual && settings.migrate) { throw new Error("auto migration is enabled, no need to run manual migration") } if (migrations.length > 0) { log("will add", migrations.length, "typeorm migrations...") } - const executedMigrations = await storageManager.Connect(migrations) + const { executedMigrations, executedMetricsMigrations } = await storageManager.Connect(migrations, metricsMigrations) if (migrations.length > 0) { log(executedMigrations.length, "of", migrations.length, "migrations were executed correctly") log(executedMigrations) + log("-------------------") + + } if (metricsMigrations.length > 0) { + log(executedMetricsMigrations.length, "of", migrations.length, "metrics migrations were executed correctly") + log(executedMetricsMigrations) } } \ No newline at end of file