standalone metrics db

.gitignore

@@ -7,5 +7,6 @@ temp/
 .env
 build/
 db.sqlite
+metrics.sqlite
 .key/
 logs

env.example

@@ -5,6 +5,7 @@ LND_MACAROON_PATH=/root/.lnd/data/chain/bitcoin/mainnet/admin.macaroon
 
 #DB
 DATABASE_FILE=db.sqlite
+METRICS_DATABASE_FILE=metrics.sqlite
 
 #LOCAL
 ADMIN_TOKEN=

metricsDatasource.js

@@ -0,0 +1,12 @@
+import { DataSource } from "typeorm"
+import { BalanceEvent } from "./build/src/services/storage/entity/BalanceEvent.js"
+import { ChannelBalanceEvent } from "./build/src/services/storage/entity/ChannelsBalanceEvent.js"
+import { RoutingEvent } from "./build/src/services/storage/entity/RoutingEvent.js"
+
+
+
+export default new DataSource({
+    type: "sqlite",
+    database: "metrics.sqlite",
+    entities: [ RoutingEvent, BalanceEvent, ChannelBalanceEvent],
+});

src/services/storage/db.ts

@@ -24,21 +24,41 @@ import { LndMetrics1703170330183 } from "./migrations/1703170330183-lnd_metrics.
 export type DbSettings = {
     databaseFile: string
     migrate: boolean
+    metricsDatabaseFile: string
 }
 export const LoadDbSettingsFromEnv = (test = false): DbSettings => {
     return {
         databaseFile: test ? ":memory:" : EnvMustBeNonEmptyString("DATABASE_FILE"),
         migrate: process.env.MIGRATE_DB === 'true' || false,
+        metricsDatabaseFile: test ? ":memory" : EnvMustBeNonEmptyString("METRICS_DATABASE_FILE")
     }
 }
 
+export const newMetricsDb = async (settings: DbSettings, metricsMigrations: Function[]): Promise<{ source: DataSource, executedMigrations: Migration[] }> => {
+    const source = await new DataSource({
+        type: "sqlite",
+        database: settings.metricsDatabaseFile,
+        entities: [ RoutingEvent, BalanceEvent, ChannelBalanceEvent],
+        migrations: metricsMigrations
+    }).initialize();
+    const log = getLogger({});
+    const pendingMigrations = await source.showMigrations()
+    if (pendingMigrations) {
+        log("Migrations found, migrating...")
+        const executedMigrations = await source.runMigrations({ transaction: 'all' })
+        return { source, executedMigrations }
+    }
+    return { source, executedMigrations: [] }
+
+}
+
 export default async (settings: DbSettings, migrations: Function[]): Promise<{ source: DataSource, executedMigrations: Migration[] }> => {
     const source = await new DataSource({
         type: "sqlite",
         database: settings.databaseFile,
         // logging: true,
         entities: [User, UserReceivingInvoice, UserReceivingAddress, AddressReceivingTransaction, UserInvoicePayment, UserTransactionPayment,
-            UserBasicAuth, UserEphemeralKey, Product, UserToUserPayment, Application, ApplicationUser, UserToUserPayment, RoutingEvent, BalanceEvent, ChannelBalanceEvent],
+            UserBasicAuth, UserEphemeralKey, Product, UserToUserPayment, Application, ApplicationUser, UserToUserPayment],
         //synchronize: true,
         migrations
     }).initialize()

src/services/storage/index.ts

@@ -25,7 +25,7 @@ export default class {
     constructor(settings: StorageSettings) {
         this.settings = settings
     }
-    async Connect(migrations: Function[]) {
+    async Connect(migrations: Function[], metricsMigrations: Function []) {
         const { source, executedMigrations } = await NewDB(this.settings.dbSettings, migrations)
         this.DB = source
         this.txQueue = new TransactionsQueue(this.DB)
@@ -33,8 +33,9 @@ export default class {
         this.productStorage = new ProductStorage(this.DB, this.txQueue)
         this.applicationStorage = new ApplicationStorage(this.DB, this.userStorage, this.txQueue)
         this.paymentStorage = new PaymentStorage(this.DB, this.userStorage, this.txQueue)
-        this.metricsStorage = new MetricsStorage(this.DB, this.txQueue)
+        this.metricsStorage = new MetricsStorage(this.settings)
-        return executedMigrations
+        const executedMetricsMigrations = await this.metricsStorage.Connect(metricsMigrations)
+        return { executedMigrations, executedMetricsMigrations };
     }
 
     StartTransaction(exec: TX<void>) {

src/services/storage/metricsStorage.ts

@@ -3,12 +3,20 @@ import { RoutingEvent } from "./entity/RoutingEvent.js"
 import { BalanceEvent } from "./entity/BalanceEvent.js"
 import { ChannelBalanceEvent } from "./entity/ChannelsBalanceEvent.js"
 import TransactionsQueue, { TX } from "./transactionsQueue.js";
+import { StorageSettings } from "./index.js";
+import { newMetricsDb } from "./db.js";
 export default class {
     DB: DataSource | EntityManager
+    settings: StorageSettings
     txQueue: TransactionsQueue
-    constructor(DB: DataSource | EntityManager, txQueue: TransactionsQueue) {
+    constructor(settings: StorageSettings) {
-        this.DB = DB
+        this.settings = settings;
-        this.txQueue = txQueue
+    }
+    async Connect(metricsMigrations: Function[]) {
+        const { source, executedMigrations } = await newMetricsDb(this.settings.dbSettings, metricsMigrations)
+        this.DB = source;
+        this.txQueue = new TransactionsQueue(this.DB)
+        return executedMigrations;
     }
     async SaveRoutingEvent(event: Partial<RoutingEvent>) {
         const entry = this.DB.getRepository(RoutingEvent).create(event)

src/services/storage/migrations/runner.ts

@@ -6,33 +6,38 @@ import { LndMetrics1703170330183 } from './1703170330183-lnd_metrics.js'
 const allMigrations = [LndMetrics1703170330183]
 export const TypeOrmMigrationRunner = async (log: PubLogger, storageManager: Storage, settings: DbSettings, arg: string | undefined): Promise<boolean> => {
     if (arg === 'initial_migration') {
-        await connectAndMigrate(log, storageManager, true, settings, [Initial1703170309875])
+        await connectAndMigrate(log, storageManager, true, settings, [Initial1703170309875], [])
         return true
     } else if (arg === 'lnd_metrics_migration') {
-        await connectAndMigrate(log, storageManager, true, settings, [LndMetrics1703170330183])
+        await connectAndMigrate(log, storageManager, true, settings, [], [LndMetrics1703170330183])
         return true
     } else if (arg === 'all_migrations') {
-        await connectAndMigrate(log, storageManager, true, settings, allMigrations)
+        await connectAndMigrate(log, storageManager, true, settings, [], allMigrations)
         return true
     } else if (settings.migrate) {
-        await connectAndMigrate(log, storageManager, false, settings, allMigrations)
+        await connectAndMigrate(log, storageManager, false, settings, [], allMigrations)
         return false
     }
-    await connectAndMigrate(log, storageManager, false, settings, [])
+    await connectAndMigrate(log, storageManager, false, settings, [], [])
     return false
 }
 
-const connectAndMigrate = async (log: PubLogger, storageManager: Storage, manual: boolean, settings: DbSettings, migrations: Function[]) => {
+const connectAndMigrate = async (log: PubLogger, storageManager: Storage, manual: boolean, settings: DbSettings, migrations: Function[], metricsMigrations: Function[]) => {
     if (manual && settings.migrate) {
         throw new Error("auto migration is enabled, no need to run manual migration")
     }
     if (migrations.length > 0) {
         log("will add", migrations.length, "typeorm migrations...")
     }
-    const executedMigrations = await storageManager.Connect(migrations)
+    const { executedMigrations, executedMetricsMigrations } = await storageManager.Connect(migrations, metricsMigrations)
     if (migrations.length > 0) {
         log(executedMigrations.length, "of", migrations.length, "migrations were executed correctly")
         log(executedMigrations)
+        log("-------------------")
+        
+    } if (metricsMigrations.length > 0) {
+        log(executedMetricsMigrations.length, "of", migrations.length, "metrics migrations were executed correctly")
+        log(executedMetricsMigrations)
     }
 
 }