compress metrics

proto/autogenerated/client.md

@@ -866,6 +866,9 @@ The nostr server will send back a message response, and inside the body there wi
   - __total_fees__: _number_
   - __users__: _[UsersInfo](#UsersInfo)_
 
+### AppUsageMetrics
+  - __app_metrics__: MAP with key: _string_ and value: _[UsageMetricTlv](#UsageMetricTlv)_
+
 ### AppUser
   - __identifier__: _string_
   - __info__: _[UserInfo](#UserInfo)_
@@ -1275,6 +1278,7 @@ The nostr server will send back a message response, and inside the body there wi
   - __update__: _[UpdateChannelPolicyRequest_update](#UpdateChannelPolicyRequest_update)_
 
 ### UsageMetric
+  - __app_id__: _string_ *this field is optional
   - __auth_in_nano__: _number_
   - __batch__: _boolean_
   - __batch_size__: _number_
@@ -1283,10 +1287,14 @@ The nostr server will send back a message response, and inside the body there wi
   - __parsed_in_nano__: _number_
   - __processed_at_ms__: _number_
   - __rpc_name__: _string_
+  - __success__: _boolean_
   - __validate_in_nano__: _number_
 
+### UsageMetricTlv
+  - __base_64_tlvs__: ARRAY of: _string_
+
 ### UsageMetrics
-  - __metrics__: ARRAY of: _[UsageMetric](#UsageMetric)_
+  - __apps__: MAP with key: _string_ and value: _[AppUsageMetrics](#AppUsageMetrics)_
 
 ### UseInviteLinkRequest
   - __invite_token__: _string_

proto/autogenerated/go/types.go

@@ -130,6 +130,9 @@ type AppMetrics struct {
 	Total_fees int64           `json:"total_fees"`
 	Users      *UsersInfo      `json:"users"`
 }
+type AppUsageMetrics struct {
+	App_metrics map[string]UsageMetricTlv `json:"app_metrics"`
+}
 type AppUser struct {
 	Identifier       string    `json:"identifier"`
 	Info             *UserInfo `json:"info"`
@@ -539,6 +542,7 @@ type UpdateChannelPolicyRequest struct {
 	Update *UpdateChannelPolicyRequest_update `json:"update"`
 }
 type UsageMetric struct {
+	App_id           string `json:"app_id"`
 	Auth_in_nano     int64  `json:"auth_in_nano"`
 	Batch            bool   `json:"batch"`
 	Batch_size       int64  `json:"batch_size"`
@@ -547,10 +551,14 @@ type UsageMetric struct {
 	Parsed_in_nano   int64  `json:"parsed_in_nano"`
 	Processed_at_ms  int64  `json:"processed_at_ms"`
 	Rpc_name         string `json:"rpc_name"`
+	Success          bool   `json:"success"`
 	Validate_in_nano int64  `json:"validate_in_nano"`
 }
+type UsageMetricTlv struct {
+	Base_64_tlvs []string `json:"base_64_tlvs"`
+}
 type UsageMetrics struct {
-	Metrics []UsageMetric `json:"metrics"`
+	Apps map[string]AppUsageMetrics `json:"apps"`
 }
 type UseInviteLinkRequest struct {
 	Invite_token string `json:"invite_token"`

proto/autogenerated/ts/types.ts

@@ -644,6 +644,28 @@ export const AppMetricsValidate = (o?: AppMetrics, opts: AppMetricsOptions = {},
     return null
 }
 
+export type AppUsageMetrics = {
+    app_metrics: Record<string, UsageMetricTlv>
+}
+export const AppUsageMetricsOptionalFields: [] = []
+export type AppUsageMetricsOptions = OptionsBaseMessage & {
+    checkOptionalsAreSet?: []
+    app_metrics_EntryOptions?: UsageMetricTlvOptions
+    app_metrics_CustomCheck?: (v: Record<string, UsageMetricTlv>) => boolean
+}
+export const AppUsageMetricsValidate = (o?: AppUsageMetrics, opts: AppUsageMetricsOptions = {}, path: string = 'AppUsageMetrics::root.'): Error | null => {
+    if (opts.checkOptionalsAreSet && opts.allOptionalsAreSet) return new Error(path + ': only one of checkOptionalsAreSet or allOptionalNonDefault can be set for each message')
+    if (typeof o !== 'object' || o === null) return new Error(path + ': object is not an instance of an object or is null')
+
+    if (typeof o.app_metrics !== 'object' || o.app_metrics === null) return new Error(`${path}.app_metrics: is not an object or is null`)
+    for (const key in o.app_metrics) {
+        const app_metricsErr = UsageMetricTlvValidate(o.app_metrics[key], opts.app_metrics_EntryOptions, `${path}.app_metrics['${key}']`)
+        if (app_metricsErr !== null) return app_metricsErr
+    }
+
+    return null
+}
+
 export type AppUser = {
     identifier: string
     info: UserInfo
@@ -3071,6 +3093,7 @@ export const UpdateChannelPolicyRequestValidate = (o?: UpdateChannelPolicyReques
 }
 
 export type UsageMetric = {
+    app_id?: string
     auth_in_nano: number
     batch: boolean
     batch_size: number
@@ -3079,11 +3102,14 @@ export type UsageMetric = {
     parsed_in_nano: number
     processed_at_ms: number
     rpc_name: string
+    success: boolean
     validate_in_nano: number
 }
-export const UsageMetricOptionalFields: [] = []
+export type UsageMetricOptionalField = 'app_id'
+export const UsageMetricOptionalFields: UsageMetricOptionalField[] = ['app_id']
 export type UsageMetricOptions = OptionsBaseMessage & {
-    checkOptionalsAreSet?: []
+    checkOptionalsAreSet?: UsageMetricOptionalField[]
+    app_id_CustomCheck?: (v?: string) => boolean
     auth_in_nano_CustomCheck?: (v: number) => boolean
     batch_CustomCheck?: (v: boolean) => boolean
     batch_size_CustomCheck?: (v: number) => boolean
@@ -3092,12 +3118,16 @@ export type UsageMetricOptions = OptionsBaseMessage & {
     parsed_in_nano_CustomCheck?: (v: number) => boolean
     processed_at_ms_CustomCheck?: (v: number) => boolean
     rpc_name_CustomCheck?: (v: string) => boolean
+    success_CustomCheck?: (v: boolean) => boolean
     validate_in_nano_CustomCheck?: (v: number) => boolean
 }
 export const UsageMetricValidate = (o?: UsageMetric, opts: UsageMetricOptions = {}, path: string = 'UsageMetric::root.'): Error | null => {
     if (opts.checkOptionalsAreSet && opts.allOptionalsAreSet) return new Error(path + ': only one of checkOptionalsAreSet or allOptionalNonDefault can be set for each message')
     if (typeof o !== 'object' || o === null) return new Error(path + ': object is not an instance of an object or is null')
 
+    if ((o.app_id || opts.allOptionalsAreSet || opts.checkOptionalsAreSet?.includes('app_id')) && typeof o.app_id !== 'string') return new Error(`${path}.app_id: is not a string`)
+    if (opts.app_id_CustomCheck && !opts.app_id_CustomCheck(o.app_id)) return new Error(`${path}.app_id: custom check failed`)
+
     if (typeof o.auth_in_nano !== 'number') return new Error(`${path}.auth_in_nano: is not a number`)
     if (opts.auth_in_nano_CustomCheck && !opts.auth_in_nano_CustomCheck(o.auth_in_nano)) return new Error(`${path}.auth_in_nano: custom check failed`)
 
@@ -3122,31 +3152,54 @@ export const UsageMetricValidate = (o?: UsageMetric, opts: UsageMetricOptions =
     if (typeof o.rpc_name !== 'string') return new Error(`${path}.rpc_name: is not a string`)
     if (opts.rpc_name_CustomCheck && !opts.rpc_name_CustomCheck(o.rpc_name)) return new Error(`${path}.rpc_name: custom check failed`)
 
+    if (typeof o.success !== 'boolean') return new Error(`${path}.success: is not a boolean`)
+    if (opts.success_CustomCheck && !opts.success_CustomCheck(o.success)) return new Error(`${path}.success: custom check failed`)
+
     if (typeof o.validate_in_nano !== 'number') return new Error(`${path}.validate_in_nano: is not a number`)
     if (opts.validate_in_nano_CustomCheck && !opts.validate_in_nano_CustomCheck(o.validate_in_nano)) return new Error(`${path}.validate_in_nano: custom check failed`)
 
     return null
 }
 
+export type UsageMetricTlv = {
+    base_64_tlvs: string[]
+}
+export const UsageMetricTlvOptionalFields: [] = []
+export type UsageMetricTlvOptions = OptionsBaseMessage & {
+    checkOptionalsAreSet?: []
+    base_64_tlvs_CustomCheck?: (v: string[]) => boolean
+}
+export const UsageMetricTlvValidate = (o?: UsageMetricTlv, opts: UsageMetricTlvOptions = {}, path: string = 'UsageMetricTlv::root.'): Error | null => {
+    if (opts.checkOptionalsAreSet && opts.allOptionalsAreSet) return new Error(path + ': only one of checkOptionalsAreSet or allOptionalNonDefault can be set for each message')
+    if (typeof o !== 'object' || o === null) return new Error(path + ': object is not an instance of an object or is null')
+
+    if (!Array.isArray(o.base_64_tlvs)) return new Error(`${path}.base_64_tlvs: is not an array`)
+    for (let index = 0; index < o.base_64_tlvs.length; index++) {
+        if (typeof o.base_64_tlvs[index] !== 'string') return new Error(`${path}.base_64_tlvs[${index}]: is not a string`)
+    }
+    if (opts.base_64_tlvs_CustomCheck && !opts.base_64_tlvs_CustomCheck(o.base_64_tlvs)) return new Error(`${path}.base_64_tlvs: custom check failed`)
+
+    return null
+}
+
 export type UsageMetrics = {
-    metrics: UsageMetric[]
+    apps: Record<string, AppUsageMetrics>
 }
 export const UsageMetricsOptionalFields: [] = []
 export type UsageMetricsOptions = OptionsBaseMessage & {
     checkOptionalsAreSet?: []
-    metrics_ItemOptions?: UsageMetricOptions
+    apps_EntryOptions?: AppUsageMetricsOptions
-    metrics_CustomCheck?: (v: UsageMetric[]) => boolean
+    apps_CustomCheck?: (v: Record<string, AppUsageMetrics>) => boolean
 }
 export const UsageMetricsValidate = (o?: UsageMetrics, opts: UsageMetricsOptions = {}, path: string = 'UsageMetrics::root.'): Error | null => {
     if (opts.checkOptionalsAreSet && opts.allOptionalsAreSet) return new Error(path + ': only one of checkOptionalsAreSet or allOptionalNonDefault can be set for each message')
     if (typeof o !== 'object' || o === null) return new Error(path + ': object is not an instance of an object or is null')
 
-    if (!Array.isArray(o.metrics)) return new Error(`${path}.metrics: is not an array`)
+    if (typeof o.apps !== 'object' || o.apps === null) return new Error(`${path}.apps: is not an object or is null`)
-    for (let index = 0; index < o.metrics.length; index++) {
+    for (const key in o.apps) {
-        const metricsErr = UsageMetricValidate(o.metrics[index], opts.metrics_ItemOptions, `${path}.metrics[${index}]`)
+        const appsErr = AppUsageMetricsValidate(o.apps[key], opts.apps_EntryOptions, `${path}.apps['${key}']`)
-        if (metricsErr !== null) return metricsErr
+        if (appsErr !== null) return appsErr
     }
-    if (opts.metrics_CustomCheck && !opts.metrics_CustomCheck(o.metrics)) return new Error(`${path}.metrics: custom check failed`)
 
     return null
 }

proto/service/structs.proto

@@ -29,10 +29,20 @@ message UsageMetric {
     bool batch = 7;
     bool nostr = 8;
     int64 batch_size = 9;
+    bool success = 10;
+    optional string app_id = 11;
+}
+
+message UsageMetricTlv {
+    repeated string base_64_tlvs = 1;
+}
+
+message AppUsageMetrics {
+    map<string,UsageMetricTlv> app_metrics = 1;
 }
 
 message UsageMetrics {
-    repeated UsageMetric metrics = 1; 
+    map<string,AppUsageMetrics> apps = 1;
 }
 
 message AppsMetricsRequest {

src/services/metrics/index.ts

@@ -7,14 +7,16 @@ import { BalanceEvent } from '../storage/entity/BalanceEvent.js'
 import { ChannelBalanceEvent } from '../storage/entity/ChannelsBalanceEvent.js'
 import LND from '../lnd/lnd.js'
 import HtlcTracker from './htlcTracker.js'
+import { encodeTLV, usageMetricsToTlv } from './tlv.js'
 const maxEvents = 100_000
+
 export default class Handler {
 
 
     storage: Storage
     lnd: LND
     htlcTracker: HtlcTracker
-    metrics: Types.UsageMetric[] = []
+    metrics: Record<string, Types.AppUsageMetrics> = {}
     constructor(storage: Storage, lnd: LND) {
         this.storage = storage
         this.lnd = lnd
@@ -63,27 +65,39 @@ export default class Handler {
     }
 
     AddMetrics(newMetrics: (Types.RequestMetric & { app_id?: string })[]) {
-        const parsed: Types.UsageMetric[] = newMetrics.map(m => ({
+        newMetrics.forEach(m => {
-            rpc_name: m.rpcName,
+            const appId = m.app_id || "_root"
-            batch: m.batch,
+            const um: Types.UsageMetric = {
-            nostr: m.nostr,
+                rpc_name: m.rpcName,
-            batch_size: m.batchSize,
+                batch: m.batch,
-            parsed_in_nano: Number(m.parse - m.start),
+                nostr: m.nostr,
-            auth_in_nano: Number(m.guard - m.parse),
+                batch_size: m.batchSize,
-            validate_in_nano: Number(m.validate - m.guard),
+                parsed_in_nano: Number(m.parse - m.start),
-            handle_in_nano: Number(m.handle - m.validate),
+                auth_in_nano: Number(m.guard - m.parse),
-            success: !m.error,
+                validate_in_nano: Number(m.validate - m.guard),
-            app_id: m.app_id ? m.app_id : "",
+                handle_in_nano: Number(m.handle - m.validate),
-            processed_at_ms: m.startMs
+                success: !m.error,
-        }))
+                app_id: m.app_id ? m.app_id : "",
-        const len = this.metrics.push(...parsed)
+                processed_at_ms: m.startMs
-        if (len > maxEvents) {
+            }
-            this.metrics.splice(0, len - maxEvents)
+            const tlv = usageMetricsToTlv(um)
-        }
+            const tlvString = Buffer.from(encodeTLV(tlv)).toString("base64")
+            if (!this.metrics[appId]) {
+                this.metrics[appId] = { app_metrics: {} }
+            }
+            if (!this.metrics[appId].app_metrics[m.rpcName]) {
+                this.metrics[appId].app_metrics[m.rpcName] = { base_64_tlvs: [] }
+            }
+            const len = this.metrics[appId].app_metrics[m.rpcName].base_64_tlvs.push(tlvString)
+            if (len > maxEvents) {
+                this.metrics[appId].app_metrics[m.rpcName].base_64_tlvs.splice(0, len - maxEvents)
+            }
+        })
     }
+
     async GetUsageMetrics(): Promise<Types.UsageMetrics> {
         return {
-            metrics: this.metrics
+            apps: this.metrics
         }
     }
     async GetAppsMetrics(req: Types.AppsMetricsRequest): Promise<Types.AppsMetrics> {

src/services/metrics/tlv.ts

@@ -0,0 +1,80 @@
+import { bytesToHex, concatBytes } from '@noble/hashes/utils'
+import * as Types from '../../../proto/autogenerated/ts/types.js'
+export const utf8Decoder: TextDecoder = new TextDecoder('utf-8')
+export const utf8Encoder: TextEncoder = new TextEncoder()
+
+export const usageMetricsToTlv = (metric: Types.UsageMetric): TLV => {
+    const tlv: TLV = {}
+    tlv[2] = [integerToUint8Array(metric.processed_at_ms)] // 6 -> 6
+    tlv[3] = [integerToUint8Array(metric.parsed_in_nano)] // 6 -> 12
+    tlv[4] = [integerToUint8Array(metric.auth_in_nano)] // 6 -> 18
+    tlv[5] = [integerToUint8Array(metric.validate_in_nano)] // 6 -> 24
+    tlv[6] = [integerToUint8Array(metric.handle_in_nano)] // 6 -> 30
+    tlv[7] = [integerToUint8Array(metric.batch_size)] // 6 -> 36
+    tlv[8] = [new Uint8Array([metric.batch ? 1 : 0])] // 3 -> 39
+    tlv[9] = [new Uint8Array([metric.nostr ? 1 : 0])] // 3 -> 42
+    tlv[10] = [new Uint8Array([metric.success ? 1 : 0])] // 3 -> 45
+    return tlv
+}
+
+export const tlvToUsageMetrics = (rpcName: string, tlv: TLV): Types.UsageMetric => {
+    const metric: Types.UsageMetric = {
+        rpc_name: rpcName,
+        processed_at_ms: parseInt(bytesToHex(tlv[2][0]), 16),
+        parsed_in_nano: parseInt(bytesToHex(tlv[3][0]), 16),
+        auth_in_nano: parseInt(bytesToHex(tlv[4][0]), 16),
+        validate_in_nano: parseInt(bytesToHex(tlv[5][0]), 16),
+        handle_in_nano: parseInt(bytesToHex(tlv[6][0]), 16),
+        batch_size: parseInt(bytesToHex(tlv[7][0]), 16),
+        batch: tlv[8][0][0] === 1,
+        nostr: tlv[9][0][0] === 1,
+        success: tlv[10][0][0] === 1,
+    }
+    return metric
+}
+
+export const integerToUint8Array = (number: number): Uint8Array => {
+    // Create a Uint8Array with enough space to hold a 32-bit integer (4 bytes).
+    const uint8Array = new Uint8Array(4)
+
+    // Use bitwise operations to extract the bytes.
+    uint8Array[0] = (number >> 24) & 0xff // Most significant byte (MSB)
+    uint8Array[1] = (number >> 16) & 0xff
+    uint8Array[2] = (number >> 8) & 0xff
+    uint8Array[3] = number & 0xff // Least significant byte (LSB)
+
+    return uint8Array
+}
+
+export type TLV = { [t: number]: Uint8Array[] }
+export const parseTLV = (data: Uint8Array): TLV => {
+    const result: TLV = {}
+    let rest = data
+    while (rest.length > 0) {
+        const t = rest[0]
+        const l = rest[1]
+        const v = rest.slice(2, 2 + l)
+        rest = rest.slice(2 + l)
+        if (v.length < l) throw new Error(`not enough data to read on TLV ${t}`)
+        result[t] = result[t] || []
+        result[t].push(v)
+    }
+    return result
+}
+
+export const encodeTLV = (tlv: TLV): Uint8Array => {
+    const entries: Uint8Array[] = []
+    Object.entries(tlv)
+        .reverse()
+        .forEach(([t, vs]) => {
+            vs.forEach(v => {
+                const entry = new Uint8Array(v.length + 2)
+                entry.set([parseInt(t)], 0)
+                entry.set([v.length], 1)
+                entry.set(v, 2)
+                entries.push(entry)
+            })
+        })
+
+    return concatBytes(...entries)
+}