diff --git a/proto/autogenerated/client.md b/proto/autogenerated/client.md index 65c78245..01572a2e 100644 --- a/proto/autogenerated/client.md +++ b/proto/autogenerated/client.md @@ -866,6 +866,9 @@ The nostr server will send back a message response, and inside the body there wi - __total_fees__: _number_ - __users__: _[UsersInfo](#UsersInfo)_ +### AppUsageMetrics + - __app_metrics__: MAP with key: _string_ and value: _[UsageMetricTlv](#UsageMetricTlv)_ + ### AppUser - __identifier__: _string_ - __info__: _[UserInfo](#UserInfo)_ @@ -1275,6 +1278,7 @@ The nostr server will send back a message response, and inside the body there wi - __update__: _[UpdateChannelPolicyRequest_update](#UpdateChannelPolicyRequest_update)_ ### UsageMetric + - __app_id__: _string_ *this field is optional - __auth_in_nano__: _number_ - __batch__: _boolean_ - __batch_size__: _number_ @@ -1283,10 +1287,14 @@ The nostr server will send back a message response, and inside the body there wi - __parsed_in_nano__: _number_ - __processed_at_ms__: _number_ - __rpc_name__: _string_ + - __success__: _boolean_ - __validate_in_nano__: _number_ +### UsageMetricTlv + - __base_64_tlvs__: ARRAY of: _string_ + ### UsageMetrics - - __metrics__: ARRAY of: _[UsageMetric](#UsageMetric)_ + - __apps__: MAP with key: _string_ and value: _[AppUsageMetrics](#AppUsageMetrics)_ ### UseInviteLinkRequest - __invite_token__: _string_ diff --git a/proto/autogenerated/go/types.go b/proto/autogenerated/go/types.go index c82ae57f..8d6ba2c2 100644 --- a/proto/autogenerated/go/types.go +++ b/proto/autogenerated/go/types.go @@ -130,6 +130,9 @@ type AppMetrics struct { Total_fees int64 `json:"total_fees"` Users *UsersInfo `json:"users"` } +type AppUsageMetrics struct { + App_metrics map[string]UsageMetricTlv `json:"app_metrics"` +} type AppUser struct { Identifier string `json:"identifier"` Info *UserInfo `json:"info"` @@ -539,6 +542,7 @@ type UpdateChannelPolicyRequest struct { Update *UpdateChannelPolicyRequest_update `json:"update"` } type UsageMetric struct { + App_id string `json:"app_id"` Auth_in_nano int64 `json:"auth_in_nano"` Batch bool `json:"batch"` Batch_size int64 `json:"batch_size"` @@ -547,10 +551,14 @@ type UsageMetric struct { Parsed_in_nano int64 `json:"parsed_in_nano"` Processed_at_ms int64 `json:"processed_at_ms"` Rpc_name string `json:"rpc_name"` + Success bool `json:"success"` Validate_in_nano int64 `json:"validate_in_nano"` } +type UsageMetricTlv struct { + Base_64_tlvs []string `json:"base_64_tlvs"` +} type UsageMetrics struct { - Metrics []UsageMetric `json:"metrics"` + Apps map[string]AppUsageMetrics `json:"apps"` } type UseInviteLinkRequest struct { Invite_token string `json:"invite_token"` diff --git a/proto/autogenerated/ts/types.ts b/proto/autogenerated/ts/types.ts index 8609d007..824204ad 100644 --- a/proto/autogenerated/ts/types.ts +++ b/proto/autogenerated/ts/types.ts @@ -644,6 +644,28 @@ export const AppMetricsValidate = (o?: AppMetrics, opts: AppMetricsOptions = {}, return null } +export type AppUsageMetrics = { + app_metrics: Record +} +export const AppUsageMetricsOptionalFields: [] = [] +export type AppUsageMetricsOptions = OptionsBaseMessage & { + checkOptionalsAreSet?: [] + app_metrics_EntryOptions?: UsageMetricTlvOptions + app_metrics_CustomCheck?: (v: Record) => boolean +} +export const AppUsageMetricsValidate = (o?: AppUsageMetrics, opts: AppUsageMetricsOptions = {}, path: string = 'AppUsageMetrics::root.'): Error | null => { + if (opts.checkOptionalsAreSet && opts.allOptionalsAreSet) return new Error(path + ': only one of checkOptionalsAreSet or allOptionalNonDefault can be set for each message') + if (typeof o !== 'object' || o === null) return new Error(path + ': object is not an instance of an object or is null') + + if (typeof o.app_metrics !== 'object' || o.app_metrics === null) return new Error(`${path}.app_metrics: is not an object or is null`) + for (const key in o.app_metrics) { + const app_metricsErr = UsageMetricTlvValidate(o.app_metrics[key], opts.app_metrics_EntryOptions, `${path}.app_metrics['${key}']`) + if (app_metricsErr !== null) return app_metricsErr + } + + return null +} + export type AppUser = { identifier: string info: UserInfo @@ -3071,6 +3093,7 @@ export const UpdateChannelPolicyRequestValidate = (o?: UpdateChannelPolicyReques } export type UsageMetric = { + app_id?: string auth_in_nano: number batch: boolean batch_size: number @@ -3079,11 +3102,14 @@ export type UsageMetric = { parsed_in_nano: number processed_at_ms: number rpc_name: string + success: boolean validate_in_nano: number } -export const UsageMetricOptionalFields: [] = [] +export type UsageMetricOptionalField = 'app_id' +export const UsageMetricOptionalFields: UsageMetricOptionalField[] = ['app_id'] export type UsageMetricOptions = OptionsBaseMessage & { - checkOptionalsAreSet?: [] + checkOptionalsAreSet?: UsageMetricOptionalField[] + app_id_CustomCheck?: (v?: string) => boolean auth_in_nano_CustomCheck?: (v: number) => boolean batch_CustomCheck?: (v: boolean) => boolean batch_size_CustomCheck?: (v: number) => boolean @@ -3092,12 +3118,16 @@ export type UsageMetricOptions = OptionsBaseMessage & { parsed_in_nano_CustomCheck?: (v: number) => boolean processed_at_ms_CustomCheck?: (v: number) => boolean rpc_name_CustomCheck?: (v: string) => boolean + success_CustomCheck?: (v: boolean) => boolean validate_in_nano_CustomCheck?: (v: number) => boolean } export const UsageMetricValidate = (o?: UsageMetric, opts: UsageMetricOptions = {}, path: string = 'UsageMetric::root.'): Error | null => { if (opts.checkOptionalsAreSet && opts.allOptionalsAreSet) return new Error(path + ': only one of checkOptionalsAreSet or allOptionalNonDefault can be set for each message') if (typeof o !== 'object' || o === null) return new Error(path + ': object is not an instance of an object or is null') + if ((o.app_id || opts.allOptionalsAreSet || opts.checkOptionalsAreSet?.includes('app_id')) && typeof o.app_id !== 'string') return new Error(`${path}.app_id: is not a string`) + if (opts.app_id_CustomCheck && !opts.app_id_CustomCheck(o.app_id)) return new Error(`${path}.app_id: custom check failed`) + if (typeof o.auth_in_nano !== 'number') return new Error(`${path}.auth_in_nano: is not a number`) if (opts.auth_in_nano_CustomCheck && !opts.auth_in_nano_CustomCheck(o.auth_in_nano)) return new Error(`${path}.auth_in_nano: custom check failed`) @@ -3122,31 +3152,54 @@ export const UsageMetricValidate = (o?: UsageMetric, opts: UsageMetricOptions = if (typeof o.rpc_name !== 'string') return new Error(`${path}.rpc_name: is not a string`) if (opts.rpc_name_CustomCheck && !opts.rpc_name_CustomCheck(o.rpc_name)) return new Error(`${path}.rpc_name: custom check failed`) + if (typeof o.success !== 'boolean') return new Error(`${path}.success: is not a boolean`) + if (opts.success_CustomCheck && !opts.success_CustomCheck(o.success)) return new Error(`${path}.success: custom check failed`) + if (typeof o.validate_in_nano !== 'number') return new Error(`${path}.validate_in_nano: is not a number`) if (opts.validate_in_nano_CustomCheck && !opts.validate_in_nano_CustomCheck(o.validate_in_nano)) return new Error(`${path}.validate_in_nano: custom check failed`) return null } +export type UsageMetricTlv = { + base_64_tlvs: string[] +} +export const UsageMetricTlvOptionalFields: [] = [] +export type UsageMetricTlvOptions = OptionsBaseMessage & { + checkOptionalsAreSet?: [] + base_64_tlvs_CustomCheck?: (v: string[]) => boolean +} +export const UsageMetricTlvValidate = (o?: UsageMetricTlv, opts: UsageMetricTlvOptions = {}, path: string = 'UsageMetricTlv::root.'): Error | null => { + if (opts.checkOptionalsAreSet && opts.allOptionalsAreSet) return new Error(path + ': only one of checkOptionalsAreSet or allOptionalNonDefault can be set for each message') + if (typeof o !== 'object' || o === null) return new Error(path + ': object is not an instance of an object or is null') + + if (!Array.isArray(o.base_64_tlvs)) return new Error(`${path}.base_64_tlvs: is not an array`) + for (let index = 0; index < o.base_64_tlvs.length; index++) { + if (typeof o.base_64_tlvs[index] !== 'string') return new Error(`${path}.base_64_tlvs[${index}]: is not a string`) + } + if (opts.base_64_tlvs_CustomCheck && !opts.base_64_tlvs_CustomCheck(o.base_64_tlvs)) return new Error(`${path}.base_64_tlvs: custom check failed`) + + return null +} + export type UsageMetrics = { - metrics: UsageMetric[] + apps: Record } export const UsageMetricsOptionalFields: [] = [] export type UsageMetricsOptions = OptionsBaseMessage & { checkOptionalsAreSet?: [] - metrics_ItemOptions?: UsageMetricOptions - metrics_CustomCheck?: (v: UsageMetric[]) => boolean + apps_EntryOptions?: AppUsageMetricsOptions + apps_CustomCheck?: (v: Record) => boolean } export const UsageMetricsValidate = (o?: UsageMetrics, opts: UsageMetricsOptions = {}, path: string = 'UsageMetrics::root.'): Error | null => { if (opts.checkOptionalsAreSet && opts.allOptionalsAreSet) return new Error(path + ': only one of checkOptionalsAreSet or allOptionalNonDefault can be set for each message') if (typeof o !== 'object' || o === null) return new Error(path + ': object is not an instance of an object or is null') - if (!Array.isArray(o.metrics)) return new Error(`${path}.metrics: is not an array`) - for (let index = 0; index < o.metrics.length; index++) { - const metricsErr = UsageMetricValidate(o.metrics[index], opts.metrics_ItemOptions, `${path}.metrics[${index}]`) - if (metricsErr !== null) return metricsErr + if (typeof o.apps !== 'object' || o.apps === null) return new Error(`${path}.apps: is not an object or is null`) + for (const key in o.apps) { + const appsErr = AppUsageMetricsValidate(o.apps[key], opts.apps_EntryOptions, `${path}.apps['${key}']`) + if (appsErr !== null) return appsErr } - if (opts.metrics_CustomCheck && !opts.metrics_CustomCheck(o.metrics)) return new Error(`${path}.metrics: custom check failed`) return null } diff --git a/proto/service/structs.proto b/proto/service/structs.proto index 53e8ecf6..bece972c 100644 --- a/proto/service/structs.proto +++ b/proto/service/structs.proto @@ -29,10 +29,20 @@ message UsageMetric { bool batch = 7; bool nostr = 8; int64 batch_size = 9; + bool success = 10; + optional string app_id = 11; +} + +message UsageMetricTlv { + repeated string base_64_tlvs = 1; +} + +message AppUsageMetrics { + map app_metrics = 1; } message UsageMetrics { - repeated UsageMetric metrics = 1; + map apps = 1; } message AppsMetricsRequest { diff --git a/src/services/metrics/index.ts b/src/services/metrics/index.ts index 84b8a355..d0894eef 100644 --- a/src/services/metrics/index.ts +++ b/src/services/metrics/index.ts @@ -7,14 +7,16 @@ import { BalanceEvent } from '../storage/entity/BalanceEvent.js' import { ChannelBalanceEvent } from '../storage/entity/ChannelsBalanceEvent.js' import LND from '../lnd/lnd.js' import HtlcTracker from './htlcTracker.js' +import { encodeTLV, usageMetricsToTlv } from './tlv.js' const maxEvents = 100_000 + export default class Handler { storage: Storage lnd: LND htlcTracker: HtlcTracker - metrics: Types.UsageMetric[] = [] + metrics: Record = {} constructor(storage: Storage, lnd: LND) { this.storage = storage this.lnd = lnd @@ -63,27 +65,39 @@ export default class Handler { } AddMetrics(newMetrics: (Types.RequestMetric & { app_id?: string })[]) { - const parsed: Types.UsageMetric[] = newMetrics.map(m => ({ - rpc_name: m.rpcName, - batch: m.batch, - nostr: m.nostr, - batch_size: m.batchSize, - parsed_in_nano: Number(m.parse - m.start), - auth_in_nano: Number(m.guard - m.parse), - validate_in_nano: Number(m.validate - m.guard), - handle_in_nano: Number(m.handle - m.validate), - success: !m.error, - app_id: m.app_id ? m.app_id : "", - processed_at_ms: m.startMs - })) - const len = this.metrics.push(...parsed) - if (len > maxEvents) { - this.metrics.splice(0, len - maxEvents) - } + newMetrics.forEach(m => { + const appId = m.app_id || "_root" + const um: Types.UsageMetric = { + rpc_name: m.rpcName, + batch: m.batch, + nostr: m.nostr, + batch_size: m.batchSize, + parsed_in_nano: Number(m.parse - m.start), + auth_in_nano: Number(m.guard - m.parse), + validate_in_nano: Number(m.validate - m.guard), + handle_in_nano: Number(m.handle - m.validate), + success: !m.error, + app_id: m.app_id ? m.app_id : "", + processed_at_ms: m.startMs + } + const tlv = usageMetricsToTlv(um) + const tlvString = Buffer.from(encodeTLV(tlv)).toString("base64") + if (!this.metrics[appId]) { + this.metrics[appId] = { app_metrics: {} } + } + if (!this.metrics[appId].app_metrics[m.rpcName]) { + this.metrics[appId].app_metrics[m.rpcName] = { base_64_tlvs: [] } + } + const len = this.metrics[appId].app_metrics[m.rpcName].base_64_tlvs.push(tlvString) + if (len > maxEvents) { + this.metrics[appId].app_metrics[m.rpcName].base_64_tlvs.splice(0, len - maxEvents) + } + }) } + async GetUsageMetrics(): Promise { return { - metrics: this.metrics + apps: this.metrics } } async GetAppsMetrics(req: Types.AppsMetricsRequest): Promise { diff --git a/src/services/metrics/tlv.ts b/src/services/metrics/tlv.ts new file mode 100644 index 00000000..af377168 --- /dev/null +++ b/src/services/metrics/tlv.ts @@ -0,0 +1,80 @@ +import { bytesToHex, concatBytes } from '@noble/hashes/utils' +import * as Types from '../../../proto/autogenerated/ts/types.js' +export const utf8Decoder: TextDecoder = new TextDecoder('utf-8') +export const utf8Encoder: TextEncoder = new TextEncoder() + +export const usageMetricsToTlv = (metric: Types.UsageMetric): TLV => { + const tlv: TLV = {} + tlv[2] = [integerToUint8Array(metric.processed_at_ms)] // 6 -> 6 + tlv[3] = [integerToUint8Array(metric.parsed_in_nano)] // 6 -> 12 + tlv[4] = [integerToUint8Array(metric.auth_in_nano)] // 6 -> 18 + tlv[5] = [integerToUint8Array(metric.validate_in_nano)] // 6 -> 24 + tlv[6] = [integerToUint8Array(metric.handle_in_nano)] // 6 -> 30 + tlv[7] = [integerToUint8Array(metric.batch_size)] // 6 -> 36 + tlv[8] = [new Uint8Array([metric.batch ? 1 : 0])] // 3 -> 39 + tlv[9] = [new Uint8Array([metric.nostr ? 1 : 0])] // 3 -> 42 + tlv[10] = [new Uint8Array([metric.success ? 1 : 0])] // 3 -> 45 + return tlv +} + +export const tlvToUsageMetrics = (rpcName: string, tlv: TLV): Types.UsageMetric => { + const metric: Types.UsageMetric = { + rpc_name: rpcName, + processed_at_ms: parseInt(bytesToHex(tlv[2][0]), 16), + parsed_in_nano: parseInt(bytesToHex(tlv[3][0]), 16), + auth_in_nano: parseInt(bytesToHex(tlv[4][0]), 16), + validate_in_nano: parseInt(bytesToHex(tlv[5][0]), 16), + handle_in_nano: parseInt(bytesToHex(tlv[6][0]), 16), + batch_size: parseInt(bytesToHex(tlv[7][0]), 16), + batch: tlv[8][0][0] === 1, + nostr: tlv[9][0][0] === 1, + success: tlv[10][0][0] === 1, + } + return metric +} + +export const integerToUint8Array = (number: number): Uint8Array => { + // Create a Uint8Array with enough space to hold a 32-bit integer (4 bytes). + const uint8Array = new Uint8Array(4) + + // Use bitwise operations to extract the bytes. + uint8Array[0] = (number >> 24) & 0xff // Most significant byte (MSB) + uint8Array[1] = (number >> 16) & 0xff + uint8Array[2] = (number >> 8) & 0xff + uint8Array[3] = number & 0xff // Least significant byte (LSB) + + return uint8Array +} + +export type TLV = { [t: number]: Uint8Array[] } +export const parseTLV = (data: Uint8Array): TLV => { + const result: TLV = {} + let rest = data + while (rest.length > 0) { + const t = rest[0] + const l = rest[1] + const v = rest.slice(2, 2 + l) + rest = rest.slice(2 + l) + if (v.length < l) throw new Error(`not enough data to read on TLV ${t}`) + result[t] = result[t] || [] + result[t].push(v) + } + return result +} + +export const encodeTLV = (tlv: TLV): Uint8Array => { + const entries: Uint8Array[] = [] + Object.entries(tlv) + .reverse() + .forEach(([t, vs]) => { + vs.forEach(v => { + const entry = new Uint8Array(v.length + 2) + entry.set([parseInt(t)], 0) + entry.set([v.length], 1) + entry.set(v, 2) + entries.push(entry) + }) + }) + + return concatBytes(...entries) +} \ No newline at end of file