From db3c27c7f29e05869382ca3492a273d7c9d8b046 Mon Sep 17 00:00:00 2001 From: boufni95 Date: Fri, 17 Jan 2025 17:54:50 +0000 Subject: [PATCH] fetch single metric --- proto/autogenerated/client.md | 13 ++++- proto/autogenerated/go/http_client.go | 9 ++-- proto/autogenerated/go/types.go | 9 ++++ proto/autogenerated/ts/express_server.ts | 7 ++- proto/autogenerated/ts/http_client.ts | 4 +- proto/autogenerated/ts/nostr_client.ts | 3 +- proto/autogenerated/ts/nostr_transport.ts | 7 ++- proto/autogenerated/ts/types.ts | 57 ++++++++++++++++++++- proto/service/methods.proto | 2 +- proto/service/structs.proto | 11 ++++ src/services/metrics/index.ts | 8 +-- src/services/serverMethods/index.ts | 4 +- src/services/storage/metricsEventStorage.ts | 22 ++++++++ 13 files changed, 137 insertions(+), 19 deletions(-) diff --git a/proto/autogenerated/client.md b/proto/autogenerated/client.md index d6a05091..cd8ab866 100644 --- a/proto/autogenerated/client.md +++ b/proto/autogenerated/client.md @@ -160,7 +160,7 @@ The nostr server will send back a message response, and inside the body there wi - GetUsageMetrics - auth type: __Metrics__ - - This methods has an __empty__ __request__ body + - input: [UsageMetricReq](#UsageMetricReq) - output: [UsageMetrics](#UsageMetrics) - GetUserInfo @@ -585,7 +585,7 @@ The nostr server will send back a message response, and inside the body there wi - auth type: __Metrics__ - http method: __post__ - http route: __/api/reports/usage__ - - This methods has an __empty__ __request__ body + - input: [UsageMetricReq](#UsageMetricReq) - output: [UsageMetrics](#UsageMetrics) - GetUserInfo @@ -1140,6 +1140,11 @@ The nostr server will send back a message response, and inside the body there wi - __payLink__: _string_ - __tag__: _string_ +### MetricsFile + - __app_id__: _string_ + - __metrics_name__: _string_ + - __page__: _number_ + ### MigrationUpdate - __closure__: _[ClosureMigration](#ClosureMigration)_ *this field is optional - __relays__: _[RelaysMigration](#RelaysMigration)_ *this field is optional @@ -1314,6 +1319,10 @@ The nostr server will send back a message response, and inside the body there wi - __success__: _boolean_ - __validate_in_nano__: _number_ +### UsageMetricReq + - __limit__: _number_ *this field is optional + - __metrics_file__: _[MetricsFile](#MetricsFile)_ *this field is optional + ### UsageMetricTlv - __available_chunks__: ARRAY of: _number_ - __base_64_tlvs__: ARRAY of: _string_ diff --git a/proto/autogenerated/go/http_client.go b/proto/autogenerated/go/http_client.go index 5ecc8996..1f5779bc 100644 --- a/proto/autogenerated/go/http_client.go +++ b/proto/autogenerated/go/http_client.go @@ -93,7 +93,7 @@ type Client struct { GetNPubLinkingState func(req GetNPubLinking) (*NPubLinking, error) GetPaymentState func(req GetPaymentStateRequest) (*PaymentState, error) GetSeed func() (*LndSeed, error) - GetUsageMetrics func() (*UsageMetrics, error) + GetUsageMetrics func(req UsageMetricReq) (*UsageMetrics, error) GetUserInfo func() (*UserInfo, error) GetUserOffer func(req OfferId) (*OfferConfig, error) GetUserOfferInvoices func(req GetUserOfferInvoicesReq) (*OfferInvoices, error) @@ -1057,13 +1057,16 @@ func NewClient(params ClientParams) *Client { } return &res, nil }, - GetUsageMetrics: func() (*UsageMetrics, error) { + GetUsageMetrics: func(req UsageMetricReq) (*UsageMetrics, error) { auth, err := params.RetrieveMetricsAuth() if err != nil { return nil, err } finalRoute := "/api/reports/usage" - body := []byte{} + body, err := json.Marshal(req) + if err != nil { + return nil, err + } resBody, err := doPostRequest(params.BaseURL+finalRoute, body, auth) if err != nil { return nil, err diff --git a/proto/autogenerated/go/types.go b/proto/autogenerated/go/types.go index 2fda35f5..c01ded0f 100644 --- a/proto/autogenerated/go/types.go +++ b/proto/autogenerated/go/types.go @@ -392,6 +392,11 @@ type LnurlWithdrawInfoResponse struct { Paylink string `json:"payLink"` Tag string `json:"tag"` } +type MetricsFile struct { + App_id string `json:"app_id"` + Metrics_name string `json:"metrics_name"` + Page int64 `json:"page"` +} type MigrationUpdate struct { Closure *ClosureMigration `json:"closure"` Relays *RelaysMigration `json:"relays"` @@ -566,6 +571,10 @@ type UsageMetric struct { Success bool `json:"success"` Validate_in_nano int64 `json:"validate_in_nano"` } +type UsageMetricReq struct { + Limit int64 `json:"limit"` + Metrics_file *MetricsFile `json:"metrics_file"` +} type UsageMetricTlv struct { Available_chunks []int64 `json:"available_chunks"` Base_64_tlvs []string `json:"base_64_tlvs"` diff --git a/proto/autogenerated/ts/express_server.ts b/proto/autogenerated/ts/express_server.ts index 9b1fd1ea..053ffc0d 100644 --- a/proto/autogenerated/ts/express_server.ts +++ b/proto/autogenerated/ts/express_server.ts @@ -1116,10 +1116,13 @@ export default (methods: Types.ServerMethods, opts: ServerOptions) => { const authContext = await opts.MetricsAuthGuard(req.headers['authorization']) authCtx = authContext stats.guard = process.hrtime.bigint() - stats.validate = stats.guard + const request = req.body + const error = Types.UsageMetricReqValidate(request) + stats.validate = process.hrtime.bigint() + if (error !== null) return logErrorAndReturnResponse(error, 'invalid request body', res, logger, { ...info, ...stats, ...authContext }, opts.metricsCallback) const query = req.query const params = req.params - const response = await methods.GetUsageMetrics({rpcName:'GetUsageMetrics', ctx:authContext }) + const response = await methods.GetUsageMetrics({rpcName:'GetUsageMetrics', ctx:authContext , req: request}) stats.handle = process.hrtime.bigint() res.json({status: 'OK', ...response}) opts.metricsCallback([{ ...info, ...stats, ...authContext }]) diff --git a/proto/autogenerated/ts/http_client.ts b/proto/autogenerated/ts/http_client.ts index 10ae0041..24cb4f89 100644 --- a/proto/autogenerated/ts/http_client.ts +++ b/proto/autogenerated/ts/http_client.ts @@ -494,11 +494,11 @@ export default (params: ClientParams) => ({ } return { status: 'ERROR', reason: 'invalid response' } }, - GetUsageMetrics: async (): Promise => { + GetUsageMetrics: async (request: Types.UsageMetricReq): Promise => { const auth = await params.retrieveMetricsAuth() if (auth === null) throw new Error('retrieveMetricsAuth() returned null') let finalRoute = '/api/reports/usage' - const { data } = await axios.post(params.baseUrl + finalRoute, {}, { headers: { 'authorization': auth } }) + const { data } = await axios.post(params.baseUrl + finalRoute, request, { headers: { 'authorization': auth } }) if (data.status === 'ERROR' && typeof data.reason === 'string') return data if (data.status === 'OK') { const result = data diff --git a/proto/autogenerated/ts/nostr_client.ts b/proto/autogenerated/ts/nostr_client.ts index 04f14158..d9087c05 100644 --- a/proto/autogenerated/ts/nostr_client.ts +++ b/proto/autogenerated/ts/nostr_client.ts @@ -422,10 +422,11 @@ export default (params: NostrClientParams, send: (to:string, message: NostrRequ } return { status: 'ERROR', reason: 'invalid response' } }, - GetUsageMetrics: async (): Promise => { + GetUsageMetrics: async (request: Types.UsageMetricReq): Promise => { const auth = await params.retrieveNostrMetricsAuth() if (auth === null) throw new Error('retrieveNostrMetricsAuth() returned null') const nostrRequest: NostrRequest = {} + nostrRequest.body = request const data = await send(params.pubDestination, {rpcName:'GetUsageMetrics',authIdentifier:auth, ...nostrRequest }) if (data.status === 'ERROR' && typeof data.reason === 'string') return data if (data.status === 'OK') { diff --git a/proto/autogenerated/ts/nostr_transport.ts b/proto/autogenerated/ts/nostr_transport.ts index 7d3f2c7d..2920b251 100644 --- a/proto/autogenerated/ts/nostr_transport.ts +++ b/proto/autogenerated/ts/nostr_transport.ts @@ -805,8 +805,11 @@ export default (methods: Types.ServerMethods, opts: NostrOptions) => { const authContext = await opts.NostrMetricsAuthGuard(req.appId, req.authIdentifier) stats.guard = process.hrtime.bigint() authCtx = authContext - stats.validate = stats.guard - const response = await methods.GetUsageMetrics({rpcName:'GetUsageMetrics', ctx:authContext }) + const request = req.body + const error = Types.UsageMetricReqValidate(request) + stats.validate = process.hrtime.bigint() + if (error !== null) return logErrorAndReturnResponse(error, 'invalid request body', res, logger, { ...info, ...stats, ...authCtx }, opts.metricsCallback) + const response = await methods.GetUsageMetrics({rpcName:'GetUsageMetrics', ctx:authContext , req: request}) stats.handle = process.hrtime.bigint() res({status: 'OK', ...response}) opts.metricsCallback([{ ...info, ...stats, ...authContext }]) diff --git a/proto/autogenerated/ts/types.ts b/proto/autogenerated/ts/types.ts index 7fbffbf1..6bc781a2 100644 --- a/proto/autogenerated/ts/types.ts +++ b/proto/autogenerated/ts/types.ts @@ -161,7 +161,7 @@ export type GetPaymentState_Output = ResultError | ({ status: 'OK' } & PaymentSt export type GetSeed_Input = {rpcName:'GetSeed'} export type GetSeed_Output = ResultError | ({ status: 'OK' } & LndSeed) -export type GetUsageMetrics_Input = {rpcName:'GetUsageMetrics'} +export type GetUsageMetrics_Input = {rpcName:'GetUsageMetrics', req: UsageMetricReq} export type GetUsageMetrics_Output = ResultError | ({ status: 'OK' } & UsageMetrics) export type GetUserInfo_Input = {rpcName:'GetUserInfo'} @@ -2234,6 +2234,34 @@ export const LnurlWithdrawInfoResponseValidate = (o?: LnurlWithdrawInfoResponse, return null } +export type MetricsFile = { + app_id: string + metrics_name: string + page: number +} +export const MetricsFileOptionalFields: [] = [] +export type MetricsFileOptions = OptionsBaseMessage & { + checkOptionalsAreSet?: [] + app_id_CustomCheck?: (v: string) => boolean + metrics_name_CustomCheck?: (v: string) => boolean + page_CustomCheck?: (v: number) => boolean +} +export const MetricsFileValidate = (o?: MetricsFile, opts: MetricsFileOptions = {}, path: string = 'MetricsFile::root.'): Error | null => { + if (opts.checkOptionalsAreSet && opts.allOptionalsAreSet) return new Error(path + ': only one of checkOptionalsAreSet or allOptionalNonDefault can be set for each message') + if (typeof o !== 'object' || o === null) return new Error(path + ': object is not an instance of an object or is null') + + if (typeof o.app_id !== 'string') return new Error(`${path}.app_id: is not a string`) + if (opts.app_id_CustomCheck && !opts.app_id_CustomCheck(o.app_id)) return new Error(`${path}.app_id: custom check failed`) + + if (typeof o.metrics_name !== 'string') return new Error(`${path}.metrics_name: is not a string`) + if (opts.metrics_name_CustomCheck && !opts.metrics_name_CustomCheck(o.metrics_name)) return new Error(`${path}.metrics_name: custom check failed`) + + if (typeof o.page !== 'number') return new Error(`${path}.page: is not a number`) + if (opts.page_CustomCheck && !opts.page_CustomCheck(o.page)) return new Error(`${path}.page: custom check failed`) + + return null +} + export type MigrationUpdate = { closure?: ClosureMigration relays?: RelaysMigration @@ -3236,6 +3264,33 @@ export const UsageMetricValidate = (o?: UsageMetric, opts: UsageMetricOptions = return null } +export type UsageMetricReq = { + limit?: number + metrics_file?: MetricsFile +} +export type UsageMetricReqOptionalField = 'limit' | 'metrics_file' +export const UsageMetricReqOptionalFields: UsageMetricReqOptionalField[] = ['limit', 'metrics_file'] +export type UsageMetricReqOptions = OptionsBaseMessage & { + checkOptionalsAreSet?: UsageMetricReqOptionalField[] + limit_CustomCheck?: (v?: number) => boolean + metrics_file_Options?: MetricsFileOptions +} +export const UsageMetricReqValidate = (o?: UsageMetricReq, opts: UsageMetricReqOptions = {}, path: string = 'UsageMetricReq::root.'): Error | null => { + if (opts.checkOptionalsAreSet && opts.allOptionalsAreSet) return new Error(path + ': only one of checkOptionalsAreSet or allOptionalNonDefault can be set for each message') + if (typeof o !== 'object' || o === null) return new Error(path + ': object is not an instance of an object or is null') + + if ((o.limit || opts.allOptionalsAreSet || opts.checkOptionalsAreSet?.includes('limit')) && typeof o.limit !== 'number') return new Error(`${path}.limit: is not a number`) + if (opts.limit_CustomCheck && !opts.limit_CustomCheck(o.limit)) return new Error(`${path}.limit: custom check failed`) + + if (typeof o.metrics_file === 'object' || opts.allOptionalsAreSet || opts.checkOptionalsAreSet?.includes('metrics_file')) { + const metrics_fileErr = MetricsFileValidate(o.metrics_file, opts.metrics_file_Options, `${path}.metrics_file`) + if (metrics_fileErr !== null) return metrics_fileErr + } + + + return null +} + export type UsageMetricTlv = { available_chunks: number[] base_64_tlvs: string[] diff --git a/proto/service/methods.proto b/proto/service/methods.proto index f628120e..1b2cea3a 100644 --- a/proto/service/methods.proto +++ b/proto/service/methods.proto @@ -172,7 +172,7 @@ service LightningPub { option (nostr) = true; } - rpc GetUsageMetrics(structs.Empty) returns (structs.UsageMetrics) { + rpc GetUsageMetrics(structs.UsageMetricReq) returns (structs.UsageMetrics) { option (auth_type) = "Metrics"; option (http_method) = "post"; option (http_route) = "/api/reports/usage"; diff --git a/proto/service/structs.proto b/proto/service/structs.proto index 94556216..9513e1ea 100644 --- a/proto/service/structs.proto +++ b/proto/service/structs.proto @@ -33,6 +33,17 @@ message ErrorStats { ErrorStat past1m = 5; } +message MetricsFile { + string app_id = 1; + string metrics_name = 2; + int64 page = 3; +} + +message UsageMetricReq { + optional int64 limit = 1; + optional MetricsFile metrics_file = 2; +} + message UsageMetric { int64 processed_at_ms = 1; int64 parsed_in_nano = 2; diff --git a/src/services/metrics/index.ts b/src/services/metrics/index.ts index 20abc735..c832c40f 100644 --- a/src/services/metrics/index.ts +++ b/src/services/metrics/index.ts @@ -69,9 +69,11 @@ export default class Handler { })) } - async GetUsageMetrics(): Promise { - const metrics = await this.storage.metricsEventStorage.LoadLatestMetrics() - return metrics + async GetUsageMetrics(req: Types.UsageMetricReq): Promise { + if (!req.metrics_file) { + return this.storage.metricsEventStorage.LoadLatestMetrics(req.limit) + } + return this.storage.metricsEventStorage.LoadMetricsFile(req.metrics_file.app_id, req.metrics_file.metrics_name, req.metrics_file.page) } async GetErrorStats(): Promise { diff --git a/src/services/serverMethods/index.ts b/src/services/serverMethods/index.ts index 316fbf92..fce92672 100644 --- a/src/services/serverMethods/index.ts +++ b/src/services/serverMethods/index.ts @@ -4,8 +4,8 @@ import main from '../main/index.js' import Main from '../main/index.js' export default (mainHandler: Main): Types.ServerMethods => { return { - GetUsageMetrics: async ({ ctx }) => { - return mainHandler.metricsManager.GetUsageMetrics() + GetUsageMetrics: async ({ ctx, req }) => { + return mainHandler.metricsManager.GetUsageMetrics(req) }, GetErrorStats: async ({ ctx }) => { return mainHandler.metricsManager.GetErrorStats() diff --git a/src/services/storage/metricsEventStorage.ts b/src/services/storage/metricsEventStorage.ts index f4b26a9d..901c6928 100644 --- a/src/services/storage/metricsEventStorage.ts +++ b/src/services/storage/metricsEventStorage.ts @@ -132,6 +132,28 @@ export default class { return metrics } + LoadMetricsFile = async (app: string, method: string, chunk: number): Promise => { + if (!this.metaReady || !this.metricsMeta[app] || !this.metricsMeta[app][method] || !this.metricsMeta[app][method].chunks.includes(chunk)) { + return { apps: {} } + } + const fullPath = [this.metricsPath, app, method, `${chunk}.mtlv`].join("/") + const tlv = fs.readFileSync(fullPath) + const decoded = decodeListTLV(parseTLV(tlv)) + return { + apps: { + [app]: { + app_metrics: { + [method]: { + base_64_tlvs: decoded.map(d => Buffer.from(d).toString('base64')), + current_chunk: chunk, + available_chunks: this.metricsMeta[app][method].chunks + } + } + } + } + } + } + persistMetrics = () => { if (!this.metaReady) { throw new Error("meta metrics not ready")