diff --git a/proto/autogenerated/client.md b/proto/autogenerated/client.md index 9b709dda..d6a05091 100644 --- a/proto/autogenerated/client.md +++ b/proto/autogenerated/client.md @@ -98,6 +98,11 @@ The nostr server will send back a message response, and inside the body there wi - This methods has an __empty__ __request__ body - output: [DebitAuthorizations](#DebitAuthorizations) +- GetErrorStats + - auth type: __Metrics__ + - This methods has an __empty__ __request__ body + - output: [ErrorStats](#ErrorStats) + - GetHttpCreds - auth type: __User__ - This methods has an __empty__ __request__ body @@ -467,6 +472,13 @@ The nostr server will send back a message response, and inside the body there wi - This methods has an __empty__ __request__ body - output: [DebitAuthorizations](#DebitAuthorizations) +- GetErrorStats + - auth type: __Metrics__ + - http method: __post__ + - http route: __/api/reports/errors__ + - This methods has an __empty__ __request__ body + - output: [ErrorStats](#ErrorStats) + - GetHttpCreds - auth type: __User__ - http method: __post__ @@ -986,6 +998,18 @@ The nostr server will send back a message response, and inside the body there wi ### EnrollAdminTokenRequest - __admin_token__: _string_ +### ErrorStat + - __errors__: _number_ + - __from_unix__: _number_ + - __total__: _number_ + +### ErrorStats + - __past10m__: _[ErrorStat](#ErrorStat)_ + - __past1h__: _[ErrorStat](#ErrorStat)_ + - __past1m__: _[ErrorStat](#ErrorStat)_ + - __past24h__: _[ErrorStat](#ErrorStat)_ + - __past6h__: _[ErrorStat](#ErrorStat)_ + ### FrequencyRule - __amount__: _number_ - __interval__: _[IntervalType](#IntervalType)_ diff --git a/proto/autogenerated/go/http_client.go b/proto/autogenerated/go/http_client.go index 522df237..5ecc8996 100644 --- a/proto/autogenerated/go/http_client.go +++ b/proto/autogenerated/go/http_client.go @@ -78,6 +78,7 @@ type Client struct { GetAppUserLNURLInfo func(req GetAppUserLNURLInfoRequest) (*LnurlPayInfoResponse, error) GetAppsMetrics func(req AppsMetricsRequest) (*AppsMetrics, error) GetDebitAuthorizations func() (*DebitAuthorizations, error) + GetErrorStats func() (*ErrorStats, error) GetHttpCreds func() (*HttpCreds, error) GetInviteLinkState func(req GetInviteTokenStateRequest) (*GetInviteTokenStateResponse, error) GetLNURLChannelLink func() (*LnurlLinkResponse, error) @@ -758,6 +759,32 @@ func NewClient(params ClientParams) *Client { } return &res, nil }, + GetErrorStats: func() (*ErrorStats, error) { + auth, err := params.RetrieveMetricsAuth() + if err != nil { + return nil, err + } + finalRoute := "/api/reports/errors" + body := []byte{} + resBody, err := doPostRequest(params.BaseURL+finalRoute, body, auth) + if err != nil { + return nil, err + } + result := ResultError{} + err = json.Unmarshal(resBody, &result) + if err != nil { + return nil, err + } + if result.Status == "ERROR" { + return nil, fmt.Errorf(result.Reason) + } + res := ErrorStats{} + err = json.Unmarshal(resBody, &res) + if err != nil { + return nil, err + } + return &res, nil + }, // server streaming method: GetHttpCreds not implemented GetInviteLinkState: func(req GetInviteTokenStateRequest) (*GetInviteTokenStateResponse, error) { auth, err := params.RetrieveAdminAuth() diff --git a/proto/autogenerated/go/types.go b/proto/autogenerated/go/types.go index a316cbdc..2fda35f5 100644 --- a/proto/autogenerated/go/types.go +++ b/proto/autogenerated/go/types.go @@ -250,6 +250,18 @@ type EncryptionExchangeRequest struct { type EnrollAdminTokenRequest struct { Admin_token string `json:"admin_token"` } +type ErrorStat struct { + Errors int64 `json:"errors"` + From_unix int64 `json:"from_unix"` + Total int64 `json:"total"` +} +type ErrorStats struct { + Past10m *ErrorStat `json:"past10m"` + Past1h *ErrorStat `json:"past1h"` + Past1m *ErrorStat `json:"past1m"` + Past24h *ErrorStat `json:"past24h"` + Past6h *ErrorStat `json:"past6h"` +} type FrequencyRule struct { Amount int64 `json:"amount"` Interval IntervalType `json:"interval"` diff --git a/proto/autogenerated/ts/express_server.ts b/proto/autogenerated/ts/express_server.ts index f76044ea..9b1fd1ea 100644 --- a/proto/autogenerated/ts/express_server.ts +++ b/proto/autogenerated/ts/express_server.ts @@ -885,6 +885,25 @@ export default (methods: Types.ServerMethods, opts: ServerOptions) => { opts.metricsCallback([{ ...info, ...stats, ...authContext }]) } catch (ex) { const e = ex as any; logErrorAndReturnResponse(e, e.message || e, res, logger, { ...info, ...stats, ...authCtx }, opts.metricsCallback); if (opts.throwErrors) throw e } }) + if (!opts.allowNotImplementedMethods && !methods.GetErrorStats) throw new Error('method: GetErrorStats is not implemented') + app.post('/api/reports/errors', async (req, res) => { + const info: Types.RequestInfo = { rpcName: 'GetErrorStats', batch: false, nostr: false, batchSize: 0} + const stats: Types.RequestStats = { startMs:req.startTimeMs || 0, start:req.startTime || 0n, parse: process.hrtime.bigint(), guard: 0n, validate: 0n, handle: 0n } + let authCtx: Types.AuthContext = {} + try { + if (!methods.GetErrorStats) throw new Error('method: GetErrorStats is not implemented') + const authContext = await opts.MetricsAuthGuard(req.headers['authorization']) + authCtx = authContext + stats.guard = process.hrtime.bigint() + stats.validate = stats.guard + const query = req.query + const params = req.params + const response = await methods.GetErrorStats({rpcName:'GetErrorStats', ctx:authContext }) + stats.handle = process.hrtime.bigint() + res.json({status: 'OK', ...response}) + opts.metricsCallback([{ ...info, ...stats, ...authContext }]) + } catch (ex) { const e = ex as any; logErrorAndReturnResponse(e, e.message || e, res, logger, { ...info, ...stats, ...authCtx }, opts.metricsCallback); if (opts.throwErrors) throw e } + }) if (!opts.allowNotImplementedMethods && !methods.GetInviteLinkState) throw new Error('method: GetInviteLinkState is not implemented') app.post('/api/admin/app/invite/get', async (req, res) => { const info: Types.RequestInfo = { rpcName: 'GetInviteLinkState', batch: false, nostr: false, batchSize: 0} diff --git a/proto/autogenerated/ts/http_client.ts b/proto/autogenerated/ts/http_client.ts index 0097d29e..10ae0041 100644 --- a/proto/autogenerated/ts/http_client.ts +++ b/proto/autogenerated/ts/http_client.ts @@ -332,6 +332,20 @@ export default (params: ClientParams) => ({ } return { status: 'ERROR', reason: 'invalid response' } }, + GetErrorStats: async (): Promise => { + const auth = await params.retrieveMetricsAuth() + if (auth === null) throw new Error('retrieveMetricsAuth() returned null') + let finalRoute = '/api/reports/errors' + const { data } = await axios.post(params.baseUrl + finalRoute, {}, { headers: { 'authorization': auth } }) + if (data.status === 'ERROR' && typeof data.reason === 'string') return data + if (data.status === 'OK') { + const result = data + if(!params.checkResult) return { status: 'OK', ...result } + const error = Types.ErrorStatsValidate(result) + if (error === null) { return { status: 'OK', ...result } } else return { status: 'ERROR', reason: error.message } + } + return { status: 'ERROR', reason: 'invalid response' } + }, GetHttpCreds: async (cb: (v:ResultError | ({ status: 'OK' }& Types.HttpCreds)) => void): Promise => { throw new Error('http streams are not supported')}, GetInviteLinkState: async (request: Types.GetInviteTokenStateRequest): Promise => { const auth = await params.retrieveAdminAuth() diff --git a/proto/autogenerated/ts/nostr_client.ts b/proto/autogenerated/ts/nostr_client.ts index 286b6414..04f14158 100644 --- a/proto/autogenerated/ts/nostr_client.ts +++ b/proto/autogenerated/ts/nostr_client.ts @@ -247,6 +247,20 @@ export default (params: NostrClientParams, send: (to:string, message: NostrRequ } return { status: 'ERROR', reason: 'invalid response' } }, + GetErrorStats: async (): Promise => { + const auth = await params.retrieveNostrMetricsAuth() + if (auth === null) throw new Error('retrieveNostrMetricsAuth() returned null') + const nostrRequest: NostrRequest = {} + const data = await send(params.pubDestination, {rpcName:'GetErrorStats',authIdentifier:auth, ...nostrRequest }) + if (data.status === 'ERROR' && typeof data.reason === 'string') return data + if (data.status === 'OK') { + const result = data + if(!params.checkResult) return { status: 'OK', ...result } + const error = Types.ErrorStatsValidate(result) + if (error === null) { return { status: 'OK', ...result } } else return { status: 'ERROR', reason: error.message } + } + return { status: 'ERROR', reason: 'invalid response' } + }, GetHttpCreds: async (cb: (res:ResultError | ({ status: 'OK' }& Types.HttpCreds)) => void): Promise => { const auth = await params.retrieveNostrUserAuth() if (auth === null) throw new Error('retrieveNostrUserAuth() returned null') diff --git a/proto/autogenerated/ts/nostr_transport.ts b/proto/autogenerated/ts/nostr_transport.ts index 96ad70e8..7d3f2c7d 100644 --- a/proto/autogenerated/ts/nostr_transport.ts +++ b/proto/autogenerated/ts/nostr_transport.ts @@ -634,6 +634,19 @@ export default (methods: Types.ServerMethods, opts: NostrOptions) => { opts.metricsCallback([{ ...info, ...stats, ...authContext }]) }catch(ex){ const e = ex as any; logErrorAndReturnResponse(e, e.message || e, res, logger, { ...info, ...stats, ...authCtx }, opts.metricsCallback); if (opts.throwErrors) throw e } break + case 'GetErrorStats': + try { + if (!methods.GetErrorStats) throw new Error('method: GetErrorStats is not implemented') + const authContext = await opts.NostrMetricsAuthGuard(req.appId, req.authIdentifier) + stats.guard = process.hrtime.bigint() + authCtx = authContext + stats.validate = stats.guard + const response = await methods.GetErrorStats({rpcName:'GetErrorStats', ctx:authContext }) + stats.handle = process.hrtime.bigint() + res({status: 'OK', ...response}) + opts.metricsCallback([{ ...info, ...stats, ...authContext }]) + }catch(ex){ const e = ex as any; logErrorAndReturnResponse(e, e.message || e, res, logger, { ...info, ...stats, ...authCtx }, opts.metricsCallback); if (opts.throwErrors) throw e } + break case 'GetHttpCreds': try { if (!methods.GetHttpCreds) throw new Error('method: GetHttpCreds is not implemented') diff --git a/proto/autogenerated/ts/types.ts b/proto/autogenerated/ts/types.ts index 6b815ab7..7fbffbf1 100644 --- a/proto/autogenerated/ts/types.ts +++ b/proto/autogenerated/ts/types.ts @@ -27,8 +27,8 @@ export type GuestWithPubMethodOutputs = LinkNPubThroughToken_Output | UseInviteL export type MetricsContext = { operator_id: string } -export type MetricsMethodInputs = GetAppsMetrics_Input | GetLndMetrics_Input | GetUsageMetrics_Input -export type MetricsMethodOutputs = GetAppsMetrics_Output | GetLndMetrics_Output | GetUsageMetrics_Output +export type MetricsMethodInputs = GetAppsMetrics_Input | GetErrorStats_Input | GetLndMetrics_Input | GetUsageMetrics_Input +export type MetricsMethodOutputs = GetAppsMetrics_Output | GetErrorStats_Output | GetLndMetrics_Output | GetUsageMetrics_Output export type UserContext = { app_id: string app_user_id: string @@ -110,6 +110,9 @@ export type GetAppsMetrics_Output = ResultError | ({ status: 'OK' } & AppsMetric export type GetDebitAuthorizations_Input = {rpcName:'GetDebitAuthorizations'} export type GetDebitAuthorizations_Output = ResultError | ({ status: 'OK' } & DebitAuthorizations) +export type GetErrorStats_Input = {rpcName:'GetErrorStats'} +export type GetErrorStats_Output = ResultError | ({ status: 'OK' } & ErrorStats) + export type GetHttpCreds_Input = {rpcName:'GetHttpCreds', cb:(res: HttpCreds, err:Error|null)=> void} export type GetHttpCreds_Output = ResultError | { status: 'OK' } @@ -300,6 +303,7 @@ export type ServerMethods = { GetAppUserLNURLInfo?: (req: GetAppUserLNURLInfo_Input & {ctx: AppContext }) => Promise GetAppsMetrics?: (req: GetAppsMetrics_Input & {ctx: MetricsContext }) => Promise GetDebitAuthorizations?: (req: GetDebitAuthorizations_Input & {ctx: UserContext }) => Promise + GetErrorStats?: (req: GetErrorStats_Input & {ctx: MetricsContext }) => Promise GetHttpCreds?: (req: GetHttpCreds_Input & {ctx: UserContext }) => Promise GetInviteLinkState?: (req: GetInviteLinkState_Input & {ctx: AdminContext }) => Promise GetLNURLChannelLink?: (req: GetLNURLChannelLink_Input & {ctx: UserContext }) => Promise @@ -1371,6 +1375,77 @@ export const EnrollAdminTokenRequestValidate = (o?: EnrollAdminTokenRequest, opt return null } +export type ErrorStat = { + errors: number + from_unix: number + total: number +} +export const ErrorStatOptionalFields: [] = [] +export type ErrorStatOptions = OptionsBaseMessage & { + checkOptionalsAreSet?: [] + errors_CustomCheck?: (v: number) => boolean + from_unix_CustomCheck?: (v: number) => boolean + total_CustomCheck?: (v: number) => boolean +} +export const ErrorStatValidate = (o?: ErrorStat, opts: ErrorStatOptions = {}, path: string = 'ErrorStat::root.'): Error | null => { + if (opts.checkOptionalsAreSet && opts.allOptionalsAreSet) return new Error(path + ': only one of checkOptionalsAreSet or allOptionalNonDefault can be set for each message') + if (typeof o !== 'object' || o === null) return new Error(path + ': object is not an instance of an object or is null') + + if (typeof o.errors !== 'number') return new Error(`${path}.errors: is not a number`) + if (opts.errors_CustomCheck && !opts.errors_CustomCheck(o.errors)) return new Error(`${path}.errors: custom check failed`) + + if (typeof o.from_unix !== 'number') return new Error(`${path}.from_unix: is not a number`) + if (opts.from_unix_CustomCheck && !opts.from_unix_CustomCheck(o.from_unix)) return new Error(`${path}.from_unix: custom check failed`) + + if (typeof o.total !== 'number') return new Error(`${path}.total: is not a number`) + if (opts.total_CustomCheck && !opts.total_CustomCheck(o.total)) return new Error(`${path}.total: custom check failed`) + + return null +} + +export type ErrorStats = { + past10m: ErrorStat + past1h: ErrorStat + past1m: ErrorStat + past24h: ErrorStat + past6h: ErrorStat +} +export const ErrorStatsOptionalFields: [] = [] +export type ErrorStatsOptions = OptionsBaseMessage & { + checkOptionalsAreSet?: [] + past10m_Options?: ErrorStatOptions + past1h_Options?: ErrorStatOptions + past1m_Options?: ErrorStatOptions + past24h_Options?: ErrorStatOptions + past6h_Options?: ErrorStatOptions +} +export const ErrorStatsValidate = (o?: ErrorStats, opts: ErrorStatsOptions = {}, path: string = 'ErrorStats::root.'): Error | null => { + if (opts.checkOptionalsAreSet && opts.allOptionalsAreSet) return new Error(path + ': only one of checkOptionalsAreSet or allOptionalNonDefault can be set for each message') + if (typeof o !== 'object' || o === null) return new Error(path + ': object is not an instance of an object or is null') + + const past10mErr = ErrorStatValidate(o.past10m, opts.past10m_Options, `${path}.past10m`) + if (past10mErr !== null) return past10mErr + + + const past1hErr = ErrorStatValidate(o.past1h, opts.past1h_Options, `${path}.past1h`) + if (past1hErr !== null) return past1hErr + + + const past1mErr = ErrorStatValidate(o.past1m, opts.past1m_Options, `${path}.past1m`) + if (past1mErr !== null) return past1mErr + + + const past24hErr = ErrorStatValidate(o.past24h, opts.past24h_Options, `${path}.past24h`) + if (past24hErr !== null) return past24hErr + + + const past6hErr = ErrorStatValidate(o.past6h, opts.past6h_Options, `${path}.past6h`) + if (past6hErr !== null) return past6hErr + + + return null +} + export type FrequencyRule = { amount: number interval: IntervalType diff --git a/proto/service/methods.proto b/proto/service/methods.proto index 9b8e9f42..f628120e 100644 --- a/proto/service/methods.proto +++ b/proto/service/methods.proto @@ -178,6 +178,12 @@ service LightningPub { option (http_route) = "/api/reports/usage"; option (nostr) = true; } + rpc GetErrorStats(structs.Empty) returns (structs.ErrorStats) { + option (auth_type) = "Metrics"; + option (http_method) = "post"; + option (http_route) = "/api/reports/errors"; + option (nostr) = true; + } rpc GetAppsMetrics(structs.AppsMetricsRequest) returns (structs.AppsMetrics) { option (auth_type) = "Metrics"; diff --git a/proto/service/structs.proto b/proto/service/structs.proto index 572f224f..94556216 100644 --- a/proto/service/structs.proto +++ b/proto/service/structs.proto @@ -19,6 +19,20 @@ message UserHealthState { string downtime_reason = 1; } +message ErrorStat { + int64 from_unix = 1; + int64 total = 2; + int64 errors = 3; +} + +message ErrorStats { + ErrorStat past24h = 1; + ErrorStat past6h = 2; + ErrorStat past1h = 3; + ErrorStat past10m = 4; + ErrorStat past1m = 5; +} + message UsageMetric { int64 processed_at_ms = 1; int64 parsed_in_nano = 2; diff --git a/src/services/metrics/index.ts b/src/services/metrics/index.ts index 943943b9..bcfcbc58 100644 --- a/src/services/metrics/index.ts +++ b/src/services/metrics/index.ts @@ -72,6 +72,48 @@ export default class Handler { return this.storage.metricsEventStorage.LoadLatestMetrics() } + async GetErrorStats(): Promise { + const last24h = this.storage.metricsEventStorage.getlast24hCache() + const nowUnix = Math.floor(Date.now() / 1000) + const stats: Types.ErrorStats = { + past24h: { errors: 0, total: 0, from_unix: nowUnix - 60 * 60 * 24 }, + past6h: { errors: 0, total: 0, from_unix: nowUnix - 60 * 60 * 6 }, + past1h: { errors: 0, total: 0, from_unix: nowUnix - 60 * 60 }, + past10m: { errors: 0, total: 0, from_unix: nowUnix - 60 * 10 }, + past1m: { errors: 0, total: 0, from_unix: nowUnix - 60 }, + } + for (let i = last24h.length; i >= 0; i--) { + const e = last24h[i] + if (e.ts < stats.past24h.from_unix) { + break + } + + stats.past24h.total += e.ok + e.fail + stats.past24h.errors += e.fail + + if (e.ts >= stats.past6h.from_unix) { + stats.past6h.total += e.ok + e.fail + stats.past6h.errors += e.fail + } + + if (e.ts >= stats.past1h.from_unix) { + stats.past1h.total += e.ok + e.fail + stats.past1h.errors += e.fail + } + + if (e.ts >= stats.past10m.from_unix) { + stats.past10m.total += e.ok + e.fail + stats.past10m.errors += e.fail + } + + if (e.ts >= stats.past1m.from_unix) { + stats.past1m.total += e.ok + e.fail + stats.past1m.errors += e.fail + } + } + return stats + } + AddMetrics(newMetrics: (Types.RequestMetric & { app_id?: string })[]) { @@ -91,7 +133,7 @@ export default class Handler { processed_at_ms: m.startMs } const tlv = usageMetricsToTlv(um) - this.storage.metricsEventStorage.AddMetricEvent(appId, m.rpcName, encodeTLV(tlv)) + this.storage.metricsEventStorage.AddMetricEvent(appId, m.rpcName, encodeTLV(tlv), !m.error) }) } diff --git a/src/services/serverMethods/index.ts b/src/services/serverMethods/index.ts index 5b17a7ee..316fbf92 100644 --- a/src/services/serverMethods/index.ts +++ b/src/services/serverMethods/index.ts @@ -7,6 +7,9 @@ export default (mainHandler: Main): Types.ServerMethods => { GetUsageMetrics: async ({ ctx }) => { return mainHandler.metricsManager.GetUsageMetrics() }, + GetErrorStats: async ({ ctx }) => { + return mainHandler.metricsManager.GetErrorStats() + }, GetAppsMetrics: async ({ ctx, req }) => { return mainHandler.metricsManager.GetAppsMetrics(req) }, diff --git a/src/services/storage/metricsEventStorage.ts b/src/services/storage/metricsEventStorage.ts index 6bf436e5..fc7017d5 100644 --- a/src/services/storage/metricsEventStorage.ts +++ b/src/services/storage/metricsEventStorage.ts @@ -6,20 +6,26 @@ const chunkSizeBytes = 128 * 1024 export default class { settings: StorageSettings metricsPath: string + cachePath: string metaReady = false metricsMeta: Record> = {} pendingMetrics: Record> = {} - last24hOk: Record = {} - last24hFail: Record = {} - lastPersisted: number = 0 + last24hCache: { ts: number, ok: number, fail: number }[] = [] + lastPersistedMetrics: number = 0 + lastPersistedCache: number = 0 constructor(settings: StorageSettings) { this.settings = settings; this.metricsPath = [settings.dataDir, "metric_events"].join("/") + this.cachePath = [settings.dataDir, "metric_cache"].join("/") this.initMetricsMeta() + this.loadCache() setInterval(() => { - if (Date.now() - this.lastPersisted > 1000 * 60 * 5) { + if (Date.now() - this.lastPersistedMetrics > 1000 * 60 * 4) { this.persistMetrics() } + if (Date.now() - this.lastPersistedCache > 1000 * 60 * 4) { + this.persistCache() + } }, 1000 * 60 * 5) process.on('exit', () => { this.persistMetrics() @@ -39,7 +45,50 @@ export default class { }); } - AddMetricEvent = (appId: string, method: string, metric: Uint8Array) => { + getlast24hCache = () => { return this.last24hCache } + + rotateCache = (nowUnix: number) => { + const yesterday = nowUnix - 60 * 60 * 24 + const latest = this.last24hCache.findIndex(c => c.ts >= yesterday) + if (latest === -1) { + this.last24hCache = [] + return + } else if (latest === 0) { + return + } + this.last24hCache = this.last24hCache.slice(latest) + } + + pushToCache = (ok: boolean) => { + const now = Math.floor(Date.now() / 1000) + this.rotateCache(now) + if (this.last24hCache.length === 0) { + this.last24hCache.push({ ts: now, ok: ok ? 1 : 0, fail: ok ? 0 : 1 }) + return + } + const last = this.last24hCache[this.last24hCache.length - 1] + if (last.ts === now) { + last.ok += ok ? 1 : 0 + last.fail += ok ? 0 : 1 + } else { + this.last24hCache.push({ ts: now, ok: ok ? 1 : 0, fail: ok ? 0 : 1 }) + } + } + + persistCache = () => { + const last24CachePath = [this.cachePath, "last24hSF.json"].join("/") + fs.writeFileSync(last24CachePath, JSON.stringify(this.last24hCache)) + } + + loadCache = () => { + const last24CachePath = [this.cachePath, "last24hSF.json"].join("/") + if (fs.existsSync(last24CachePath)) { + this.last24hCache = JSON.parse(fs.readFileSync(last24CachePath, 'utf-8')) + this.rotateCache(Math.floor(Date.now() / 1000)) + } + } + + AddMetricEvent = (appId: string, method: string, metric: Uint8Array, success: boolean) => { if (!this.metaReady) { throw new Error("meta metrics not ready") } @@ -50,6 +99,8 @@ export default class { this.pendingMetrics[appId][method] = { tlvs: [] } } this.pendingMetrics[appId][method].tlvs.push(metric) + this.pushToCache(success) + } LoadLatestMetrics = async (): Promise => { @@ -78,7 +129,7 @@ export default class { if (!this.metaReady) { throw new Error("meta metrics not ready") } - this.lastPersisted = Date.now() + this.lastPersistedMetrics = Date.now() const tosync = this.pendingMetrics this.pendingMetrics = {} const apps = Object.keys(tosync)