From 9e6df5c2e62cedfd8b04dde2be99fe17f0ccff9f Mon Sep 17 00:00:00 2001 From: boufni95 Date: Mon, 12 May 2025 16:49:35 +0000 Subject: [PATCH 1/4] http upgrade + fix unlock --- .gitignore | 3 ++- env.example | 1 + proto/autogenerated/go/http_client.go | 27 ++++++++++++++++++++- proto/autogenerated/ts/express_server.ts | 29 +++++++++++++++++++++++ proto/autogenerated/ts/http_client.ts | 15 +++++++++++- proto/autogenerated/ts/nostr_client.ts | 21 ++++++++-------- proto/autogenerated/ts/nostr_transport.ts | 16 ++++++++++--- proto/autogenerated/ts/types.ts | 10 ++++---- proto/service/methods.proto | 2 +- src/services/main/appUserManager.ts | 10 ++++++++ src/services/main/settings.ts | 4 +++- src/services/main/unlocker.ts | 11 ++++++--- src/services/nostr/handler.ts | 2 ++ src/services/serverMethods/index.ts | 5 +++- 14 files changed, 128 insertions(+), 28 deletions(-) diff --git a/.gitignore b/.gitignore index 3dfafe60..23387198 100644 --- a/.gitignore +++ b/.gitignore @@ -25,4 +25,5 @@ proto/autogenerated/debug.txt metrics_cache/ metric_cache/ metrics_events/ -bundler_events/ \ No newline at end of file +bundler_events/ +metric_events/ \ No newline at end of file diff --git a/env.example b/env.example index 6a03789b..319585fb 100644 --- a/env.example +++ b/env.example @@ -104,6 +104,7 @@ LSP_MAX_FEE_BPS=100 # Disable outbound payments aka honeypot mode #DISABLE_EXTERNAL_PAYMENTS=false #ALLOW_RESET_METRICS_STORAGES=false +ALLOW_HTTP_UPGRADE=false #WATCHDOG SECURITY # A last line of defense against 0-day drainage attacks diff --git a/proto/autogenerated/go/http_client.go b/proto/autogenerated/go/http_client.go index 970a39d0..97f6ff04 100644 --- a/proto/autogenerated/go/http_client.go +++ b/proto/autogenerated/go/http_client.go @@ -821,7 +821,32 @@ func NewClient(params ClientParams) *Client { } return &res, nil }, - // server streaming method: GetHttpCreds not implemented + GetHttpCreds: func() (*HttpCreds, error) { + auth, err := params.RetrieveUserAuth() + if err != nil { + return nil, err + } + finalRoute := "/api/user/http_creds" + body := []byte{} + resBody, err := doPostRequest(params.BaseURL+finalRoute, body, auth) + if err != nil { + return nil, err + } + result := ResultError{} + err = json.Unmarshal(resBody, &result) + if err != nil { + return nil, err + } + if result.Status == "ERROR" { + return nil, fmt.Errorf(result.Reason) + } + res := HttpCreds{} + err = json.Unmarshal(resBody, &res) + if err != nil { + return nil, err + } + return &res, nil + }, GetInviteLinkState: func(req GetInviteTokenStateRequest) (*GetInviteTokenStateResponse, error) { auth, err := params.RetrieveAdminAuth() if err != nil { diff --git a/proto/autogenerated/ts/express_server.ts b/proto/autogenerated/ts/express_server.ts index 92f54ba2..9be3037d 100644 --- a/proto/autogenerated/ts/express_server.ts +++ b/proto/autogenerated/ts/express_server.ts @@ -403,6 +403,16 @@ export default (methods: Types.ServerMethods, opts: ServerOptions) => { callsMetrics.push({ ...opInfo, ...opStats, ...ctx }) } break + case 'GetHttpCreds': + if (!methods.GetHttpCreds) { + throw new Error('method GetHttpCreds not found' ) + } else { + opStats.validate = opStats.guard + const res = await methods.GetHttpCreds({...operation, ctx}); responses.push({ status: 'OK', ...res }) + opStats.handle = process.hrtime.bigint() + callsMetrics.push({ ...opInfo, ...opStats, ...ctx }) + } + break case 'GetLNURLChannelLink': if (!methods.GetLNURLChannelLink) { throw new Error('method GetLNURLChannelLink not found' ) @@ -926,6 +936,25 @@ export default (methods: Types.ServerMethods, opts: ServerOptions) => { opts.metricsCallback([{ ...info, ...stats, ...authContext }]) } catch (ex) { const e = ex as any; logErrorAndReturnResponse(e, e.message || e, res, logger, { ...info, ...stats, ...authCtx }, opts.metricsCallback); if (opts.throwErrors) throw e } }) + if (!opts.allowNotImplementedMethods && !methods.GetHttpCreds) throw new Error('method: GetHttpCreds is not implemented') + app.post('/api/user/http_creds', async (req, res) => { + const info: Types.RequestInfo = { rpcName: 'GetHttpCreds', batch: false, nostr: false, batchSize: 0} + const stats: Types.RequestStats = { startMs:req.startTimeMs || 0, start:req.startTime || 0n, parse: process.hrtime.bigint(), guard: 0n, validate: 0n, handle: 0n } + let authCtx: Types.AuthContext = {} + try { + if (!methods.GetHttpCreds) throw new Error('method: GetHttpCreds is not implemented') + const authContext = await opts.UserAuthGuard(req.headers['authorization']) + authCtx = authContext + stats.guard = process.hrtime.bigint() + stats.validate = stats.guard + const query = req.query + const params = req.params + const response = await methods.GetHttpCreds({rpcName:'GetHttpCreds', ctx:authContext }) + stats.handle = process.hrtime.bigint() + res.json({status: 'OK', ...response}) + opts.metricsCallback([{ ...info, ...stats, ...authContext }]) + } catch (ex) { const e = ex as any; logErrorAndReturnResponse(e, e.message || e, res, logger, { ...info, ...stats, ...authCtx }, opts.metricsCallback); if (opts.throwErrors) throw e } + }) if (!opts.allowNotImplementedMethods && !methods.GetInviteLinkState) throw new Error('method: GetInviteLinkState is not implemented') app.post('/api/admin/app/invite/get', async (req, res) => { const info: Types.RequestInfo = { rpcName: 'GetInviteLinkState', batch: false, nostr: false, batchSize: 0} diff --git a/proto/autogenerated/ts/http_client.ts b/proto/autogenerated/ts/http_client.ts index 8ba73a0f..9fa96276 100644 --- a/proto/autogenerated/ts/http_client.ts +++ b/proto/autogenerated/ts/http_client.ts @@ -360,7 +360,20 @@ export default (params: ClientParams) => ({ } return { status: 'ERROR', reason: 'invalid response' } }, - GetHttpCreds: async (cb: (v:ResultError | ({ status: 'OK' }& Types.HttpCreds)) => void): Promise => { throw new Error('http streams are not supported')}, + GetHttpCreds: async (): Promise => { + const auth = await params.retrieveUserAuth() + if (auth === null) throw new Error('retrieveUserAuth() returned null') + let finalRoute = '/api/user/http_creds' + const { data } = await axios.post(params.baseUrl + finalRoute, {}, { headers: { 'authorization': auth } }) + if (data.status === 'ERROR' && typeof data.reason === 'string') return data + if (data.status === 'OK') { + const result = data + if(!params.checkResult) return { status: 'OK', ...result } + const error = Types.HttpCredsValidate(result) + if (error === null) { return { status: 'OK', ...result } } else return { status: 'ERROR', reason: error.message } + } + return { status: 'ERROR', reason: 'invalid response' } + }, GetInviteLinkState: async (request: Types.GetInviteTokenStateRequest): Promise => { const auth = await params.retrieveAdminAuth() if (auth === null) throw new Error('retrieveAdminAuth() returned null') diff --git a/proto/autogenerated/ts/nostr_client.ts b/proto/autogenerated/ts/nostr_client.ts index d7dc1d06..f6dcf1dd 100644 --- a/proto/autogenerated/ts/nostr_client.ts +++ b/proto/autogenerated/ts/nostr_client.ts @@ -276,20 +276,19 @@ export default (params: NostrClientParams, send: (to:string, message: NostrRequ } return { status: 'ERROR', reason: 'invalid response' } }, - GetHttpCreds: async (cb: (res:ResultError | ({ status: 'OK' }& Types.HttpCreds)) => void): Promise => { + GetHttpCreds: async (): Promise => { const auth = await params.retrieveNostrUserAuth() if (auth === null) throw new Error('retrieveNostrUserAuth() returned null') const nostrRequest: NostrRequest = {} - subscribe(params.pubDestination, {rpcName:'GetHttpCreds',authIdentifier:auth, ...nostrRequest }, (data) => { - if (data.status === 'ERROR' && typeof data.reason === 'string') return cb(data) - if (data.status === 'OK') { - const result = data - if(!params.checkResult) return cb({ status: 'OK', ...result }) - const error = Types.HttpCredsValidate(result) - if (error === null) { return cb({ status: 'OK', ...result }) } else return cb({ status: 'ERROR', reason: error.message }) - } - return cb({ status: 'ERROR', reason: 'invalid response' }) - }) + const data = await send(params.pubDestination, {rpcName:'GetHttpCreds',authIdentifier:auth, ...nostrRequest }) + if (data.status === 'ERROR' && typeof data.reason === 'string') return data + if (data.status === 'OK') { + const result = data + if(!params.checkResult) return { status: 'OK', ...result } + const error = Types.HttpCredsValidate(result) + if (error === null) { return { status: 'OK', ...result } } else return { status: 'ERROR', reason: error.message } + } + return { status: 'ERROR', reason: 'invalid response' } }, GetInviteLinkState: async (request: Types.GetInviteTokenStateRequest): Promise => { const auth = await params.retrieveNostrAdminAuth() diff --git a/proto/autogenerated/ts/nostr_transport.ts b/proto/autogenerated/ts/nostr_transport.ts index 99d7ec86..4f48a701 100644 --- a/proto/autogenerated/ts/nostr_transport.ts +++ b/proto/autogenerated/ts/nostr_transport.ts @@ -285,6 +285,16 @@ export default (methods: Types.ServerMethods, opts: NostrOptions) => { callsMetrics.push({ ...opInfo, ...opStats, ...ctx }) } break + case 'GetHttpCreds': + if (!methods.GetHttpCreds) { + throw new Error('method not defined: GetHttpCreds') + } else { + opStats.validate = opStats.guard + const res = await methods.GetHttpCreds({...operation, ctx}); responses.push({ status: 'OK', ...res }) + opStats.handle = process.hrtime.bigint() + callsMetrics.push({ ...opInfo, ...opStats, ...ctx }) + } + break case 'GetLNURLChannelLink': if (!methods.GetLNURLChannelLink) { throw new Error('method not defined: GetLNURLChannelLink') @@ -670,10 +680,10 @@ export default (methods: Types.ServerMethods, opts: NostrOptions) => { stats.guard = process.hrtime.bigint() authCtx = authContext stats.validate = stats.guard - methods.GetHttpCreds({rpcName:'GetHttpCreds', ctx:authContext ,cb: (response, err) => { + const response = await methods.GetHttpCreds({rpcName:'GetHttpCreds', ctx:authContext }) stats.handle = process.hrtime.bigint() - if (err) { logErrorAndReturnResponse(err, err.message, res, logger, { ...info, ...stats, ...authContext }, opts.metricsCallback)} else { res({status: 'OK', ...response});opts.metricsCallback([{ ...info, ...stats, ...authContext }])} - }}) + res({status: 'OK', ...response}) + opts.metricsCallback([{ ...info, ...stats, ...authContext }]) }catch(ex){ const e = ex as any; logErrorAndReturnResponse(e, e.message || e, res, logger, { ...info, ...stats, ...authCtx }, opts.metricsCallback); if (opts.throwErrors) throw e } break case 'GetInviteLinkState': diff --git a/proto/autogenerated/ts/types.ts b/proto/autogenerated/ts/types.ts index 6bdaa118..ae494f22 100644 --- a/proto/autogenerated/ts/types.ts +++ b/proto/autogenerated/ts/types.ts @@ -35,8 +35,8 @@ export type UserContext = { app_user_id: string user_id: string } -export type UserMethodInputs = AddProduct_Input | AddUserOffer_Input | AuthorizeDebit_Input | BanDebit_Input | DecodeInvoice_Input | DeleteUserOffer_Input | EditDebit_Input | EnrollAdminToken_Input | GetDebitAuthorizations_Input | GetLNURLChannelLink_Input | GetLnurlPayLink_Input | GetLnurlWithdrawLink_Input | GetPaymentState_Input | GetUserInfo_Input | GetUserOffer_Input | GetUserOfferInvoices_Input | GetUserOffers_Input | GetUserOperations_Input | NewAddress_Input | NewInvoice_Input | NewProductInvoice_Input | PayAddress_Input | PayInvoice_Input | ResetDebit_Input | RespondToDebit_Input | UpdateCallbackUrl_Input | UpdateUserOffer_Input | UserHealth_Input -export type UserMethodOutputs = AddProduct_Output | AddUserOffer_Output | AuthorizeDebit_Output | BanDebit_Output | DecodeInvoice_Output | DeleteUserOffer_Output | EditDebit_Output | EnrollAdminToken_Output | GetDebitAuthorizations_Output | GetLNURLChannelLink_Output | GetLnurlPayLink_Output | GetLnurlWithdrawLink_Output | GetPaymentState_Output | GetUserInfo_Output | GetUserOffer_Output | GetUserOfferInvoices_Output | GetUserOffers_Output | GetUserOperations_Output | NewAddress_Output | NewInvoice_Output | NewProductInvoice_Output | PayAddress_Output | PayInvoice_Output | ResetDebit_Output | RespondToDebit_Output | UpdateCallbackUrl_Output | UpdateUserOffer_Output | UserHealth_Output +export type UserMethodInputs = AddProduct_Input | AddUserOffer_Input | AuthorizeDebit_Input | BanDebit_Input | DecodeInvoice_Input | DeleteUserOffer_Input | EditDebit_Input | EnrollAdminToken_Input | GetDebitAuthorizations_Input | GetHttpCreds_Input | GetLNURLChannelLink_Input | GetLnurlPayLink_Input | GetLnurlWithdrawLink_Input | GetPaymentState_Input | GetUserInfo_Input | GetUserOffer_Input | GetUserOfferInvoices_Input | GetUserOffers_Input | GetUserOperations_Input | NewAddress_Input | NewInvoice_Input | NewProductInvoice_Input | PayAddress_Input | PayInvoice_Input | ResetDebit_Input | RespondToDebit_Input | UpdateCallbackUrl_Input | UpdateUserOffer_Input | UserHealth_Input +export type UserMethodOutputs = AddProduct_Output | AddUserOffer_Output | AuthorizeDebit_Output | BanDebit_Output | DecodeInvoice_Output | DeleteUserOffer_Output | EditDebit_Output | EnrollAdminToken_Output | GetDebitAuthorizations_Output | GetHttpCreds_Output | GetLNURLChannelLink_Output | GetLnurlPayLink_Output | GetLnurlWithdrawLink_Output | GetPaymentState_Output | GetUserInfo_Output | GetUserOffer_Output | GetUserOfferInvoices_Output | GetUserOffers_Output | GetUserOperations_Output | NewAddress_Output | NewInvoice_Output | NewProductInvoice_Output | PayAddress_Output | PayInvoice_Output | ResetDebit_Output | RespondToDebit_Output | UpdateCallbackUrl_Output | UpdateUserOffer_Output | UserHealth_Output export type AuthContext = AdminContext | AppContext | GuestContext | GuestWithPubContext | MetricsContext | UserContext export type AddApp_Input = {rpcName:'AddApp', req: AddAppRequest} @@ -117,8 +117,8 @@ export type GetDebitAuthorizations_Output = ResultError | ({ status: 'OK' } & De export type GetErrorStats_Input = {rpcName:'GetErrorStats'} export type GetErrorStats_Output = ResultError | ({ status: 'OK' } & ErrorStats) -export type GetHttpCreds_Input = {rpcName:'GetHttpCreds', cb:(res: HttpCreds, err:Error|null)=> void} -export type GetHttpCreds_Output = ResultError | { status: 'OK' } +export type GetHttpCreds_Input = {rpcName:'GetHttpCreds'} +export type GetHttpCreds_Output = ResultError | ({ status: 'OK' } & HttpCreds) export type GetInviteLinkState_Input = {rpcName:'GetInviteLinkState', req: GetInviteTokenStateRequest} export type GetInviteLinkState_Output = ResultError | ({ status: 'OK' } & GetInviteTokenStateResponse) @@ -327,7 +327,7 @@ export type ServerMethods = { GetBundleMetrics?: (req: GetBundleMetrics_Input & {ctx: MetricsContext }) => Promise GetDebitAuthorizations?: (req: GetDebitAuthorizations_Input & {ctx: UserContext }) => Promise GetErrorStats?: (req: GetErrorStats_Input & {ctx: MetricsContext }) => Promise - GetHttpCreds?: (req: GetHttpCreds_Input & {ctx: UserContext }) => Promise + GetHttpCreds?: (req: GetHttpCreds_Input & {ctx: UserContext }) => Promise GetInviteLinkState?: (req: GetInviteLinkState_Input & {ctx: AdminContext }) => Promise GetLNURLChannelLink?: (req: GetLNURLChannelLink_Input & {ctx: UserContext }) => Promise GetLiveDebitRequests?: (req: GetLiveDebitRequests_Input & {ctx: UserContext }) => Promise diff --git a/proto/service/methods.proto b/proto/service/methods.proto index 0d129d29..81a9165e 100644 --- a/proto/service/methods.proto +++ b/proto/service/methods.proto @@ -618,7 +618,7 @@ service LightningPub { option (http_route) = "/api/user/migrations/sub"; option (nostr) = true; } - rpc GetHttpCreds(structs.Empty) returns (stream structs.HttpCreds){ + rpc GetHttpCreds(structs.Empty) returns (structs.HttpCreds){ option (auth_type) = "User"; option (http_method) = "post"; option (http_route) = "/api/user/http_creds"; diff --git a/src/services/main/appUserManager.ts b/src/services/main/appUserManager.ts index de130262..502860c9 100644 --- a/src/services/main/appUserManager.ts +++ b/src/services/main/appUserManager.ts @@ -35,6 +35,16 @@ export default class { return decoded } + GetHttpCreds(ctx: Types.UserContext): Types.HttpCreds { + if (!this.settings.allowHttpUpgrade) { + throw new Error("http upgrade not allowed") + } + return { + url: this.settings.serviceUrl, + token: this.SignUserToken(ctx.user_id, ctx.app_id, ctx.app_user_id) + } + } + async BanUser(userId: string): Promise { const banned = await this.storage.userStorage.BanUser(userId) const appUsers = await this.storage.applicationStorage.GetAllAppUsersFromUser(userId) diff --git a/src/services/main/settings.ts b/src/services/main/settings.ts index d412e658..fe1d6b7d 100644 --- a/src/services/main/settings.ts +++ b/src/services/main/settings.ts @@ -36,6 +36,7 @@ export type MainSettings = { lnurlMetaText: string, bridgeUrl: string, allowResetMetricsStorages: boolean + allowHttpUpgrade: boolean } export type BitcoinCoreSettings = { @@ -76,7 +77,8 @@ export const LoadMainSettingsFromEnv = (): MainSettings => { pushBackupsToNostr: process.env.PUSH_BACKUPS_TO_NOSTR === 'true' || false, lnurlMetaText: process.env.LNURL_META_TEXT || "LNURL via Lightning.pub", bridgeUrl: process.env.BRIDGE_URL || "https://shockwallet.app", - allowResetMetricsStorages: process.env.ALLOW_RESET_METRICS_STORAGES === 'true' || false + allowResetMetricsStorages: process.env.ALLOW_RESET_METRICS_STORAGES === 'true' || false, + allowHttpUpgrade: process.env.ALLOW_HTTP_UPGRADE === 'true' || false } } diff --git a/src/services/main/unlocker.ts b/src/services/main/unlocker.ts index 1a5cd361..084fb533 100644 --- a/src/services/main/unlocker.ts +++ b/src/services/main/unlocker.ts @@ -81,9 +81,14 @@ export class Unlocker { const unlocker = this.GetUnlockerClient(lndCert) const walletPassword = this.GetWalletPassword() await unlocker.unlockWallet({ walletPassword, recoveryWindow: 0, statelessInit: false, channelBackups: undefined }, DeadLineMetadata()) - const infoAfter = await this.GetLndInfo(ln) + let infoAfter = await this.GetLndInfo(ln) if (!infoAfter.ok) { - throw new Error("failed to unlock lnd wallet " + infoAfter.failure) + this.log("failed to unlock lnd wallet, retrying in 5 seconds...") + await new Promise(resolve => setTimeout(resolve, 5000)) + infoAfter = await this.GetLndInfo(ln) + if (!infoAfter.ok) { + throw new Error("failed to unlock lnd wallet " + infoAfter.failure) + } } this.log("unlocked wallet with pub:", infoAfter.pub) this.nodePub = infoAfter.pub @@ -152,7 +157,7 @@ export class Unlocker { const info = await ln.getInfo({}, DeadLineMetadata()) return { ok: true, pub: info.response.identityPubkey } } catch (err: any) { - if (err.message === '2 UNKNOWN: wallet locked, unlock it to enable full RPC access') { + if (err.message === 'wallet locked, unlock it to enable full RPC access') { this.log("wallet is locked") return { ok: false, failure: 'locked' } } else if (err.message === '2 UNKNOWN: the RPC server is in the process of starting up, but not yet ready to accept calls') { diff --git a/src/services/nostr/handler.ts b/src/services/nostr/handler.ts index 6174deb1..46b10145 100644 --- a/src/services/nostr/handler.ts +++ b/src/services/nostr/handler.ts @@ -252,6 +252,8 @@ export default class Handler { })) if (!sent) { log("failed to send event") + } else { + log("sent event") } } diff --git a/src/services/serverMethods/index.ts b/src/services/serverMethods/index.ts index d9f767df..015e66e6 100644 --- a/src/services/serverMethods/index.ts +++ b/src/services/serverMethods/index.ts @@ -400,6 +400,9 @@ export default (mainHandler: Main): Types.ServerMethods => { }) if (err != null) throw new Error(err.message) return mainHandler.offerManager.GetUserOfferInvoices(ctx, req) - } + }, + GetHttpCreds: async ({ ctx }) => { + return mainHandler.appUserManager.GetHttpCreds(ctx) + }, } } \ No newline at end of file From 74e6d5b3ca16e2d606ca162489d725add7528d21 Mon Sep 17 00:00:00 2001 From: boufni95 Date: Mon, 12 May 2025 17:11:05 +0000 Subject: [PATCH 2/4] better unlock fix --- src/services/main/unlocker.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/services/main/unlocker.ts b/src/services/main/unlocker.ts index 084fb533..2d0d623d 100644 --- a/src/services/main/unlocker.ts +++ b/src/services/main/unlocker.ts @@ -157,10 +157,10 @@ export class Unlocker { const info = await ln.getInfo({}, DeadLineMetadata()) return { ok: true, pub: info.response.identityPubkey } } catch (err: any) { - if (err.message === 'wallet locked, unlock it to enable full RPC access') { + if ((err.message as string).includes('wallet locked, unlock it to enable full RPC access')) { this.log("wallet is locked") return { ok: false, failure: 'locked' } - } else if (err.message === '2 UNKNOWN: the RPC server is in the process of starting up, but not yet ready to accept calls') { + } else if ((err.message as string).includes('the RPC server is in the process of starting up, but not yet ready to accept calls')) { this.log("lnd is not ready yet, waiting...") await new Promise((res) => setTimeout(res, 1000)) } else { From ed6036ce1ec416d895cd1978d55680e987b680d1 Mon Sep 17 00:00:00 2001 From: boufni95 Date: Wed, 14 May 2025 18:59:00 +0000 Subject: [PATCH 3/4] add alerter endpoints --- proto/autogenerated/client.md | 32 ++++++++++ proto/autogenerated/go/http_client.go | 49 +++++++++++++++ proto/autogenerated/go/types.go | 8 +++ proto/autogenerated/ts/express_server.ts | 38 +++++++++++ proto/autogenerated/ts/http_client.ts | 25 ++++++++ proto/autogenerated/ts/nostr_client.ts | 25 ++++++++ proto/autogenerated/ts/nostr_transport.ts | 26 ++++++++ proto/autogenerated/ts/types.ts | 63 ++++++++++++++++++- proto/service/methods.proto | 13 ++++ proto/service/structs.proto | 14 ++++- src/index.ts | 3 +- src/nostrMiddleware.ts | 4 +- src/services/main/index.ts | 32 ++++++++++ src/services/metrics/index.ts | 12 ++++ src/services/nostr/handler.ts | 16 ++++- src/services/nostr/index.ts | 12 ++++ src/services/storage/db/storageInterface.ts | 7 +++ src/services/storage/db/storageProcessor.ts | 19 +++++- .../storage/tlv/tlvFilesStorageFactory.ts | 8 ++- .../storage/tlv/tlvFilesStorageProcessor.ts | 19 +++++- 20 files changed, 414 insertions(+), 11 deletions(-) diff --git a/proto/autogenerated/client.md b/proto/autogenerated/client.md index 93fb4efc..2596a38e 100644 --- a/proto/autogenerated/client.md +++ b/proto/autogenerated/client.md @@ -158,6 +158,11 @@ The nostr server will send back a message response, and inside the body there wi - input: [GetPaymentStateRequest](#GetPaymentStateRequest) - output: [PaymentState](#PaymentState) +- GetProvidersDisruption + - auth type: __Metrics__ + - This methods has an __empty__ __request__ body + - output: [ProvidersDisruption](#ProvidersDisruption) + - GetSeed - auth type: __Admin__ - This methods has an __empty__ __request__ body @@ -250,6 +255,11 @@ The nostr server will send back a message response, and inside the body there wi - input: [PayInvoiceRequest](#PayInvoiceRequest) - output: [PayInvoiceResponse](#PayInvoiceResponse) +- PingSubProcesses + - auth type: __Metrics__ + - This methods has an __empty__ __request__ body + - This methods has an __empty__ __response__ body + - ResetDebit - auth type: __User__ - input: [DebitOperation](#DebitOperation) @@ -617,6 +627,13 @@ The nostr server will send back a message response, and inside the body there wi - input: [GetPaymentStateRequest](#GetPaymentStateRequest) - output: [PaymentState](#PaymentState) +- GetProvidersDisruption + - auth type: __Metrics__ + - http method: __post__ + - http route: __/api/metrics/providers/disruption__ + - This methods has an __empty__ __request__ body + - output: [ProvidersDisruption](#ProvidersDisruption) + - GetSeed - auth type: __Admin__ - http method: __get__ @@ -790,6 +807,13 @@ The nostr server will send back a message response, and inside the body there wi - input: [PayInvoiceRequest](#PayInvoiceRequest) - output: [PayInvoiceResponse](#PayInvoiceResponse) +- PingSubProcesses + - auth type: __Metrics__ + - http method: __post__ + - http route: __/api/metrics/ping__ + - This methods has an __empty__ __request__ body + - This methods has an __empty__ __response__ body + - RequestNPubLinkingToken - auth type: __App__ - http method: __post__ @@ -1352,6 +1376,14 @@ The nostr server will send back a message response, and inside the body there wi - __noffer__: _string_ - __price_sats__: _number_ +### ProviderDisruption + - __provider_pubkey__: _string_ + - __provider_type__: _string_ + - __since_unix__: _number_ + +### ProvidersDisruption + - __disruptions__: ARRAY of: _[ProviderDisruption](#ProviderDisruption)_ + ### RelaysMigration - __relays__: ARRAY of: _string_ diff --git a/proto/autogenerated/go/http_client.go b/proto/autogenerated/go/http_client.go index 97f6ff04..1f29f872 100644 --- a/proto/autogenerated/go/http_client.go +++ b/proto/autogenerated/go/http_client.go @@ -93,6 +93,7 @@ type Client struct { GetMigrationUpdate func() (*MigrationUpdate, error) GetNPubLinkingState func(req GetNPubLinking) (*NPubLinking, error) GetPaymentState func(req GetPaymentStateRequest) (*PaymentState, error) + GetProvidersDisruption func() (*ProvidersDisruption, error) GetSeed func() (*LndSeed, error) GetSingleBundleMetrics func(req SingleMetricReq) (*BundleData, error) GetSingleUsageMetrics func(req SingleMetricReq) (*UsageMetricTlv, error) @@ -116,6 +117,7 @@ type Client struct { PayAddress func(req PayAddressRequest) (*PayAddressResponse, error) PayAppUserInvoice func(req PayAppUserInvoiceRequest) (*PayInvoiceResponse, error) PayInvoice func(req PayInvoiceRequest) (*PayInvoiceResponse, error) + PingSubProcesses func() error RequestNPubLinkingToken func(req RequestNPubLinkingTokenRequest) (*RequestNPubLinkingTokenResponse, error) ResetDebit func(req DebitOperation) error ResetMetricsStorages func() error @@ -1096,6 +1098,32 @@ func NewClient(params ClientParams) *Client { } return &res, nil }, + GetProvidersDisruption: func() (*ProvidersDisruption, error) { + auth, err := params.RetrieveMetricsAuth() + if err != nil { + return nil, err + } + finalRoute := "/api/metrics/providers/disruption" + body := []byte{} + resBody, err := doPostRequest(params.BaseURL+finalRoute, body, auth) + if err != nil { + return nil, err + } + result := ResultError{} + err = json.Unmarshal(resBody, &result) + if err != nil { + return nil, err + } + if result.Status == "ERROR" { + return nil, fmt.Errorf(result.Reason) + } + res := ProvidersDisruption{} + err = json.Unmarshal(resBody, &res) + if err != nil { + return nil, err + } + return &res, nil + }, GetSeed: func() (*LndSeed, error) { auth, err := params.RetrieveAdminAuth() if err != nil { @@ -1726,6 +1754,27 @@ func NewClient(params ClientParams) *Client { } return &res, nil }, + PingSubProcesses: func() error { + auth, err := params.RetrieveMetricsAuth() + if err != nil { + return err + } + finalRoute := "/api/metrics/ping" + body := []byte{} + resBody, err := doPostRequest(params.BaseURL+finalRoute, body, auth) + if err != nil { + return err + } + result := ResultError{} + err = json.Unmarshal(resBody, &result) + if err != nil { + return err + } + if result.Status == "ERROR" { + return fmt.Errorf(result.Reason) + } + return nil + }, RequestNPubLinkingToken: func(req RequestNPubLinkingTokenRequest) (*RequestNPubLinkingTokenResponse, error) { auth, err := params.RetrieveAppAuth() if err != nil { diff --git a/proto/autogenerated/go/types.go b/proto/autogenerated/go/types.go index 2fa9b31d..68d35a25 100644 --- a/proto/autogenerated/go/types.go +++ b/proto/autogenerated/go/types.go @@ -527,6 +527,14 @@ type Product struct { Noffer string `json:"noffer"` Price_sats int64 `json:"price_sats"` } +type ProviderDisruption struct { + Provider_pubkey string `json:"provider_pubkey"` + Provider_type string `json:"provider_type"` + Since_unix int64 `json:"since_unix"` +} +type ProvidersDisruption struct { + Disruptions []ProviderDisruption `json:"disruptions"` +} type RelaysMigration struct { Relays []string `json:"relays"` } diff --git a/proto/autogenerated/ts/express_server.ts b/proto/autogenerated/ts/express_server.ts index 9be3037d..7a3116ef 100644 --- a/proto/autogenerated/ts/express_server.ts +++ b/proto/autogenerated/ts/express_server.ts @@ -1138,6 +1138,25 @@ export default (methods: Types.ServerMethods, opts: ServerOptions) => { opts.metricsCallback([{ ...info, ...stats, ...authContext }]) } catch (ex) { const e = ex as any; logErrorAndReturnResponse(e, e.message || e, res, logger, { ...info, ...stats, ...authCtx }, opts.metricsCallback); if (opts.throwErrors) throw e } }) + if (!opts.allowNotImplementedMethods && !methods.GetProvidersDisruption) throw new Error('method: GetProvidersDisruption is not implemented') + app.post('/api/metrics/providers/disruption', async (req, res) => { + const info: Types.RequestInfo = { rpcName: 'GetProvidersDisruption', batch: false, nostr: false, batchSize: 0} + const stats: Types.RequestStats = { startMs:req.startTimeMs || 0, start:req.startTime || 0n, parse: process.hrtime.bigint(), guard: 0n, validate: 0n, handle: 0n } + let authCtx: Types.AuthContext = {} + try { + if (!methods.GetProvidersDisruption) throw new Error('method: GetProvidersDisruption is not implemented') + const authContext = await opts.MetricsAuthGuard(req.headers['authorization']) + authCtx = authContext + stats.guard = process.hrtime.bigint() + stats.validate = stats.guard + const query = req.query + const params = req.params + const response = await methods.GetProvidersDisruption({rpcName:'GetProvidersDisruption', ctx:authContext }) + stats.handle = process.hrtime.bigint() + res.json({status: 'OK', ...response}) + opts.metricsCallback([{ ...info, ...stats, ...authContext }]) + } catch (ex) { const e = ex as any; logErrorAndReturnResponse(e, e.message || e, res, logger, { ...info, ...stats, ...authCtx }, opts.metricsCallback); if (opts.throwErrors) throw e } + }) if (!opts.allowNotImplementedMethods && !methods.GetSeed) throw new Error('method: GetSeed is not implemented') app.get('/api/admin/seed', async (req, res) => { const info: Types.RequestInfo = { rpcName: 'GetSeed', batch: false, nostr: false, batchSize: 0} @@ -1617,6 +1636,25 @@ export default (methods: Types.ServerMethods, opts: ServerOptions) => { opts.metricsCallback([{ ...info, ...stats, ...authContext }]) } catch (ex) { const e = ex as any; logErrorAndReturnResponse(e, e.message || e, res, logger, { ...info, ...stats, ...authCtx }, opts.metricsCallback); if (opts.throwErrors) throw e } }) + if (!opts.allowNotImplementedMethods && !methods.PingSubProcesses) throw new Error('method: PingSubProcesses is not implemented') + app.post('/api/metrics/ping', async (req, res) => { + const info: Types.RequestInfo = { rpcName: 'PingSubProcesses', batch: false, nostr: false, batchSize: 0} + const stats: Types.RequestStats = { startMs:req.startTimeMs || 0, start:req.startTime || 0n, parse: process.hrtime.bigint(), guard: 0n, validate: 0n, handle: 0n } + let authCtx: Types.AuthContext = {} + try { + if (!methods.PingSubProcesses) throw new Error('method: PingSubProcesses is not implemented') + const authContext = await opts.MetricsAuthGuard(req.headers['authorization']) + authCtx = authContext + stats.guard = process.hrtime.bigint() + stats.validate = stats.guard + const query = req.query + const params = req.params + await methods.PingSubProcesses({rpcName:'PingSubProcesses', ctx:authContext }) + stats.handle = process.hrtime.bigint() + res.json({status: 'OK'}) + opts.metricsCallback([{ ...info, ...stats, ...authContext }]) + } catch (ex) { const e = ex as any; logErrorAndReturnResponse(e, e.message || e, res, logger, { ...info, ...stats, ...authCtx }, opts.metricsCallback); if (opts.throwErrors) throw e } + }) if (!opts.allowNotImplementedMethods && !methods.RequestNPubLinkingToken) throw new Error('method: RequestNPubLinkingToken is not implemented') app.post('/api/app/user/npub/token', async (req, res) => { const info: Types.RequestInfo = { rpcName: 'RequestNPubLinkingToken', batch: false, nostr: false, batchSize: 0} diff --git a/proto/autogenerated/ts/http_client.ts b/proto/autogenerated/ts/http_client.ts index 9fa96276..0afa72aa 100644 --- a/proto/autogenerated/ts/http_client.ts +++ b/proto/autogenerated/ts/http_client.ts @@ -507,6 +507,20 @@ export default (params: ClientParams) => ({ } return { status: 'ERROR', reason: 'invalid response' } }, + GetProvidersDisruption: async (): Promise => { + const auth = await params.retrieveMetricsAuth() + if (auth === null) throw new Error('retrieveMetricsAuth() returned null') + let finalRoute = '/api/metrics/providers/disruption' + const { data } = await axios.post(params.baseUrl + finalRoute, {}, { headers: { 'authorization': auth } }) + if (data.status === 'ERROR' && typeof data.reason === 'string') return data + if (data.status === 'OK') { + const result = data + if(!params.checkResult) return { status: 'OK', ...result } + const error = Types.ProvidersDisruptionValidate(result) + if (error === null) { return { status: 'OK', ...result } } else return { status: 'ERROR', reason: error.message } + } + return { status: 'ERROR', reason: 'invalid response' } + }, GetSeed: async (): Promise => { const auth = await params.retrieveAdminAuth() if (auth === null) throw new Error('retrieveAdminAuth() returned null') @@ -827,6 +841,17 @@ export default (params: ClientParams) => ({ } return { status: 'ERROR', reason: 'invalid response' } }, + PingSubProcesses: async (): Promise => { + const auth = await params.retrieveMetricsAuth() + if (auth === null) throw new Error('retrieveMetricsAuth() returned null') + let finalRoute = '/api/metrics/ping' + const { data } = await axios.post(params.baseUrl + finalRoute, {}, { headers: { 'authorization': auth } }) + if (data.status === 'ERROR' && typeof data.reason === 'string') return data + if (data.status === 'OK') { + return data + } + return { status: 'ERROR', reason: 'invalid response' } + }, RequestNPubLinkingToken: async (request: Types.RequestNPubLinkingTokenRequest): Promise => { const auth = await params.retrieveAppAuth() if (auth === null) throw new Error('retrieveAppAuth() returned null') diff --git a/proto/autogenerated/ts/nostr_client.ts b/proto/autogenerated/ts/nostr_client.ts index f6dcf1dd..6813815d 100644 --- a/proto/autogenerated/ts/nostr_client.ts +++ b/proto/autogenerated/ts/nostr_client.ts @@ -422,6 +422,20 @@ export default (params: NostrClientParams, send: (to:string, message: NostrRequ } return { status: 'ERROR', reason: 'invalid response' } }, + GetProvidersDisruption: async (): Promise => { + const auth = await params.retrieveNostrMetricsAuth() + if (auth === null) throw new Error('retrieveNostrMetricsAuth() returned null') + const nostrRequest: NostrRequest = {} + const data = await send(params.pubDestination, {rpcName:'GetProvidersDisruption',authIdentifier:auth, ...nostrRequest }) + if (data.status === 'ERROR' && typeof data.reason === 'string') return data + if (data.status === 'OK') { + const result = data + if(!params.checkResult) return { status: 'OK', ...result } + const error = Types.ProvidersDisruptionValidate(result) + if (error === null) { return { status: 'OK', ...result } } else return { status: 'ERROR', reason: error.message } + } + return { status: 'ERROR', reason: 'invalid response' } + }, GetSeed: async (): Promise => { const auth = await params.retrieveNostrAdminAuth() if (auth === null) throw new Error('retrieveNostrAdminAuth() returned null') @@ -685,6 +699,17 @@ export default (params: NostrClientParams, send: (to:string, message: NostrRequ } return { status: 'ERROR', reason: 'invalid response' } }, + PingSubProcesses: async (): Promise => { + const auth = await params.retrieveNostrMetricsAuth() + if (auth === null) throw new Error('retrieveNostrMetricsAuth() returned null') + const nostrRequest: NostrRequest = {} + const data = await send(params.pubDestination, {rpcName:'PingSubProcesses',authIdentifier:auth, ...nostrRequest }) + if (data.status === 'ERROR' && typeof data.reason === 'string') return data + if (data.status === 'OK') { + return data + } + return { status: 'ERROR', reason: 'invalid response' } + }, ResetDebit: async (request: Types.DebitOperation): Promise => { const auth = await params.retrieveNostrUserAuth() if (auth === null) throw new Error('retrieveNostrUserAuth() returned null') diff --git a/proto/autogenerated/ts/nostr_transport.ts b/proto/autogenerated/ts/nostr_transport.ts index 4f48a701..55553eb6 100644 --- a/proto/autogenerated/ts/nostr_transport.ts +++ b/proto/autogenerated/ts/nostr_transport.ts @@ -812,6 +812,19 @@ export default (methods: Types.ServerMethods, opts: NostrOptions) => { opts.metricsCallback([{ ...info, ...stats, ...authContext }]) }catch(ex){ const e = ex as any; logErrorAndReturnResponse(e, e.message || e, res, logger, { ...info, ...stats, ...authCtx }, opts.metricsCallback); if (opts.throwErrors) throw e } break + case 'GetProvidersDisruption': + try { + if (!methods.GetProvidersDisruption) throw new Error('method: GetProvidersDisruption is not implemented') + const authContext = await opts.NostrMetricsAuthGuard(req.appId, req.authIdentifier) + stats.guard = process.hrtime.bigint() + authCtx = authContext + stats.validate = stats.guard + const response = await methods.GetProvidersDisruption({rpcName:'GetProvidersDisruption', ctx:authContext }) + stats.handle = process.hrtime.bigint() + res({status: 'OK', ...response}) + opts.metricsCallback([{ ...info, ...stats, ...authContext }]) + }catch(ex){ const e = ex as any; logErrorAndReturnResponse(e, e.message || e, res, logger, { ...info, ...stats, ...authCtx }, opts.metricsCallback); if (opts.throwErrors) throw e } + break case 'GetSeed': try { if (!methods.GetSeed) throw new Error('method: GetSeed is not implemented') @@ -1085,6 +1098,19 @@ export default (methods: Types.ServerMethods, opts: NostrOptions) => { opts.metricsCallback([{ ...info, ...stats, ...authContext }]) }catch(ex){ const e = ex as any; logErrorAndReturnResponse(e, e.message || e, res, logger, { ...info, ...stats, ...authCtx }, opts.metricsCallback); if (opts.throwErrors) throw e } break + case 'PingSubProcesses': + try { + if (!methods.PingSubProcesses) throw new Error('method: PingSubProcesses is not implemented') + const authContext = await opts.NostrMetricsAuthGuard(req.appId, req.authIdentifier) + stats.guard = process.hrtime.bigint() + authCtx = authContext + stats.validate = stats.guard + await methods.PingSubProcesses({rpcName:'PingSubProcesses', ctx:authContext }) + stats.handle = process.hrtime.bigint() + res({status: 'OK'}) + opts.metricsCallback([{ ...info, ...stats, ...authContext }]) + }catch(ex){ const e = ex as any; logErrorAndReturnResponse(e, e.message || e, res, logger, { ...info, ...stats, ...authCtx }, opts.metricsCallback); if (opts.throwErrors) throw e } + break case 'ResetDebit': try { if (!methods.ResetDebit) throw new Error('method: ResetDebit is not implemented') diff --git a/proto/autogenerated/ts/types.ts b/proto/autogenerated/ts/types.ts index ae494f22..dff4ae99 100644 --- a/proto/autogenerated/ts/types.ts +++ b/proto/autogenerated/ts/types.ts @@ -28,8 +28,8 @@ export type MetricsContext = { app_id: string operator_id: string } -export type MetricsMethodInputs = GetAppsMetrics_Input | GetBundleMetrics_Input | GetErrorStats_Input | GetLndMetrics_Input | GetSingleBundleMetrics_Input | GetSingleUsageMetrics_Input | GetUsageMetrics_Input | ResetMetricsStorages_Input | SubmitWebRtcMessage_Input | ZipMetricsStorages_Input -export type MetricsMethodOutputs = GetAppsMetrics_Output | GetBundleMetrics_Output | GetErrorStats_Output | GetLndMetrics_Output | GetSingleBundleMetrics_Output | GetSingleUsageMetrics_Output | GetUsageMetrics_Output | ResetMetricsStorages_Output | SubmitWebRtcMessage_Output | ZipMetricsStorages_Output +export type MetricsMethodInputs = GetAppsMetrics_Input | GetBundleMetrics_Input | GetErrorStats_Input | GetLndMetrics_Input | GetProvidersDisruption_Input | GetSingleBundleMetrics_Input | GetSingleUsageMetrics_Input | GetUsageMetrics_Input | PingSubProcesses_Input | ResetMetricsStorages_Input | SubmitWebRtcMessage_Input | ZipMetricsStorages_Input +export type MetricsMethodOutputs = GetAppsMetrics_Output | GetBundleMetrics_Output | GetErrorStats_Output | GetLndMetrics_Output | GetProvidersDisruption_Output | GetSingleBundleMetrics_Output | GetSingleUsageMetrics_Output | GetUsageMetrics_Output | PingSubProcesses_Output | ResetMetricsStorages_Output | SubmitWebRtcMessage_Output | ZipMetricsStorages_Output export type UserContext = { app_id: string app_user_id: string @@ -162,6 +162,9 @@ export type GetNPubLinkingState_Output = ResultError | ({ status: 'OK' } & NPubL export type GetPaymentState_Input = {rpcName:'GetPaymentState', req: GetPaymentStateRequest} export type GetPaymentState_Output = ResultError | ({ status: 'OK' } & PaymentState) +export type GetProvidersDisruption_Input = {rpcName:'GetProvidersDisruption'} +export type GetProvidersDisruption_Output = ResultError | ({ status: 'OK' } & ProvidersDisruption) + export type GetSeed_Input = {rpcName:'GetSeed'} export type GetSeed_Output = ResultError | ({ status: 'OK' } & LndSeed) @@ -247,6 +250,9 @@ export type PayAppUserInvoice_Output = ResultError | ({ status: 'OK' } & PayInvo export type PayInvoice_Input = {rpcName:'PayInvoice', req: PayInvoiceRequest} export type PayInvoice_Output = ResultError | ({ status: 'OK' } & PayInvoiceResponse) +export type PingSubProcesses_Input = {rpcName:'PingSubProcesses'} +export type PingSubProcesses_Output = ResultError | { status: 'OK' } + export type RequestNPubLinkingToken_Input = {rpcName:'RequestNPubLinkingToken', req: RequestNPubLinkingTokenRequest} export type RequestNPubLinkingToken_Output = ResultError | ({ status: 'OK' } & RequestNPubLinkingTokenResponse) @@ -340,6 +346,7 @@ export type ServerMethods = { GetMigrationUpdate?: (req: GetMigrationUpdate_Input & {ctx: UserContext }) => Promise GetNPubLinkingState?: (req: GetNPubLinkingState_Input & {ctx: AppContext }) => Promise GetPaymentState?: (req: GetPaymentState_Input & {ctx: UserContext }) => Promise + GetProvidersDisruption?: (req: GetProvidersDisruption_Input & {ctx: MetricsContext }) => Promise GetSeed?: (req: GetSeed_Input & {ctx: AdminContext }) => Promise GetSingleBundleMetrics?: (req: GetSingleBundleMetrics_Input & {ctx: MetricsContext }) => Promise GetSingleUsageMetrics?: (req: GetSingleUsageMetrics_Input & {ctx: MetricsContext }) => Promise @@ -363,6 +370,7 @@ export type ServerMethods = { PayAddress?: (req: PayAddress_Input & {ctx: UserContext }) => Promise PayAppUserInvoice?: (req: PayAppUserInvoice_Input & {ctx: AppContext }) => Promise PayInvoice?: (req: PayInvoice_Input & {ctx: UserContext }) => Promise + PingSubProcesses?: (req: PingSubProcesses_Input & {ctx: MetricsContext }) => Promise RequestNPubLinkingToken?: (req: RequestNPubLinkingToken_Input & {ctx: AppContext }) => Promise ResetDebit?: (req: ResetDebit_Input & {ctx: UserContext }) => Promise ResetMetricsStorages?: (req: ResetMetricsStorages_Input & {ctx: MetricsContext }) => Promise @@ -3030,6 +3038,57 @@ export const ProductValidate = (o?: Product, opts: ProductOptions = {}, path: st return null } +export type ProviderDisruption = { + provider_pubkey: string + provider_type: string + since_unix: number +} +export const ProviderDisruptionOptionalFields: [] = [] +export type ProviderDisruptionOptions = OptionsBaseMessage & { + checkOptionalsAreSet?: [] + provider_pubkey_CustomCheck?: (v: string) => boolean + provider_type_CustomCheck?: (v: string) => boolean + since_unix_CustomCheck?: (v: number) => boolean +} +export const ProviderDisruptionValidate = (o?: ProviderDisruption, opts: ProviderDisruptionOptions = {}, path: string = 'ProviderDisruption::root.'): Error | null => { + if (opts.checkOptionalsAreSet && opts.allOptionalsAreSet) return new Error(path + ': only one of checkOptionalsAreSet or allOptionalNonDefault can be set for each message') + if (typeof o !== 'object' || o === null) return new Error(path + ': object is not an instance of an object or is null') + + if (typeof o.provider_pubkey !== 'string') return new Error(`${path}.provider_pubkey: is not a string`) + if (opts.provider_pubkey_CustomCheck && !opts.provider_pubkey_CustomCheck(o.provider_pubkey)) return new Error(`${path}.provider_pubkey: custom check failed`) + + if (typeof o.provider_type !== 'string') return new Error(`${path}.provider_type: is not a string`) + if (opts.provider_type_CustomCheck && !opts.provider_type_CustomCheck(o.provider_type)) return new Error(`${path}.provider_type: custom check failed`) + + if (typeof o.since_unix !== 'number') return new Error(`${path}.since_unix: is not a number`) + if (opts.since_unix_CustomCheck && !opts.since_unix_CustomCheck(o.since_unix)) return new Error(`${path}.since_unix: custom check failed`) + + return null +} + +export type ProvidersDisruption = { + disruptions: ProviderDisruption[] +} +export const ProvidersDisruptionOptionalFields: [] = [] +export type ProvidersDisruptionOptions = OptionsBaseMessage & { + checkOptionalsAreSet?: [] + disruptions_ItemOptions?: ProviderDisruptionOptions + disruptions_CustomCheck?: (v: ProviderDisruption[]) => boolean +} +export const ProvidersDisruptionValidate = (o?: ProvidersDisruption, opts: ProvidersDisruptionOptions = {}, path: string = 'ProvidersDisruption::root.'): Error | null => { + if (opts.checkOptionalsAreSet && opts.allOptionalsAreSet) return new Error(path + ': only one of checkOptionalsAreSet or allOptionalNonDefault can be set for each message') + if (typeof o !== 'object' || o === null) return new Error(path + ': object is not an instance of an object or is null') + + if (!Array.isArray(o.disruptions)) return new Error(`${path}.disruptions: is not an array`) + for (let index = 0; index < o.disruptions.length; index++) { + const disruptionsErr = ProviderDisruptionValidate(o.disruptions[index], opts.disruptions_ItemOptions, `${path}.disruptions[${index}]`) + if (disruptionsErr !== null) return disruptionsErr + } + if (opts.disruptions_CustomCheck && !opts.disruptions_CustomCheck(o.disruptions)) return new Error(`${path}.disruptions: custom check failed`) + + return null +} + export type RelaysMigration = { relays: string[] } diff --git a/proto/service/methods.proto b/proto/service/methods.proto index 81a9165e..7ebe175f 100644 --- a/proto/service/methods.proto +++ b/proto/service/methods.proto @@ -240,6 +240,19 @@ service LightningPub { option (http_route) = "/api/metrics/zip"; option (nostr) = true; } + rpc PingSubProcesses(structs.Empty) returns (structs.Empty) { + option (auth_type) = "Metrics"; + option (http_method) = "post"; + option (http_route) = "/api/metrics/ping"; + option (nostr) = true; + } + + rpc GetProvidersDisruption(structs.Empty) returns (structs.ProvidersDisruption) { + option (auth_type) = "Metrics"; + option (http_method) = "post"; + option (http_route) = "/api/metrics/providers/disruption"; + option (nostr) = true; + } rpc ResetMetricsStorages(structs.Empty) returns (structs.Empty) { option (auth_type) = "Metrics"; diff --git a/proto/service/structs.proto b/proto/service/structs.proto index f7693ce7..068576c5 100644 --- a/proto/service/structs.proto +++ b/proto/service/structs.proto @@ -749,4 +749,16 @@ message OfferInvoice { int64 paid_at_unix = 3; int64 amount = 4; map data = 5; -} \ No newline at end of file +} + +message ProviderDisruption { + string provider_pubkey = 1; + string provider_type = 2; + int64 since_unix = 3; +} + +message ProvidersDisruption { + repeated ProviderDisruption disruptions = 1; +} + + diff --git a/src/index.ts b/src/index.ts index 19464c04..3dd3db76 100644 --- a/src/index.ts +++ b/src/index.ts @@ -24,13 +24,14 @@ const start = async () => { const serverMethods = GetServerMethods(mainHandler) const nostrSettings = LoadNosrtSettingsFromEnv() log("initializing nostr middleware") - const { Send, Stop } = nostrMiddleware(serverMethods, mainHandler, + const { Send, Stop, Ping } = nostrMiddleware(serverMethods, mainHandler, { ...nostrSettings, apps, clients: [liquidityProviderInfo] }, (e, p) => mainHandler.liquidityProvider.onEvent(e, p) ) exitHandler(() => { Stop() }) log("starting server") mainHandler.attachNostrSend(Send) + mainHandler.attachNostrProcessPing(Ping) mainHandler.StartBeacons() const appNprofile = nprofileEncode({ pubkey: liquidityProviderInfo.publicKey, relays: nostrSettings.relays }) if (wizard) { diff --git a/src/nostrMiddleware.ts b/src/nostrMiddleware.ts index 4b6f6cef..ec8c6726 100644 --- a/src/nostrMiddleware.ts +++ b/src/nostrMiddleware.ts @@ -7,7 +7,7 @@ import { ERROR, getLogger } from "./services/helpers/logger.js"; import { NdebitData } from "nostr-tools/lib/types/nip68.js"; import { NofferData } from "nostr-tools/lib/types/nip69.js"; -export default (serverMethods: Types.ServerMethods, mainHandler: Main, nostrSettings: NostrSettings, onClientEvent: (e: { requestId: string }, fromPub: string) => void): { Stop: () => void, Send: NostrSend } => { +export default (serverMethods: Types.ServerMethods, mainHandler: Main, nostrSettings: NostrSettings, onClientEvent: (e: { requestId: string }, fromPub: string) => void): { Stop: () => void, Send: NostrSend, Ping: () => Promise } => { const log = getLogger({}) const nostrTransport = NewNostrTransport(serverMethods, { NostrUserAuthGuard: async (appId, pub) => { @@ -68,7 +68,7 @@ export default (serverMethods: Types.ServerMethods, mainHandler: Main, nostrSett nostr.Send({ type: 'app', appId: event.appId }, { type: 'content', pub: event.pub, content: JSON.stringify({ ...res, requestId: j.requestId }) }) }, event.startAtNano, event.startAtMs) }) - return { Stop: () => nostr.Stop, Send: (...args) => nostr.Send(...args) } + return { Stop: () => nostr.Stop, Send: (...args) => nostr.Send(...args), Ping: () => nostr.Ping() } } diff --git a/src/services/main/index.ts b/src/services/main/index.ts index 39eb235f..2ee3754e 100644 --- a/src/services/main/index.ts +++ b/src/services/main/index.ts @@ -57,6 +57,7 @@ export default class { unlocker: Unlocker //webRTC: webRTC nostrSend: NostrSend = () => { getLogger({})("nostr send not initialized yet") } + nostrProcessPing: (() => Promise) | null = null constructor(settings: MainSettings, storage: Storage, adminManager: AdminManager, utils: Utils, unlocker: Unlocker) { this.settings = settings this.storage = storage @@ -102,6 +103,37 @@ export default class { //this.webRTC.attachNostrSend(f) } + attachNostrProcessPing(f: () => Promise) { + this.nostrProcessPing = f + } + + async pingSubProcesses() { + if (!this.nostrProcessPing) { + throw new Error("nostr process ping not initialized") + } + const storageP = this.storage.dbs.Ping() + const metricsP = this.storage.metricsStorage.dbs.Ping() + const tlvP = this.utils.tlvStorageFactory.Ping() + const nostrP = this.nostrProcessPing() + const timeout = new Promise(res => setTimeout(() => res(true), 2 * 1000)) + const storageFail = await Promise.race([storageP, timeout]) + const metricsFail = await Promise.race([metricsP, timeout]) + const tlvFail = await Promise.race([tlvP, timeout]) + const nostrFail = await Promise.race([nostrP, timeout]) + if (storageFail) { + throw new Error("storage ping failed") + } + if (metricsFail) { + throw new Error("metrics ping failed") + } + if (tlvFail) { + throw new Error("tlv ping failed") + } + if (nostrFail) { + throw new Error("nostr ping failed") + } + } + htlcCb: HtlcCb = (e) => { this.metricsManager.HtlcCb(e) } diff --git a/src/services/metrics/index.ts b/src/services/metrics/index.ts index 8321c2dc..8396a556 100644 --- a/src/services/metrics/index.ts +++ b/src/services/metrics/index.ts @@ -27,6 +27,18 @@ export default class Handler { } + async GetProvidersDisruption(): Promise { + const providers = await this.storage.liquidityStorage.GetTrackedProviders() + const disruptions = providers.filter(p => p.latest_distruption_at_unix > 0) + return { + disruptions: disruptions.map(d => ({ + provider_pubkey: d.provider_pubkey, + provider_type: d.provider_type, + since_unix: d.latest_distruption_at_unix + })) + } + } + async HtlcCb(htlc: HtlcEvent) { diff --git a/src/services/nostr/handler.ts b/src/services/nostr/handler.ts index 46b10145..6e8c8ce5 100644 --- a/src/services/nostr/handler.ts +++ b/src/services/nostr/handler.ts @@ -38,12 +38,21 @@ type SettingsRequest = { settings: NostrSettings } +type PingRequest = { + type: 'ping' +} + type SendRequest = { type: 'send' initiator: SendInitiator data: SendData relays?: string[] } + +type PingResponse = { + type: 'pong' +} + type ReadyResponse = { type: 'ready' } @@ -56,8 +65,8 @@ type ProcessMetricsResponse = { metrics: ProcessMetrics } -export type ChildProcessRequest = SettingsRequest | SendRequest -export type ChildProcessResponse = ReadyResponse | EventResponse | ProcessMetricsResponse +export type ChildProcessRequest = SettingsRequest | SendRequest | PingRequest +export type ChildProcessResponse = ReadyResponse | EventResponse | ProcessMetricsResponse | PingResponse const send = (message: ChildProcessResponse) => { if (process.send) { process.send(message, undefined, undefined, err => { @@ -77,6 +86,9 @@ process.on("message", (message: ChildProcessRequest) => { case 'send': sendToNostr(message.initiator, message.data, message.relays) break + case 'ping': + send({ type: 'pong' }) + break default: getLogger({ component: "nostrMiddleware" })(ERROR, "unknown nostr request", message) break diff --git a/src/services/nostr/index.ts b/src/services/nostr/index.ts index cbe55034..9b9745e3 100644 --- a/src/services/nostr/index.ts +++ b/src/services/nostr/index.ts @@ -19,6 +19,7 @@ export default class NostrSubprocess { settings: NostrSettings childProcess: ChildProcess utils: Utils + awaitingPongs: (() => void)[] = [] constructor(settings: NostrSettings, utils: Utils, eventCallback: EventCallback) { this.utils = utils this.childProcess = fork("./build/src/services/nostr/handler") @@ -34,6 +35,10 @@ export default class NostrSubprocess { case 'processMetrics': this.utils.tlvStorageFactory.ProcessMetrics(message.metrics, 'nostr') break + case 'pong': + this.awaitingPongs.forEach(resolve => resolve()) + this.awaitingPongs = [] + break default: console.error("unknown nostr event response", message) break; @@ -44,6 +49,13 @@ export default class NostrSubprocess { this.childProcess.send(message) } + Ping() { + this.sendToChildProcess({ type: 'ping' }) + return new Promise((resolve) => { + this.awaitingPongs.push(resolve) + }) + } + Send(initiator: SendInitiator, data: SendData, relays?: string[]) { this.sendToChildProcess({ type: 'send', data, initiator, relays }) } diff --git a/src/services/storage/db/storageInterface.ts b/src/services/storage/db/storageInterface.ts index 00794f08..81da2270 100644 --- a/src/services/storage/db/storageInterface.ts +++ b/src/services/storage/db/storageInterface.ts @@ -12,6 +12,7 @@ import { SumOperation, DBNames, SuccessOperationResponse, + PingOperation, } from './storageProcessor.js'; import { PickKeysByType } from 'typeorm/common/PickKeysByType.js'; import { serializeRequest, WhereCondition } from './serializationHelpers.js'; @@ -67,6 +68,12 @@ export class StorageInterface extends EventEmitter { this.isConnected = true; } + Ping(): Promise { + const opId = Math.random().toString() + const pingOp: PingOperation = { type: 'ping', opId } + return this.handleOp(pingOp) + } + Connect(settings: DbSettings, dbType: 'main' | 'metrics'): Promise { const opId = Math.random().toString() this.dbType = dbType diff --git a/src/services/storage/db/storageProcessor.ts b/src/services/storage/db/storageProcessor.ts index 20c2cc07..05940629 100644 --- a/src/services/storage/db/storageProcessor.ts +++ b/src/services/storage/db/storageProcessor.ts @@ -22,6 +22,12 @@ export type ConnectOperation = { debug?: boolean } +export type PingOperation = { + type: 'ping' + opId: string + debug?: boolean +} + export type StartTxOperation = { type: 'startTx' opId: string @@ -133,7 +139,7 @@ export interface IStorageOperation { } export type StorageOperation = ConnectOperation | StartTxOperation | EndTxOperation | DeleteOperation | RemoveOperation | UpdateOperation | - FindOneOperation | FindOperation | CreateAndSaveOperation | IncrementOperation | DecrementOperation | SumOperation + FindOneOperation | FindOperation | CreateAndSaveOperation | IncrementOperation | DecrementOperation | SumOperation | PingOperation export type SuccessOperationResponse = { success: true, type: string, data: T, opId: string } export type OperationResponse = SuccessOperationResponse | ErrorOperationResponse @@ -222,6 +228,9 @@ class StorageProcessor { case 'createAndSave': await this.handleCreateAndSave(operation); break; + case 'ping': + await this.handlePing(operation); + break; default: this.sendResponse({ success: false, @@ -239,6 +248,14 @@ class StorageProcessor { } } + private async handlePing(operation: PingOperation) { + this.sendResponse({ + success: true, + type: 'ping', + data: null, + opId: operation.opId + }); + } private async handleConnect(operation: ConnectOperation) { let migrationsExecuted = 0 if (this.mode !== '') { diff --git a/src/services/storage/tlv/tlvFilesStorageFactory.ts b/src/services/storage/tlv/tlvFilesStorageFactory.ts index 9840c6ad..6a7c9bfd 100644 --- a/src/services/storage/tlv/tlvFilesStorageFactory.ts +++ b/src/services/storage/tlv/tlvFilesStorageFactory.ts @@ -1,6 +1,6 @@ import { ChildProcess, fork } from 'child_process'; import { EventEmitter } from 'events'; -import { AddTlvOperation, ITlvStorageOperation, SuccessTlvOperationResponse, LoadLatestTlvOperation, LoadTlvFileOperation, NewTlvStorageOperation, SerializableLatestData, SerializableTlvFile, TlvOperationResponse, TlvStorageSettings, WebRtcMessageOperation, ProcessMetricsTlvOperation, ZipStoragesOperation, ResetTlvStorageOperation } from './tlvFilesStorageProcessor'; +import { AddTlvOperation, ITlvStorageOperation, SuccessTlvOperationResponse, LoadLatestTlvOperation, LoadTlvFileOperation, NewTlvStorageOperation, SerializableLatestData, SerializableTlvFile, TlvOperationResponse, TlvStorageSettings, WebRtcMessageOperation, ProcessMetricsTlvOperation, ZipStoragesOperation, ResetTlvStorageOperation, PingTlvOperation } from './tlvFilesStorageProcessor'; import { LatestData, TlvFile } from './tlvFilesStorage'; import { NostrSend, SendData, SendInitiator } from '../../nostr/handler'; import { WebRtcUserInfo } from '../../webRTC'; @@ -63,6 +63,12 @@ export class TlvStorageFactory extends EventEmitter { this.isConnected = true; } + Ping(): Promise { + const opId = Math.random().toString() + const op: PingTlvOperation = { type: 'ping', opId } + return this.handleOp(op) + } + ZipStorages(): Promise { const opId = Math.random().toString() const op: ZipStoragesOperation = { type: 'zipStorages', opId } diff --git a/src/services/storage/tlv/tlvFilesStorageProcessor.ts b/src/services/storage/tlv/tlvFilesStorageProcessor.ts index 1beeeddc..1d725e37 100644 --- a/src/services/storage/tlv/tlvFilesStorageProcessor.ts +++ b/src/services/storage/tlv/tlvFilesStorageProcessor.ts @@ -80,6 +80,12 @@ export type ProcessMetricsTlvOperation = { debug?: boolean } +export type PingTlvOperation = { + type: 'ping' + opId: string + debug?: boolean +} + export type ErrorTlvOperationResponse = { success: false, error: string, opId: string } export interface ITlvStorageOperation { @@ -88,7 +94,7 @@ export interface ITlvStorageOperation { debug?: boolean } -export type TlvStorageOperation = NewTlvStorageOperation | AddTlvOperation | LoadLatestTlvOperation | LoadTlvFileOperation | WebRtcMessageOperation | ProcessMetricsTlvOperation | ResetTlvStorageOperation | ZipStoragesOperation +export type TlvStorageOperation = NewTlvStorageOperation | AddTlvOperation | LoadLatestTlvOperation | LoadTlvFileOperation | WebRtcMessageOperation | ProcessMetricsTlvOperation | ResetTlvStorageOperation | ZipStoragesOperation | PingTlvOperation export type SuccessTlvOperationResponse = { success: true, type: string, data: T, opId: string } export type TlvOperationResponse = SuccessTlvOperationResponse | ErrorTlvOperationResponse @@ -186,6 +192,9 @@ class TlvFilesStorageProcessor { case 'zipStorages': await this.handleZipStorages(operation); break; + case 'ping': + await this.handlePing(operation); + break; default: this.sendResponse({ success: false, @@ -203,6 +212,14 @@ class TlvFilesStorageProcessor { } } + private async handlePing(operation: PingTlvOperation) { + this.sendResponse({ + success: true, + type: 'ping', + data: null, + opId: operation.opId + }); + } private async handleResetStorage(operation: ResetTlvStorageOperation) { for (const storageName in this.storages) { this.storages[storageName].Reset() From e75f13db13782b5067ebf0ff408e5e24e9a9429c Mon Sep 17 00:00:00 2001 From: boufni95 Date: Wed, 14 May 2025 19:10:12 +0000 Subject: [PATCH 4/4] up --- src/services/serverMethods/index.ts | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/src/services/serverMethods/index.ts b/src/services/serverMethods/index.ts index 015e66e6..65d36ddf 100644 --- a/src/services/serverMethods/index.ts +++ b/src/services/serverMethods/index.ts @@ -91,6 +91,12 @@ export default (mainHandler: Main): Types.ServerMethods => { if (err != null) throw new Error(err.message) return mainHandler.adminManager.CloseChannel(req) }, + GetProvidersDisruption: async () => { + return mainHandler.metricsManager.GetProvidersDisruption() + }, + PingSubProcesses: async () => { + await mainHandler.pingSubProcesses() + }, EncryptionExchange: async () => { }, Health: async () => { await mainHandler.lnd.Health() }, LndGetInfo: async ({ ctx }) => {