From e7aef1e3abd3713f07073364c9c5d8c209c3b8c9 Mon Sep 17 00:00:00 2001 From: boufni95 Date: Mon, 27 Jan 2025 21:16:53 +0000 Subject: [PATCH 01/19] wrtc test --- proto/autogenerated/client.md | 34 +++++++ proto/autogenerated/go/http_client.go | 32 +++++++ proto/autogenerated/go/types.go | 22 +++++ proto/autogenerated/ts/express_server.ts | 22 +++++ proto/autogenerated/ts/http_client.ts | 15 ++++ proto/autogenerated/ts/nostr_client.ts | 30 +++++++ proto/autogenerated/ts/nostr_transport.ts | 29 ++++++ proto/autogenerated/ts/types.ts | 104 +++++++++++++++++++++- proto/service/methods.proto | 19 +++- proto/service/structs.proto | 15 ++++ src/auth.ts | 2 +- src/nostrMiddleware.ts | 2 +- src/services/main/index.ts | 4 + src/services/serverMethods/index.ts | 10 +++ src/services/webRTC/index.ts | 89 ++++++++++++++++++ 15 files changed, 423 insertions(+), 6 deletions(-) create mode 100644 src/services/webRTC/index.ts diff --git a/proto/autogenerated/client.md b/proto/autogenerated/client.md index 80d81868..e648b2b6 100644 --- a/proto/autogenerated/client.md +++ b/proto/autogenerated/client.md @@ -250,6 +250,16 @@ The nostr server will send back a message response, and inside the body there wi - input: [DebitResponse](#DebitResponse) - This methods has an __empty__ __response__ body +- SubToWebRtcCandidates + - auth type: __Metrics__ + - This methods has an __empty__ __request__ body + - output: [WebRtcCandidate](#WebRtcCandidate) + +- SubmitWebRtcMessage + - auth type: __Metrics__ + - input: [WebRtcMessage](#WebRtcMessage) + - output: [WebRtcAnswer](#WebRtcAnswer) + - UpdateCallbackUrl - auth type: __User__ - input: [CallbackUrl](#CallbackUrl) @@ -298,6 +308,7 @@ The nostr server will send back a message response, and inside the body there wi - __Metrics__: - expected context content + - __app_id__: _string_ - __operator_id__: _string_ - __User__: @@ -808,6 +819,20 @@ The nostr server will send back a message response, and inside the body there wi - input: [SetMockInvoiceAsPaidRequest](#SetMockInvoiceAsPaidRequest) - This methods has an __empty__ __response__ body +- SubToWebRtcCandidates + - auth type: __Metrics__ + - http method: __post__ + - http route: __/api/upgrade/wrtc/candidates__ + - This methods has an __empty__ __request__ body + - output: [WebRtcCandidate](#WebRtcCandidate) + +- SubmitWebRtcMessage + - auth type: __Metrics__ + - http method: __post__ + - http route: __/api/upgrade/wrtc__ + - input: [WebRtcMessage](#WebRtcMessage) + - output: [WebRtcAnswer](#WebRtcAnswer) + - UpdateCallbackUrl - auth type: __User__ - http method: __post__ @@ -1391,6 +1416,15 @@ The nostr server will send back a message response, and inside the body there wi - __negative_balance__: _number_ - __no_balance__: _number_ - __total__: _number_ + +### WebRtcAnswer + - __answer__: _string_ *this field is optional + +### WebRtcCandidate + - __candidate__: _string_ + +### WebRtcMessage + - __message__: _[WebRtcMessage_message](#WebRtcMessage_message)_ ## Enums ### The enumerators used in the messages diff --git a/proto/autogenerated/go/http_client.go b/proto/autogenerated/go/http_client.go index be0129d4..78889c60 100644 --- a/proto/autogenerated/go/http_client.go +++ b/proto/autogenerated/go/http_client.go @@ -123,6 +123,8 @@ type Client struct { SetMockAppBalance func(req SetMockAppBalanceRequest) error SetMockAppUserBalance func(req SetMockAppUserBalanceRequest) error SetMockInvoiceAsPaid func(req SetMockInvoiceAsPaidRequest) error + SubToWebRtcCandidates func() (*WebRtcCandidate, error) + SubmitWebRtcMessage func(req WebRtcMessage) (*WebRtcAnswer, error) UpdateCallbackUrl func(req CallbackUrl) (*CallbackUrl, error) UpdateChannelPolicy func(req UpdateChannelPolicyRequest) error UpdateUserOffer func(req OfferConfig) error @@ -1863,6 +1865,36 @@ func NewClient(params ClientParams) *Client { } return nil }, + // server streaming method: SubToWebRtcCandidates not implemented + SubmitWebRtcMessage: func(req WebRtcMessage) (*WebRtcAnswer, error) { + auth, err := params.RetrieveMetricsAuth() + if err != nil { + return nil, err + } + finalRoute := "/api/upgrade/wrtc" + body, err := json.Marshal(req) + if err != nil { + return nil, err + } + resBody, err := doPostRequest(params.BaseURL+finalRoute, body, auth) + if err != nil { + return nil, err + } + result := ResultError{} + err = json.Unmarshal(resBody, &result) + if err != nil { + return nil, err + } + if result.Status == "ERROR" { + return nil, fmt.Errorf(result.Reason) + } + res := WebRtcAnswer{} + err = json.Unmarshal(resBody, &res) + if err != nil { + return nil, err + } + return &res, nil + }, UpdateCallbackUrl: func(req CallbackUrl) (*CallbackUrl, error) { auth, err := params.RetrieveUserAuth() if err != nil { diff --git a/proto/autogenerated/go/types.go b/proto/autogenerated/go/types.go index 720a4071..b4479edb 100644 --- a/proto/autogenerated/go/types.go +++ b/proto/autogenerated/go/types.go @@ -19,6 +19,7 @@ type GuestWithPubContext struct { Pub string `json:"pub"` } type MetricsContext struct { + App_id string `json:"app_id"` Operator_id string `json:"operator_id"` } type UserContext struct { @@ -632,6 +633,15 @@ type UsersInfo struct { No_balance int64 `json:"no_balance"` Total int64 `json:"total"` } +type WebRtcAnswer struct { + Answer string `json:"answer"` +} +type WebRtcCandidate struct { + Candidate string `json:"candidate"` +} +type WebRtcMessage struct { + Message *WebRtcMessage_message `json:"message"` +} type DebitResponse_response_type string const ( @@ -696,3 +706,15 @@ type UpdateChannelPolicyRequest_update struct { All *Empty `json:"all"` Channel_point *string `json:"channel_point"` } +type WebRtcMessage_message_type string + +const ( + CANDIDATE WebRtcMessage_message_type = "candidate" + OFFER WebRtcMessage_message_type = "offer" +) + +type WebRtcMessage_message struct { + Type WebRtcMessage_message_type `json:"type"` + Candidate *string `json:"candidate"` + Offer *string `json:"offer"` +} diff --git a/proto/autogenerated/ts/express_server.ts b/proto/autogenerated/ts/express_server.ts index a0102fe0..5c1da226 100644 --- a/proto/autogenerated/ts/express_server.ts +++ b/proto/autogenerated/ts/express_server.ts @@ -1742,6 +1742,28 @@ export default (methods: Types.ServerMethods, opts: ServerOptions) => { opts.metricsCallback([{ ...info, ...stats, ...authContext }]) } catch (ex) { const e = ex as any; logErrorAndReturnResponse(e, e.message || e, res, logger, { ...info, ...stats, ...authCtx }, opts.metricsCallback); if (opts.throwErrors) throw e } }) + if (!opts.allowNotImplementedMethods && !methods.SubmitWebRtcMessage) throw new Error('method: SubmitWebRtcMessage is not implemented') + app.post('/api/upgrade/wrtc', async (req, res) => { + const info: Types.RequestInfo = { rpcName: 'SubmitWebRtcMessage', batch: false, nostr: false, batchSize: 0} + const stats: Types.RequestStats = { startMs:req.startTimeMs || 0, start:req.startTime || 0n, parse: process.hrtime.bigint(), guard: 0n, validate: 0n, handle: 0n } + let authCtx: Types.AuthContext = {} + try { + if (!methods.SubmitWebRtcMessage) throw new Error('method: SubmitWebRtcMessage is not implemented') + const authContext = await opts.MetricsAuthGuard(req.headers['authorization']) + authCtx = authContext + stats.guard = process.hrtime.bigint() + const request = req.body + const error = Types.WebRtcMessageValidate(request) + stats.validate = process.hrtime.bigint() + if (error !== null) return logErrorAndReturnResponse(error, 'invalid request body', res, logger, { ...info, ...stats, ...authContext }, opts.metricsCallback) + const query = req.query + const params = req.params + const response = await methods.SubmitWebRtcMessage({rpcName:'SubmitWebRtcMessage', ctx:authContext , req: request}) + stats.handle = process.hrtime.bigint() + res.json({status: 'OK', ...response}) + opts.metricsCallback([{ ...info, ...stats, ...authContext }]) + } catch (ex) { const e = ex as any; logErrorAndReturnResponse(e, e.message || e, res, logger, { ...info, ...stats, ...authCtx }, opts.metricsCallback); if (opts.throwErrors) throw e } + }) if (!opts.allowNotImplementedMethods && !methods.UpdateCallbackUrl) throw new Error('method: UpdateCallbackUrl is not implemented') app.post('/api/user/cb/update', async (req, res) => { const info: Types.RequestInfo = { rpcName: 'UpdateCallbackUrl', batch: false, nostr: false, batchSize: 0} diff --git a/proto/autogenerated/ts/http_client.ts b/proto/autogenerated/ts/http_client.ts index adc2d270..2c408f93 100644 --- a/proto/autogenerated/ts/http_client.ts +++ b/proto/autogenerated/ts/http_client.ts @@ -891,6 +891,21 @@ export default (params: ClientParams) => ({ } return { status: 'ERROR', reason: 'invalid response' } }, + SubToWebRtcCandidates: async (cb: (v:ResultError | ({ status: 'OK' }& Types.WebRtcCandidate)) => void): Promise => { throw new Error('http streams are not supported')}, + SubmitWebRtcMessage: async (request: Types.WebRtcMessage): Promise => { + const auth = await params.retrieveMetricsAuth() + if (auth === null) throw new Error('retrieveMetricsAuth() returned null') + let finalRoute = '/api/upgrade/wrtc' + const { data } = await axios.post(params.baseUrl + finalRoute, request, { headers: { 'authorization': auth } }) + if (data.status === 'ERROR' && typeof data.reason === 'string') return data + if (data.status === 'OK') { + const result = data + if(!params.checkResult) return { status: 'OK', ...result } + const error = Types.WebRtcAnswerValidate(result) + if (error === null) { return { status: 'OK', ...result } } else return { status: 'ERROR', reason: error.message } + } + return { status: 'ERROR', reason: 'invalid response' } + }, UpdateCallbackUrl: async (request: Types.CallbackUrl): Promise => { const auth = await params.retrieveUserAuth() if (auth === null) throw new Error('retrieveUserAuth() returned null') diff --git a/proto/autogenerated/ts/nostr_client.ts b/proto/autogenerated/ts/nostr_client.ts index 48dd15b9..b2cc12c7 100644 --- a/proto/autogenerated/ts/nostr_client.ts +++ b/proto/autogenerated/ts/nostr_client.ts @@ -680,6 +680,36 @@ export default (params: NostrClientParams, send: (to:string, message: NostrRequ } return { status: 'ERROR', reason: 'invalid response' } }, + SubToWebRtcCandidates: async (cb: (res:ResultError | ({ status: 'OK' }& Types.WebRtcCandidate)) => void): Promise => { + const auth = await params.retrieveNostrMetricsAuth() + if (auth === null) throw new Error('retrieveNostrMetricsAuth() returned null') + const nostrRequest: NostrRequest = {} + subscribe(params.pubDestination, {rpcName:'SubToWebRtcCandidates',authIdentifier:auth, ...nostrRequest }, (data) => { + if (data.status === 'ERROR' && typeof data.reason === 'string') return cb(data) + if (data.status === 'OK') { + const result = data + if(!params.checkResult) return cb({ status: 'OK', ...result }) + const error = Types.WebRtcCandidateValidate(result) + if (error === null) { return cb({ status: 'OK', ...result }) } else return cb({ status: 'ERROR', reason: error.message }) + } + return cb({ status: 'ERROR', reason: 'invalid response' }) + }) + }, + SubmitWebRtcMessage: async (request: Types.WebRtcMessage): Promise => { + const auth = await params.retrieveNostrMetricsAuth() + if (auth === null) throw new Error('retrieveNostrMetricsAuth() returned null') + const nostrRequest: NostrRequest = {} + nostrRequest.body = request + const data = await send(params.pubDestination, {rpcName:'SubmitWebRtcMessage',authIdentifier:auth, ...nostrRequest }) + if (data.status === 'ERROR' && typeof data.reason === 'string') return data + if (data.status === 'OK') { + const result = data + if(!params.checkResult) return { status: 'OK', ...result } + const error = Types.WebRtcAnswerValidate(result) + if (error === null) { return { status: 'OK', ...result } } else return { status: 'ERROR', reason: error.message } + } + return { status: 'ERROR', reason: 'invalid response' } + }, UpdateCallbackUrl: async (request: Types.CallbackUrl): Promise => { const auth = await params.retrieveNostrUserAuth() if (auth === null) throw new Error('retrieveNostrUserAuth() returned null') diff --git a/proto/autogenerated/ts/nostr_transport.ts b/proto/autogenerated/ts/nostr_transport.ts index 412e02e8..2c4ff956 100644 --- a/proto/autogenerated/ts/nostr_transport.ts +++ b/proto/autogenerated/ts/nostr_transport.ts @@ -1075,6 +1075,35 @@ export default (methods: Types.ServerMethods, opts: NostrOptions) => { opts.metricsCallback([{ ...info, ...stats, ...authContext }]) }catch(ex){ const e = ex as any; logErrorAndReturnResponse(e, e.message || e, res, logger, { ...info, ...stats, ...authCtx }, opts.metricsCallback); if (opts.throwErrors) throw e } break + case 'SubToWebRtcCandidates': + try { + if (!methods.SubToWebRtcCandidates) throw new Error('method: SubToWebRtcCandidates is not implemented') + const authContext = await opts.NostrMetricsAuthGuard(req.appId, req.authIdentifier) + stats.guard = process.hrtime.bigint() + authCtx = authContext + stats.validate = stats.guard + methods.SubToWebRtcCandidates({rpcName:'SubToWebRtcCandidates', ctx:authContext ,cb: (response, err) => { + stats.handle = process.hrtime.bigint() + if (err) { logErrorAndReturnResponse(err, err.message, res, logger, { ...info, ...stats, ...authContext }, opts.metricsCallback)} else { res({status: 'OK', ...response});opts.metricsCallback([{ ...info, ...stats, ...authContext }])} + }}) + }catch(ex){ const e = ex as any; logErrorAndReturnResponse(e, e.message || e, res, logger, { ...info, ...stats, ...authCtx }, opts.metricsCallback); if (opts.throwErrors) throw e } + break + case 'SubmitWebRtcMessage': + try { + if (!methods.SubmitWebRtcMessage) throw new Error('method: SubmitWebRtcMessage is not implemented') + const authContext = await opts.NostrMetricsAuthGuard(req.appId, req.authIdentifier) + stats.guard = process.hrtime.bigint() + authCtx = authContext + const request = req.body + const error = Types.WebRtcMessageValidate(request) + stats.validate = process.hrtime.bigint() + if (error !== null) return logErrorAndReturnResponse(error, 'invalid request body', res, logger, { ...info, ...stats, ...authCtx }, opts.metricsCallback) + const response = await methods.SubmitWebRtcMessage({rpcName:'SubmitWebRtcMessage', ctx:authContext , req: request}) + stats.handle = process.hrtime.bigint() + res({status: 'OK', ...response}) + opts.metricsCallback([{ ...info, ...stats, ...authContext }]) + }catch(ex){ const e = ex as any; logErrorAndReturnResponse(e, e.message || e, res, logger, { ...info, ...stats, ...authCtx }, opts.metricsCallback); if (opts.throwErrors) throw e } + break case 'UpdateCallbackUrl': try { if (!methods.UpdateCallbackUrl) throw new Error('method: UpdateCallbackUrl is not implemented') diff --git a/proto/autogenerated/ts/types.ts b/proto/autogenerated/ts/types.ts index 767710c5..042ebb35 100644 --- a/proto/autogenerated/ts/types.ts +++ b/proto/autogenerated/ts/types.ts @@ -25,10 +25,11 @@ export type GuestWithPubContext = { export type GuestWithPubMethodInputs = LinkNPubThroughToken_Input | UseInviteLink_Input export type GuestWithPubMethodOutputs = LinkNPubThroughToken_Output | UseInviteLink_Output export type MetricsContext = { + app_id: string operator_id: string } -export type MetricsMethodInputs = GetAppsMetrics_Input | GetErrorStats_Input | GetLndMetrics_Input | GetSingleUsageMetrics_Input | GetUsageMetrics_Input -export type MetricsMethodOutputs = GetAppsMetrics_Output | GetErrorStats_Output | GetLndMetrics_Output | GetSingleUsageMetrics_Output | GetUsageMetrics_Output +export type MetricsMethodInputs = GetAppsMetrics_Input | GetErrorStats_Input | GetLndMetrics_Input | GetSingleUsageMetrics_Input | GetUsageMetrics_Input | SubmitWebRtcMessage_Input +export type MetricsMethodOutputs = GetAppsMetrics_Output | GetErrorStats_Output | GetLndMetrics_Output | GetSingleUsageMetrics_Output | GetUsageMetrics_Output | SubmitWebRtcMessage_Output export type UserContext = { app_id: string app_user_id: string @@ -267,6 +268,12 @@ export type SetMockAppUserBalance_Output = ResultError | { status: 'OK' } export type SetMockInvoiceAsPaid_Input = {rpcName:'SetMockInvoiceAsPaid', req: SetMockInvoiceAsPaidRequest} export type SetMockInvoiceAsPaid_Output = ResultError | { status: 'OK' } +export type SubToWebRtcCandidates_Input = {rpcName:'SubToWebRtcCandidates', cb:(res: WebRtcCandidate, err:Error|null)=> void} +export type SubToWebRtcCandidates_Output = ResultError | { status: 'OK' } + +export type SubmitWebRtcMessage_Input = {rpcName:'SubmitWebRtcMessage', req: WebRtcMessage} +export type SubmitWebRtcMessage_Output = ResultError | ({ status: 'OK' } & WebRtcAnswer) + export type UpdateCallbackUrl_Input = {rpcName:'UpdateCallbackUrl', req: CallbackUrl} export type UpdateCallbackUrl_Output = ResultError | ({ status: 'OK' } & CallbackUrl) @@ -351,6 +358,8 @@ export type ServerMethods = { SetMockAppBalance?: (req: SetMockAppBalance_Input & {ctx: AppContext }) => Promise SetMockAppUserBalance?: (req: SetMockAppUserBalance_Input & {ctx: AppContext }) => Promise SetMockInvoiceAsPaid?: (req: SetMockInvoiceAsPaid_Input & {ctx: GuestContext }) => Promise + SubToWebRtcCandidates?: (req: SubToWebRtcCandidates_Input & {ctx: MetricsContext }) => Promise + SubmitWebRtcMessage?: (req: SubmitWebRtcMessage_Input & {ctx: MetricsContext }) => Promise UpdateCallbackUrl?: (req: UpdateCallbackUrl_Input & {ctx: UserContext }) => Promise UpdateChannelPolicy?: (req: UpdateChannelPolicy_Input & {ctx: AdminContext }) => Promise UpdateUserOffer?: (req: UpdateUserOffer_Input & {ctx: UserContext }) => Promise @@ -3627,6 +3636,62 @@ export const UsersInfoValidate = (o?: UsersInfo, opts: UsersInfoOptions = {}, pa return null } +export type WebRtcAnswer = { + answer?: string +} +export type WebRtcAnswerOptionalField = 'answer' +export const WebRtcAnswerOptionalFields: WebRtcAnswerOptionalField[] = ['answer'] +export type WebRtcAnswerOptions = OptionsBaseMessage & { + checkOptionalsAreSet?: WebRtcAnswerOptionalField[] + answer_CustomCheck?: (v?: string) => boolean +} +export const WebRtcAnswerValidate = (o?: WebRtcAnswer, opts: WebRtcAnswerOptions = {}, path: string = 'WebRtcAnswer::root.'): Error | null => { + if (opts.checkOptionalsAreSet && opts.allOptionalsAreSet) return new Error(path + ': only one of checkOptionalsAreSet or allOptionalNonDefault can be set for each message') + if (typeof o !== 'object' || o === null) return new Error(path + ': object is not an instance of an object or is null') + + if ((o.answer || opts.allOptionalsAreSet || opts.checkOptionalsAreSet?.includes('answer')) && typeof o.answer !== 'string') return new Error(`${path}.answer: is not a string`) + if (opts.answer_CustomCheck && !opts.answer_CustomCheck(o.answer)) return new Error(`${path}.answer: custom check failed`) + + return null +} + +export type WebRtcCandidate = { + candidate: string +} +export const WebRtcCandidateOptionalFields: [] = [] +export type WebRtcCandidateOptions = OptionsBaseMessage & { + checkOptionalsAreSet?: [] + candidate_CustomCheck?: (v: string) => boolean +} +export const WebRtcCandidateValidate = (o?: WebRtcCandidate, opts: WebRtcCandidateOptions = {}, path: string = 'WebRtcCandidate::root.'): Error | null => { + if (opts.checkOptionalsAreSet && opts.allOptionalsAreSet) return new Error(path + ': only one of checkOptionalsAreSet or allOptionalNonDefault can be set for each message') + if (typeof o !== 'object' || o === null) return new Error(path + ': object is not an instance of an object or is null') + + if (typeof o.candidate !== 'string') return new Error(`${path}.candidate: is not a string`) + if (opts.candidate_CustomCheck && !opts.candidate_CustomCheck(o.candidate)) return new Error(`${path}.candidate: custom check failed`) + + return null +} + +export type WebRtcMessage = { + message: WebRtcMessage_message +} +export const WebRtcMessageOptionalFields: [] = [] +export type WebRtcMessageOptions = OptionsBaseMessage & { + checkOptionalsAreSet?: [] + message_Options?: WebRtcMessage_messageOptions +} +export const WebRtcMessageValidate = (o?: WebRtcMessage, opts: WebRtcMessageOptions = {}, path: string = 'WebRtcMessage::root.'): Error | null => { + if (opts.checkOptionalsAreSet && opts.allOptionalsAreSet) return new Error(path + ': only one of checkOptionalsAreSet or allOptionalNonDefault can be set for each message') + if (typeof o !== 'object' || o === null) return new Error(path + ': object is not an instance of an object or is null') + + const messageErr = WebRtcMessage_messageValidate(o.message, opts.message_Options, `${path}.message`) + if (messageErr !== null) return messageErr + + + return null +} + export enum DebitResponse_response_type { DENIED = 'denied', INVOICE = 'invoice', @@ -3825,3 +3890,38 @@ export const UpdateChannelPolicyRequest_updateValidate = (o?: UpdateChannelPolic } return null } +export enum WebRtcMessage_message_type { + CANDIDATE = 'candidate', + OFFER = 'offer', +} +export const enumCheckWebRtcMessage_message_type = (e?: WebRtcMessage_message_type): boolean => { + for (const v in WebRtcMessage_message_type) if (e === v) return true + return false +} +export type WebRtcMessage_message = + {type:WebRtcMessage_message_type.CANDIDATE, candidate:string}| + {type:WebRtcMessage_message_type.OFFER, offer:string} + +export type WebRtcMessage_messageOptions = { + candidate_CustomCheck?: (v: string) => boolean + offer_CustomCheck?: (v: string) => boolean +} +export const WebRtcMessage_messageValidate = (o?: WebRtcMessage_message, opts:WebRtcMessage_messageOptions = {}, path: string = 'WebRtcMessage_message::root.'): Error | null => { + if (typeof o !== 'object' || o === null) return new Error(path + ': object is not an instance of an object or is null') + const stringType: string = o.type + switch (o.type) { + case WebRtcMessage_message_type.CANDIDATE: + if (typeof o.candidate !== 'string') return new Error(`${path}.candidate: is not a string`) + if (opts.candidate_CustomCheck && !opts.candidate_CustomCheck(o.candidate)) return new Error(`${path}.candidate: custom check failed`) + + break + case WebRtcMessage_message_type.OFFER: + if (typeof o.offer !== 'string') return new Error(`${path}.offer: is not a string`) + if (opts.offer_CustomCheck && !opts.offer_CustomCheck(o.offer)) return new Error(`${path}.offer: custom check failed`) + + break + default: + return new Error(path + ': unknown type '+ stringType) + } + return null +} diff --git a/proto/service/methods.proto b/proto/service/methods.proto index 2b15c83e..1b404f2b 100644 --- a/proto/service/methods.proto +++ b/proto/service/methods.proto @@ -41,10 +41,13 @@ option (file_options) = { id: "metrics", name: "Metrics", //encrypted:true, - context:{ + context:[{ key:"operator_id", value:"string" - } + },{ + key:"app_id", + value:"string" + }] }, { id:"app", @@ -204,6 +207,18 @@ service LightningPub { option (http_route) = "/api/reports/lnd"; option (nostr) = true; } + rpc SubmitWebRtcMessage(structs.WebRtcMessage) returns (structs.WebRtcAnswer) { + option (auth_type) = "Metrics"; + option (http_method) = "post"; + option (http_route) = "/api/upgrade/wrtc"; + option (nostr) = true; + } + rpc SubToWebRtcCandidates(structs.Empty) returns (stream structs.WebRtcCandidate) { + option (auth_type) = "Metrics"; + option (http_method) = "post"; + option (http_route) = "/api/upgrade/wrtc/candidates"; + option (nostr) = true; + } rpc CreateOneTimeInviteLink(structs.CreateOneTimeInviteLinkRequest) returns (structs.CreateOneTimeInviteLinkResponse) { option (auth_type) = "Admin"; diff --git a/proto/service/structs.proto b/proto/service/structs.proto index 60960d8f..4fa295e8 100644 --- a/proto/service/structs.proto +++ b/proto/service/structs.proto @@ -206,6 +206,21 @@ message LndChannels { repeated OpenChannel open_channels = 1; } +message WebRtcMessage { + oneof message { + string offer = 1; + string candidate = 2; + } +} + +message WebRtcAnswer { + optional string answer = 1; +} + +message WebRtcCandidate { + string candidate = 1; +} + message OpenChannelRequest{ diff --git a/src/auth.ts b/src/auth.ts index aba27bff..e1534b34 100644 --- a/src/auth.ts +++ b/src/auth.ts @@ -67,6 +67,6 @@ const metricsAuth = async (header: string | undefined): Promise if (h !== metricsToken) { throw new Error("metrics token invalid") } - return { operator_id: "metrics1" } + return { operator_id: "http_operator", app_id: "" } } export default serverOptions \ No newline at end of file diff --git a/src/nostrMiddleware.ts b/src/nostrMiddleware.ts index 541bb772..39d79605 100644 --- a/src/nostrMiddleware.ts +++ b/src/nostrMiddleware.ts @@ -27,7 +27,7 @@ export default (serverMethods: Types.ServerMethods, mainHandler: Main, nostrSett if (!adminNpub) { throw new Error("admin access not configured") } if (pub !== adminNpub) { throw new Error("Metrics unavailable") } log("operator access from", pub) - return { operator_id: pub } + return { operator_id: pub, app_id: appId || "" } }, metricsCallback: metrics => mainHandler.settings.recordPerformance ? mainHandler.metricsManager.AddMetrics(metrics) : null, NostrGuestWithPubAuthGuard: async (appId, pub) => { diff --git a/src/services/main/index.ts b/src/services/main/index.ts index e2f4eb92..e7c8e41d 100644 --- a/src/services/main/index.ts +++ b/src/services/main/index.ts @@ -25,6 +25,7 @@ import { defaultInvoiceExpiry } from "../storage/paymentStorage.js" import { DebitManager } from "./debitManager.js" import { NofferData } from "nostr-tools/lib/types/nip69.js" import { OfferManager } from "./offerManager.js" +import webRTC from "../webRTC/index.js" type UserOperationsSub = { id: string @@ -54,6 +55,7 @@ export default class { utils: Utils rugPullTracker: RugPullTracker unlocker: Unlocker + webRTC: webRTC nostrSend: NostrSend = () => { getLogger({})("nostr send not initialized yet") } constructor(settings: MainSettings, storage: Storage, adminManager: AdminManager, utils: Utils, unlocker: Unlocker) { this.settings = settings @@ -74,6 +76,7 @@ export default class { this.appUserManager = new AppUserManager(this.storage, this.settings, this.applicationManager) this.debitManager = new DebitManager(this.storage, this.lnd, this.applicationManager) this.offerManager = new OfferManager(this.storage, this.lnd, this.applicationManager, this.productManager) + this.webRTC = new webRTC() } @@ -94,6 +97,7 @@ export default class { this.liquidityProvider.attachNostrSend(f) this.debitManager.attachNostrSend(f) this.offerManager.attachNostrSend(f) + this.webRTC.attachNostrSend(f) } htlcCb: HtlcCb = (e) => { diff --git a/src/services/serverMethods/index.ts b/src/services/serverMethods/index.ts index d01ab79b..246985bf 100644 --- a/src/services/serverMethods/index.ts +++ b/src/services/serverMethods/index.ts @@ -4,6 +4,16 @@ import main from '../main/index.js' import Main from '../main/index.js' export default (mainHandler: Main): Types.ServerMethods => { return { + SubmitWebRtcMessage: async ({ ctx, req }) => { + const err = Types.WebRtcMessageValidate(req, { + message_Options: { + candidate_CustomCheck: candidate => candidate !== '', + offer_CustomCheck: offer => offer !== '', + } + }) + return mainHandler.webRTC.OnMessage({ userPub: ctx.operator_id, appId: ctx.app_id }, req.message) + }, + SubToWebRtcCandidates: async ({ ctx }) => { }, GetUsageMetrics: async ({ ctx, req }) => { return mainHandler.metricsManager.GetUsageMetrics(req) }, diff --git a/src/services/webRTC/index.ts b/src/services/webRTC/index.ts new file mode 100644 index 00000000..4be68c1e --- /dev/null +++ b/src/services/webRTC/index.ts @@ -0,0 +1,89 @@ +import { getLogger } from "../helpers/logger.js" +import * as Types from '../../../proto/autogenerated/ts/types.js' +import { NostrSend, SendData, SendInitiator } from "../nostr/handler.js" +type IceCandidate = { type: string, candidate?: string, sdpMid?: string, sdpMLineIndex?: number } +const configuration = { 'iceServers': [{ 'urls': 'stun:stun.l.google.com:19302' }] } +type UserInfo = { userPub: string, appId: string } +export default class webRTC { + + private log = getLogger({ component: 'webRTC' }) + private connections: Record = {} + private _nostrSend: NostrSend + attachNostrSend(f: NostrSend) { + this._nostrSend = f + } + private nostrSend: NostrSend = (initiator: SendInitiator, data: SendData, relays?: string[] | undefined) => { + if (!this._nostrSend) { + throw new Error("No nostrSend attached") + } + this._nostrSend(initiator, data, relays) + } + + private sendCandidate = (u: UserInfo, candidate: string) => { + const message: Types.WebRtcCandidate & { requestId: string, status: 'OK' } = { candidate, requestId: "SubToWebRtcCandidates", status: 'OK' } + this.nostrSend({ type: 'app', appId: u.appId }, { type: 'content', content: JSON.stringify(message), pub: u.userPub }) + } + + OnMessage = async (u: UserInfo, message: Types.WebRtcMessage_message): Promise => { + if (message.type === Types.WebRtcMessage_message_type.OFFER) { + return this.connect(u, message.offer) + } else if (message.type === Types.WebRtcMessage_message_type.CANDIDATE) { + return this.onCandidate(u, message.candidate) + } + return {} + + } + private onCandidate = async (u: UserInfo, candidate: string): Promise => { + const key = this.getConnectionsKey(u) + if (!this.connections[key]) { + throw new Error('Connection not found') + } + const conn = this.connections[key] + const iceCandidate: IceCandidate = JSON.parse(candidate) + if (!iceCandidate.candidate) { + await conn.addIceCandidate(null); + } else { + await conn.addIceCandidate(iceCandidate); + } + return {} + } + private connect = async (u: UserInfo, offer: string): Promise => { + const key = this.getConnectionsKey(u) + if (this.connections[key]) { + throw new Error('Connection already exists') + } + const conn = new RTCPeerConnection(configuration) + conn.onicecandidate = (event) => { + const message: IceCandidate = { + type: 'candidate' + }; + if (event.candidate) { + message.candidate = event.candidate.candidate; + message.sdpMid = event.candidate.sdpMid || undefined; + message.sdpMLineIndex = event.candidate.sdpMLineIndex || undefined; + } + this.sendCandidate(u, JSON.stringify(message)) + } + conn.onconnectionstatechange = (event) => { + console.log('onconnectionstatechange', event) + } + conn.ondatachannel = (event) => { + console.log('ondatachannel', event) + } + conn.oniceconnectionstatechange = (event) => { + console.log('oniceconnectionstatechange', event) + } + conn.onicegatheringstatechange = (event) => { + console.log('onicegatheringstatechange', event) + } + await conn.setRemoteDescription(JSON.parse(offer)) + const answer = await conn.createAnswer() + await conn.setLocalDescription(answer) + this.connections[key] = conn + return { answer: JSON.stringify(answer) } + } + + getConnectionsKey = (u: UserInfo) => { + return u.appId + ":" + u.userPub + } +} \ No newline at end of file From 0553d9cd6a0b70f48f7d758bf876d4211d1fdfe0 Mon Sep 17 00:00:00 2001 From: boufni95 Date: Mon, 27 Jan 2025 21:18:24 +0000 Subject: [PATCH 02/19] fix --- src/services/webRTC/index.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/services/webRTC/index.ts b/src/services/webRTC/index.ts index 4be68c1e..9efd59df 100644 --- a/src/services/webRTC/index.ts +++ b/src/services/webRTC/index.ts @@ -41,7 +41,7 @@ export default class webRTC { const conn = this.connections[key] const iceCandidate: IceCandidate = JSON.parse(candidate) if (!iceCandidate.candidate) { - await conn.addIceCandidate(null); + await conn.addIceCandidate(undefined); } else { await conn.addIceCandidate(iceCandidate); } From d3c728d6be35b690da407b792c67c2202fa85903 Mon Sep 17 00:00:00 2001 From: boufni95 Date: Mon, 27 Jan 2025 21:24:12 +0000 Subject: [PATCH 03/19] deb --- src/services/webRTC/index.ts | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/services/webRTC/index.ts b/src/services/webRTC/index.ts index 9efd59df..22d39386 100644 --- a/src/services/webRTC/index.ts +++ b/src/services/webRTC/index.ts @@ -31,7 +31,6 @@ export default class webRTC { return this.onCandidate(u, message.candidate) } return {} - } private onCandidate = async (u: UserInfo, candidate: string): Promise => { const key = this.getConnectionsKey(u) @@ -49,6 +48,7 @@ export default class webRTC { } private connect = async (u: UserInfo, offer: string): Promise => { const key = this.getConnectionsKey(u) + console.log("connect", key) if (this.connections[key]) { throw new Error('Connection already exists') } @@ -80,6 +80,7 @@ export default class webRTC { const answer = await conn.createAnswer() await conn.setLocalDescription(answer) this.connections[key] = conn + console.log("answer", answer) return { answer: JSON.stringify(answer) } } From 001c289d241e2ee7c945555f76590799537d91ee Mon Sep 17 00:00:00 2001 From: boufni95 Date: Mon, 27 Jan 2025 21:53:04 +0000 Subject: [PATCH 04/19] wrtc dep --- package-lock.json | 38 ++++++++++++++++++++++++++++++++++++ package.json | 3 ++- src/services/webRTC/index.ts | 4 +++- 3 files changed, 43 insertions(+), 2 deletions(-) diff --git a/package-lock.json b/package-lock.json index 223d36de..e1d5285c 100644 --- a/package-lock.json +++ b/package-lock.json @@ -47,6 +47,7 @@ "websocket": "^1.0.34", "websocket-polyfill": "^0.0.3", "why-is-node-running": "^3.2.0", + "wrtc": "^0.4.7", "ws": "^8.18.0" }, "devDependencies": { @@ -1882,6 +1883,24 @@ "node": ">=8" } }, + "node_modules/domexception": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/domexception/-/domexception-1.0.1.tgz", + "integrity": "sha512-raigMkn7CJNNo6Ihro1fzG7wr3fHuYVytzquZKX5n0yizGsTcYgzdIUwj1X9pK0VvjeihV+XiclP+DjwbsSKug==", + "deprecated": "Use your platform's native DOMException instead", + "license": "MIT", + "optional": true, + "dependencies": { + "webidl-conversions": "^4.0.2" + } + }, + "node_modules/domexception/node_modules/webidl-conversions": { + "version": "4.0.2", + "resolved": "https://registry.npmjs.org/webidl-conversions/-/webidl-conversions-4.0.2.tgz", + "integrity": "sha512-YQ+BmxuTgd6UXZW3+ICGfyqRyHXVlD5GtQr5+qjiNW7bF0cqrzX500HVXPBOvgXb5YnzDd+h0zqyv61KUD7+Sg==", + "license": "BSD-2-Clause", + "optional": true + }, "node_modules/dotenv": { "version": "16.4.5", "resolved": "https://registry.npmjs.org/dotenv/-/dotenv-16.4.5.tgz", @@ -5742,6 +5761,25 @@ "resolved": "https://registry.npmjs.org/wrappy/-/wrappy-1.0.2.tgz", "integrity": "sha512-l4Sp/DRseor9wL6EvV2+TuQn63dMkPjZ/sp9XkghTEbV9KlPS1xUsZ3u7/IQO4wxtcFB4bgpQPRcR3QCvezPcQ==" }, + "node_modules/wrtc": { + "version": "0.4.7", + "resolved": "https://registry.npmjs.org/wrtc/-/wrtc-0.4.7.tgz", + "integrity": "sha512-P6Hn7VT4lfSH49HxLHcHhDq+aFf/jd9dPY7lDHeFhZ22N3858EKuwm2jmnlPzpsRGEPaoF6XwkcxY5SYnt4f/g==", + "bundleDependencies": [ + "node-pre-gyp" + ], + "hasInstallScript": true, + "license": "BSD-2-Clause", + "dependencies": { + "node-pre-gyp": "^0.13.0" + }, + "engines": { + "node": "^8.11.2 || >=10.0.0" + }, + "optionalDependencies": { + "domexception": "^1.0.1" + } + }, "node_modules/ws": { "version": "8.18.0", "resolved": "https://registry.npmjs.org/ws/-/ws-8.18.0.tgz", diff --git a/package.json b/package.json index 8720fc99..3a6c5f48 100644 --- a/package.json +++ b/package.json @@ -64,6 +64,7 @@ "websocket": "^1.0.34", "websocket-polyfill": "^0.0.3", "why-is-node-running": "^3.2.0", + "wrtc": "^0.4.7", "ws": "^8.18.0" }, "devDependencies": { @@ -82,4 +83,4 @@ "ts-node": "10.7.0", "typescript": "5.5.4" } -} \ No newline at end of file +} diff --git a/src/services/webRTC/index.ts b/src/services/webRTC/index.ts index 22d39386..721ef8e9 100644 --- a/src/services/webRTC/index.ts +++ b/src/services/webRTC/index.ts @@ -1,3 +1,5 @@ +//@ts-ignore +import wrtc from 'wrtc' import { getLogger } from "../helpers/logger.js" import * as Types from '../../../proto/autogenerated/ts/types.js' import { NostrSend, SendData, SendInitiator } from "../nostr/handler.js" @@ -52,7 +54,7 @@ export default class webRTC { if (this.connections[key]) { throw new Error('Connection already exists') } - const conn = new RTCPeerConnection(configuration) + const conn = new wrtc.RTCPeerConnection(configuration) as RTCPeerConnection conn.onicecandidate = (event) => { const message: IceCandidate = { type: 'candidate' From 4a55bd6b7776929cecbf40558b4eb50057c92076 Mon Sep 17 00:00:00 2001 From: boufni95 Date: Mon, 27 Jan 2025 22:05:44 +0000 Subject: [PATCH 05/19] up --- src/services/webRTC/index.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/services/webRTC/index.ts b/src/services/webRTC/index.ts index 721ef8e9..9e684e79 100644 --- a/src/services/webRTC/index.ts +++ b/src/services/webRTC/index.ts @@ -52,7 +52,7 @@ export default class webRTC { const key = this.getConnectionsKey(u) console.log("connect", key) if (this.connections[key]) { - throw new Error('Connection already exists') + this.connections[key].close() } const conn = new wrtc.RTCPeerConnection(configuration) as RTCPeerConnection conn.onicecandidate = (event) => { From bf161ac7baf50f092b13579af0e66579b26f862b Mon Sep 17 00:00:00 2001 From: boufni95 Date: Mon, 27 Jan 2025 22:12:43 +0000 Subject: [PATCH 06/19] up --- src/services/webRTC/index.ts | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/services/webRTC/index.ts b/src/services/webRTC/index.ts index 9e684e79..1c494a7f 100644 --- a/src/services/webRTC/index.ts +++ b/src/services/webRTC/index.ts @@ -71,6 +71,11 @@ export default class webRTC { } conn.ondatachannel = (event) => { console.log('ondatachannel', event) + const channel = event.channel + channel.addEventListener('message', (event) => { + console.log('message', event) + channel.send(event.data + " to you!") + }) } conn.oniceconnectionstatechange = (event) => { console.log('oniceconnectionstatechange', event) From 78a2bdc4b7c909b7a110d564ee771bff9a488e0c Mon Sep 17 00:00:00 2001 From: boufni95 Date: Tue, 28 Jan 2025 17:29:27 +0000 Subject: [PATCH 07/19] up --- src/services/webRTC/index.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/services/webRTC/index.ts b/src/services/webRTC/index.ts index 1c494a7f..419444ec 100644 --- a/src/services/webRTC/index.ts +++ b/src/services/webRTC/index.ts @@ -67,7 +67,7 @@ export default class webRTC { this.sendCandidate(u, JSON.stringify(message)) } conn.onconnectionstatechange = (event) => { - console.log('onconnectionstatechange', event) + console.log('onconnectionstatechange', event, conn.connectionState) } conn.ondatachannel = (event) => { console.log('ondatachannel', event) From 565da33c5ad28a8f304652f6aee2edadfce2840f Mon Sep 17 00:00:00 2001 From: boufni95 Date: Wed, 29 Jan 2025 15:13:21 +0000 Subject: [PATCH 08/19] send metric events --- src/services/helpers/tlv.ts | 19 ++++++++ src/services/storage/metricsEventStorage.ts | 7 ++- src/services/webRTC/index.ts | 53 ++++++++++++++++----- 3 files changed, 64 insertions(+), 15 deletions(-) diff --git a/src/services/helpers/tlv.ts b/src/services/helpers/tlv.ts index 7e8d6b08..df215090 100644 --- a/src/services/helpers/tlv.ts +++ b/src/services/helpers/tlv.ts @@ -2,6 +2,25 @@ import { bytesToHex, concatBytes } from '@noble/hashes/utils' import * as Types from '../../../proto/autogenerated/ts/types.js' export const utf8Decoder: TextDecoder = new TextDecoder('utf-8') export const utf8Encoder: TextEncoder = new TextEncoder() +export type DataPacket = { dataId: number, packetNum: number, totalPackets: number, data: Uint8Array } +export const encodeTLVDataPacket = (packInfo: DataPacket): TLV => { + const { data, dataId, packetNum, totalPackets } = packInfo + const tlv: TLV = {} + tlv[2] = [integerToUint8Array(dataId)] + tlv[3] = [integerToUint8Array(packetNum)] + tlv[4] = [integerToUint8Array(totalPackets)] + tlv[5] = [data] + return tlv +} + +export const decodeTLVDataPacket = (tlv: TLV): DataPacket => { + return { + dataId: parseInt(bytesToHex(tlv[2][0]), 16), + packetNum: parseInt(bytesToHex(tlv[3][0]), 16), + totalPackets: parseInt(bytesToHex(tlv[4][0]), 16), + data: tlv[5][0] + } +} export const encodeListTLV = (list: Uint8Array[]): TLV => { const tlv: TLV = {} diff --git a/src/services/storage/metricsEventStorage.ts b/src/services/storage/metricsEventStorage.ts index a1447883..619f903c 100644 --- a/src/services/storage/metricsEventStorage.ts +++ b/src/services/storage/metricsEventStorage.ts @@ -132,12 +132,15 @@ export default class { return metrics } - LoadMetricsFile = async (app: string, method: string, chunk: number): Promise => { + LoadRawMetricsFile = async (app: string, method: string, chunk: number): Promise => { if (!this.metaReady || !this.metricsMeta[app] || !this.metricsMeta[app][method] || !this.metricsMeta[app][method].chunks.includes(chunk)) { throw new Error("metrics not found") } const fullPath = [this.metricsPath, app, method, `${chunk}.mtlv`].join("/") - const tlv = fs.readFileSync(fullPath) + return fs.readFileSync(fullPath) + } + LoadMetricsFile = async (app: string, method: string, chunk: number): Promise => { + const tlv = await this.LoadRawMetricsFile(app, method, chunk) const decoded = decodeListTLV(parseTLV(tlv)) return { base_64_tlvs: decoded.map(d => Buffer.from(d).toString('base64')), diff --git a/src/services/webRTC/index.ts b/src/services/webRTC/index.ts index 419444ec..03b472fc 100644 --- a/src/services/webRTC/index.ts +++ b/src/services/webRTC/index.ts @@ -1,16 +1,21 @@ //@ts-ignore import wrtc from 'wrtc' -import { getLogger } from "../helpers/logger.js" +import Storage from '../storage/index.js' +import { ERROR, getLogger } from "../helpers/logger.js" import * as Types from '../../../proto/autogenerated/ts/types.js' import { NostrSend, SendData, SendInitiator } from "../nostr/handler.js" +import { encodeTLV, encodeTLVDataPacket } from '../helpers/tlv.js' type IceCandidate = { type: string, candidate?: string, sdpMid?: string, sdpMLineIndex?: number } -const configuration = { 'iceServers': [{ 'urls': 'stun:stun.l.google.com:19302' }] } +const configuration = { 'iceServers': [{ 'urls': 'stun.stunprotocol.org:3478' }] } type UserInfo = { userPub: string, appId: string } export default class webRTC { - + private storage: Storage private log = getLogger({ component: 'webRTC' }) private connections: Record = {} private _nostrSend: NostrSend + constructor(storage: Storage) { + this.storage = storage + } attachNostrSend(f: NostrSend) { this._nostrSend = f } @@ -67,22 +72,44 @@ export default class webRTC { this.sendCandidate(u, JSON.stringify(message)) } conn.onconnectionstatechange = (event) => { - console.log('onconnectionstatechange', event, conn.connectionState) + console.log('onconnectionstatechange', conn.connectionState) } conn.ondatachannel = (event) => { console.log('ondatachannel', event) const channel = event.channel - channel.addEventListener('message', (event) => { - console.log('message', event) - channel.send(event.data + " to you!") + channel.addEventListener('message', async (event) => { + const j = JSON.parse(event.data) as Types.SingleUsageMetricReq + const err = Types.SingleUsageMetricReqValidate(j, { + app_id_CustomCheck: id => id === u.appId, + metrics_name_CustomCheck: name => name !== "" + }) + if (err) { + this.log(ERROR, 'SingleUsageMetricReqValidate', err) + return + } + const res = await this.storage.metricsEventStorage.LoadRawMetricsFile(j.app_id, j.metrics_name, j.page) + const id = Math.floor(Math.random() * 2_000_000_000) + let i = 0 + const packets: Buffer[] = [] + while (i < res.length) { + const chunk = res.slice(i, Math.min(i + 15_000, res.length)) + packets.push(chunk) + i += 15_000 + } + for (let i = 0; i < packets.length; i++) { + const packet = packets[i] + const tlv = encodeTLVDataPacket({ dataId: id, packetNum: i + 1, totalPackets: packets.length, data: packet }) + const bytes = encodeTLV(tlv) + channel.send(bytes) + } }) } - conn.oniceconnectionstatechange = (event) => { - console.log('oniceconnectionstatechange', event) - } - conn.onicegatheringstatechange = (event) => { - console.log('onicegatheringstatechange', event) - } + /* conn.oniceconnectionstatechange = (event) => { + console.log('oniceconnectionstatechange', event) + } + conn.onicegatheringstatechange = (event) => { + console.log('onicegatheringstatechange', event) + } */ await conn.setRemoteDescription(JSON.parse(offer)) const answer = await conn.createAnswer() await conn.setLocalDescription(answer) From 5655e1692e91bce5d4fec17aff9b231f73fd9082 Mon Sep 17 00:00:00 2001 From: boufni95 Date: Wed, 29 Jan 2025 15:14:37 +0000 Subject: [PATCH 09/19] fix --- src/services/main/index.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/services/main/index.ts b/src/services/main/index.ts index e7c8e41d..08001081 100644 --- a/src/services/main/index.ts +++ b/src/services/main/index.ts @@ -76,7 +76,7 @@ export default class { this.appUserManager = new AppUserManager(this.storage, this.settings, this.applicationManager) this.debitManager = new DebitManager(this.storage, this.lnd, this.applicationManager) this.offerManager = new OfferManager(this.storage, this.lnd, this.applicationManager, this.productManager) - this.webRTC = new webRTC() + this.webRTC = new webRTC(this.storage) } From d11f83b05ad6f396b7d09df46c3ab00cb76475f6 Mon Sep 17 00:00:00 2001 From: boufni95 Date: Wed, 29 Jan 2025 15:21:14 +0000 Subject: [PATCH 10/19] deb --- src/services/webRTC/index.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/services/webRTC/index.ts b/src/services/webRTC/index.ts index 03b472fc..7eb5d78c 100644 --- a/src/services/webRTC/index.ts +++ b/src/services/webRTC/index.ts @@ -6,7 +6,7 @@ import * as Types from '../../../proto/autogenerated/ts/types.js' import { NostrSend, SendData, SendInitiator } from "../nostr/handler.js" import { encodeTLV, encodeTLVDataPacket } from '../helpers/tlv.js' type IceCandidate = { type: string, candidate?: string, sdpMid?: string, sdpMLineIndex?: number } -const configuration = { 'iceServers': [{ 'urls': 'stun.stunprotocol.org:3478' }] } +const configuration = { 'iceServers': [{ 'urls': 'stun:stun.l.google.com:19302' }] } type UserInfo = { userPub: string, appId: string } export default class webRTC { private storage: Storage From 3b2e376ffab088e5404484006bb5d6b57aad1470 Mon Sep 17 00:00:00 2001 From: boufni95 Date: Wed, 29 Jan 2025 15:25:05 +0000 Subject: [PATCH 11/19] try catch --- src/services/webRTC/index.ts | 50 +++++++++++++++++++----------------- 1 file changed, 27 insertions(+), 23 deletions(-) diff --git a/src/services/webRTC/index.ts b/src/services/webRTC/index.ts index 7eb5d78c..5bdf2ba6 100644 --- a/src/services/webRTC/index.ts +++ b/src/services/webRTC/index.ts @@ -78,29 +78,33 @@ export default class webRTC { console.log('ondatachannel', event) const channel = event.channel channel.addEventListener('message', async (event) => { - const j = JSON.parse(event.data) as Types.SingleUsageMetricReq - const err = Types.SingleUsageMetricReqValidate(j, { - app_id_CustomCheck: id => id === u.appId, - metrics_name_CustomCheck: name => name !== "" - }) - if (err) { - this.log(ERROR, 'SingleUsageMetricReqValidate', err) - return - } - const res = await this.storage.metricsEventStorage.LoadRawMetricsFile(j.app_id, j.metrics_name, j.page) - const id = Math.floor(Math.random() * 2_000_000_000) - let i = 0 - const packets: Buffer[] = [] - while (i < res.length) { - const chunk = res.slice(i, Math.min(i + 15_000, res.length)) - packets.push(chunk) - i += 15_000 - } - for (let i = 0; i < packets.length; i++) { - const packet = packets[i] - const tlv = encodeTLVDataPacket({ dataId: id, packetNum: i + 1, totalPackets: packets.length, data: packet }) - const bytes = encodeTLV(tlv) - channel.send(bytes) + try { + const j = JSON.parse(event.data) as Types.SingleUsageMetricReq + const err = Types.SingleUsageMetricReqValidate(j, { + app_id_CustomCheck: id => id === u.appId, + metrics_name_CustomCheck: name => name !== "" + }) + if (err) { + this.log(ERROR, 'SingleUsageMetricReqValidate', err) + return + } + const res = await this.storage.metricsEventStorage.LoadRawMetricsFile(j.app_id, j.metrics_name, j.page) + const id = Math.floor(Math.random() * 2_000_000_000) + let i = 0 + const packets: Buffer[] = [] + while (i < res.length) { + const chunk = res.slice(i, Math.min(i + 15_000, res.length)) + packets.push(chunk) + i += 15_000 + } + for (let i = 0; i < packets.length; i++) { + const packet = packets[i] + const tlv = encodeTLVDataPacket({ dataId: id, packetNum: i + 1, totalPackets: packets.length, data: packet }) + const bytes = encodeTLV(tlv) + channel.send(bytes) + } + } catch (e: any) { + this.log(ERROR, 'ondatachannel', e.message || e) } }) } From 21d6ed19669108c705d8506d1d7acfd6bafcc0b3 Mon Sep 17 00:00:00 2001 From: boufni95 Date: Wed, 29 Jan 2025 16:03:07 +0000 Subject: [PATCH 12/19] req id --- proto/autogenerated/client.md | 1 + proto/autogenerated/go/types.go | 1 + proto/autogenerated/ts/types.ts | 10 ++++++++-- proto/service/structs.proto | 1 + src/services/webRTC/index.ts | 2 +- 5 files changed, 12 insertions(+), 3 deletions(-) diff --git a/proto/autogenerated/client.md b/proto/autogenerated/client.md index e648b2b6..277807ee 100644 --- a/proto/autogenerated/client.md +++ b/proto/autogenerated/client.md @@ -1343,6 +1343,7 @@ The nostr server will send back a message response, and inside the body there wi - __app_id__: _string_ - __metrics_name__: _string_ - __page__: _number_ + - __request_id__: _number_ *this field is optional ### UpdateChannelPolicyRequest - __policy__: _[ChannelPolicy](#ChannelPolicy)_ diff --git a/proto/autogenerated/go/types.go b/proto/autogenerated/go/types.go index b4479edb..11158b1b 100644 --- a/proto/autogenerated/go/types.go +++ b/proto/autogenerated/go/types.go @@ -559,6 +559,7 @@ type SingleUsageMetricReq struct { App_id string `json:"app_id"` Metrics_name string `json:"metrics_name"` Page int64 `json:"page"` + Request_id int64 `json:"request_id"` } type UpdateChannelPolicyRequest struct { Policy *ChannelPolicy `json:"policy"` diff --git a/proto/autogenerated/ts/types.ts b/proto/autogenerated/ts/types.ts index 042ebb35..76655f9a 100644 --- a/proto/autogenerated/ts/types.ts +++ b/proto/autogenerated/ts/types.ts @@ -3191,13 +3191,16 @@ export type SingleUsageMetricReq = { app_id: string metrics_name: string page: number + request_id?: number } -export const SingleUsageMetricReqOptionalFields: [] = [] +export type SingleUsageMetricReqOptionalField = 'request_id' +export const SingleUsageMetricReqOptionalFields: SingleUsageMetricReqOptionalField[] = ['request_id'] export type SingleUsageMetricReqOptions = OptionsBaseMessage & { - checkOptionalsAreSet?: [] + checkOptionalsAreSet?: SingleUsageMetricReqOptionalField[] app_id_CustomCheck?: (v: string) => boolean metrics_name_CustomCheck?: (v: string) => boolean page_CustomCheck?: (v: number) => boolean + request_id_CustomCheck?: (v?: number) => boolean } export const SingleUsageMetricReqValidate = (o?: SingleUsageMetricReq, opts: SingleUsageMetricReqOptions = {}, path: string = 'SingleUsageMetricReq::root.'): Error | null => { if (opts.checkOptionalsAreSet && opts.allOptionalsAreSet) return new Error(path + ': only one of checkOptionalsAreSet or allOptionalNonDefault can be set for each message') @@ -3212,6 +3215,9 @@ export const SingleUsageMetricReqValidate = (o?: SingleUsageMetricReq, opts: Sin if (typeof o.page !== 'number') return new Error(`${path}.page: is not a number`) if (opts.page_CustomCheck && !opts.page_CustomCheck(o.page)) return new Error(`${path}.page: custom check failed`) + if ((o.request_id || opts.allOptionalsAreSet || opts.checkOptionalsAreSet?.includes('request_id')) && typeof o.request_id !== 'number') return new Error(`${path}.request_id: is not a number`) + if (opts.request_id_CustomCheck && !opts.request_id_CustomCheck(o.request_id)) return new Error(`${path}.request_id: custom check failed`) + return null } diff --git a/proto/service/structs.proto b/proto/service/structs.proto index 4fa295e8..2e74b8fa 100644 --- a/proto/service/structs.proto +++ b/proto/service/structs.proto @@ -45,6 +45,7 @@ message SingleUsageMetricReq { string app_id = 1; string metrics_name = 2; int64 page = 3; + optional int64 request_id = 4; } message UsageMetric { diff --git a/src/services/webRTC/index.ts b/src/services/webRTC/index.ts index 5bdf2ba6..74c8b655 100644 --- a/src/services/webRTC/index.ts +++ b/src/services/webRTC/index.ts @@ -89,7 +89,7 @@ export default class webRTC { return } const res = await this.storage.metricsEventStorage.LoadRawMetricsFile(j.app_id, j.metrics_name, j.page) - const id = Math.floor(Math.random() * 2_000_000_000) + const id = j.request_id || Math.floor(Math.random() * 2_000_000_000) let i = 0 const packets: Buffer[] = [] while (i < res.length) { From 02a522a1884fb50044fec0fad50ff0ec538625a7 Mon Sep 17 00:00:00 2001 From: boufni95 Date: Wed, 29 Jan 2025 16:31:33 +0000 Subject: [PATCH 13/19] deb --- src/services/webRTC/index.ts | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/services/webRTC/index.ts b/src/services/webRTC/index.ts index 74c8b655..652ad445 100644 --- a/src/services/webRTC/index.ts +++ b/src/services/webRTC/index.ts @@ -89,7 +89,8 @@ export default class webRTC { return } const res = await this.storage.metricsEventStorage.LoadRawMetricsFile(j.app_id, j.metrics_name, j.page) - const id = j.request_id || Math.floor(Math.random() * 2_000_000_000) + const id = j.request_id || Math.floor(Math.random() * 100_000_000) + console.log("processing req:", j, "id:", id) let i = 0 const packets: Buffer[] = [] while (i < res.length) { From c0459de3acc9923fb5d774a13a51725da5f810ac Mon Sep 17 00:00:00 2001 From: boufni95 Date: Wed, 29 Jan 2025 16:57:43 +0000 Subject: [PATCH 14/19] deb --- src/services/webRTC/index.ts | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/services/webRTC/index.ts b/src/services/webRTC/index.ts index 652ad445..c0f18af7 100644 --- a/src/services/webRTC/index.ts +++ b/src/services/webRTC/index.ts @@ -102,6 +102,9 @@ export default class webRTC { const packet = packets[i] const tlv = encodeTLVDataPacket({ dataId: id, packetNum: i + 1, totalPackets: packets.length, data: packet }) const bytes = encodeTLV(tlv) + if (i === 0) { + console.log("sending first packet", bytes) + } channel.send(bytes) } } catch (e: any) { From 9a8213591aaa9f25df91f4ddad348ad069e07ab9 Mon Sep 17 00:00:00 2001 From: boufni95 Date: Wed, 29 Jan 2025 17:31:25 +0000 Subject: [PATCH 15/19] TLbV --- src/services/helpers/tlv.ts | 36 +++++++++++++++++++++++++++++++++++- src/services/webRTC/index.ts | 6 +++--- 2 files changed, 38 insertions(+), 4 deletions(-) diff --git a/src/services/helpers/tlv.ts b/src/services/helpers/tlv.ts index df215090..e86b9ada 100644 --- a/src/services/helpers/tlv.ts +++ b/src/services/helpers/tlv.ts @@ -76,12 +76,14 @@ export const integerToUint8Array = (number: number): Uint8Array => { } export type TLV = { [t: number]: Uint8Array[] } -export const parseTLV = (data: Uint8Array): TLV => { +export type TLbV = { [t: number]: Uint8Array[] } +export const parseTLV = (data: Uint8Array, log = false): TLV => { const result: TLV = {} let rest = data while (rest.length > 0) { const t = rest[0] const l = rest[1] + if (log) console.log({ t, l }) const v = rest.slice(2, 2 + l) rest = rest.slice(2 + l) if (v.length < l) throw new Error(`not enough data to read on TLV ${t}`) @@ -97,6 +99,7 @@ export const encodeTLV = (tlv: TLV): Uint8Array => { .reverse() .forEach(([t, vs]) => { vs.forEach(v => { + if (v.length > 255) throw new Error(`value too long to encode in TLV ${t}`) const entry = new Uint8Array(v.length + 2) entry.set([parseInt(t)], 0) entry.set([v.length], 1) @@ -106,4 +109,35 @@ export const encodeTLV = (tlv: TLV): Uint8Array => { }) return concatBytes(...entries) +} +export const parseTLbV = (data: Uint8Array, log = false): TLV => { + const result: TLV = {} + let rest = data + while (rest.length > 0) { + const t = rest[0] + const l = parseInt(bytesToHex(rest.slice(1, 5)), 16) + if (log) console.log({ t, l }) + const v = rest.slice(5, 5 + l) + rest = rest.slice(5 + l) + if (v.length < l) throw new Error(`not enough data to read on TLV ${t}`) + result[t] = result[t] || [] + result[t].push(v) + } + return result +} + +export const encodeTLbV = (tlv: TLV): Uint8Array => { + const entries: Uint8Array[] = [] + Object.entries(tlv) + .reverse() + .forEach(([t, vs]) => { + vs.forEach(v => { + const entry = new Uint8Array(v.length + 5) + entry.set([parseInt(t)], 0) + entry.set(integerToUint8Array(v.length), 1) + entry.set(v, 5) + entries.push(entry) + }) + }) + return concatBytes(...entries) } \ No newline at end of file diff --git a/src/services/webRTC/index.ts b/src/services/webRTC/index.ts index c0f18af7..f3e70db0 100644 --- a/src/services/webRTC/index.ts +++ b/src/services/webRTC/index.ts @@ -4,7 +4,7 @@ import Storage from '../storage/index.js' import { ERROR, getLogger } from "../helpers/logger.js" import * as Types from '../../../proto/autogenerated/ts/types.js' import { NostrSend, SendData, SendInitiator } from "../nostr/handler.js" -import { encodeTLV, encodeTLVDataPacket } from '../helpers/tlv.js' +import { encodeTLbV, encodeTLV, encodeTLVDataPacket } from '../helpers/tlv.js' type IceCandidate = { type: string, candidate?: string, sdpMid?: string, sdpMLineIndex?: number } const configuration = { 'iceServers': [{ 'urls': 'stun:stun.l.google.com:19302' }] } type UserInfo = { userPub: string, appId: string } @@ -101,9 +101,9 @@ export default class webRTC { for (let i = 0; i < packets.length; i++) { const packet = packets[i] const tlv = encodeTLVDataPacket({ dataId: id, packetNum: i + 1, totalPackets: packets.length, data: packet }) - const bytes = encodeTLV(tlv) + const bytes = encodeTLbV(tlv) if (i === 0) { - console.log("sending first packet", bytes) + console.log("sending first packet", tlv) } channel.send(bytes) } From e6c3e9654e23fa28971d8bb03b009f7d22e877f6 Mon Sep 17 00:00:00 2001 From: boufni95 Date: Wed, 29 Jan 2025 17:46:07 +0000 Subject: [PATCH 16/19] less logs --- src/services/main/appUserManager.ts | 1 - src/services/main/debitManager.ts | 2 -- src/services/nostr/handler.ts | 1 - src/services/webRTC/index.ts | 21 ++++++--------------- 4 files changed, 6 insertions(+), 19 deletions(-) diff --git a/src/services/main/appUserManager.ts b/src/services/main/appUserManager.ts index 89f25494..de130262 100644 --- a/src/services/main/appUserManager.ts +++ b/src/services/main/appUserManager.ts @@ -53,7 +53,6 @@ export default class { const user = await this.storage.userStorage.GetUser(ctx.user_id) const app = await this.storage.applicationStorage.GetApplication(ctx.app_id) const appUser = await this.storage.applicationStorage.GetAppUserFromUser(app, user.user_id) - console.log("User Identifier/pointer here", appUser?.identifier) if (!appUser) { throw new Error(`app user ${ctx.user_id} not found`) // TODO: fix logs doxing diff --git a/src/services/main/debitManager.ts b/src/services/main/debitManager.ts index 881039e8..23fd8b2c 100644 --- a/src/services/main/debitManager.ts +++ b/src/services/main/debitManager.ts @@ -194,9 +194,7 @@ export class DebitManager { if (!this._nostrSend) { throw new Error("No nostrSend attached") } - console.log({ pointerdata, event }) const res = await this.payNdebitInvoice(event, pointerdata) - console.log({ debitRes: res }) if (res.status === 'fail' || res.status === 'authOk') { const e = newNdebitResponse(JSON.stringify(res.debitRes), event) this.nostrSend({ type: 'app', appId: event.appId }, { type: 'event', event: e, encrypt: { toPub: event.pub } }) diff --git a/src/services/nostr/handler.ts b/src/services/nostr/handler.ts index 33d1263b..c84a2c5f 100644 --- a/src/services/nostr/handler.ts +++ b/src/services/nostr/handler.ts @@ -212,7 +212,6 @@ export default class Handler { tags: [['p', data.pub]], } } else { - console.log(data) toSign = data.event if (data.encrypt) { try { diff --git a/src/services/webRTC/index.ts b/src/services/webRTC/index.ts index f3e70db0..0358e398 100644 --- a/src/services/webRTC/index.ts +++ b/src/services/webRTC/index.ts @@ -55,7 +55,7 @@ export default class webRTC { } private connect = async (u: UserInfo, offer: string): Promise => { const key = this.getConnectionsKey(u) - console.log("connect", key) + this.log("connect", key) if (this.connections[key]) { this.connections[key].close() } @@ -71,11 +71,13 @@ export default class webRTC { } this.sendCandidate(u, JSON.stringify(message)) } - conn.onconnectionstatechange = (event) => { - console.log('onconnectionstatechange', conn.connectionState) + conn.onconnectionstatechange = () => { + if (conn.connectionState === 'disconnected') { + conn.close() + delete this.connections[key] + } } conn.ondatachannel = (event) => { - console.log('ondatachannel', event) const channel = event.channel channel.addEventListener('message', async (event) => { try { @@ -90,7 +92,6 @@ export default class webRTC { } const res = await this.storage.metricsEventStorage.LoadRawMetricsFile(j.app_id, j.metrics_name, j.page) const id = j.request_id || Math.floor(Math.random() * 100_000_000) - console.log("processing req:", j, "id:", id) let i = 0 const packets: Buffer[] = [] while (i < res.length) { @@ -102,9 +103,6 @@ export default class webRTC { const packet = packets[i] const tlv = encodeTLVDataPacket({ dataId: id, packetNum: i + 1, totalPackets: packets.length, data: packet }) const bytes = encodeTLbV(tlv) - if (i === 0) { - console.log("sending first packet", tlv) - } channel.send(bytes) } } catch (e: any) { @@ -112,17 +110,10 @@ export default class webRTC { } }) } - /* conn.oniceconnectionstatechange = (event) => { - console.log('oniceconnectionstatechange', event) - } - conn.onicegatheringstatechange = (event) => { - console.log('onicegatheringstatechange', event) - } */ await conn.setRemoteDescription(JSON.parse(offer)) const answer = await conn.createAnswer() await conn.setLocalDescription(answer) this.connections[key] = conn - console.log("answer", answer) return { answer: JSON.stringify(answer) } } From 7dc6cffd1dc17657ee8c75c6c200f610fadc3473 Mon Sep 17 00:00:00 2001 From: boufni95 Date: Wed, 29 Jan 2025 17:51:02 +0000 Subject: [PATCH 17/19] stun update --- src/services/webRTC/index.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/services/webRTC/index.ts b/src/services/webRTC/index.ts index 0358e398..72561a22 100644 --- a/src/services/webRTC/index.ts +++ b/src/services/webRTC/index.ts @@ -6,7 +6,7 @@ import * as Types from '../../../proto/autogenerated/ts/types.js' import { NostrSend, SendData, SendInitiator } from "../nostr/handler.js" import { encodeTLbV, encodeTLV, encodeTLVDataPacket } from '../helpers/tlv.js' type IceCandidate = { type: string, candidate?: string, sdpMid?: string, sdpMLineIndex?: number } -const configuration = { 'iceServers': [{ 'urls': 'stun:stun.l.google.com:19302' }] } +const configuration = { 'iceServers': [{ 'urls': 'stunserver2024.stunprotocol.org' }] } type UserInfo = { userPub: string, appId: string } export default class webRTC { private storage: Storage From 635168516512805bddd22aee88cfbc9ab4232fa0 Mon Sep 17 00:00:00 2001 From: boufni95 Date: Wed, 29 Jan 2025 17:54:02 +0000 Subject: [PATCH 18/19] up --- src/services/webRTC/index.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/services/webRTC/index.ts b/src/services/webRTC/index.ts index 72561a22..8944c0f4 100644 --- a/src/services/webRTC/index.ts +++ b/src/services/webRTC/index.ts @@ -6,7 +6,7 @@ import * as Types from '../../../proto/autogenerated/ts/types.js' import { NostrSend, SendData, SendInitiator } from "../nostr/handler.js" import { encodeTLbV, encodeTLV, encodeTLVDataPacket } from '../helpers/tlv.js' type IceCandidate = { type: string, candidate?: string, sdpMid?: string, sdpMLineIndex?: number } -const configuration = { 'iceServers': [{ 'urls': 'stunserver2024.stunprotocol.org' }] } +const configuration = { 'iceServers': [{ 'urls': 'relay.webwormhole.io' }] } type UserInfo = { userPub: string, appId: string } export default class webRTC { private storage: Storage From d25450e022309940a0f38a991fb2a5d32d49baa2 Mon Sep 17 00:00:00 2001 From: boufni95 Date: Wed, 29 Jan 2025 17:56:41 +0000 Subject: [PATCH 19/19] prefix? --- src/services/webRTC/index.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/services/webRTC/index.ts b/src/services/webRTC/index.ts index 8944c0f4..bc90a0a6 100644 --- a/src/services/webRTC/index.ts +++ b/src/services/webRTC/index.ts @@ -6,7 +6,7 @@ import * as Types from '../../../proto/autogenerated/ts/types.js' import { NostrSend, SendData, SendInitiator } from "../nostr/handler.js" import { encodeTLbV, encodeTLV, encodeTLVDataPacket } from '../helpers/tlv.js' type IceCandidate = { type: string, candidate?: string, sdpMid?: string, sdpMLineIndex?: number } -const configuration = { 'iceServers': [{ 'urls': 'relay.webwormhole.io' }] } +const configuration = { 'iceServers': [{ 'urls': 'stun:relay.webwormhole.io' }] } type UserInfo = { userPub: string, appId: string } export default class webRTC { private storage: Storage