aiolabs/lightning-pub
2
0
Fork 0
Code Issues 9 Pull requests 4 Projects Releases Packages Wiki Activity Actions

reset metrics b4 stress test and export after

Browse source
This commit is contained in:
boufni95 2025-04-10 16:47:26 +00:00
parent f2445dd850
commit 67c1260748
22 changed files with 1241 additions and 34 deletions
Download patch file Download diff file Expand all files Collapse all files

865
package-lock.json generated
View file

File diff suppressed because it is too large Load diff

3
package.json
View file

@ -65,7 +65,8 @@
"websocket-polyfill": "^0.0.3",
"why-is-node-running": "^3.2.0",
"wrtc": "^0.4.7",
"ws": "^8.18.0"
"ws": "^8.18.0",
"zip-a-folder": "^3.1.9"
},
"devDependencies": {
"@types/chai": "^4.3.4",

27
proto/autogenerated/client.md
View file

@ -255,6 +255,11 @@ The nostr server will send back a message response, and inside the body there wi
- input: [DebitOperation](#DebitOperation)
- This methods has an __empty__ __response__ body
- ResetMetricsStorages
- auth type: __Admin__
- This methods has an __empty__ __request__ body
- This methods has an __empty__ __response__ body
- RespondToDebit
- auth type: __User__
- input: [DebitResponse](#DebitResponse)
@ -295,6 +300,11 @@ The nostr server will send back a message response, and inside the body there wi
- This methods has an __empty__ __request__ body
- output: [UserHealthState](#UserHealthState)
- ZipMetricsStorages
- auth type: __Admin__
- This methods has an __empty__ __request__ body
- output: [ZippedMetrics](#ZippedMetrics)
# HTTP API DEFINITION
## Supported HTTP Auths
@ -794,6 +804,13 @@ The nostr server will send back a message response, and inside the body there wi
- input: [DebitOperation](#DebitOperation)
- This methods has an __empty__ __response__ body
- ResetMetricsStorages
- auth type: __Admin__
- http method: __post__
- http route: __/api/admin/metrics/reset__
- This methods has an __empty__ __request__ body
- This methods has an __empty__ __response__ body
- ResetNPubLinkingToken
- auth type: __App__
- http method: __post__
@ -892,6 +909,13 @@ The nostr server will send back a message response, and inside the body there wi
- This methods has an __empty__ __request__ body
- output: [UserHealthState](#UserHealthState)
- ZipMetricsStorages
- auth type: __Admin__
- http method: __post__
- http route: __/api/admin/metrics/zip__
- This methods has an __empty__ __request__ body
- output: [ZippedMetrics](#ZippedMetrics)
# INPUTS AND OUTPUTS
## Messages
@ -1465,6 +1489,9 @@ The nostr server will send back a message response, and inside the body there wi
### WebRtcMessage
- __message__: _[WebRtcMessage_message](#WebRtcMessage_message)_
### ZippedMetrics
- __path__: _string_
## Enums
### The enumerators used in the messages

49
proto/autogenerated/go/http_client.go
View file

@ -118,6 +118,7 @@ type Client struct {
PayInvoice func(req PayInvoiceRequest) (*PayInvoiceResponse, error)
RequestNPubLinkingToken func(req RequestNPubLinkingTokenRequest) (*RequestNPubLinkingTokenResponse, error)
ResetDebit func(req DebitOperation) error
ResetMetricsStorages func() error
ResetNPubLinkingToken func(req RequestNPubLinkingTokenRequest) (*RequestNPubLinkingTokenResponse, error)
RespondToDebit func(req DebitResponse) error
SendAppUserToAppPayment func(req SendAppUserToAppPaymentRequest) error
@ -132,6 +133,7 @@ type Client struct {
UpdateUserOffer func(req OfferConfig) error
UseInviteLink func(req UseInviteLinkRequest) error
UserHealth func() (*UserHealthState, error)
ZipMetricsStorages func() (*ZippedMetrics, error)
}
func NewClient(params ClientParams) *Client {
@ -1752,6 +1754,27 @@ func NewClient(params ClientParams) *Client {
}
return nil
},
ResetMetricsStorages: func() error {
auth, err := params.RetrieveAdminAuth()
if err != nil {
return err
}
finalRoute := "/api/admin/metrics/reset"
body := []byte{}
resBody, err := doPostRequest(params.BaseURL+finalRoute, body, auth)
if err != nil {
return err
}
result := ResultError{}
err = json.Unmarshal(resBody, &result)
if err != nil {
return err
}
if result.Status == "ERROR" {
return fmt.Errorf(result.Reason)
}
return nil
},
ResetNPubLinkingToken: func(req RequestNPubLinkingTokenRequest) (*RequestNPubLinkingTokenResponse, error) {
auth, err := params.RetrieveAppAuth()
if err != nil {
@ -2082,5 +2105,31 @@ func NewClient(params ClientParams) *Client {
}
return &res, nil
},
ZipMetricsStorages: func() (*ZippedMetrics, error) {
auth, err := params.RetrieveAdminAuth()
if err != nil {
return nil, err
}
finalRoute := "/api/admin/metrics/zip"
body := []byte{}
resBody, err := doPostRequest(params.BaseURL+finalRoute, body, auth)
if err != nil {
return nil, err
}
result := ResultError{}
err = json.Unmarshal(resBody, &result)
if err != nil {
return nil, err
}
if result.Status == "ERROR" {
return nil, fmt.Errorf(result.Reason)
}
res := ZippedMetrics{}
err = json.Unmarshal(resBody, &res)
if err != nil {
return nil, err
}
return &res, nil
},
}
}

3
proto/autogenerated/go/types.go
View file

@ -665,6 +665,9 @@ type WebRtcCandidate struct {
type WebRtcMessage struct {
Message *WebRtcMessage_message `json:"message"`
}
type ZippedMetrics struct {
Path string `json:"path"`
}
type DebitResponse_response_type string
const (

38
proto/autogenerated/ts/express_server.ts
View file

@ -1632,6 +1632,25 @@ export default (methods: Types.ServerMethods, opts: ServerOptions) => {
opts.metricsCallback([{ ...info, ...stats, ...authContext }])
} catch (ex) { const e = ex as any; logErrorAndReturnResponse(e, e.message || e, res, logger, { ...info, ...stats, ...authCtx }, opts.metricsCallback); if (opts.throwErrors) throw e }
})
if (!opts.allowNotImplementedMethods && !methods.ResetMetricsStorages) throw new Error('method: ResetMetricsStorages is not implemented')
app.post('/api/admin/metrics/reset', async (req, res) => {
const info: Types.RequestInfo = { rpcName: 'ResetMetricsStorages', batch: false, nostr: false, batchSize: 0}
const stats: Types.RequestStats = { startMs:req.startTimeMs || 0, start:req.startTime || 0n, parse: process.hrtime.bigint(), guard: 0n, validate: 0n, handle: 0n }
let authCtx: Types.AuthContext = {}
try {
if (!methods.ResetMetricsStorages) throw new Error('method: ResetMetricsStorages is not implemented')
const authContext = await opts.AdminAuthGuard(req.headers['authorization'])
authCtx = authContext
stats.guard = process.hrtime.bigint()
stats.validate = stats.guard
const query = req.query
const params = req.params
await methods.ResetMetricsStorages({rpcName:'ResetMetricsStorages', ctx:authContext })
stats.handle = process.hrtime.bigint()
res.json({status: 'OK'})
opts.metricsCallback([{ ...info, ...stats, ...authContext }])
} catch (ex) { const e = ex as any; logErrorAndReturnResponse(e, e.message || e, res, logger, { ...info, ...stats, ...authCtx }, opts.metricsCallback); if (opts.throwErrors) throw e }
})
if (!opts.allowNotImplementedMethods && !methods.ResetNPubLinkingToken) throw new Error('method: ResetNPubLinkingToken is not implemented')
app.post('/api/app/user/npub/token/reset', async (req, res) => {
const info: Types.RequestInfo = { rpcName: 'ResetNPubLinkingToken', batch: false, nostr: false, batchSize: 0}
@ -1915,6 +1934,25 @@ export default (methods: Types.ServerMethods, opts: ServerOptions) => {
opts.metricsCallback([{ ...info, ...stats, ...authContext }])
} catch (ex) { const e = ex as any; logErrorAndReturnResponse(e, e.message || e, res, logger, { ...info, ...stats, ...authCtx }, opts.metricsCallback); if (opts.throwErrors) throw e }
})
if (!opts.allowNotImplementedMethods && !methods.ZipMetricsStorages) throw new Error('method: ZipMetricsStorages is not implemented')
app.post('/api/admin/metrics/zip', async (req, res) => {
const info: Types.RequestInfo = { rpcName: 'ZipMetricsStorages', batch: false, nostr: false, batchSize: 0}
const stats: Types.RequestStats = { startMs:req.startTimeMs || 0, start:req.startTime || 0n, parse: process.hrtime.bigint(), guard: 0n, validate: 0n, handle: 0n }
let authCtx: Types.AuthContext = {}
try {
if (!methods.ZipMetricsStorages) throw new Error('method: ZipMetricsStorages is not implemented')
const authContext = await opts.AdminAuthGuard(req.headers['authorization'])
authCtx = authContext
stats.guard = process.hrtime.bigint()
stats.validate = stats.guard
const query = req.query
const params = req.params
const response = await methods.ZipMetricsStorages({rpcName:'ZipMetricsStorages', ctx:authContext })
stats.handle = process.hrtime.bigint()
res.json({status: 'OK', ...response})
opts.metricsCallback([{ ...info, ...stats, ...authContext }])
} catch (ex) { const e = ex as any; logErrorAndReturnResponse(e, e.message || e, res, logger, { ...info, ...stats, ...authCtx }, opts.metricsCallback); if (opts.throwErrors) throw e }
})
if (opts.staticFiles) {
app.use(express.static(opts.staticFiles))
app.get('*', function (_, res) { res.sendFile('index.html', { root: opts.staticFiles })})

25
proto/autogenerated/ts/http_client.ts
View file

@ -839,6 +839,17 @@ export default (params: ClientParams) => ({
}
return { status: 'ERROR', reason: 'invalid response' }
},
ResetMetricsStorages: async (): Promise<ResultError | ({ status: 'OK' })> => {
const auth = await params.retrieveAdminAuth()
if (auth === null) throw new Error('retrieveAdminAuth() returned null')
let finalRoute = '/api/admin/metrics/reset'
const { data } = await axios.post(params.baseUrl + finalRoute, {}, { headers: { 'authorization': auth } })
if (data.status === 'ERROR' && typeof data.reason === 'string') return data
if (data.status === 'OK') {
return data
}
return { status: 'ERROR', reason: 'invalid response' }
},
ResetNPubLinkingToken: async (request: Types.RequestNPubLinkingTokenRequest): Promise<ResultError | ({ status: 'OK' }& Types.RequestNPubLinkingTokenResponse)> => {
const auth = await params.retrieveAppAuth()
if (auth === null) throw new Error('retrieveAppAuth() returned null')
@ -995,4 +1006,18 @@ export default (params: ClientParams) => ({
}
return { status: 'ERROR', reason: 'invalid response' }
},
ZipMetricsStorages: async (): Promise<ResultError | ({ status: 'OK' }& Types.ZippedMetrics)> => {
const auth = await params.retrieveAdminAuth()
if (auth === null) throw new Error('retrieveAdminAuth() returned null')
let finalRoute = '/api/admin/metrics/zip'
const { data } = await axios.post(params.baseUrl + finalRoute, {}, { headers: { 'authorization': auth } })
if (data.status === 'ERROR' && typeof data.reason === 'string') return data
if (data.status === 'OK') {
const result = data
if(!params.checkResult) return { status: 'OK', ...result }
const error = Types.ZippedMetricsValidate(result)
if (error === null) { return { status: 'OK', ...result } } else return { status: 'ERROR', reason: error.message }
}
return { status: 'ERROR', reason: 'invalid response' }
},
})

25
proto/autogenerated/ts/nostr_client.ts
View file

@ -698,6 +698,17 @@ export default (params: NostrClientParams, send: (to:string, message: NostrRequ
}
return { status: 'ERROR', reason: 'invalid response' }
},
ResetMetricsStorages: async (): Promise<ResultError | ({ status: 'OK' })> => {
const auth = await params.retrieveNostrAdminAuth()
if (auth === null) throw new Error('retrieveNostrAdminAuth() returned null')
const nostrRequest: NostrRequest = {}
const data = await send(params.pubDestination, {rpcName:'ResetMetricsStorages',authIdentifier:auth, ...nostrRequest })
if (data.status === 'ERROR' && typeof data.reason === 'string') return data
if (data.status === 'OK') {
return data
}
return { status: 'ERROR', reason: 'invalid response' }
},
RespondToDebit: async (request: Types.DebitResponse): Promise<ResultError | ({ status: 'OK' })> => {
const auth = await params.retrieveNostrUserAuth()
if (auth === null) throw new Error('retrieveNostrUserAuth() returned null')
@ -805,4 +816,18 @@ export default (params: NostrClientParams, send: (to:string, message: NostrRequ
}
return { status: 'ERROR', reason: 'invalid response' }
},
ZipMetricsStorages: async (): Promise<ResultError | ({ status: 'OK' }& Types.ZippedMetrics)> => {
const auth = await params.retrieveNostrAdminAuth()
if (auth === null) throw new Error('retrieveNostrAdminAuth() returned null')
const nostrRequest: NostrRequest = {}
const data = await send(params.pubDestination, {rpcName:'ZipMetricsStorages',authIdentifier:auth, ...nostrRequest })
if (data.status === 'ERROR' && typeof data.reason === 'string') return data
if (data.status === 'OK') {
const result = data
if(!params.checkResult) return { status: 'OK', ...result }
const error = Types.ZippedMetricsValidate(result)
if (error === null) { return { status: 'OK', ...result } } else return { status: 'ERROR', reason: error.message }
}
return { status: 'ERROR', reason: 'invalid response' }
},
})

26
proto/autogenerated/ts/nostr_transport.ts
View file

@ -1091,6 +1091,19 @@ export default (methods: Types.ServerMethods, opts: NostrOptions) => {
opts.metricsCallback([{ ...info, ...stats, ...authContext }])
}catch(ex){ const e = ex as any; logErrorAndReturnResponse(e, e.message || e, res, logger, { ...info, ...stats, ...authCtx }, opts.metricsCallback); if (opts.throwErrors) throw e }
break
case 'ResetMetricsStorages':
try {
if (!methods.ResetMetricsStorages) throw new Error('method: ResetMetricsStorages is not implemented')
const authContext = await opts.NostrAdminAuthGuard(req.appId, req.authIdentifier)
stats.guard = process.hrtime.bigint()
authCtx = authContext
stats.validate = stats.guard
await methods.ResetMetricsStorages({rpcName:'ResetMetricsStorages', ctx:authContext })
stats.handle = process.hrtime.bigint()
res({status: 'OK'})
opts.metricsCallback([{ ...info, ...stats, ...authContext }])
}catch(ex){ const e = ex as any; logErrorAndReturnResponse(e, e.message || e, res, logger, { ...info, ...stats, ...authCtx }, opts.metricsCallback); if (opts.throwErrors) throw e }
break
case 'RespondToDebit':
try {
if (!methods.RespondToDebit) throw new Error('method: RespondToDebit is not implemented')
@ -1213,6 +1226,19 @@ export default (methods: Types.ServerMethods, opts: NostrOptions) => {
opts.metricsCallback([{ ...info, ...stats, ...authContext }])
}catch(ex){ const e = ex as any; logErrorAndReturnResponse(e, e.message || e, res, logger, { ...info, ...stats, ...authCtx }, opts.metricsCallback); if (opts.throwErrors) throw e }
break
case 'ZipMetricsStorages':
try {
if (!methods.ZipMetricsStorages) throw new Error('method: ZipMetricsStorages is not implemented')
const authContext = await opts.NostrAdminAuthGuard(req.appId, req.authIdentifier)
stats.guard = process.hrtime.bigint()
authCtx = authContext
stats.validate = stats.guard
const response = await methods.ZipMetricsStorages({rpcName:'ZipMetricsStorages', ctx:authContext })
stats.handle = process.hrtime.bigint()
res({status: 'OK', ...response})
opts.metricsCallback([{ ...info, ...stats, ...authContext }])
}catch(ex){ const e = ex as any; logErrorAndReturnResponse(e, e.message || e, res, logger, { ...info, ...stats, ...authCtx }, opts.metricsCallback); if (opts.throwErrors) throw e }
break
default: logger.error('unknown rpc call name from nostr event:'+req.rpcName)
}
}

30
proto/autogenerated/ts/types.ts
View file

@ -7,8 +7,8 @@ export type RequestMetric = AuthContext & RequestInfo & RequestStats & { error?:
export type AdminContext = {
admin_id: string
}
export type AdminMethodInputs = AddApp_Input | AddPeer_Input | AuthApp_Input | BanUser_Input | CloseChannel_Input | CreateOneTimeInviteLink_Input | GetInviteLinkState_Input | GetSeed_Input | ListChannels_Input | LndGetInfo_Input | OpenChannel_Input | UpdateChannelPolicy_Input
export type AdminMethodOutputs = AddApp_Output | AddPeer_Output | AuthApp_Output | BanUser_Output | CloseChannel_Output | CreateOneTimeInviteLink_Output | GetInviteLinkState_Output | GetSeed_Output | ListChannels_Output | LndGetInfo_Output | OpenChannel_Output | UpdateChannelPolicy_Output
export type AdminMethodInputs = AddApp_Input | AddPeer_Input | AuthApp_Input | BanUser_Input | CloseChannel_Input | CreateOneTimeInviteLink_Input | GetInviteLinkState_Input | GetSeed_Input | ListChannels_Input | LndGetInfo_Input | OpenChannel_Input | ResetMetricsStorages_Input | UpdateChannelPolicy_Input | ZipMetricsStorages_Input
export type AdminMethodOutputs = AddApp_Output | AddPeer_Output | AuthApp_Output | BanUser_Output | CloseChannel_Output | CreateOneTimeInviteLink_Output | GetInviteLinkState_Output | GetSeed_Output | ListChannels_Output | LndGetInfo_Output | OpenChannel_Output | ResetMetricsStorages_Output | UpdateChannelPolicy_Output | ZipMetricsStorages_Output
export type AppContext = {
app_id: string
}
@ -253,6 +253,9 @@ export type RequestNPubLinkingToken_Output = ResultError | ({ status: 'OK' } & R
export type ResetDebit_Input = {rpcName:'ResetDebit', req: DebitOperation}
export type ResetDebit_Output = ResultError | { status: 'OK' }
export type ResetMetricsStorages_Input = {rpcName:'ResetMetricsStorages'}
export type ResetMetricsStorages_Output = ResultError | { status: 'OK' }
export type ResetNPubLinkingToken_Input = {rpcName:'ResetNPubLinkingToken', req: RequestNPubLinkingTokenRequest}
export type ResetNPubLinkingToken_Output = ResultError | ({ status: 'OK' } & RequestNPubLinkingTokenResponse)
@ -295,6 +298,9 @@ export type UseInviteLink_Output = ResultError | { status: 'OK' }
export type UserHealth_Input = {rpcName:'UserHealth'}
export type UserHealth_Output = ResultError | ({ status: 'OK' } & UserHealthState)
export type ZipMetricsStorages_Input = {rpcName:'ZipMetricsStorages'}
export type ZipMetricsStorages_Output = ResultError | ({ status: 'OK' } & ZippedMetrics)
export type ServerMethods = {
AddApp?: (req: AddApp_Input & {ctx: AdminContext }) => Promise<AuthApp>
AddAppInvoice?: (req: AddAppInvoice_Input & {ctx: AppContext }) => Promise<NewInvoiceResponse>
@ -359,6 +365,7 @@ export type ServerMethods = {
PayInvoice?: (req: PayInvoice_Input & {ctx: UserContext }) => Promise<PayInvoiceResponse>
RequestNPubLinkingToken?: (req: RequestNPubLinkingToken_Input & {ctx: AppContext }) => Promise<RequestNPubLinkingTokenResponse>
ResetDebit?: (req: ResetDebit_Input & {ctx: UserContext }) => Promise<void>
ResetMetricsStorages?: (req: ResetMetricsStorages_Input & {ctx: AdminContext }) => Promise<void>
ResetNPubLinkingToken?: (req: ResetNPubLinkingToken_Input & {ctx: AppContext }) => Promise<RequestNPubLinkingTokenResponse>
RespondToDebit?: (req: RespondToDebit_Input & {ctx: UserContext }) => Promise<void>
SendAppUserToAppPayment?: (req: SendAppUserToAppPayment_Input & {ctx: AppContext }) => Promise<void>
@ -373,6 +380,7 @@ export type ServerMethods = {
UpdateUserOffer?: (req: UpdateUserOffer_Input & {ctx: UserContext }) => Promise<void>
UseInviteLink?: (req: UseInviteLink_Input & {ctx: GuestWithPubContext }) => Promise<void>
UserHealth?: (req: UserHealth_Input & {ctx: UserContext }) => Promise<UserHealthState>
ZipMetricsStorages?: (req: ZipMetricsStorages_Input & {ctx: AdminContext }) => Promise<ZippedMetrics>
}
export enum AddressType {
@ -3816,6 +3824,24 @@ export const WebRtcMessageValidate = (o?: WebRtcMessage, opts: WebRtcMessageOpti
return null
}
export type ZippedMetrics = {
path: string
}
export const ZippedMetricsOptionalFields: [] = []
export type ZippedMetricsOptions = OptionsBaseMessage & {
checkOptionalsAreSet?: []
path_CustomCheck?: (v: string) => boolean
}
export const ZippedMetricsValidate = (o?: ZippedMetrics, opts: ZippedMetricsOptions = {}, path: string = 'ZippedMetrics::root.'): Error | null => {
if (opts.checkOptionalsAreSet && opts.allOptionalsAreSet) return new Error(path + ': only one of checkOptionalsAreSet or allOptionalNonDefault can be set for each message')
if (typeof o !== 'object' || o === null) return new Error(path + ': object is not an instance of an object or is null')
if (typeof o.path !== 'string') return new Error(`${path}.path: is not a string`)
if (opts.path_CustomCheck && !opts.path_CustomCheck(o.path)) return new Error(`${path}.path: custom check failed`)
return null
}
export enum DebitResponse_response_type {
DENIED = 'denied',
INVOICE = 'invoice',

14
proto/service/methods.proto
View file

@ -234,6 +234,20 @@ service LightningPub {
option (nostr) = true;
}
rpc ZipMetricsStorages(structs.Empty) returns (structs.ZippedMetrics) {
option (auth_type) = "Admin";
option (http_method) = "post";
option (http_route) = "/api/admin/metrics/zip";
option (nostr) = true;
}
rpc ResetMetricsStorages(structs.Empty) returns (structs.Empty) {
option (auth_type) = "Admin";
option (http_method) = "post";
option (http_route) = "/api/admin/metrics/reset";
option (nostr) = true;
}
rpc CreateOneTimeInviteLink(structs.CreateOneTimeInviteLinkRequest) returns (structs.CreateOneTimeInviteLinkResponse) {
option (auth_type) = "Admin";
option (http_method) = "post";

4
proto/service/structs.proto
View file

@ -10,6 +10,10 @@ message LndSeed {
repeated string seed = 1;
}
message ZippedMetrics {
string path = 1;
}
message EncryptionExchangeRequest {
string publicKey = 1;
string deviceId = 2;

8
src/services/helpers/utilsWrapper.ts
View file

@ -5,15 +5,15 @@ import { NostrSend } from "../nostr/handler.js";
import { ProcessMetricsCollector } from "../storage/tlv/processMetricsCollector.js";
type UtilsSettings = {
noCollector?: boolean
dataDir: string
dataDir: string,
allowResetMetricsStorages: boolean
}
export class Utils {
tlvStorageFactory: TlvStorageFactory
stateBundler: StateBundler
settings: MainSettings
_nostrSend: NostrSend = () => { throw new Error('nostr send not initialized yet') }
constructor({ noCollector, dataDir }: UtilsSettings) {
this.tlvStorageFactory = new TlvStorageFactory()
constructor({ noCollector, dataDir, allowResetMetricsStorages }: UtilsSettings) {
this.tlvStorageFactory = new TlvStorageFactory(allowResetMetricsStorages)
this.stateBundler = new StateBundler(dataDir, this.tlvStorageFactory)
if (!noCollector) {
new ProcessMetricsCollector((metrics) => {

2
src/services/main/init.ts
View file

@ -17,7 +17,7 @@ export type AppData = {
name: string;
}
export const initMainHandler = async (log: PubLogger, mainSettings: MainSettings) => {
const utils = new Utils({ dataDir: mainSettings.storageSettings.dataDir })
const utils = new Utils({ dataDir: mainSettings.storageSettings.dataDir, allowResetMetricsStorages: mainSettings.allowResetMetricsStorages })
const storageManager = new Storage(mainSettings.storageSettings, utils)
await storageManager.Connect(log)
/* const manualMigration = await TypeOrmMigrationRunner(log, storageManager, mainSettings.storageSettings.dbSettings, process.argv[2])

6
src/services/main/settings.ts
View file

@ -34,7 +34,8 @@ export type MainSettings = {
defaultAppName: string
pushBackupsToNostr: boolean
lnurlMetaText: string,
bridgeUrl: string
bridgeUrl: string,
allowResetMetricsStorages: boolean
}
export type BitcoinCoreSettings = {
@ -74,7 +75,8 @@ export const LoadMainSettingsFromEnv = (): MainSettings => {
defaultAppName: process.env.DEFAULT_APP_NAME || "wallet",
pushBackupsToNostr: process.env.PUSH_BACKUPS_TO_NOSTR === 'true' || false,
lnurlMetaText: process.env.LNURL_META_TEXT || "LNURL via Lightning.pub",
bridgeUrl: process.env.BRIDGE_URL || "https://shockwallet.app"
bridgeUrl: process.env.BRIDGE_URL || "https://shockwallet.app",
allowResetMetricsStorages: process.env.ALLOW_RESET_METRICS_STORAGES === 'true' || false
}
}

7
src/services/serverMethods/index.ts
View file

@ -46,6 +46,13 @@ export default (mainHandler: Main): Types.ServerMethods => {
GetLndMetrics: async ({ ctx, req }) => {
return mainHandler.metricsManager.GetLndMetrics(req)
},
ResetMetricsStorages: async ({ ctx }) => {
return mainHandler.utils.tlvStorageFactory.ResetStorages()
},
ZipMetricsStorages: async ({ ctx }) => {
const path = await mainHandler.utils.tlvStorageFactory.ZipStorages()
return { path }
},
ListChannels: async ({ ctx }) => {
return mainHandler.adminManager.ListChannels()
},

8
src/services/storage/tlv/processMetricsCollector.ts
View file

@ -5,20 +5,28 @@ export type ProcessMetrics = {
memory_heap_total_kb?: number
memory_heap_used_kb?: number
memory_external_kb?: number
cpu_user_ms?: number
cpu_system_ms?: number
}
export class ProcessMetricsCollector {
reportLog = getLogger({ component: 'ProcessMetricsCollector' })
prevValues: Record<string, number> = {}
interval: NodeJS.Timeout
cpuUsage: { user: number, system: number }
constructor(cb: (metrics: ProcessMetrics) => void) {
this.cpuUsage = process.cpuUsage()
this.interval = setInterval(() => {
const mem = process.memoryUsage()
const diff = process.cpuUsage(this.cpuUsage)
this.cpuUsage = process.cpuUsage()
const metrics: ProcessMetrics = {
memory_rss_kb: this.AddValue('memory_rss_kb', Math.ceil(mem.rss / 1000 || 0), true),
memory_buffer_kb: this.AddValue('memory_buffer_kb', Math.ceil(mem.arrayBuffers / 1000 || 0), true),
memory_heap_total_kb: this.AddValue('memory_heap_total_kb', Math.ceil(mem.heapTotal / 1000 || 0), true),
memory_heap_used_kb: this.AddValue('memory_heap_used_kb', Math.ceil(mem.heapUsed / 1000 || 0), true),
memory_external_kb: this.AddValue('memory_external_kb', Math.ceil(mem.external / 1000 || 0), true),
cpu_user_ms: this.AddValue('cpu_user_ms', Math.ceil(diff.user / 1000), true),
cpu_system_ms: this.AddValue('cpu_system_ms', Math.ceil(diff.system / 1000), true),
}
cb(metrics)

23
src/services/storage/tlv/tlvFilesStorage.ts
View file

@ -9,20 +9,35 @@ export class TlvFilesStorage {
private meta: Record<string, Record<string, { chunks: number[] }>> = {}
private pending: Record<string, Record<string, { tlvs: Uint8Array[] }>> = {}
private metaReady = false
private interval: NodeJS.Timeout
constructor(storagePath: string) {
this.storagePath = storagePath
if (!fs.existsSync(this.storagePath)) {
fs.mkdirSync(this.storagePath, { recursive: true });
}
this.init()
process.on('exit', () => {
this.persist()
});
}
GetStoragePath = () => {
return this.storagePath
}
init = () => {
this.initMeta()
setInterval(() => {
this.interval = setInterval(() => {
if (Date.now() - this.lastPersisted > 1000 * 60 * 4) {
this.persist()
}
}, 1000 * 60 * 5)
process.on('exit', () => {
this.persist()
});
}
Reset = () => {
clearInterval(this.interval)
fs.rmSync(this.storagePath, { recursive: true, force: true })
this.init()
}
LoadFile = (app: string, dataName: string, chunk: number): TlvFile => {

21
src/services/storage/tlv/tlvFilesStorageFactory.ts
View file

@ -1,6 +1,6 @@
import { ChildProcess, fork } from 'child_process';
import { EventEmitter } from 'events';
import { AddTlvOperation, ITlvStorageOperation, SuccessTlvOperationResponse, LoadLatestTlvOperation, LoadTlvFileOperation, NewTlvStorageOperation, SerializableLatestData, SerializableTlvFile, TlvOperationResponse, TlvStorageSettings, WebRtcMessageOperation, ProcessMetricsTlvOperation } from './tlvFilesStorageProcessor';
import { AddTlvOperation, ITlvStorageOperation, SuccessTlvOperationResponse, LoadLatestTlvOperation, LoadTlvFileOperation, NewTlvStorageOperation, SerializableLatestData, SerializableTlvFile, TlvOperationResponse, TlvStorageSettings, WebRtcMessageOperation, ProcessMetricsTlvOperation, ZipStoragesOperation, ResetTlvStorageOperation } from './tlvFilesStorageProcessor';
import { LatestData, TlvFile } from './tlvFilesStorage';
import { NostrSend, SendData, SendInitiator } from '../../nostr/handler';
import { WebRtcUserInfo } from '../../webRTC';
@ -17,8 +17,10 @@ export class TlvStorageFactory extends EventEmitter {
private isConnected: boolean = false;
private debug: boolean = false;
private _nostrSend: NostrSend = () => { throw new Error('nostr send not initialized yet') }
constructor() {
private allowResetMetricsStorages: boolean
constructor(allowResetMetricsStorages: boolean) {
super();
this.allowResetMetricsStorages = allowResetMetricsStorages
this.initializeSubprocess();
}
@ -61,6 +63,21 @@ export class TlvStorageFactory extends EventEmitter {
this.isConnected = true;
}
ZipStorages(): Promise<string> {
const opId = Math.random().toString()
const op: ZipStoragesOperation = { type: 'zipStorages', opId }
return this.handleOp<string>(op)
}
ResetStorages(): Promise<void> {
if (!this.allowResetMetricsStorages) {
throw new Error('Resetting metrics storages is not allowed')
}
const opId = Math.random().toString()
const op: ResetTlvStorageOperation = { type: 'resetStorage', opId }
return this.handleOp<void>(op)
}
NewStorage(settings: TlvStorageSettings): TlvStorageInterface {
const opId = Math.random().toString()
const op: NewTlvStorageOperation = { type: 'newStorage', opId, settings }

77
src/services/storage/tlv/tlvFilesStorageProcessor.ts
View file

@ -6,6 +6,8 @@ import { SendData } from '../../nostr/handler.js';
import { SendInitiator } from '../../nostr/handler.js';
import { ProcessMetrics, ProcessMetricsCollector } from './processMetricsCollector.js';
import { integerToUint8Array } from '../../helpers/tlv.js';
import { zip } from 'zip-a-folder'
import crypto from 'crypto'
export type SerializableLatestData = Record<string, Record<string, { base64tlvs: string[], current_chunk: number, available_chunks: number[] }>>
export type SerializableTlvFile = { base64fileData: string, chunks: number[] }
export const usageStorageName = 'usage'
@ -15,6 +17,18 @@ export type TlvStorageSettings = {
name: typeof usageStorageName | typeof bundlerStorageName
}
export type ZipStoragesOperation = {
type: 'zipStorages'
opId: string
debug?: boolean
}
export type ResetTlvStorageOperation = {
type: 'resetStorage'
opId: string
debug?: boolean
}
export type NewTlvStorageOperation = {
type: 'newStorage'
opId: string
@ -74,7 +88,7 @@ export interface ITlvStorageOperation {
debug?: boolean
}
export type TlvStorageOperation = NewTlvStorageOperation | AddTlvOperation | LoadLatestTlvOperation | LoadTlvFileOperation | WebRtcMessageOperation | ProcessMetricsTlvOperation
export type TlvStorageOperation = NewTlvStorageOperation | AddTlvOperation | LoadLatestTlvOperation | LoadTlvFileOperation | WebRtcMessageOperation | ProcessMetricsTlvOperation | ResetTlvStorageOperation | ZipStoragesOperation
export type SuccessTlvOperationResponse<T> = { success: true, type: string, data: T, opId: string }
export type TlvOperationResponse<T> = SuccessTlvOperationResponse<T> | ErrorTlvOperationResponse
@ -132,11 +146,15 @@ class TlvFilesStorageProcessor {
console.log('no bundle storage yet')
return
}
if (pMetrics.memory_rss_kb) this.storages[bundlerStorageName].AddTlv('_root', 'memory_rss_kb' + pName, this.serializeNowTlv(pMetrics.memory_rss_kb))
for (const key in pMetrics) {
const v = pMetrics[key as keyof ProcessMetrics]
if (v) this.storages[bundlerStorageName].AddTlv('_root', key + pName, this.serializeNowTlv(v))
}
/* if (pMetrics.memory_rss_kb) this.storages[bundlerStorageName].AddTlv('_root', 'memory_rss_kb' + pName, this.serializeNowTlv(pMetrics.memory_rss_kb))
if (pMetrics.memory_buffer_kb) this.storages[bundlerStorageName].AddTlv('_root', 'memory_buffer_kb' + pName, this.serializeNowTlv(pMetrics.memory_buffer_kb))
if (pMetrics.memory_heap_total_kb) this.storages[bundlerStorageName].AddTlv('_root', 'memory_heap_total_kb' + pName, this.serializeNowTlv(pMetrics.memory_heap_total_kb))
if (pMetrics.memory_heap_used_kb) this.storages[bundlerStorageName].AddTlv('_root', 'memory_heap_used_kb' + pName, this.serializeNowTlv(pMetrics.memory_heap_used_kb))
if (pMetrics.memory_external_kb) this.storages[bundlerStorageName].AddTlv('_root', 'memory_external_kb' + pName, this.serializeNowTlv(pMetrics.memory_external_kb))
if (pMetrics.memory_external_kb) this.storages[bundlerStorageName].AddTlv('_root', 'memory_external_kb' + pName, this.serializeNowTlv(pMetrics.memory_external_kb)) */
}
private async handleOperation(operation: TlvStorageOperation) {
@ -144,6 +162,9 @@ class TlvFilesStorageProcessor {
const opId = operation.opId;
if (operation.debug) console.log('handleOperation', operation)
switch (operation.type) {
case 'resetStorage':
await this.handleResetStorage(operation);
break;
case 'newStorage':
await this.handleNewStorage(operation);
break;
@ -162,6 +183,9 @@ class TlvFilesStorageProcessor {
case 'processMetrics':
await this.handleProcessMetrics(operation);
break;
case 'zipStorages':
await this.handleZipStorages(operation);
break;
default:
this.sendResponse({
success: false,
@ -179,6 +203,19 @@ class TlvFilesStorageProcessor {
}
}
private async handleResetStorage(operation: ResetTlvStorageOperation) {
for (const storageName in this.storages) {
this.storages[storageName].Reset()
}
this.sendResponse({
success: true,
type: 'resetStorage',
data: null,
opId: operation.opId
});
}
private async handleNewStorage(operation: NewTlvStorageOperation) {
if (this.storages[operation.settings.name]) {
this.sendResponse({
@ -279,6 +316,40 @@ class TlvFilesStorageProcessor {
});
}
private async handleZipStorages(operation: ZipStoragesOperation) {
const paths = []
for (const storageName in this.storages) {
paths.push(this.storages[storageName].GetStoragePath())
}
if (paths.length === 0) {
this.sendResponse({
success: false,
error: 'No storages to zip',
opId: operation.opId
})
return
}
const name = crypto.randomBytes(16).toString('hex') + '.zip'
const path = paths.join(', ')
const err = await zip(path, name)
if (err) {
this.sendResponse({
success: false,
error: err.message,
opId: operation.opId
})
return
}
this.sendResponse<string>({
success: true,
type: 'zipStorages',
data: name,
opId: operation.opId
});
}
private sendResponse<T>(response: TlvOperationResponse<T>) {
if (process.send) {
process.send(response);

4
src/tests/networkSetup.ts
View file

@ -3,14 +3,12 @@ import { BitcoinCoreWrapper } from "./bitcoinCore.js"
import LND from '../services/lnd/lnd.js'
import { LiquidityProvider } from "../services/main/liquidityProvider.js"
import { Utils } from "../services/helpers/utilsWrapper.js"
import { TlvStorageFactory } from "../services/storage/tlv/tlvFilesStorageFactory.js"
export const setupNetwork = async () => {
const settings = LoadTestSettingsFromEnv()
const core = new BitcoinCoreWrapper(settings)
await core.InitAddress()
await core.Mine(1)
const tlvStorageFactory = new TlvStorageFactory()
const setupUtils = new Utils({ dataDir: settings.storageSettings.dataDir })
const setupUtils = new Utils({ dataDir: settings.storageSettings.dataDir, allowResetMetricsStorages: settings.allowResetMetricsStorages })
const alice = new LND(settings.lndSettings, new LiquidityProvider("", setupUtils, async () => { }, async () => { }), setupUtils, async () => { }, async () => { }, () => { }, () => { })
const bob = new LND({ ...settings.lndSettings, mainNode: settings.lndSettings.otherNode }, new LiquidityProvider("", setupUtils, async () => { }, async () => { }), setupUtils, async () => { }, async () => { }, () => { }, () => { })
await tryUntil<void>(async i => {

4
src/tests/testBase.ts
View file

@ -44,7 +44,7 @@ export type StorageTestBase = {
export const setupStorageTest = async (d: Describe): Promise<StorageTestBase> => {
const settings = GetTestStorageSettings()
const utils = new Utils({ dataDir: settings.dataDir })
const utils = new Utils({ dataDir: settings.dataDir, allowResetMetricsStorages: true })
const storageManager = new Storage(settings, utils)
await storageManager.Connect(console.log)
return {
@ -71,7 +71,7 @@ export const SetupTest = async (d: Describe): Promise<TestBase> => {
const user1 = { userId: u1.info.userId, appUserIdentifier: u1.identifier, appId: app.appId }
const user2 = { userId: u2.info.userId, appUserIdentifier: u2.identifier, appId: app.appId }
const extermnalUtils = new Utils({ dataDir: settings.storageSettings.dataDir })
const extermnalUtils = new Utils({ dataDir: settings.storageSettings.dataDir, allowResetMetricsStorages: settings.allowResetMetricsStorages })
const externalAccessToMainLnd = new LND(settings.lndSettings, new LiquidityProvider("", extermnalUtils, async () => { }, async () => { }), extermnalUtils, async () => { }, async () => { }, () => { }, () => { })
await externalAccessToMainLnd.Warmup()