reset metrics b4 stress test and export after

package.json

@@ -65,7 +65,8 @@
     "websocket-polyfill": "^0.0.3",
     "why-is-node-running": "^3.2.0",
     "wrtc": "^0.4.7",
-    "ws": "^8.18.0"
+    "ws": "^8.18.0",
+    "zip-a-folder": "^3.1.9"
   },
   "devDependencies": {
     "@types/chai": "^4.3.4",

proto/autogenerated/client.md

@@ -255,6 +255,11 @@ The nostr server will send back a message response, and inside the body there wi
   - input: [DebitOperation](#DebitOperation)
   - This methods has an __empty__ __response__ body
 
+- ResetMetricsStorages
+  - auth type: __Admin__
+  - This methods has an __empty__ __request__ body
+  - This methods has an __empty__ __response__ body
+
 - RespondToDebit
   - auth type: __User__
   - input: [DebitResponse](#DebitResponse)
@@ -295,6 +300,11 @@ The nostr server will send back a message response, and inside the body there wi
   - This methods has an __empty__ __request__ body
   - output: [UserHealthState](#UserHealthState)
 
+- ZipMetricsStorages
+  - auth type: __Admin__
+  - This methods has an __empty__ __request__ body
+  - output: [ZippedMetrics](#ZippedMetrics)
+
 # HTTP API DEFINITION
 
 ## Supported HTTP Auths
@@ -794,6 +804,13 @@ The nostr server will send back a message response, and inside the body there wi
   - input: [DebitOperation](#DebitOperation)
   - This methods has an __empty__ __response__ body
 
+- ResetMetricsStorages
+  - auth type: __Admin__
+  - http method: __post__
+  - http route: __/api/admin/metrics/reset__
+  - This methods has an __empty__ __request__ body
+  - This methods has an __empty__ __response__ body
+
 - ResetNPubLinkingToken
   - auth type: __App__
   - http method: __post__
@@ -892,6 +909,13 @@ The nostr server will send back a message response, and inside the body there wi
   - This methods has an __empty__ __request__ body
   - output: [UserHealthState](#UserHealthState)
 
+- ZipMetricsStorages
+  - auth type: __Admin__
+  - http method: __post__
+  - http route: __/api/admin/metrics/zip__
+  - This methods has an __empty__ __request__ body
+  - output: [ZippedMetrics](#ZippedMetrics)
+
 # INPUTS AND OUTPUTS
 
 ## Messages
@@ -1465,6 +1489,9 @@ The nostr server will send back a message response, and inside the body there wi
 
 ### WebRtcMessage
   - __message__: _[WebRtcMessage_message](#WebRtcMessage_message)_
+
+### ZippedMetrics
+  - __path__: _string_
 ## Enums
 ### The enumerators used in the messages
 

proto/autogenerated/go/http_client.go

@@ -118,6 +118,7 @@ type Client struct {
 	PayInvoice                  func(req PayInvoiceRequest) (*PayInvoiceResponse, error)
 	RequestNPubLinkingToken     func(req RequestNPubLinkingTokenRequest) (*RequestNPubLinkingTokenResponse, error)
 	ResetDebit                  func(req DebitOperation) error
+	ResetMetricsStorages        func() error
 	ResetNPubLinkingToken       func(req RequestNPubLinkingTokenRequest) (*RequestNPubLinkingTokenResponse, error)
 	RespondToDebit              func(req DebitResponse) error
 	SendAppUserToAppPayment     func(req SendAppUserToAppPaymentRequest) error
@@ -132,6 +133,7 @@ type Client struct {
 	UpdateUserOffer             func(req OfferConfig) error
 	UseInviteLink               func(req UseInviteLinkRequest) error
 	UserHealth                  func() (*UserHealthState, error)
+	ZipMetricsStorages          func() (*ZippedMetrics, error)
 }
 
 func NewClient(params ClientParams) *Client {
@@ -1752,6 +1754,27 @@ func NewClient(params ClientParams) *Client {
 			}
 			return nil
 		},
+		ResetMetricsStorages: func() error {
+			auth, err := params.RetrieveAdminAuth()
+			if err != nil {
+				return err
+			}
+			finalRoute := "/api/admin/metrics/reset"
+			body := []byte{}
+			resBody, err := doPostRequest(params.BaseURL+finalRoute, body, auth)
+			if err != nil {
+				return err
+			}
+			result := ResultError{}
+			err = json.Unmarshal(resBody, &result)
+			if err != nil {
+				return err
+			}
+			if result.Status == "ERROR" {
+				return fmt.Errorf(result.Reason)
+			}
+			return nil
+		},
 		ResetNPubLinkingToken: func(req RequestNPubLinkingTokenRequest) (*RequestNPubLinkingTokenResponse, error) {
 			auth, err := params.RetrieveAppAuth()
 			if err != nil {
@@ -2082,5 +2105,31 @@ func NewClient(params ClientParams) *Client {
 			}
 			return &res, nil
 		},
+		ZipMetricsStorages: func() (*ZippedMetrics, error) {
+			auth, err := params.RetrieveAdminAuth()
+			if err != nil {
+				return nil, err
+			}
+			finalRoute := "/api/admin/metrics/zip"
+			body := []byte{}
+			resBody, err := doPostRequest(params.BaseURL+finalRoute, body, auth)
+			if err != nil {
+				return nil, err
+			}
+			result := ResultError{}
+			err = json.Unmarshal(resBody, &result)
+			if err != nil {
+				return nil, err
+			}
+			if result.Status == "ERROR" {
+				return nil, fmt.Errorf(result.Reason)
+			}
+			res := ZippedMetrics{}
+			err = json.Unmarshal(resBody, &res)
+			if err != nil {
+				return nil, err
+			}
+			return &res, nil
+		},
 	}
 }

proto/autogenerated/go/types.go

@@ -665,6 +665,9 @@ type WebRtcCandidate struct {
 type WebRtcMessage struct {
 	Message *WebRtcMessage_message `json:"message"`
 }
+type ZippedMetrics struct {
+	Path string `json:"path"`
+}
 type DebitResponse_response_type string
 
 const (

proto/autogenerated/ts/express_server.ts

@@ -1632,6 +1632,25 @@ export default (methods: Types.ServerMethods, opts: ServerOptions) => {
             opts.metricsCallback([{ ...info, ...stats, ...authContext }])
         } catch (ex) { const e = ex as any; logErrorAndReturnResponse(e, e.message || e, res, logger, { ...info, ...stats, ...authCtx }, opts.metricsCallback); if (opts.throwErrors) throw e }
     })
+    if (!opts.allowNotImplementedMethods && !methods.ResetMetricsStorages) throw new Error('method: ResetMetricsStorages is not implemented')
+    app.post('/api/admin/metrics/reset', async (req, res) => {
+        const info: Types.RequestInfo = { rpcName: 'ResetMetricsStorages', batch: false, nostr: false, batchSize: 0}
+        const stats: Types.RequestStats = { startMs:req.startTimeMs || 0, start:req.startTime || 0n, parse: process.hrtime.bigint(), guard: 0n, validate: 0n, handle: 0n }
+        let authCtx: Types.AuthContext = {}
+        try {
+            if (!methods.ResetMetricsStorages) throw new Error('method: ResetMetricsStorages is not implemented')
+            const authContext = await opts.AdminAuthGuard(req.headers['authorization'])
+            authCtx = authContext
+            stats.guard = process.hrtime.bigint()
+            stats.validate = stats.guard
+            const query = req.query
+            const params = req.params
+             await methods.ResetMetricsStorages({rpcName:'ResetMetricsStorages', ctx:authContext })
+            stats.handle = process.hrtime.bigint()
+            res.json({status: 'OK'})
+            opts.metricsCallback([{ ...info, ...stats, ...authContext }])
+        } catch (ex) { const e = ex as any; logErrorAndReturnResponse(e, e.message || e, res, logger, { ...info, ...stats, ...authCtx }, opts.metricsCallback); if (opts.throwErrors) throw e }
+    })
     if (!opts.allowNotImplementedMethods && !methods.ResetNPubLinkingToken) throw new Error('method: ResetNPubLinkingToken is not implemented')
     app.post('/api/app/user/npub/token/reset', async (req, res) => {
         const info: Types.RequestInfo = { rpcName: 'ResetNPubLinkingToken', batch: false, nostr: false, batchSize: 0}
@@ -1915,6 +1934,25 @@ export default (methods: Types.ServerMethods, opts: ServerOptions) => {
             opts.metricsCallback([{ ...info, ...stats, ...authContext }])
         } catch (ex) { const e = ex as any; logErrorAndReturnResponse(e, e.message || e, res, logger, { ...info, ...stats, ...authCtx }, opts.metricsCallback); if (opts.throwErrors) throw e }
     })
+    if (!opts.allowNotImplementedMethods && !methods.ZipMetricsStorages) throw new Error('method: ZipMetricsStorages is not implemented')
+    app.post('/api/admin/metrics/zip', async (req, res) => {
+        const info: Types.RequestInfo = { rpcName: 'ZipMetricsStorages', batch: false, nostr: false, batchSize: 0}
+        const stats: Types.RequestStats = { startMs:req.startTimeMs || 0, start:req.startTime || 0n, parse: process.hrtime.bigint(), guard: 0n, validate: 0n, handle: 0n }
+        let authCtx: Types.AuthContext = {}
+        try {
+            if (!methods.ZipMetricsStorages) throw new Error('method: ZipMetricsStorages is not implemented')
+            const authContext = await opts.AdminAuthGuard(req.headers['authorization'])
+            authCtx = authContext
+            stats.guard = process.hrtime.bigint()
+            stats.validate = stats.guard
+            const query = req.query
+            const params = req.params
+            const response =  await methods.ZipMetricsStorages({rpcName:'ZipMetricsStorages', ctx:authContext })
+            stats.handle = process.hrtime.bigint()
+            res.json({status: 'OK', ...response})
+            opts.metricsCallback([{ ...info, ...stats, ...authContext }])
+        } catch (ex) { const e = ex as any; logErrorAndReturnResponse(e, e.message || e, res, logger, { ...info, ...stats, ...authCtx }, opts.metricsCallback); if (opts.throwErrors) throw e }
+    })
     if (opts.staticFiles) {
             app.use(express.static(opts.staticFiles))
             app.get('*', function (_, res) {	res.sendFile('index.html', { root: opts.staticFiles })})

proto/autogenerated/ts/http_client.ts

@@ -839,6 +839,17 @@ export default (params: ClientParams) => ({
         }
         return { status: 'ERROR', reason: 'invalid response' }
     },
+    ResetMetricsStorages: async (): Promise<ResultError | ({ status: 'OK' })> => {
+        const auth = await params.retrieveAdminAuth()
+        if (auth === null) throw new Error('retrieveAdminAuth() returned null')
+        let finalRoute = '/api/admin/metrics/reset'
+        const { data } = await axios.post(params.baseUrl + finalRoute, {}, { headers: { 'authorization': auth } })
+        if (data.status === 'ERROR' && typeof data.reason === 'string') return data
+        if (data.status === 'OK') { 
+            return data
+        }
+        return { status: 'ERROR', reason: 'invalid response' }
+    },
     ResetNPubLinkingToken: async (request: Types.RequestNPubLinkingTokenRequest): Promise<ResultError | ({ status: 'OK' }& Types.RequestNPubLinkingTokenResponse)> => {
         const auth = await params.retrieveAppAuth()
         if (auth === null) throw new Error('retrieveAppAuth() returned null')
@@ -995,4 +1006,18 @@ export default (params: ClientParams) => ({
         }
         return { status: 'ERROR', reason: 'invalid response' }
     },
+    ZipMetricsStorages: async (): Promise<ResultError | ({ status: 'OK' }& Types.ZippedMetrics)> => {
+        const auth = await params.retrieveAdminAuth()
+        if (auth === null) throw new Error('retrieveAdminAuth() returned null')
+        let finalRoute = '/api/admin/metrics/zip'
+        const { data } = await axios.post(params.baseUrl + finalRoute, {}, { headers: { 'authorization': auth } })
+        if (data.status === 'ERROR' && typeof data.reason === 'string') return data
+        if (data.status === 'OK') { 
+            const result = data
+            if(!params.checkResult) return { status: 'OK', ...result }
+            const error = Types.ZippedMetricsValidate(result)
+            if (error === null) { return { status: 'OK', ...result } } else return { status: 'ERROR', reason: error.message }
+        }
+        return { status: 'ERROR', reason: 'invalid response' }
+    },
 })

proto/autogenerated/ts/nostr_client.ts

@@ -698,6 +698,17 @@ export default (params: NostrClientParams,  send: (to:string, message: NostrRequ
         }
         return { status: 'ERROR', reason: 'invalid response' }
     },
+    ResetMetricsStorages: async (): Promise<ResultError | ({ status: 'OK' })> => {
+        const auth = await params.retrieveNostrAdminAuth()
+        if (auth === null) throw new Error('retrieveNostrAdminAuth() returned null')
+        const nostrRequest: NostrRequest = {}
+        const data = await send(params.pubDestination, {rpcName:'ResetMetricsStorages',authIdentifier:auth, ...nostrRequest }) 
+        if (data.status === 'ERROR' && typeof data.reason === 'string') return data
+        if (data.status === 'OK') { 
+            return data
+        }
+        return { status: 'ERROR', reason: 'invalid response' }
+    },
     RespondToDebit: async (request: Types.DebitResponse): Promise<ResultError | ({ status: 'OK' })> => {
         const auth = await params.retrieveNostrUserAuth()
         if (auth === null) throw new Error('retrieveNostrUserAuth() returned null')
@@ -805,4 +816,18 @@ export default (params: NostrClientParams,  send: (to:string, message: NostrRequ
         }
         return { status: 'ERROR', reason: 'invalid response' }
     },
+    ZipMetricsStorages: async (): Promise<ResultError | ({ status: 'OK' }& Types.ZippedMetrics)> => {
+        const auth = await params.retrieveNostrAdminAuth()
+        if (auth === null) throw new Error('retrieveNostrAdminAuth() returned null')
+        const nostrRequest: NostrRequest = {}
+        const data = await send(params.pubDestination, {rpcName:'ZipMetricsStorages',authIdentifier:auth, ...nostrRequest }) 
+        if (data.status === 'ERROR' && typeof data.reason === 'string') return data
+        if (data.status === 'OK') { 
+            const result = data
+            if(!params.checkResult) return { status: 'OK', ...result }
+            const error = Types.ZippedMetricsValidate(result)
+            if (error === null) { return { status: 'OK', ...result } } else return { status: 'ERROR', reason: error.message }
+        }
+        return { status: 'ERROR', reason: 'invalid response' }
+    },
 })

proto/autogenerated/ts/nostr_transport.ts

@@ -1091,6 +1091,19 @@ export default (methods: Types.ServerMethods, opts: NostrOptions) => {
                     opts.metricsCallback([{ ...info, ...stats, ...authContext }])
                 }catch(ex){ const e = ex as any; logErrorAndReturnResponse(e, e.message || e, res, logger, { ...info, ...stats, ...authCtx }, opts.metricsCallback); if (opts.throwErrors) throw e }
                 break
+            case 'ResetMetricsStorages':
+                try {
+                    if (!methods.ResetMetricsStorages) throw new Error('method: ResetMetricsStorages is not implemented')
+                    const authContext = await opts.NostrAdminAuthGuard(req.appId, req.authIdentifier)
+                    stats.guard = process.hrtime.bigint()
+                    authCtx = authContext
+                    stats.validate = stats.guard
+                    await methods.ResetMetricsStorages({rpcName:'ResetMetricsStorages', ctx:authContext })
+                    stats.handle = process.hrtime.bigint()
+                    res({status: 'OK'})
+                    opts.metricsCallback([{ ...info, ...stats, ...authContext }])
+                }catch(ex){ const e = ex as any; logErrorAndReturnResponse(e, e.message || e, res, logger, { ...info, ...stats, ...authCtx }, opts.metricsCallback); if (opts.throwErrors) throw e }
+                break
             case 'RespondToDebit':
                 try {
                     if (!methods.RespondToDebit) throw new Error('method: RespondToDebit is not implemented')
@@ -1213,6 +1226,19 @@ export default (methods: Types.ServerMethods, opts: NostrOptions) => {
                     opts.metricsCallback([{ ...info, ...stats, ...authContext }])
                 }catch(ex){ const e = ex as any; logErrorAndReturnResponse(e, e.message || e, res, logger, { ...info, ...stats, ...authCtx }, opts.metricsCallback); if (opts.throwErrors) throw e }
                 break
+            case 'ZipMetricsStorages':
+                try {
+                    if (!methods.ZipMetricsStorages) throw new Error('method: ZipMetricsStorages is not implemented')
+                    const authContext = await opts.NostrAdminAuthGuard(req.appId, req.authIdentifier)
+                    stats.guard = process.hrtime.bigint()
+                    authCtx = authContext
+                    stats.validate = stats.guard
+                    const response = await methods.ZipMetricsStorages({rpcName:'ZipMetricsStorages', ctx:authContext })
+                    stats.handle = process.hrtime.bigint()
+                    res({status: 'OK', ...response})
+                    opts.metricsCallback([{ ...info, ...stats, ...authContext }])
+                }catch(ex){ const e = ex as any; logErrorAndReturnResponse(e, e.message || e, res, logger, { ...info, ...stats, ...authCtx }, opts.metricsCallback); if (opts.throwErrors) throw e }
+                break
             default: logger.error('unknown rpc call name from nostr event:'+req.rpcName) 
         }
     }

proto/autogenerated/ts/types.ts

@@ -7,8 +7,8 @@ export type RequestMetric = AuthContext & RequestInfo & RequestStats & { error?:
 export type AdminContext = {
     admin_id: string
 }
-export type AdminMethodInputs = AddApp_Input | AddPeer_Input | AuthApp_Input | BanUser_Input | CloseChannel_Input | CreateOneTimeInviteLink_Input | GetInviteLinkState_Input | GetSeed_Input | ListChannels_Input | LndGetInfo_Input | OpenChannel_Input | UpdateChannelPolicy_Input
+export type AdminMethodInputs = AddApp_Input | AddPeer_Input | AuthApp_Input | BanUser_Input | CloseChannel_Input | CreateOneTimeInviteLink_Input | GetInviteLinkState_Input | GetSeed_Input | ListChannels_Input | LndGetInfo_Input | OpenChannel_Input | ResetMetricsStorages_Input | UpdateChannelPolicy_Input | ZipMetricsStorages_Input
-export type AdminMethodOutputs = AddApp_Output | AddPeer_Output | AuthApp_Output | BanUser_Output | CloseChannel_Output | CreateOneTimeInviteLink_Output | GetInviteLinkState_Output | GetSeed_Output | ListChannels_Output | LndGetInfo_Output | OpenChannel_Output | UpdateChannelPolicy_Output
+export type AdminMethodOutputs = AddApp_Output | AddPeer_Output | AuthApp_Output | BanUser_Output | CloseChannel_Output | CreateOneTimeInviteLink_Output | GetInviteLinkState_Output | GetSeed_Output | ListChannels_Output | LndGetInfo_Output | OpenChannel_Output | ResetMetricsStorages_Output | UpdateChannelPolicy_Output | ZipMetricsStorages_Output
 export type AppContext = {
     app_id: string
 }
@@ -253,6 +253,9 @@ export type RequestNPubLinkingToken_Output = ResultError | ({ status: 'OK' } & R
 export type ResetDebit_Input = {rpcName:'ResetDebit', req: DebitOperation}
 export type ResetDebit_Output = ResultError | { status: 'OK' }
 
+export type ResetMetricsStorages_Input = {rpcName:'ResetMetricsStorages'}
+export type ResetMetricsStorages_Output = ResultError | { status: 'OK' }
+
 export type ResetNPubLinkingToken_Input = {rpcName:'ResetNPubLinkingToken', req: RequestNPubLinkingTokenRequest}
 export type ResetNPubLinkingToken_Output = ResultError | ({ status: 'OK' } & RequestNPubLinkingTokenResponse)
 
@@ -295,6 +298,9 @@ export type UseInviteLink_Output = ResultError | { status: 'OK' }
 export type UserHealth_Input = {rpcName:'UserHealth'}
 export type UserHealth_Output = ResultError | ({ status: 'OK' } & UserHealthState)
 
+export type ZipMetricsStorages_Input = {rpcName:'ZipMetricsStorages'}
+export type ZipMetricsStorages_Output = ResultError | ({ status: 'OK' } & ZippedMetrics)
+
 export type ServerMethods = {
     AddApp?: (req: AddApp_Input & {ctx: AdminContext }) => Promise<AuthApp>
     AddAppInvoice?: (req: AddAppInvoice_Input & {ctx: AppContext }) => Promise<NewInvoiceResponse>
@@ -359,6 +365,7 @@ export type ServerMethods = {
     PayInvoice?: (req: PayInvoice_Input & {ctx: UserContext }) => Promise<PayInvoiceResponse>
     RequestNPubLinkingToken?: (req: RequestNPubLinkingToken_Input & {ctx: AppContext }) => Promise<RequestNPubLinkingTokenResponse>
     ResetDebit?: (req: ResetDebit_Input & {ctx: UserContext }) => Promise<void>
+    ResetMetricsStorages?: (req: ResetMetricsStorages_Input & {ctx: AdminContext }) => Promise<void>
     ResetNPubLinkingToken?: (req: ResetNPubLinkingToken_Input & {ctx: AppContext }) => Promise<RequestNPubLinkingTokenResponse>
     RespondToDebit?: (req: RespondToDebit_Input & {ctx: UserContext }) => Promise<void>
     SendAppUserToAppPayment?: (req: SendAppUserToAppPayment_Input & {ctx: AppContext }) => Promise<void>
@@ -373,6 +380,7 @@ export type ServerMethods = {
     UpdateUserOffer?: (req: UpdateUserOffer_Input & {ctx: UserContext }) => Promise<void>
     UseInviteLink?: (req: UseInviteLink_Input & {ctx: GuestWithPubContext }) => Promise<void>
     UserHealth?: (req: UserHealth_Input & {ctx: UserContext }) => Promise<UserHealthState>
+    ZipMetricsStorages?: (req: ZipMetricsStorages_Input & {ctx: AdminContext }) => Promise<ZippedMetrics>
 }
 
 export enum AddressType {
@@ -3816,6 +3824,24 @@ export const WebRtcMessageValidate = (o?: WebRtcMessage, opts: WebRtcMessageOpti
     return null
 }
 
+export type ZippedMetrics = {
+    path: string
+}
+export const ZippedMetricsOptionalFields: [] = []
+export type ZippedMetricsOptions = OptionsBaseMessage & {
+    checkOptionalsAreSet?: []
+    path_CustomCheck?: (v: string) => boolean
+}
+export const ZippedMetricsValidate = (o?: ZippedMetrics, opts: ZippedMetricsOptions = {}, path: string = 'ZippedMetrics::root.'): Error | null => {
+    if (opts.checkOptionalsAreSet && opts.allOptionalsAreSet) return new Error(path + ': only one of checkOptionalsAreSet or allOptionalNonDefault can be set for each message')
+    if (typeof o !== 'object' || o === null) return new Error(path + ': object is not an instance of an object or is null')
+
+    if (typeof o.path !== 'string') return new Error(`${path}.path: is not a string`)
+    if (opts.path_CustomCheck && !opts.path_CustomCheck(o.path)) return new Error(`${path}.path: custom check failed`)
+
+    return null
+}
+
 export enum DebitResponse_response_type {
     DENIED = 'denied',
     INVOICE = 'invoice',

proto/service/methods.proto

@@ -234,6 +234,20 @@ service LightningPub {
         option (nostr) = true;
     }
 
+    rpc ZipMetricsStorages(structs.Empty) returns (structs.ZippedMetrics) {
+        option (auth_type)  = "Admin";
+        option (http_method) = "post";
+        option (http_route) = "/api/admin/metrics/zip";
+        option (nostr) = true;
+    }
+
+    rpc ResetMetricsStorages(structs.Empty) returns (structs.Empty) {
+        option (auth_type)  = "Admin";
+        option (http_method) = "post";
+        option (http_route) = "/api/admin/metrics/reset";
+        option (nostr) = true;
+    }
+
     rpc CreateOneTimeInviteLink(structs.CreateOneTimeInviteLinkRequest) returns (structs.CreateOneTimeInviteLinkResponse) {
         option (auth_type) = "Admin";
         option (http_method) = "post";

proto/service/structs.proto

@@ -10,6 +10,10 @@ message LndSeed {
     repeated string seed = 1;
 }
 
+message ZippedMetrics {
+    string path = 1;
+}
+
 message EncryptionExchangeRequest {
     string publicKey = 1;
     string deviceId = 2;

src/services/helpers/utilsWrapper.ts

@@ -5,15 +5,15 @@ import { NostrSend } from "../nostr/handler.js";
 import { ProcessMetricsCollector } from "../storage/tlv/processMetricsCollector.js";
 type UtilsSettings = {
     noCollector?: boolean
-    dataDir: string
+    dataDir: string,
+    allowResetMetricsStorages: boolean
 }
 export class Utils {
     tlvStorageFactory: TlvStorageFactory
     stateBundler: StateBundler
-    settings: MainSettings
     _nostrSend: NostrSend = () => { throw new Error('nostr send not initialized yet') }
-    constructor({ noCollector, dataDir }: UtilsSettings) {
+    constructor({ noCollector, dataDir, allowResetMetricsStorages }: UtilsSettings) {
-        this.tlvStorageFactory = new TlvStorageFactory()
+        this.tlvStorageFactory = new TlvStorageFactory(allowResetMetricsStorages)
         this.stateBundler = new StateBundler(dataDir, this.tlvStorageFactory)
         if (!noCollector) {
             new ProcessMetricsCollector((metrics) => {

src/services/main/init.ts

@@ -17,7 +17,7 @@ export type AppData = {
     name: string;
 }
 export const initMainHandler = async (log: PubLogger, mainSettings: MainSettings) => {
-    const utils = new Utils({ dataDir: mainSettings.storageSettings.dataDir })
+    const utils = new Utils({ dataDir: mainSettings.storageSettings.dataDir, allowResetMetricsStorages: mainSettings.allowResetMetricsStorages })
     const storageManager = new Storage(mainSettings.storageSettings, utils)
     await storageManager.Connect(log)
     /*     const manualMigration = await TypeOrmMigrationRunner(log, storageManager, mainSettings.storageSettings.dbSettings, process.argv[2])

src/services/main/settings.ts

@@ -34,7 +34,8 @@ export type MainSettings = {
     defaultAppName: string
     pushBackupsToNostr: boolean
     lnurlMetaText: string,
-    bridgeUrl: string
+    bridgeUrl: string,
+    allowResetMetricsStorages: boolean
 }
 
 export type BitcoinCoreSettings = {
@@ -74,7 +75,8 @@ export const LoadMainSettingsFromEnv = (): MainSettings => {
         defaultAppName: process.env.DEFAULT_APP_NAME || "wallet",
         pushBackupsToNostr: process.env.PUSH_BACKUPS_TO_NOSTR === 'true' || false,
         lnurlMetaText: process.env.LNURL_META_TEXT || "LNURL via Lightning.pub",
-        bridgeUrl: process.env.BRIDGE_URL || "https://shockwallet.app"
+        bridgeUrl: process.env.BRIDGE_URL || "https://shockwallet.app",
+        allowResetMetricsStorages: process.env.ALLOW_RESET_METRICS_STORAGES === 'true' || false
     }
 }
 

src/services/serverMethods/index.ts

@@ -46,6 +46,13 @@ export default (mainHandler: Main): Types.ServerMethods => {
         GetLndMetrics: async ({ ctx, req }) => {
             return mainHandler.metricsManager.GetLndMetrics(req)
         },
+        ResetMetricsStorages: async ({ ctx }) => {
+            return mainHandler.utils.tlvStorageFactory.ResetStorages()
+        },
+        ZipMetricsStorages: async ({ ctx }) => {
+            const path = await mainHandler.utils.tlvStorageFactory.ZipStorages()
+            return { path }
+        },
         ListChannels: async ({ ctx }) => {
             return mainHandler.adminManager.ListChannels()
         },

src/services/storage/tlv/processMetricsCollector.ts

@@ -5,20 +5,28 @@ export type ProcessMetrics = {
     memory_heap_total_kb?: number
     memory_heap_used_kb?: number
     memory_external_kb?: number
+    cpu_user_ms?: number
+    cpu_system_ms?: number
 }
 export class ProcessMetricsCollector {
     reportLog = getLogger({ component: 'ProcessMetricsCollector' })
     prevValues: Record<string, number> = {}
     interval: NodeJS.Timeout
+    cpuUsage: { user: number, system: number }
     constructor(cb: (metrics: ProcessMetrics) => void) {
+        this.cpuUsage = process.cpuUsage()
         this.interval = setInterval(() => {
             const mem = process.memoryUsage()
+            const diff = process.cpuUsage(this.cpuUsage)
+            this.cpuUsage = process.cpuUsage()
             const metrics: ProcessMetrics = {
                 memory_rss_kb: this.AddValue('memory_rss_kb', Math.ceil(mem.rss / 1000 || 0), true),
                 memory_buffer_kb: this.AddValue('memory_buffer_kb', Math.ceil(mem.arrayBuffers / 1000 || 0), true),
                 memory_heap_total_kb: this.AddValue('memory_heap_total_kb', Math.ceil(mem.heapTotal / 1000 || 0), true),
                 memory_heap_used_kb: this.AddValue('memory_heap_used_kb', Math.ceil(mem.heapUsed / 1000 || 0), true),
                 memory_external_kb: this.AddValue('memory_external_kb', Math.ceil(mem.external / 1000 || 0), true),
+                cpu_user_ms: this.AddValue('cpu_user_ms', Math.ceil(diff.user / 1000), true),
+                cpu_system_ms: this.AddValue('cpu_system_ms', Math.ceil(diff.system / 1000), true),
             }
 
             cb(metrics)

src/services/storage/tlv/tlvFilesStorage.ts

@@ -9,20 +9,35 @@ export class TlvFilesStorage {
     private meta: Record<string, Record<string, { chunks: number[] }>> = {}
     private pending: Record<string, Record<string, { tlvs: Uint8Array[] }>> = {}
     private metaReady = false
+    private interval: NodeJS.Timeout
     constructor(storagePath: string) {
         this.storagePath = storagePath
         if (!fs.existsSync(this.storagePath)) {
             fs.mkdirSync(this.storagePath, { recursive: true });
         }
+        this.init()
+        process.on('exit', () => {
+            this.persist()
+        });
+    }
+
+    GetStoragePath = () => {
+        return this.storagePath
+    }
+
+    init = () => {
         this.initMeta()
-        setInterval(() => {
+        this.interval = setInterval(() => {
             if (Date.now() - this.lastPersisted > 1000 * 60 * 4) {
                 this.persist()
             }
         }, 1000 * 60 * 5)
-        process.on('exit', () => {
+    }
-            this.persist()
+
-        });
+    Reset = () => {
+        clearInterval(this.interval)
+        fs.rmSync(this.storagePath, { recursive: true, force: true })
+        this.init()
     }
 
     LoadFile = (app: string, dataName: string, chunk: number): TlvFile => {

src/services/storage/tlv/tlvFilesStorageFactory.ts

@@ -1,6 +1,6 @@
 import { ChildProcess, fork } from 'child_process';
 import { EventEmitter } from 'events';
-import { AddTlvOperation, ITlvStorageOperation, SuccessTlvOperationResponse, LoadLatestTlvOperation, LoadTlvFileOperation, NewTlvStorageOperation, SerializableLatestData, SerializableTlvFile, TlvOperationResponse, TlvStorageSettings, WebRtcMessageOperation, ProcessMetricsTlvOperation } from './tlvFilesStorageProcessor';
+import { AddTlvOperation, ITlvStorageOperation, SuccessTlvOperationResponse, LoadLatestTlvOperation, LoadTlvFileOperation, NewTlvStorageOperation, SerializableLatestData, SerializableTlvFile, TlvOperationResponse, TlvStorageSettings, WebRtcMessageOperation, ProcessMetricsTlvOperation, ZipStoragesOperation, ResetTlvStorageOperation } from './tlvFilesStorageProcessor';
 import { LatestData, TlvFile } from './tlvFilesStorage';
 import { NostrSend, SendData, SendInitiator } from '../../nostr/handler';
 import { WebRtcUserInfo } from '../../webRTC';
@@ -17,8 +17,10 @@ export class TlvStorageFactory extends EventEmitter {
     private isConnected: boolean = false;
     private debug: boolean = false;
     private _nostrSend: NostrSend = () => { throw new Error('nostr send not initialized yet') }
-    constructor() {
+    private allowResetMetricsStorages: boolean
+    constructor(allowResetMetricsStorages: boolean) {
         super();
+        this.allowResetMetricsStorages = allowResetMetricsStorages
         this.initializeSubprocess();
     }
 
@@ -61,6 +63,21 @@ export class TlvStorageFactory extends EventEmitter {
         this.isConnected = true;
     }
 
+    ZipStorages(): Promise<string> {
+        const opId = Math.random().toString()
+        const op: ZipStoragesOperation = { type: 'zipStorages', opId }
+        return this.handleOp<string>(op)
+    }
+
+    ResetStorages(): Promise<void> {
+        if (!this.allowResetMetricsStorages) {
+            throw new Error('Resetting metrics storages is not allowed')
+        }
+        const opId = Math.random().toString()
+        const op: ResetTlvStorageOperation = { type: 'resetStorage', opId }
+        return this.handleOp<void>(op)
+    }
+
     NewStorage(settings: TlvStorageSettings): TlvStorageInterface {
         const opId = Math.random().toString()
         const op: NewTlvStorageOperation = { type: 'newStorage', opId, settings }

src/services/storage/tlv/tlvFilesStorageProcessor.ts

@@ -6,6 +6,8 @@ import { SendData } from '../../nostr/handler.js';
 import { SendInitiator } from '../../nostr/handler.js';
 import { ProcessMetrics, ProcessMetricsCollector } from './processMetricsCollector.js';
 import { integerToUint8Array } from '../../helpers/tlv.js';
+import { zip } from 'zip-a-folder'
+import crypto from 'crypto'
 export type SerializableLatestData = Record<string, Record<string, { base64tlvs: string[], current_chunk: number, available_chunks: number[] }>>
 export type SerializableTlvFile = { base64fileData: string, chunks: number[] }
 export const usageStorageName = 'usage'
@@ -15,6 +17,18 @@ export type TlvStorageSettings = {
     name: typeof usageStorageName | typeof bundlerStorageName
 }
 
+export type ZipStoragesOperation = {
+    type: 'zipStorages'
+    opId: string
+    debug?: boolean
+}
+
+export type ResetTlvStorageOperation = {
+    type: 'resetStorage'
+    opId: string
+    debug?: boolean
+}
+
 export type NewTlvStorageOperation = {
     type: 'newStorage'
     opId: string
@@ -74,7 +88,7 @@ export interface ITlvStorageOperation {
     debug?: boolean
 }
 
-export type TlvStorageOperation = NewTlvStorageOperation | AddTlvOperation | LoadLatestTlvOperation | LoadTlvFileOperation | WebRtcMessageOperation | ProcessMetricsTlvOperation
+export type TlvStorageOperation = NewTlvStorageOperation | AddTlvOperation | LoadLatestTlvOperation | LoadTlvFileOperation | WebRtcMessageOperation | ProcessMetricsTlvOperation | ResetTlvStorageOperation | ZipStoragesOperation
 
 export type SuccessTlvOperationResponse<T> = { success: true, type: string, data: T, opId: string }
 export type TlvOperationResponse<T> = SuccessTlvOperationResponse<T> | ErrorTlvOperationResponse
@@ -132,11 +146,15 @@ class TlvFilesStorageProcessor {
             console.log('no bundle storage yet')
             return
         }
-        if (pMetrics.memory_rss_kb) this.storages[bundlerStorageName].AddTlv('_root', 'memory_rss_kb' + pName, this.serializeNowTlv(pMetrics.memory_rss_kb))
+        for (const key in pMetrics) {
+            const v = pMetrics[key as keyof ProcessMetrics]
+            if (v) this.storages[bundlerStorageName].AddTlv('_root', key + pName, this.serializeNowTlv(v))
+        }
+        /*         if (pMetrics.memory_rss_kb) this.storages[bundlerStorageName].AddTlv('_root', 'memory_rss_kb' + pName, this.serializeNowTlv(pMetrics.memory_rss_kb))
                 if (pMetrics.memory_buffer_kb) this.storages[bundlerStorageName].AddTlv('_root', 'memory_buffer_kb' + pName, this.serializeNowTlv(pMetrics.memory_buffer_kb))
                 if (pMetrics.memory_heap_total_kb) this.storages[bundlerStorageName].AddTlv('_root', 'memory_heap_total_kb' + pName, this.serializeNowTlv(pMetrics.memory_heap_total_kb))
                 if (pMetrics.memory_heap_used_kb) this.storages[bundlerStorageName].AddTlv('_root', 'memory_heap_used_kb' + pName, this.serializeNowTlv(pMetrics.memory_heap_used_kb))
-        if (pMetrics.memory_external_kb) this.storages[bundlerStorageName].AddTlv('_root', 'memory_external_kb' + pName, this.serializeNowTlv(pMetrics.memory_external_kb))
+                if (pMetrics.memory_external_kb) this.storages[bundlerStorageName].AddTlv('_root', 'memory_external_kb' + pName, this.serializeNowTlv(pMetrics.memory_external_kb)) */
     }
 
     private async handleOperation(operation: TlvStorageOperation) {
@@ -144,6 +162,9 @@ class TlvFilesStorageProcessor {
             const opId = operation.opId;
             if (operation.debug) console.log('handleOperation', operation)
             switch (operation.type) {
+                case 'resetStorage':
+                    await this.handleResetStorage(operation);
+                    break;
                 case 'newStorage':
                     await this.handleNewStorage(operation);
                     break;
@@ -162,6 +183,9 @@ class TlvFilesStorageProcessor {
                 case 'processMetrics':
                     await this.handleProcessMetrics(operation);
                     break;
+                case 'zipStorages':
+                    await this.handleZipStorages(operation);
+                    break;
                 default:
                     this.sendResponse({
                         success: false,
@@ -179,6 +203,19 @@ class TlvFilesStorageProcessor {
         }
     }
 
+    private async handleResetStorage(operation: ResetTlvStorageOperation) {
+        for (const storageName in this.storages) {
+            this.storages[storageName].Reset()
+        }
+        this.sendResponse({
+            success: true,
+            type: 'resetStorage',
+            data: null,
+            opId: operation.opId
+        });
+    }
+
+
     private async handleNewStorage(operation: NewTlvStorageOperation) {
         if (this.storages[operation.settings.name]) {
             this.sendResponse({
@@ -279,6 +316,40 @@ class TlvFilesStorageProcessor {
         });
     }
 
+    private async handleZipStorages(operation: ZipStoragesOperation) {
+        const paths = []
+        for (const storageName in this.storages) {
+            paths.push(this.storages[storageName].GetStoragePath())
+        }
+        if (paths.length === 0) {
+            this.sendResponse({
+                success: false,
+                error: 'No storages to zip',
+                opId: operation.opId
+            })
+            return
+        }
+        const name = crypto.randomBytes(16).toString('hex') + '.zip'
+        const path = paths.join(', ')
+        const err = await zip(path, name)
+        if (err) {
+            this.sendResponse({
+                success: false,
+                error: err.message,
+                opId: operation.opId
+            })
+            return
+        }
+        this.sendResponse<string>({
+            success: true,
+            type: 'zipStorages',
+            data: name,
+            opId: operation.opId
+        });
+
+
+    }
+
     private sendResponse<T>(response: TlvOperationResponse<T>) {
         if (process.send) {
             process.send(response);

src/tests/networkSetup.ts

@@ -3,14 +3,12 @@ import { BitcoinCoreWrapper } from "./bitcoinCore.js"
 import LND from '../services/lnd/lnd.js'
 import { LiquidityProvider } from "../services/main/liquidityProvider.js"
 import { Utils } from "../services/helpers/utilsWrapper.js"
-import { TlvStorageFactory } from "../services/storage/tlv/tlvFilesStorageFactory.js"
 export const setupNetwork = async () => {
     const settings = LoadTestSettingsFromEnv()
     const core = new BitcoinCoreWrapper(settings)
     await core.InitAddress()
     await core.Mine(1)
-    const tlvStorageFactory = new TlvStorageFactory()
+    const setupUtils = new Utils({ dataDir: settings.storageSettings.dataDir, allowResetMetricsStorages: settings.allowResetMetricsStorages })
-    const setupUtils = new Utils({ dataDir: settings.storageSettings.dataDir })
     const alice = new LND(settings.lndSettings, new LiquidityProvider("", setupUtils, async () => { }, async () => { }), setupUtils, async () => { }, async () => { }, () => { }, () => { })
     const bob = new LND({ ...settings.lndSettings, mainNode: settings.lndSettings.otherNode }, new LiquidityProvider("", setupUtils, async () => { }, async () => { }), setupUtils, async () => { }, async () => { }, () => { }, () => { })
     await tryUntil<void>(async i => {

src/tests/testBase.ts

@@ -44,7 +44,7 @@ export type StorageTestBase = {
 
 export const setupStorageTest = async (d: Describe): Promise<StorageTestBase> => {
     const settings = GetTestStorageSettings()
-    const utils = new Utils({ dataDir: settings.dataDir })
+    const utils = new Utils({ dataDir: settings.dataDir, allowResetMetricsStorages: true })
     const storageManager = new Storage(settings, utils)
     await storageManager.Connect(console.log)
     return {
@@ -71,7 +71,7 @@ export const SetupTest = async (d: Describe): Promise<TestBase> => {
     const user1 = { userId: u1.info.userId, appUserIdentifier: u1.identifier, appId: app.appId }
     const user2 = { userId: u2.info.userId, appUserIdentifier: u2.identifier, appId: app.appId }
 
-    const extermnalUtils = new Utils({ dataDir: settings.storageSettings.dataDir })
+    const extermnalUtils = new Utils({ dataDir: settings.storageSettings.dataDir, allowResetMetricsStorages: settings.allowResetMetricsStorages })
     const externalAccessToMainLnd = new LND(settings.lndSettings, new LiquidityProvider("", extermnalUtils, async () => { }, async () => { }), extermnalUtils, async () => { }, async () => { }, () => { }, () => { })
     await externalAccessToMainLnd.Warmup()