From 67c12607487df65cd0686f447a66747204e8b5d1 Mon Sep 17 00:00:00 2001 From: boufni95 Date: Thu, 10 Apr 2025 16:47:26 +0000 Subject: [PATCH 01/19] reset metrics b4 stress test and export after --- package-lock.json | 865 +++++++++++++++++- package.json | 3 +- proto/autogenerated/client.md | 27 + proto/autogenerated/go/http_client.go | 49 + proto/autogenerated/go/types.go | 3 + proto/autogenerated/ts/express_server.ts | 38 + proto/autogenerated/ts/http_client.ts | 25 + proto/autogenerated/ts/nostr_client.ts | 25 + proto/autogenerated/ts/nostr_transport.ts | 26 + proto/autogenerated/ts/types.ts | 30 +- proto/service/methods.proto | 14 + proto/service/structs.proto | 4 + src/services/helpers/utilsWrapper.ts | 8 +- src/services/main/init.ts | 2 +- src/services/main/settings.ts | 6 +- src/services/serverMethods/index.ts | 7 + .../storage/tlv/processMetricsCollector.ts | 8 + src/services/storage/tlv/tlvFilesStorage.ts | 23 +- .../storage/tlv/tlvFilesStorageFactory.ts | 21 +- .../storage/tlv/tlvFilesStorageProcessor.ts | 83 +- src/tests/networkSetup.ts | 4 +- src/tests/testBase.ts | 4 +- 22 files changed, 1241 insertions(+), 34 deletions(-) diff --git a/package-lock.json b/package-lock.json index e1d5285c..db73cdc2 100644 --- a/package-lock.json +++ b/package-lock.json @@ -48,7 +48,8 @@ "websocket-polyfill": "^0.0.3", "why-is-node-running": "^3.2.0", "wrtc": "^0.4.7", - "ws": "^8.18.0" + "ws": "^8.18.0", + "zip-a-folder": "^3.1.9" }, "devDependencies": { "@types/chai": "^4.3.4", @@ -166,6 +167,102 @@ "node": ">=12" } }, + "node_modules/@isaacs/cliui": { + "version": "8.0.2", + "resolved": "https://registry.npmjs.org/@isaacs/cliui/-/cliui-8.0.2.tgz", + "integrity": "sha512-O8jcjabXaleOG9DQ0+ARXWZBTfnP4WNAqzuiJK7ll44AmxGKv/J2M4TPjxjY3znBCfvBXFzucm1twdyFybFqEA==", + "license": "ISC", + "dependencies": { + "string-width": "^5.1.2", + "string-width-cjs": "npm:string-width@^4.2.0", + "strip-ansi": "^7.0.1", + "strip-ansi-cjs": "npm:strip-ansi@^6.0.1", + "wrap-ansi": "^8.1.0", + "wrap-ansi-cjs": "npm:wrap-ansi@^7.0.0" + }, + "engines": { + "node": ">=12" + } + }, + "node_modules/@isaacs/cliui/node_modules/ansi-regex": { + "version": "6.1.0", + "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-6.1.0.tgz", + "integrity": "sha512-7HSX4QQb4CspciLpVFwyRe79O3xsIZDDLER21kERQ71oaPodF8jL725AgJMFAYbooIqolJoRLuM81SpeUkpkvA==", + "license": "MIT", + "engines": { + "node": ">=12" + }, + "funding": { + "url": "https://github.com/chalk/ansi-regex?sponsor=1" + } + }, + "node_modules/@isaacs/cliui/node_modules/ansi-styles": { + "version": "6.2.1", + "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-6.2.1.tgz", + "integrity": "sha512-bN798gFfQX+viw3R7yrGWRqnrN2oRkEkUjjl4JNn4E8GxxbjtG3FbrEIIY3l8/hrwUwIeCZvi4QuOTP4MErVug==", + "license": "MIT", + "engines": { + "node": ">=12" + }, + "funding": { + "url": "https://github.com/chalk/ansi-styles?sponsor=1" + } + }, + "node_modules/@isaacs/cliui/node_modules/emoji-regex": { + "version": "9.2.2", + "resolved": "https://registry.npmjs.org/emoji-regex/-/emoji-regex-9.2.2.tgz", + "integrity": "sha512-L18DaJsXSUk2+42pv8mLs5jJT2hqFkFE4j21wOmgbUqsZ2hL72NsUU785g9RXgo3s0ZNgVl42TiHp3ZtOv/Vyg==", + "license": "MIT" + }, + "node_modules/@isaacs/cliui/node_modules/string-width": { + "version": "5.1.2", + "resolved": "https://registry.npmjs.org/string-width/-/string-width-5.1.2.tgz", + "integrity": "sha512-HnLOCR3vjcY8beoNLtcjZ5/nxn2afmME6lhrDrebokqMap+XbeW8n9TXpPDOqdGK5qcI3oT0GKTW6wC7EMiVqA==", + "license": "MIT", + "dependencies": { + "eastasianwidth": "^0.2.0", + "emoji-regex": "^9.2.2", + "strip-ansi": "^7.0.1" + }, + "engines": { + "node": ">=12" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/@isaacs/cliui/node_modules/strip-ansi": { + "version": "7.1.0", + "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-7.1.0.tgz", + "integrity": "sha512-iq6eVVI64nQQTRYq2KtEg2d2uU7LElhTJwsH4YzIHZshxlgZms/wIc4VoDQTlG/IvVIrBKG06CrZnp0qv7hkcQ==", + "license": "MIT", + "dependencies": { + "ansi-regex": "^6.0.1" + }, + "engines": { + "node": ">=12" + }, + "funding": { + "url": "https://github.com/chalk/strip-ansi?sponsor=1" + } + }, + "node_modules/@isaacs/cliui/node_modules/wrap-ansi": { + "version": "8.1.0", + "resolved": "https://registry.npmjs.org/wrap-ansi/-/wrap-ansi-8.1.0.tgz", + "integrity": "sha512-si7QWI6zUMq56bESFvagtmzMdGOtoxfR+Sez11Mobfc7tm+VkUckk9bW2UeffTGVUbOksxmSw0AA2gs8g71NCQ==", + "license": "MIT", + "dependencies": { + "ansi-styles": "^6.1.0", + "string-width": "^5.0.1", + "strip-ansi": "^7.0.1" + }, + "engines": { + "node": ">=12" + }, + "funding": { + "url": "https://github.com/chalk/wrap-ansi?sponsor=1" + } + }, "node_modules/@js-sdsl/ordered-map": { "version": "4.4.2", "resolved": "https://registry.npmjs.org/@js-sdsl/ordered-map/-/ordered-map-4.4.2.tgz", @@ -295,6 +392,16 @@ "node": ">=10" } }, + "node_modules/@pkgjs/parseargs": { + "version": "0.11.0", + "resolved": "https://registry.npmjs.org/@pkgjs/parseargs/-/parseargs-0.11.0.tgz", + "integrity": "sha512-+1VkjdD0QBLPodGrJUeqarH8VAIvQODIbwh9XpP5Syisf7YoQgsJKPNFoqqLQlu+VQ/tVSshMR6loPMn8U+dPg==", + "license": "MIT", + "optional": true, + "engines": { + "node": ">=14" + } + }, "node_modules/@protobuf-ts/grpc-transport": { "version": "2.9.4", "resolved": "https://registry.npmjs.org/@protobuf-ts/grpc-transport/-/grpc-transport-2.9.4.tgz", @@ -790,6 +897,18 @@ "resolved": "https://registry.npmjs.org/abbrev/-/abbrev-1.1.1.tgz", "integrity": "sha512-nne9/IiQ/hzIhY6pdDnbBtz7DjPTKrY00P/zvPSm5pOFkl6xuGrGnXn/VtTNNfNtAfZ9/1RtehkszU9qcTii0Q==" }, + "node_modules/abort-controller": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/abort-controller/-/abort-controller-3.0.0.tgz", + "integrity": "sha512-h8lQ8tacZYnR3vNQTgibj+tODHI5/+l06Au2Pcriv/Gmet0eaj4TwWH41sO9wnHDiQsEj19q0drzdWdeAHtweg==", + "license": "MIT", + "dependencies": { + "event-target-shim": "^5.0.0" + }, + "engines": { + "node": ">=6.5" + } + }, "node_modules/accepts": { "version": "1.3.8", "resolved": "https://registry.npmjs.org/accepts/-/accepts-1.3.8.tgz", @@ -981,6 +1100,156 @@ "resolved": "https://registry.npmjs.org/aproba/-/aproba-2.0.0.tgz", "integrity": "sha512-lYe4Gx7QT+MKGbDsA+Z+he/Wtef0BiwDOlK/XkBrdfsh9J/jPPXbX0tE9x9cl27Tmu5gg3QUbUrQYa/y+KOHPQ==" }, + "node_modules/archiver": { + "version": "7.0.1", + "resolved": "https://registry.npmjs.org/archiver/-/archiver-7.0.1.tgz", + "integrity": "sha512-ZcbTaIqJOfCc03QwD468Unz/5Ir8ATtvAHsK+FdXbDIbGfihqh9mrvdcYunQzqn4HrvWWaFyaxJhGZagaJJpPQ==", + "license": "MIT", + "dependencies": { + "archiver-utils": "^5.0.2", + "async": "^3.2.4", + "buffer-crc32": "^1.0.0", + "readable-stream": "^4.0.0", + "readdir-glob": "^1.1.2", + "tar-stream": "^3.0.0", + "zip-stream": "^6.0.1" + }, + "engines": { + "node": ">= 14" + } + }, + "node_modules/archiver-utils": { + "version": "5.0.2", + "resolved": "https://registry.npmjs.org/archiver-utils/-/archiver-utils-5.0.2.tgz", + "integrity": "sha512-wuLJMmIBQYCsGZgYLTy5FIB2pF6Lfb6cXMSF8Qywwk3t20zWnAi7zLcQFdKQmIB8wyZpY5ER38x08GbwtR2cLA==", + "license": "MIT", + "dependencies": { + "glob": "^10.0.0", + "graceful-fs": "^4.2.0", + "is-stream": "^2.0.1", + "lazystream": "^1.0.0", + "lodash": "^4.17.15", + "normalize-path": "^3.0.0", + "readable-stream": "^4.0.0" + }, + "engines": { + "node": ">= 14" + } + }, + "node_modules/archiver-utils/node_modules/brace-expansion": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-2.0.1.tgz", + "integrity": "sha512-XnAIvQ8eM+kC6aULx6wuQiwVsnzsi9d3WxzV3FpWTGA19F621kwdbsAcFKXgKUHZWsy+mY6iL1sHTxWEFCytDA==", + "license": "MIT", + "dependencies": { + "balanced-match": "^1.0.0" + } + }, + "node_modules/archiver-utils/node_modules/glob": { + "version": "10.4.5", + "resolved": "https://registry.npmjs.org/glob/-/glob-10.4.5.tgz", + "integrity": "sha512-7Bv8RF0k6xjo7d4A/PxYLbUCfb6c+Vpd2/mB2yRDlew7Jb5hEXiCD9ibfO7wpk8i4sevK6DFny9h7EYbM3/sHg==", + "license": "ISC", + "dependencies": { + "foreground-child": "^3.1.0", + "jackspeak": "^3.1.2", + "minimatch": "^9.0.4", + "minipass": "^7.1.2", + "package-json-from-dist": "^1.0.0", + "path-scurry": "^1.11.1" + }, + "bin": { + "glob": "dist/esm/bin.mjs" + }, + "funding": { + "url": "https://github.com/sponsors/isaacs" + } + }, + "node_modules/archiver-utils/node_modules/minimatch": { + "version": "9.0.5", + "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-9.0.5.tgz", + "integrity": "sha512-G6T0ZX48xgozx7587koeX9Ys2NYy6Gmv//P89sEte9V9whIapMNF4idKxnW2QtCcLiTWlb/wfCabAtAFWhhBow==", + "license": "ISC", + "dependencies": { + "brace-expansion": "^2.0.1" + }, + "engines": { + "node": ">=16 || 14 >=14.17" + }, + "funding": { + "url": "https://github.com/sponsors/isaacs" + } + }, + "node_modules/archiver-utils/node_modules/minipass": { + "version": "7.1.2", + "resolved": "https://registry.npmjs.org/minipass/-/minipass-7.1.2.tgz", + "integrity": "sha512-qOOzS1cBTWYF4BH8fVePDBOO9iptMnGUEZwNc/cMWnTV2nVLZ7VoNWEPHkYczZA0pdoA7dl6e7FL659nX9S2aw==", + "license": "ISC", + "engines": { + "node": ">=16 || 14 >=14.17" + } + }, + "node_modules/archiver-utils/node_modules/readable-stream": { + "version": "4.7.0", + "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-4.7.0.tgz", + "integrity": "sha512-oIGGmcpTLwPga8Bn6/Z75SVaH1z5dUut2ibSyAMVhmUggWpmDn2dapB0n7f8nwaSiRtepAsfJyfXIO5DCVAODg==", + "license": "MIT", + "dependencies": { + "abort-controller": "^3.0.0", + "buffer": "^6.0.3", + "events": "^3.3.0", + "process": "^0.11.10", + "string_decoder": "^1.3.0" + }, + "engines": { + "node": "^12.22.0 || ^14.17.0 || >=16.0.0" + } + }, + "node_modules/archiver-utils/node_modules/string_decoder": { + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.3.0.tgz", + "integrity": "sha512-hkRX8U1WjJFd8LsDJ2yQ/wWWxaopEsABU1XfkM8A+j0+85JAGppt16cr1Whg6KIbb4okU6Mql6BOj+uup/wKeA==", + "license": "MIT", + "dependencies": { + "safe-buffer": "~5.2.0" + } + }, + "node_modules/archiver/node_modules/readable-stream": { + "version": "4.7.0", + "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-4.7.0.tgz", + "integrity": "sha512-oIGGmcpTLwPga8Bn6/Z75SVaH1z5dUut2ibSyAMVhmUggWpmDn2dapB0n7f8nwaSiRtepAsfJyfXIO5DCVAODg==", + "license": "MIT", + "dependencies": { + "abort-controller": "^3.0.0", + "buffer": "^6.0.3", + "events": "^3.3.0", + "process": "^0.11.10", + "string_decoder": "^1.3.0" + }, + "engines": { + "node": "^12.22.0 || ^14.17.0 || >=16.0.0" + } + }, + "node_modules/archiver/node_modules/string_decoder": { + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.3.0.tgz", + "integrity": "sha512-hkRX8U1WjJFd8LsDJ2yQ/wWWxaopEsABU1XfkM8A+j0+85JAGppt16cr1Whg6KIbb4okU6Mql6BOj+uup/wKeA==", + "license": "MIT", + "dependencies": { + "safe-buffer": "~5.2.0" + } + }, + "node_modules/archiver/node_modules/tar-stream": { + "version": "3.1.7", + "resolved": "https://registry.npmjs.org/tar-stream/-/tar-stream-3.1.7.tgz", + "integrity": "sha512-qJj60CXt7IU1Ffyc3NJMjh6EkuCFej46zUqJ4J7pqYlThyd9bO0XBTmcOIhSzZJVWfsLks0+nle/j538YAW9RQ==", + "license": "MIT", + "dependencies": { + "b4a": "^1.6.4", + "fast-fifo": "^1.2.0", + "streamx": "^2.15.0" + } + }, "node_modules/are-we-there-yet": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/are-we-there-yet/-/are-we-there-yet-2.0.0.tgz", @@ -1049,6 +1318,12 @@ "node": "*" } }, + "node_modules/async": { + "version": "3.2.6", + "resolved": "https://registry.npmjs.org/async/-/async-3.2.6.tgz", + "integrity": "sha512-htCUDlxyyCLMgaM3xXg0C0LW2xqfuQ6p05pCEIsXuyQ+a1koYKTuBMzRNwmybfLgvJDMd0r1LTn4+E0Ti6C2AA==", + "license": "MIT" + }, "node_modules/asynckit": { "version": "0.4.0", "resolved": "https://registry.npmjs.org/asynckit/-/asynckit-0.4.0.tgz", @@ -1077,11 +1352,24 @@ "proxy-from-env": "^1.1.0" } }, + "node_modules/b4a": { + "version": "1.6.7", + "resolved": "https://registry.npmjs.org/b4a/-/b4a-1.6.7.tgz", + "integrity": "sha512-OnAYlL5b7LEkALw87fUVafQw5rVR9RjwGd4KUwNQ6DrrNmaVaUCgLipfVlzrPQ4tWOR9P0IXGNOx50jYCCdSJg==", + "license": "Apache-2.0" + }, "node_modules/balanced-match": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/balanced-match/-/balanced-match-1.0.2.tgz", "integrity": "sha512-3oSeUO0TMV67hN1AmbXsK4yaqU7tjiHlbxRDZOpH0KW9+CeX4bRAaX0Anxt0tx2MrpRpWwQaPwIlISEJhYU5Pw==" }, + "node_modules/bare-events": { + "version": "2.5.4", + "resolved": "https://registry.npmjs.org/bare-events/-/bare-events-2.5.4.tgz", + "integrity": "sha512-+gFfDkR8pj4/TrWCGUGWmJIkBwuxPS5F+a5yWjOHQt2hHvNZd5YLzadjmDUtFmMM4y429bnKLa8bYBMHcYdnQA==", + "license": "Apache-2.0", + "optional": true + }, "node_modules/base64-js": { "version": "1.5.1", "resolved": "https://registry.npmjs.org/base64-js/-/base64-js-1.5.1.tgz", @@ -1317,6 +1605,15 @@ "ieee754": "^1.2.1" } }, + "node_modules/buffer-crc32": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/buffer-crc32/-/buffer-crc32-1.0.0.tgz", + "integrity": "sha512-Db1SbgBS/fg/392AblrMJk97KggmvYhr4pB5ZIMTWtaivCPMWLkmb7m21cJvpvgK+J3nsU2CmmixNBZx4vFj/w==", + "license": "MIT", + "engines": { + "node": ">=8.0.0" + } + }, "node_modules/buffer-equal-constant-time": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/buffer-equal-constant-time/-/buffer-equal-constant-time-1.0.1.tgz", @@ -1592,6 +1889,47 @@ "node": ">= 0.8" } }, + "node_modules/compress-commons": { + "version": "6.0.2", + "resolved": "https://registry.npmjs.org/compress-commons/-/compress-commons-6.0.2.tgz", + "integrity": "sha512-6FqVXeETqWPoGcfzrXb37E50NP0LXT8kAMu5ooZayhWWdgEY4lBEEcbQNXtkuKQsGduxiIcI4gOTsxTmuq/bSg==", + "license": "MIT", + "dependencies": { + "crc-32": "^1.2.0", + "crc32-stream": "^6.0.0", + "is-stream": "^2.0.1", + "normalize-path": "^3.0.0", + "readable-stream": "^4.0.0" + }, + "engines": { + "node": ">= 14" + } + }, + "node_modules/compress-commons/node_modules/readable-stream": { + "version": "4.7.0", + "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-4.7.0.tgz", + "integrity": "sha512-oIGGmcpTLwPga8Bn6/Z75SVaH1z5dUut2ibSyAMVhmUggWpmDn2dapB0n7f8nwaSiRtepAsfJyfXIO5DCVAODg==", + "license": "MIT", + "dependencies": { + "abort-controller": "^3.0.0", + "buffer": "^6.0.3", + "events": "^3.3.0", + "process": "^0.11.10", + "string_decoder": "^1.3.0" + }, + "engines": { + "node": "^12.22.0 || ^14.17.0 || >=16.0.0" + } + }, + "node_modules/compress-commons/node_modules/string_decoder": { + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.3.0.tgz", + "integrity": "sha512-hkRX8U1WjJFd8LsDJ2yQ/wWWxaopEsABU1XfkM8A+j0+85JAGppt16cr1Whg6KIbb4okU6Mql6BOj+uup/wKeA==", + "license": "MIT", + "dependencies": { + "safe-buffer": "~5.2.0" + } + }, "node_modules/concat-map": { "version": "0.0.1", "resolved": "https://registry.npmjs.org/concat-map/-/concat-map-0.0.1.tgz", @@ -1669,6 +2007,56 @@ "node": ">= 0.10" } }, + "node_modules/crc-32": { + "version": "1.2.2", + "resolved": "https://registry.npmjs.org/crc-32/-/crc-32-1.2.2.tgz", + "integrity": "sha512-ROmzCKrTnOwybPcJApAA6WBWij23HVfGVNKqqrZpuyZOHqK2CwHSvpGuyt/UNNvaIjEd8X5IFGp4Mh+Ie1IHJQ==", + "license": "Apache-2.0", + "bin": { + "crc32": "bin/crc32.njs" + }, + "engines": { + "node": ">=0.8" + } + }, + "node_modules/crc32-stream": { + "version": "6.0.0", + "resolved": "https://registry.npmjs.org/crc32-stream/-/crc32-stream-6.0.0.tgz", + "integrity": "sha512-piICUB6ei4IlTv1+653yq5+KoqfBYmj9bw6LqXoOneTMDXk5nM1qt12mFW1caG3LlJXEKW1Bp0WggEmIfQB34g==", + "license": "MIT", + "dependencies": { + "crc-32": "^1.2.0", + "readable-stream": "^4.0.0" + }, + "engines": { + "node": ">= 14" + } + }, + "node_modules/crc32-stream/node_modules/readable-stream": { + "version": "4.7.0", + "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-4.7.0.tgz", + "integrity": "sha512-oIGGmcpTLwPga8Bn6/Z75SVaH1z5dUut2ibSyAMVhmUggWpmDn2dapB0n7f8nwaSiRtepAsfJyfXIO5DCVAODg==", + "license": "MIT", + "dependencies": { + "abort-controller": "^3.0.0", + "buffer": "^6.0.3", + "events": "^3.3.0", + "process": "^0.11.10", + "string_decoder": "^1.3.0" + }, + "engines": { + "node": "^12.22.0 || ^14.17.0 || >=16.0.0" + } + }, + "node_modules/crc32-stream/node_modules/string_decoder": { + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.3.0.tgz", + "integrity": "sha512-hkRX8U1WjJFd8LsDJ2yQ/wWWxaopEsABU1XfkM8A+j0+85JAGppt16cr1Whg6KIbb4okU6Mql6BOj+uup/wKeA==", + "license": "MIT", + "dependencies": { + "safe-buffer": "~5.2.0" + } + }, "node_modules/create-hash": { "version": "1.2.0", "resolved": "https://registry.npmjs.org/create-hash/-/create-hash-1.2.0.tgz", @@ -1702,6 +2090,20 @@ "integrity": "sha512-dcKFX3jn0MpIaXjisoRvexIJVEKzaq7z2rZKxf+MSr9TkdmHmsU4m2lcLojrj/FHl8mk5VxMmYA+ftRkP/3oKQ==", "devOptional": true }, + "node_modules/cross-spawn": { + "version": "7.0.6", + "resolved": "https://registry.npmjs.org/cross-spawn/-/cross-spawn-7.0.6.tgz", + "integrity": "sha512-uV2QOWP2nWzsy2aMp8aRibhi9dlzF5Hgh5SHaB9OiTGEyDTiJJyx0uy51QXdyWbtAHNua4XJzUKca3OzKUd3vA==", + "license": "MIT", + "dependencies": { + "path-key": "^3.1.0", + "shebang-command": "^2.0.0", + "which": "^2.0.1" + }, + "engines": { + "node": ">= 8" + } + }, "node_modules/csv": { "version": "6.3.8", "resolved": "https://registry.npmjs.org/csv/-/csv-6.3.8.tgz", @@ -1958,6 +2360,12 @@ "node": ">=0.10" } }, + "node_modules/eastasianwidth": { + "version": "0.2.0", + "resolved": "https://registry.npmjs.org/eastasianwidth/-/eastasianwidth-0.2.0.tgz", + "integrity": "sha512-I88TYZWc9XiYHRQ4/3c5rjjfgkjhLyW2luGIheGERbNQ6OY7yTybanSpDXZa8y7VUP9YmDcYa+eyq4ca7iLqWA==", + "license": "MIT" + }, "node_modules/ecc-jsbn": { "version": "0.1.2", "resolved": "https://registry.npmjs.org/ecc-jsbn/-/ecc-jsbn-0.1.2.tgz", @@ -2193,6 +2601,24 @@ "es5-ext": "~0.10.14" } }, + "node_modules/event-target-shim": { + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/event-target-shim/-/event-target-shim-5.0.1.tgz", + "integrity": "sha512-i/2XbnSz/uxRCU6+NdVJgKWDTM427+MqYbkQzD321DuCQJUqOuJKIA0IM2+W2xtYHdKOmZ4dR6fExsd4SXL+WQ==", + "license": "MIT", + "engines": { + "node": ">=6" + } + }, + "node_modules/events": { + "version": "3.3.0", + "resolved": "https://registry.npmjs.org/events/-/events-3.3.0.tgz", + "integrity": "sha512-mQw+2fkQbALzQ7V0MY0IqdnXNOeTtP4r0lN9z7AAawCXgqea7bDii20AYrIBrFd/Hx0M2Ocz6S111CaFkUcb0Q==", + "license": "MIT", + "engines": { + "node": ">=0.8.x" + } + }, "node_modules/evp_bytestokey": { "version": "1.0.3", "resolved": "https://registry.npmjs.org/evp_bytestokey/-/evp_bytestokey-1.0.3.tgz", @@ -2291,6 +2717,12 @@ "resolved": "https://registry.npmjs.org/fast-deep-equal/-/fast-deep-equal-3.1.3.tgz", "integrity": "sha512-f3qQ9oQy9j2AhBe/H9VC91wLmKBCCU/gDOnKNAYG5hswO7BLKj09Hc5HYNz9cGI++xlpDCIgDaitVs03ATR84Q==" }, + "node_modules/fast-fifo": { + "version": "1.3.2", + "resolved": "https://registry.npmjs.org/fast-fifo/-/fast-fifo-1.3.2.tgz", + "integrity": "sha512-/d9sfos4yxzpwkDkuN7k2SqFKtYNmCTzgfEpz82x34IM9/zc8KGxQoXg1liNC/izpRM/MBdt44Nmx41ZWqk+FQ==", + "license": "MIT" + }, "node_modules/fast-glob": { "version": "3.2.12", "resolved": "https://registry.npmjs.org/fast-glob/-/fast-glob-3.2.12.tgz", @@ -2379,6 +2811,34 @@ } } }, + "node_modules/foreground-child": { + "version": "3.3.1", + "resolved": "https://registry.npmjs.org/foreground-child/-/foreground-child-3.3.1.tgz", + "integrity": "sha512-gIXjKqtFuWEgzFRJA9WCQeSJLZDjgJUOMCMzxtvFq/37KojM1BFGufqsCy0r4qSQmYLsZYMeyRqzIWOMup03sw==", + "license": "ISC", + "dependencies": { + "cross-spawn": "^7.0.6", + "signal-exit": "^4.0.1" + }, + "engines": { + "node": ">=14" + }, + "funding": { + "url": "https://github.com/sponsors/isaacs" + } + }, + "node_modules/foreground-child/node_modules/signal-exit": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/signal-exit/-/signal-exit-4.1.0.tgz", + "integrity": "sha512-bzyZ1e88w9O1iNJbKnOlvYTrWPDl46O1bG0D3XInv+9tkPrxrN8jUUTiFlDkkmKWgn1M6CfIA13SuGqOa9Korw==", + "license": "ISC", + "engines": { + "node": ">=14" + }, + "funding": { + "url": "https://github.com/sponsors/isaacs" + } + }, "node_modules/forever-agent": { "version": "0.6.1", "resolved": "https://registry.npmjs.org/forever-agent/-/forever-agent-0.6.1.tgz", @@ -2573,8 +3033,7 @@ "node_modules/graceful-fs": { "version": "4.2.10", "resolved": "https://registry.npmjs.org/graceful-fs/-/graceful-fs-4.2.10.tgz", - "integrity": "sha512-9ByhssR2fPVsNZj478qUUbKfmL0+t5BDVyjShtyZZLiK7ZDAArFFfopyOTj0M05wE2tJPisA4iTnnXl2YoPvOA==", - "optional": true + "integrity": "sha512-9ByhssR2fPVsNZj478qUUbKfmL0+t5BDVyjShtyZZLiK7ZDAArFFfopyOTj0M05wE2tJPisA4iTnnXl2YoPvOA==" }, "node_modules/grpc-tools": { "version": "1.12.4", @@ -2999,6 +3458,18 @@ "node": ">=0.12.0" } }, + "node_modules/is-stream": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/is-stream/-/is-stream-2.0.1.tgz", + "integrity": "sha512-hFoiJiTl63nn+kstHGBtewWSKnQLpyb155KHheA1l39uvtO9nWIop1p3udqPcUd/xbF1VLMO4n7OI6p7RbngDg==", + "license": "MIT", + "engines": { + "node": ">=8" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, "node_modules/is-typedarray": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/is-typedarray/-/is-typedarray-1.0.0.tgz", @@ -3012,14 +3483,28 @@ "node_modules/isexe": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/isexe/-/isexe-2.0.0.tgz", - "integrity": "sha512-RHxMLp9lnKHGHRng9QFhRCMbYAcVpn69smSGcq3f36xjgVVWThj4qqLbTLlq7Ssj8B+fIQ1EuCEGI2lKsyQeIw==", - "optional": true + "integrity": "sha512-RHxMLp9lnKHGHRng9QFhRCMbYAcVpn69smSGcq3f36xjgVVWThj4qqLbTLlq7Ssj8B+fIQ1EuCEGI2lKsyQeIw==" }, "node_modules/isstream": { "version": "0.1.2", "resolved": "https://registry.npmjs.org/isstream/-/isstream-0.1.2.tgz", "integrity": "sha512-Yljz7ffyPbrLpLngrMtZ7NduUgVvi6wG9RJ9IUcyCd59YQ911PBJphODUcbOVbqYfxe1wuYf/LJ8PauMRwsM/g==" }, + "node_modules/jackspeak": { + "version": "3.4.3", + "resolved": "https://registry.npmjs.org/jackspeak/-/jackspeak-3.4.3.tgz", + "integrity": "sha512-OGlZQpz2yfahA/Rd1Y8Cd9SIEsqvXkLVoSw/cgwhnhFMDbsQFeZYoJJ7bIZBS9BcamUW96asq/npPWugM+RQBw==", + "license": "BlueOak-1.0.0", + "dependencies": { + "@isaacs/cliui": "^8.0.2" + }, + "funding": { + "url": "https://github.com/sponsors/isaacs" + }, + "optionalDependencies": { + "@pkgjs/parseargs": "^0.11.0" + } + }, "node_modules/jsbn": { "version": "0.1.1", "resolved": "https://registry.npmjs.org/jsbn/-/jsbn-0.1.1.tgz", @@ -3107,6 +3592,54 @@ "safe-buffer": "^5.0.1" } }, + "node_modules/lazystream": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/lazystream/-/lazystream-1.0.1.tgz", + "integrity": "sha512-b94GiNHQNy6JNTrt5w6zNyffMrNkXZb3KTkCZJb2V1xaEGCk093vkZ2jk3tpaeP33/OiXC+WvK9AxUebnf5nbw==", + "license": "MIT", + "dependencies": { + "readable-stream": "^2.0.5" + }, + "engines": { + "node": ">= 0.6.3" + } + }, + "node_modules/lazystream/node_modules/isarray": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/isarray/-/isarray-1.0.0.tgz", + "integrity": "sha512-VLghIWNM6ELQzo7zwmcg0NmTVyWKYjvIeM83yjp0wRDTmUnrM678fQbcKBo6n2CJEF0szoG//ytg+TKla89ALQ==", + "license": "MIT" + }, + "node_modules/lazystream/node_modules/readable-stream": { + "version": "2.3.8", + "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-2.3.8.tgz", + "integrity": "sha512-8p0AUk4XODgIewSi0l8Epjs+EVnWiK7NoDIEGU0HhE7+ZyY8D1IMY7odu5lRrFXGg71L15KG8QrPmum45RTtdA==", + "license": "MIT", + "dependencies": { + "core-util-is": "~1.0.0", + "inherits": "~2.0.3", + "isarray": "~1.0.0", + "process-nextick-args": "~2.0.0", + "safe-buffer": "~5.1.1", + "string_decoder": "~1.1.1", + "util-deprecate": "~1.0.1" + } + }, + "node_modules/lazystream/node_modules/safe-buffer": { + "version": "5.1.2", + "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.1.2.tgz", + "integrity": "sha512-Gd2UZBJDkXlY7GbJxfsE8/nvKkUEU1G38c1siN6QP6a9PT9MmHB8GnpscSmMJSoF8LOIrt8ud/wPtojys4G6+g==", + "license": "MIT" + }, + "node_modules/lazystream/node_modules/string_decoder": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.1.1.tgz", + "integrity": "sha512-n/ShnvDi6FHbbVfviro+WojiFzv+s8MPMHBczVePfUpDJLwoLT0ht1l4YwBCbi8pJAveEEdnkHyPyTP/mzRfwg==", + "license": "MIT", + "dependencies": { + "safe-buffer": "~5.1.0" + } + }, "node_modules/lodash": { "version": "4.17.21", "resolved": "https://registry.npmjs.org/lodash/-/lodash-4.17.21.tgz", @@ -3809,7 +4342,6 @@ "version": "3.0.0", "resolved": "https://registry.npmjs.org/normalize-path/-/normalize-path-3.0.0.tgz", "integrity": "sha512-6eZs5Ls3WtCisHWp9S2GUy8dqkpGi4BVSz3GaqiE6ezub0512ESztXUwUB6C6IKbQkY2Pnb/mD4WYojCRwcwLA==", - "dev": true, "engines": { "node": ">=0.10.0" } @@ -3926,6 +4458,12 @@ "url": "https://github.com/sponsors/sindresorhus" } }, + "node_modules/package-json-from-dist": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/package-json-from-dist/-/package-json-from-dist-1.0.1.tgz", + "integrity": "sha512-UEZIS3/by4OC8vL3P2dTXRETpebLI2NiI5vIrjaD/5UtrkFX/tNbwjTSRAGC/+7CAo2pIcBaRgWmcBBHcsaCIw==", + "license": "BlueOak-1.0.0" + }, "node_modules/packet-reader": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/packet-reader/-/packet-reader-1.0.0.tgz", @@ -3965,6 +4503,46 @@ "node": ">=0.10.0" } }, + "node_modules/path-key": { + "version": "3.1.1", + "resolved": "https://registry.npmjs.org/path-key/-/path-key-3.1.1.tgz", + "integrity": "sha512-ojmeN0qd+y0jszEtoY48r0Peq5dwMEkIlCOu6Q5f41lfkswXuKtYrhgoTpLnyIcHm24Uhqx+5Tqm2InSwLhE6Q==", + "license": "MIT", + "engines": { + "node": ">=8" + } + }, + "node_modules/path-scurry": { + "version": "1.11.1", + "resolved": "https://registry.npmjs.org/path-scurry/-/path-scurry-1.11.1.tgz", + "integrity": "sha512-Xa4Nw17FS9ApQFJ9umLiJS4orGjm7ZzwUrwamcGQuHSzDyth9boKDaycYdDcZDuqYATXw4HFXgaqWTctW/v1HA==", + "license": "BlueOak-1.0.0", + "dependencies": { + "lru-cache": "^10.2.0", + "minipass": "^5.0.0 || ^6.0.2 || ^7.0.0" + }, + "engines": { + "node": ">=16 || 14 >=14.18" + }, + "funding": { + "url": "https://github.com/sponsors/isaacs" + } + }, + "node_modules/path-scurry/node_modules/lru-cache": { + "version": "10.4.3", + "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-10.4.3.tgz", + "integrity": "sha512-JNAzZcXrCt42VGLuYz0zfAzDfAvJWW6AfYlDBQyDV5DClI2m5sAmK+OIO7s59XfsRsWHp02jAJrRadPRGTt6SQ==", + "license": "ISC" + }, + "node_modules/path-scurry/node_modules/minipass": { + "version": "7.1.2", + "resolved": "https://registry.npmjs.org/minipass/-/minipass-7.1.2.tgz", + "integrity": "sha512-qOOzS1cBTWYF4BH8fVePDBOO9iptMnGUEZwNc/cMWnTV2nVLZ7VoNWEPHkYczZA0pdoA7dl6e7FL659nX9S2aw==", + "license": "ISC", + "engines": { + "node": ">=16 || 14 >=14.17" + } + }, "node_modules/path-to-regexp": { "version": "0.1.10", "resolved": "https://registry.npmjs.org/path-to-regexp/-/path-to-regexp-0.1.10.tgz", @@ -4136,6 +4714,15 @@ "node": ">=10" } }, + "node_modules/process": { + "version": "0.11.10", + "resolved": "https://registry.npmjs.org/process/-/process-0.11.10.tgz", + "integrity": "sha512-cdGef/drWFoydD1JsMzuFf8100nZl+GT+yacc2bEced5f9Rjk4z+WtFUTBu9PhOi9j/jfmBPu0mMEY4wIdAF8A==", + "license": "MIT", + "engines": { + "node": ">= 0.6.0" + } + }, "node_modules/process-nextick-args": { "version": "2.0.1", "resolved": "https://registry.npmjs.org/process-nextick-args/-/process-nextick-args-2.0.1.tgz", @@ -4313,6 +4900,36 @@ "string_decoder": "~0.10.x" } }, + "node_modules/readdir-glob": { + "version": "1.1.3", + "resolved": "https://registry.npmjs.org/readdir-glob/-/readdir-glob-1.1.3.tgz", + "integrity": "sha512-v05I2k7xN8zXvPD9N+z/uhXPaj0sUFCe2rcWZIpBsqxfP7xXFQ0tipAd/wjj1YxWyWtUS5IDJpOG82JKt2EAVA==", + "license": "Apache-2.0", + "dependencies": { + "minimatch": "^5.1.0" + } + }, + "node_modules/readdir-glob/node_modules/brace-expansion": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-2.0.1.tgz", + "integrity": "sha512-XnAIvQ8eM+kC6aULx6wuQiwVsnzsi9d3WxzV3FpWTGA19F621kwdbsAcFKXgKUHZWsy+mY6iL1sHTxWEFCytDA==", + "license": "MIT", + "dependencies": { + "balanced-match": "^1.0.0" + } + }, + "node_modules/readdir-glob/node_modules/minimatch": { + "version": "5.1.6", + "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-5.1.6.tgz", + "integrity": "sha512-lKwV/1brpG6mBUFHtb7NUmtABCb2WZZmm2wNiOA5hAb8VdCS4B3dtMWyvcoViccwAW/COERjXLt0zP1zXUN26g==", + "license": "ISC", + "dependencies": { + "brace-expansion": "^2.0.1" + }, + "engines": { + "node": ">=10" + } + }, "node_modules/readdirp": { "version": "3.6.0", "resolved": "https://registry.npmjs.org/readdirp/-/readdirp-3.6.0.tgz", @@ -4617,6 +5234,27 @@ "sha.js": "bin.js" } }, + "node_modules/shebang-command": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/shebang-command/-/shebang-command-2.0.0.tgz", + "integrity": "sha512-kHxr2zZpYtdmrN1qDjrrX/Z1rR1kG8Dx+gkpK1G4eXmvXswmcE1hTWBWYUzlraYw1/yZp6YuDY77YtvbN0dmDA==", + "license": "MIT", + "dependencies": { + "shebang-regex": "^3.0.0" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/shebang-regex": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/shebang-regex/-/shebang-regex-3.0.0.tgz", + "integrity": "sha512-7++dFhtcx3353uBaq8DDR4NuxBetBzC7ZQOhmTQInHEd6bSrXdiEyzCvG07Z44UYdLShWUyXt5M/yhz8ekcb1A==", + "license": "MIT", + "engines": { + "node": ">=8" + } + }, "node_modules/side-channel": { "version": "1.0.6", "resolved": "https://registry.npmjs.org/side-channel/-/side-channel-1.0.6.tgz", @@ -4865,6 +5503,19 @@ "resolved": "https://registry.npmjs.org/stream-transform/-/stream-transform-3.3.1.tgz", "integrity": "sha512-BL8pv9QL8Ikd11oZwlRDp1qYMhGR0i50zI9ltoijKGc4ubQWal/Rc4p6SYJp1TBOGpE0uAGchwbxOZ1ycwTuqQ==" }, + "node_modules/streamx": { + "version": "2.22.0", + "resolved": "https://registry.npmjs.org/streamx/-/streamx-2.22.0.tgz", + "integrity": "sha512-sLh1evHOzBy/iWRiR6d1zRcLao4gGZr3C1kzNz4fopCOKJb6xD9ub8Mpi9Mr1R6id5o43S+d93fI48UC5uM9aw==", + "license": "MIT", + "dependencies": { + "fast-fifo": "^1.3.2", + "text-decoder": "^1.1.0" + }, + "optionalDependencies": { + "bare-events": "^2.2.0" + } + }, "node_modules/string_decoder": { "version": "0.10.31", "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-0.10.31.tgz", @@ -4883,6 +5534,21 @@ "node": ">=8" } }, + "node_modules/string-width-cjs": { + "name": "string-width", + "version": "4.2.3", + "resolved": "https://registry.npmjs.org/string-width/-/string-width-4.2.3.tgz", + "integrity": "sha512-wKyQRQpjJ0sIp62ErSZdGsjMJWsap5oRNihHhu6G7JVO/9jIB6UyevL+tXuOqrng8j/cxKTWyWUwvSTriiZz/g==", + "license": "MIT", + "dependencies": { + "emoji-regex": "^8.0.0", + "is-fullwidth-code-point": "^3.0.0", + "strip-ansi": "^6.0.1" + }, + "engines": { + "node": ">=8" + } + }, "node_modules/strip-ansi": { "version": "6.0.1", "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-6.0.1.tgz", @@ -4894,6 +5560,19 @@ "node": ">=8" } }, + "node_modules/strip-ansi-cjs": { + "name": "strip-ansi", + "version": "6.0.1", + "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-6.0.1.tgz", + "integrity": "sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A==", + "license": "MIT", + "dependencies": { + "ansi-regex": "^5.0.1" + }, + "engines": { + "node": ">=8" + } + }, "node_modules/strip-json-comments": { "version": "2.0.1", "resolved": "https://registry.npmjs.org/strip-json-comments/-/strip-json-comments-2.0.1.tgz", @@ -4989,6 +5668,15 @@ "node": ">=8" } }, + "node_modules/text-decoder": { + "version": "1.2.3", + "resolved": "https://registry.npmjs.org/text-decoder/-/text-decoder-1.2.3.tgz", + "integrity": "sha512-3/o9z3X0X0fTupwsYvR03pJ/DjWuqqrfwBgTQzdWDiQSm9KitAyz/9WqsT2JQW7KV2m+bC2ol/zqpW37NHxLaA==", + "license": "Apache-2.0", + "dependencies": { + "b4a": "^1.6.4" + } + }, "node_modules/thenify": { "version": "3.3.1", "resolved": "https://registry.npmjs.org/thenify/-/thenify-3.3.1.tgz", @@ -5710,7 +6398,6 @@ "version": "2.0.2", "resolved": "https://registry.npmjs.org/which/-/which-2.0.2.tgz", "integrity": "sha512-BLI3Tl1TW3Pvl70l3yq3Y64i+awpwXqsGBYWkkqMtnbXgrMD+yj7rhW0kuEDxzJaYXGjEW5ogapKNMEKNMjibA==", - "optional": true, "dependencies": { "isexe": "^2.0.0" }, @@ -5756,6 +6443,24 @@ "url": "https://github.com/chalk/wrap-ansi?sponsor=1" } }, + "node_modules/wrap-ansi-cjs": { + "name": "wrap-ansi", + "version": "7.0.0", + "resolved": "https://registry.npmjs.org/wrap-ansi/-/wrap-ansi-7.0.0.tgz", + "integrity": "sha512-YVGIj2kamLSTxw6NsZjoBxfSwsn0ycdesmc4p+Q21c5zPuZ1pl+NfxVdxPtdHvmNVOQ6XSYG4AUtyt/Fi7D16Q==", + "license": "MIT", + "dependencies": { + "ansi-styles": "^4.0.0", + "string-width": "^4.1.0", + "strip-ansi": "^6.0.0" + }, + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/chalk/wrap-ansi?sponsor=1" + } + }, "node_modules/wrappy": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/wrappy/-/wrappy-1.0.2.tgz", @@ -5863,6 +6568,152 @@ "engines": { "node": ">=6" } + }, + "node_modules/zip-a-folder": { + "version": "3.1.9", + "resolved": "https://registry.npmjs.org/zip-a-folder/-/zip-a-folder-3.1.9.tgz", + "integrity": "sha512-0TPP3eK5mbZxHnOE8w/Jg6gwxsxZOrA3hXHMfC3I4mcTvyJwNt7GZP8i6uiAMVNu43QTmVz0ngEMKcjgpLZLmQ==", + "license": "MIT", + "dependencies": { + "archiver": "^7.0.1", + "glob": "^11.0.1", + "is-glob": "^4.0.3" + } + }, + "node_modules/zip-a-folder/node_modules/brace-expansion": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-2.0.1.tgz", + "integrity": "sha512-XnAIvQ8eM+kC6aULx6wuQiwVsnzsi9d3WxzV3FpWTGA19F621kwdbsAcFKXgKUHZWsy+mY6iL1sHTxWEFCytDA==", + "license": "MIT", + "dependencies": { + "balanced-match": "^1.0.0" + } + }, + "node_modules/zip-a-folder/node_modules/glob": { + "version": "11.0.1", + "resolved": "https://registry.npmjs.org/glob/-/glob-11.0.1.tgz", + "integrity": "sha512-zrQDm8XPnYEKawJScsnM0QzobJxlT/kHOOlRTio8IH/GrmxRE5fjllkzdaHclIuNjUQTJYH2xHNIGfdpJkDJUw==", + "license": "ISC", + "dependencies": { + "foreground-child": "^3.1.0", + "jackspeak": "^4.0.1", + "minimatch": "^10.0.0", + "minipass": "^7.1.2", + "package-json-from-dist": "^1.0.0", + "path-scurry": "^2.0.0" + }, + "bin": { + "glob": "dist/esm/bin.mjs" + }, + "engines": { + "node": "20 || >=22" + }, + "funding": { + "url": "https://github.com/sponsors/isaacs" + } + }, + "node_modules/zip-a-folder/node_modules/jackspeak": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/jackspeak/-/jackspeak-4.1.0.tgz", + "integrity": "sha512-9DDdhb5j6cpeitCbvLO7n7J4IxnbM6hoF6O1g4HQ5TfhvvKN8ywDM7668ZhMHRqVmxqhps/F6syWK2KcPxYlkw==", + "license": "BlueOak-1.0.0", + "dependencies": { + "@isaacs/cliui": "^8.0.2" + }, + "engines": { + "node": "20 || >=22" + }, + "funding": { + "url": "https://github.com/sponsors/isaacs" + } + }, + "node_modules/zip-a-folder/node_modules/lru-cache": { + "version": "11.1.0", + "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-11.1.0.tgz", + "integrity": "sha512-QIXZUBJUx+2zHUdQujWejBkcD9+cs94tLn0+YL8UrCh+D5sCXZ4c7LaEH48pNwRY3MLDgqUFyhlCyjJPf1WP0A==", + "license": "ISC", + "engines": { + "node": "20 || >=22" + } + }, + "node_modules/zip-a-folder/node_modules/minimatch": { + "version": "10.0.1", + "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-10.0.1.tgz", + "integrity": "sha512-ethXTt3SGGR+95gudmqJ1eNhRO7eGEGIgYA9vnPatK4/etz2MEVDno5GMCibdMTuBMyElzIlgxMna3K94XDIDQ==", + "license": "ISC", + "dependencies": { + "brace-expansion": "^2.0.1" + }, + "engines": { + "node": "20 || >=22" + }, + "funding": { + "url": "https://github.com/sponsors/isaacs" + } + }, + "node_modules/zip-a-folder/node_modules/minipass": { + "version": "7.1.2", + "resolved": "https://registry.npmjs.org/minipass/-/minipass-7.1.2.tgz", + "integrity": "sha512-qOOzS1cBTWYF4BH8fVePDBOO9iptMnGUEZwNc/cMWnTV2nVLZ7VoNWEPHkYczZA0pdoA7dl6e7FL659nX9S2aw==", + "license": "ISC", + "engines": { + "node": ">=16 || 14 >=14.17" + } + }, + "node_modules/zip-a-folder/node_modules/path-scurry": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/path-scurry/-/path-scurry-2.0.0.tgz", + "integrity": "sha512-ypGJsmGtdXUOeM5u93TyeIEfEhM6s+ljAhrk5vAvSx8uyY/02OvrZnA0YNGUrPXfpJMgI1ODd3nwz8Npx4O4cg==", + "license": "BlueOak-1.0.0", + "dependencies": { + "lru-cache": "^11.0.0", + "minipass": "^7.1.2" + }, + "engines": { + "node": "20 || >=22" + }, + "funding": { + "url": "https://github.com/sponsors/isaacs" + } + }, + "node_modules/zip-stream": { + "version": "6.0.1", + "resolved": "https://registry.npmjs.org/zip-stream/-/zip-stream-6.0.1.tgz", + "integrity": "sha512-zK7YHHz4ZXpW89AHXUPbQVGKI7uvkd3hzusTdotCg1UxyaVtg0zFJSTfW/Dq5f7OBBVnq6cZIaC8Ti4hb6dtCA==", + "license": "MIT", + "dependencies": { + "archiver-utils": "^5.0.0", + "compress-commons": "^6.0.2", + "readable-stream": "^4.0.0" + }, + "engines": { + "node": ">= 14" + } + }, + "node_modules/zip-stream/node_modules/readable-stream": { + "version": "4.7.0", + "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-4.7.0.tgz", + "integrity": "sha512-oIGGmcpTLwPga8Bn6/Z75SVaH1z5dUut2ibSyAMVhmUggWpmDn2dapB0n7f8nwaSiRtepAsfJyfXIO5DCVAODg==", + "license": "MIT", + "dependencies": { + "abort-controller": "^3.0.0", + "buffer": "^6.0.3", + "events": "^3.3.0", + "process": "^0.11.10", + "string_decoder": "^1.3.0" + }, + "engines": { + "node": "^12.22.0 || ^14.17.0 || >=16.0.0" + } + }, + "node_modules/zip-stream/node_modules/string_decoder": { + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.3.0.tgz", + "integrity": "sha512-hkRX8U1WjJFd8LsDJ2yQ/wWWxaopEsABU1XfkM8A+j0+85JAGppt16cr1Whg6KIbb4okU6Mql6BOj+uup/wKeA==", + "license": "MIT", + "dependencies": { + "safe-buffer": "~5.2.0" + } } } } diff --git a/package.json b/package.json index 3a6c5f48..d9774a82 100644 --- a/package.json +++ b/package.json @@ -65,7 +65,8 @@ "websocket-polyfill": "^0.0.3", "why-is-node-running": "^3.2.0", "wrtc": "^0.4.7", - "ws": "^8.18.0" + "ws": "^8.18.0", + "zip-a-folder": "^3.1.9" }, "devDependencies": { "@types/chai": "^4.3.4", diff --git a/proto/autogenerated/client.md b/proto/autogenerated/client.md index 29433f5c..38eb55c5 100644 --- a/proto/autogenerated/client.md +++ b/proto/autogenerated/client.md @@ -255,6 +255,11 @@ The nostr server will send back a message response, and inside the body there wi - input: [DebitOperation](#DebitOperation) - This methods has an __empty__ __response__ body +- ResetMetricsStorages + - auth type: __Admin__ + - This methods has an __empty__ __request__ body + - This methods has an __empty__ __response__ body + - RespondToDebit - auth type: __User__ - input: [DebitResponse](#DebitResponse) @@ -295,6 +300,11 @@ The nostr server will send back a message response, and inside the body there wi - This methods has an __empty__ __request__ body - output: [UserHealthState](#UserHealthState) +- ZipMetricsStorages + - auth type: __Admin__ + - This methods has an __empty__ __request__ body + - output: [ZippedMetrics](#ZippedMetrics) + # HTTP API DEFINITION ## Supported HTTP Auths @@ -794,6 +804,13 @@ The nostr server will send back a message response, and inside the body there wi - input: [DebitOperation](#DebitOperation) - This methods has an __empty__ __response__ body +- ResetMetricsStorages + - auth type: __Admin__ + - http method: __post__ + - http route: __/api/admin/metrics/reset__ + - This methods has an __empty__ __request__ body + - This methods has an __empty__ __response__ body + - ResetNPubLinkingToken - auth type: __App__ - http method: __post__ @@ -892,6 +909,13 @@ The nostr server will send back a message response, and inside the body there wi - This methods has an __empty__ __request__ body - output: [UserHealthState](#UserHealthState) +- ZipMetricsStorages + - auth type: __Admin__ + - http method: __post__ + - http route: __/api/admin/metrics/zip__ + - This methods has an __empty__ __request__ body + - output: [ZippedMetrics](#ZippedMetrics) + # INPUTS AND OUTPUTS ## Messages @@ -1465,6 +1489,9 @@ The nostr server will send back a message response, and inside the body there wi ### WebRtcMessage - __message__: _[WebRtcMessage_message](#WebRtcMessage_message)_ + +### ZippedMetrics + - __path__: _string_ ## Enums ### The enumerators used in the messages diff --git a/proto/autogenerated/go/http_client.go b/proto/autogenerated/go/http_client.go index 5f6a5a35..652a0838 100644 --- a/proto/autogenerated/go/http_client.go +++ b/proto/autogenerated/go/http_client.go @@ -118,6 +118,7 @@ type Client struct { PayInvoice func(req PayInvoiceRequest) (*PayInvoiceResponse, error) RequestNPubLinkingToken func(req RequestNPubLinkingTokenRequest) (*RequestNPubLinkingTokenResponse, error) ResetDebit func(req DebitOperation) error + ResetMetricsStorages func() error ResetNPubLinkingToken func(req RequestNPubLinkingTokenRequest) (*RequestNPubLinkingTokenResponse, error) RespondToDebit func(req DebitResponse) error SendAppUserToAppPayment func(req SendAppUserToAppPaymentRequest) error @@ -132,6 +133,7 @@ type Client struct { UpdateUserOffer func(req OfferConfig) error UseInviteLink func(req UseInviteLinkRequest) error UserHealth func() (*UserHealthState, error) + ZipMetricsStorages func() (*ZippedMetrics, error) } func NewClient(params ClientParams) *Client { @@ -1752,6 +1754,27 @@ func NewClient(params ClientParams) *Client { } return nil }, + ResetMetricsStorages: func() error { + auth, err := params.RetrieveAdminAuth() + if err != nil { + return err + } + finalRoute := "/api/admin/metrics/reset" + body := []byte{} + resBody, err := doPostRequest(params.BaseURL+finalRoute, body, auth) + if err != nil { + return err + } + result := ResultError{} + err = json.Unmarshal(resBody, &result) + if err != nil { + return err + } + if result.Status == "ERROR" { + return fmt.Errorf(result.Reason) + } + return nil + }, ResetNPubLinkingToken: func(req RequestNPubLinkingTokenRequest) (*RequestNPubLinkingTokenResponse, error) { auth, err := params.RetrieveAppAuth() if err != nil { @@ -2082,5 +2105,31 @@ func NewClient(params ClientParams) *Client { } return &res, nil }, + ZipMetricsStorages: func() (*ZippedMetrics, error) { + auth, err := params.RetrieveAdminAuth() + if err != nil { + return nil, err + } + finalRoute := "/api/admin/metrics/zip" + body := []byte{} + resBody, err := doPostRequest(params.BaseURL+finalRoute, body, auth) + if err != nil { + return nil, err + } + result := ResultError{} + err = json.Unmarshal(resBody, &result) + if err != nil { + return nil, err + } + if result.Status == "ERROR" { + return nil, fmt.Errorf(result.Reason) + } + res := ZippedMetrics{} + err = json.Unmarshal(resBody, &res) + if err != nil { + return nil, err + } + return &res, nil + }, } } diff --git a/proto/autogenerated/go/types.go b/proto/autogenerated/go/types.go index 3a6bf98b..2fa9b31d 100644 --- a/proto/autogenerated/go/types.go +++ b/proto/autogenerated/go/types.go @@ -665,6 +665,9 @@ type WebRtcCandidate struct { type WebRtcMessage struct { Message *WebRtcMessage_message `json:"message"` } +type ZippedMetrics struct { + Path string `json:"path"` +} type DebitResponse_response_type string const ( diff --git a/proto/autogenerated/ts/express_server.ts b/proto/autogenerated/ts/express_server.ts index 6f57f62c..9bed5af5 100644 --- a/proto/autogenerated/ts/express_server.ts +++ b/proto/autogenerated/ts/express_server.ts @@ -1632,6 +1632,25 @@ export default (methods: Types.ServerMethods, opts: ServerOptions) => { opts.metricsCallback([{ ...info, ...stats, ...authContext }]) } catch (ex) { const e = ex as any; logErrorAndReturnResponse(e, e.message || e, res, logger, { ...info, ...stats, ...authCtx }, opts.metricsCallback); if (opts.throwErrors) throw e } }) + if (!opts.allowNotImplementedMethods && !methods.ResetMetricsStorages) throw new Error('method: ResetMetricsStorages is not implemented') + app.post('/api/admin/metrics/reset', async (req, res) => { + const info: Types.RequestInfo = { rpcName: 'ResetMetricsStorages', batch: false, nostr: false, batchSize: 0} + const stats: Types.RequestStats = { startMs:req.startTimeMs || 0, start:req.startTime || 0n, parse: process.hrtime.bigint(), guard: 0n, validate: 0n, handle: 0n } + let authCtx: Types.AuthContext = {} + try { + if (!methods.ResetMetricsStorages) throw new Error('method: ResetMetricsStorages is not implemented') + const authContext = await opts.AdminAuthGuard(req.headers['authorization']) + authCtx = authContext + stats.guard = process.hrtime.bigint() + stats.validate = stats.guard + const query = req.query + const params = req.params + await methods.ResetMetricsStorages({rpcName:'ResetMetricsStorages', ctx:authContext }) + stats.handle = process.hrtime.bigint() + res.json({status: 'OK'}) + opts.metricsCallback([{ ...info, ...stats, ...authContext }]) + } catch (ex) { const e = ex as any; logErrorAndReturnResponse(e, e.message || e, res, logger, { ...info, ...stats, ...authCtx }, opts.metricsCallback); if (opts.throwErrors) throw e } + }) if (!opts.allowNotImplementedMethods && !methods.ResetNPubLinkingToken) throw new Error('method: ResetNPubLinkingToken is not implemented') app.post('/api/app/user/npub/token/reset', async (req, res) => { const info: Types.RequestInfo = { rpcName: 'ResetNPubLinkingToken', batch: false, nostr: false, batchSize: 0} @@ -1915,6 +1934,25 @@ export default (methods: Types.ServerMethods, opts: ServerOptions) => { opts.metricsCallback([{ ...info, ...stats, ...authContext }]) } catch (ex) { const e = ex as any; logErrorAndReturnResponse(e, e.message || e, res, logger, { ...info, ...stats, ...authCtx }, opts.metricsCallback); if (opts.throwErrors) throw e } }) + if (!opts.allowNotImplementedMethods && !methods.ZipMetricsStorages) throw new Error('method: ZipMetricsStorages is not implemented') + app.post('/api/admin/metrics/zip', async (req, res) => { + const info: Types.RequestInfo = { rpcName: 'ZipMetricsStorages', batch: false, nostr: false, batchSize: 0} + const stats: Types.RequestStats = { startMs:req.startTimeMs || 0, start:req.startTime || 0n, parse: process.hrtime.bigint(), guard: 0n, validate: 0n, handle: 0n } + let authCtx: Types.AuthContext = {} + try { + if (!methods.ZipMetricsStorages) throw new Error('method: ZipMetricsStorages is not implemented') + const authContext = await opts.AdminAuthGuard(req.headers['authorization']) + authCtx = authContext + stats.guard = process.hrtime.bigint() + stats.validate = stats.guard + const query = req.query + const params = req.params + const response = await methods.ZipMetricsStorages({rpcName:'ZipMetricsStorages', ctx:authContext }) + stats.handle = process.hrtime.bigint() + res.json({status: 'OK', ...response}) + opts.metricsCallback([{ ...info, ...stats, ...authContext }]) + } catch (ex) { const e = ex as any; logErrorAndReturnResponse(e, e.message || e, res, logger, { ...info, ...stats, ...authCtx }, opts.metricsCallback); if (opts.throwErrors) throw e } + }) if (opts.staticFiles) { app.use(express.static(opts.staticFiles)) app.get('*', function (_, res) { res.sendFile('index.html', { root: opts.staticFiles })}) diff --git a/proto/autogenerated/ts/http_client.ts b/proto/autogenerated/ts/http_client.ts index c5a273fa..afe6ea6a 100644 --- a/proto/autogenerated/ts/http_client.ts +++ b/proto/autogenerated/ts/http_client.ts @@ -839,6 +839,17 @@ export default (params: ClientParams) => ({ } return { status: 'ERROR', reason: 'invalid response' } }, + ResetMetricsStorages: async (): Promise => { + const auth = await params.retrieveAdminAuth() + if (auth === null) throw new Error('retrieveAdminAuth() returned null') + let finalRoute = '/api/admin/metrics/reset' + const { data } = await axios.post(params.baseUrl + finalRoute, {}, { headers: { 'authorization': auth } }) + if (data.status === 'ERROR' && typeof data.reason === 'string') return data + if (data.status === 'OK') { + return data + } + return { status: 'ERROR', reason: 'invalid response' } + }, ResetNPubLinkingToken: async (request: Types.RequestNPubLinkingTokenRequest): Promise => { const auth = await params.retrieveAppAuth() if (auth === null) throw new Error('retrieveAppAuth() returned null') @@ -995,4 +1006,18 @@ export default (params: ClientParams) => ({ } return { status: 'ERROR', reason: 'invalid response' } }, + ZipMetricsStorages: async (): Promise => { + const auth = await params.retrieveAdminAuth() + if (auth === null) throw new Error('retrieveAdminAuth() returned null') + let finalRoute = '/api/admin/metrics/zip' + const { data } = await axios.post(params.baseUrl + finalRoute, {}, { headers: { 'authorization': auth } }) + if (data.status === 'ERROR' && typeof data.reason === 'string') return data + if (data.status === 'OK') { + const result = data + if(!params.checkResult) return { status: 'OK', ...result } + const error = Types.ZippedMetricsValidate(result) + if (error === null) { return { status: 'OK', ...result } } else return { status: 'ERROR', reason: error.message } + } + return { status: 'ERROR', reason: 'invalid response' } + }, }) diff --git a/proto/autogenerated/ts/nostr_client.ts b/proto/autogenerated/ts/nostr_client.ts index 53a9b167..2a2cf713 100644 --- a/proto/autogenerated/ts/nostr_client.ts +++ b/proto/autogenerated/ts/nostr_client.ts @@ -698,6 +698,17 @@ export default (params: NostrClientParams, send: (to:string, message: NostrRequ } return { status: 'ERROR', reason: 'invalid response' } }, + ResetMetricsStorages: async (): Promise => { + const auth = await params.retrieveNostrAdminAuth() + if (auth === null) throw new Error('retrieveNostrAdminAuth() returned null') + const nostrRequest: NostrRequest = {} + const data = await send(params.pubDestination, {rpcName:'ResetMetricsStorages',authIdentifier:auth, ...nostrRequest }) + if (data.status === 'ERROR' && typeof data.reason === 'string') return data + if (data.status === 'OK') { + return data + } + return { status: 'ERROR', reason: 'invalid response' } + }, RespondToDebit: async (request: Types.DebitResponse): Promise => { const auth = await params.retrieveNostrUserAuth() if (auth === null) throw new Error('retrieveNostrUserAuth() returned null') @@ -805,4 +816,18 @@ export default (params: NostrClientParams, send: (to:string, message: NostrRequ } return { status: 'ERROR', reason: 'invalid response' } }, + ZipMetricsStorages: async (): Promise => { + const auth = await params.retrieveNostrAdminAuth() + if (auth === null) throw new Error('retrieveNostrAdminAuth() returned null') + const nostrRequest: NostrRequest = {} + const data = await send(params.pubDestination, {rpcName:'ZipMetricsStorages',authIdentifier:auth, ...nostrRequest }) + if (data.status === 'ERROR' && typeof data.reason === 'string') return data + if (data.status === 'OK') { + const result = data + if(!params.checkResult) return { status: 'OK', ...result } + const error = Types.ZippedMetricsValidate(result) + if (error === null) { return { status: 'OK', ...result } } else return { status: 'ERROR', reason: error.message } + } + return { status: 'ERROR', reason: 'invalid response' } + }, }) diff --git a/proto/autogenerated/ts/nostr_transport.ts b/proto/autogenerated/ts/nostr_transport.ts index e5efd03a..6603f973 100644 --- a/proto/autogenerated/ts/nostr_transport.ts +++ b/proto/autogenerated/ts/nostr_transport.ts @@ -1091,6 +1091,19 @@ export default (methods: Types.ServerMethods, opts: NostrOptions) => { opts.metricsCallback([{ ...info, ...stats, ...authContext }]) }catch(ex){ const e = ex as any; logErrorAndReturnResponse(e, e.message || e, res, logger, { ...info, ...stats, ...authCtx }, opts.metricsCallback); if (opts.throwErrors) throw e } break + case 'ResetMetricsStorages': + try { + if (!methods.ResetMetricsStorages) throw new Error('method: ResetMetricsStorages is not implemented') + const authContext = await opts.NostrAdminAuthGuard(req.appId, req.authIdentifier) + stats.guard = process.hrtime.bigint() + authCtx = authContext + stats.validate = stats.guard + await methods.ResetMetricsStorages({rpcName:'ResetMetricsStorages', ctx:authContext }) + stats.handle = process.hrtime.bigint() + res({status: 'OK'}) + opts.metricsCallback([{ ...info, ...stats, ...authContext }]) + }catch(ex){ const e = ex as any; logErrorAndReturnResponse(e, e.message || e, res, logger, { ...info, ...stats, ...authCtx }, opts.metricsCallback); if (opts.throwErrors) throw e } + break case 'RespondToDebit': try { if (!methods.RespondToDebit) throw new Error('method: RespondToDebit is not implemented') @@ -1213,6 +1226,19 @@ export default (methods: Types.ServerMethods, opts: NostrOptions) => { opts.metricsCallback([{ ...info, ...stats, ...authContext }]) }catch(ex){ const e = ex as any; logErrorAndReturnResponse(e, e.message || e, res, logger, { ...info, ...stats, ...authCtx }, opts.metricsCallback); if (opts.throwErrors) throw e } break + case 'ZipMetricsStorages': + try { + if (!methods.ZipMetricsStorages) throw new Error('method: ZipMetricsStorages is not implemented') + const authContext = await opts.NostrAdminAuthGuard(req.appId, req.authIdentifier) + stats.guard = process.hrtime.bigint() + authCtx = authContext + stats.validate = stats.guard + const response = await methods.ZipMetricsStorages({rpcName:'ZipMetricsStorages', ctx:authContext }) + stats.handle = process.hrtime.bigint() + res({status: 'OK', ...response}) + opts.metricsCallback([{ ...info, ...stats, ...authContext }]) + }catch(ex){ const e = ex as any; logErrorAndReturnResponse(e, e.message || e, res, logger, { ...info, ...stats, ...authCtx }, opts.metricsCallback); if (opts.throwErrors) throw e } + break default: logger.error('unknown rpc call name from nostr event:'+req.rpcName) } } diff --git a/proto/autogenerated/ts/types.ts b/proto/autogenerated/ts/types.ts index c24874f3..6997ecef 100644 --- a/proto/autogenerated/ts/types.ts +++ b/proto/autogenerated/ts/types.ts @@ -7,8 +7,8 @@ export type RequestMetric = AuthContext & RequestInfo & RequestStats & { error?: export type AdminContext = { admin_id: string } -export type AdminMethodInputs = AddApp_Input | AddPeer_Input | AuthApp_Input | BanUser_Input | CloseChannel_Input | CreateOneTimeInviteLink_Input | GetInviteLinkState_Input | GetSeed_Input | ListChannels_Input | LndGetInfo_Input | OpenChannel_Input | UpdateChannelPolicy_Input -export type AdminMethodOutputs = AddApp_Output | AddPeer_Output | AuthApp_Output | BanUser_Output | CloseChannel_Output | CreateOneTimeInviteLink_Output | GetInviteLinkState_Output | GetSeed_Output | ListChannels_Output | LndGetInfo_Output | OpenChannel_Output | UpdateChannelPolicy_Output +export type AdminMethodInputs = AddApp_Input | AddPeer_Input | AuthApp_Input | BanUser_Input | CloseChannel_Input | CreateOneTimeInviteLink_Input | GetInviteLinkState_Input | GetSeed_Input | ListChannels_Input | LndGetInfo_Input | OpenChannel_Input | ResetMetricsStorages_Input | UpdateChannelPolicy_Input | ZipMetricsStorages_Input +export type AdminMethodOutputs = AddApp_Output | AddPeer_Output | AuthApp_Output | BanUser_Output | CloseChannel_Output | CreateOneTimeInviteLink_Output | GetInviteLinkState_Output | GetSeed_Output | ListChannels_Output | LndGetInfo_Output | OpenChannel_Output | ResetMetricsStorages_Output | UpdateChannelPolicy_Output | ZipMetricsStorages_Output export type AppContext = { app_id: string } @@ -253,6 +253,9 @@ export type RequestNPubLinkingToken_Output = ResultError | ({ status: 'OK' } & R export type ResetDebit_Input = {rpcName:'ResetDebit', req: DebitOperation} export type ResetDebit_Output = ResultError | { status: 'OK' } +export type ResetMetricsStorages_Input = {rpcName:'ResetMetricsStorages'} +export type ResetMetricsStorages_Output = ResultError | { status: 'OK' } + export type ResetNPubLinkingToken_Input = {rpcName:'ResetNPubLinkingToken', req: RequestNPubLinkingTokenRequest} export type ResetNPubLinkingToken_Output = ResultError | ({ status: 'OK' } & RequestNPubLinkingTokenResponse) @@ -295,6 +298,9 @@ export type UseInviteLink_Output = ResultError | { status: 'OK' } export type UserHealth_Input = {rpcName:'UserHealth'} export type UserHealth_Output = ResultError | ({ status: 'OK' } & UserHealthState) +export type ZipMetricsStorages_Input = {rpcName:'ZipMetricsStorages'} +export type ZipMetricsStorages_Output = ResultError | ({ status: 'OK' } & ZippedMetrics) + export type ServerMethods = { AddApp?: (req: AddApp_Input & {ctx: AdminContext }) => Promise AddAppInvoice?: (req: AddAppInvoice_Input & {ctx: AppContext }) => Promise @@ -359,6 +365,7 @@ export type ServerMethods = { PayInvoice?: (req: PayInvoice_Input & {ctx: UserContext }) => Promise RequestNPubLinkingToken?: (req: RequestNPubLinkingToken_Input & {ctx: AppContext }) => Promise ResetDebit?: (req: ResetDebit_Input & {ctx: UserContext }) => Promise + ResetMetricsStorages?: (req: ResetMetricsStorages_Input & {ctx: AdminContext }) => Promise ResetNPubLinkingToken?: (req: ResetNPubLinkingToken_Input & {ctx: AppContext }) => Promise RespondToDebit?: (req: RespondToDebit_Input & {ctx: UserContext }) => Promise SendAppUserToAppPayment?: (req: SendAppUserToAppPayment_Input & {ctx: AppContext }) => Promise @@ -373,6 +380,7 @@ export type ServerMethods = { UpdateUserOffer?: (req: UpdateUserOffer_Input & {ctx: UserContext }) => Promise UseInviteLink?: (req: UseInviteLink_Input & {ctx: GuestWithPubContext }) => Promise UserHealth?: (req: UserHealth_Input & {ctx: UserContext }) => Promise + ZipMetricsStorages?: (req: ZipMetricsStorages_Input & {ctx: AdminContext }) => Promise } export enum AddressType { @@ -3816,6 +3824,24 @@ export const WebRtcMessageValidate = (o?: WebRtcMessage, opts: WebRtcMessageOpti return null } +export type ZippedMetrics = { + path: string +} +export const ZippedMetricsOptionalFields: [] = [] +export type ZippedMetricsOptions = OptionsBaseMessage & { + checkOptionalsAreSet?: [] + path_CustomCheck?: (v: string) => boolean +} +export const ZippedMetricsValidate = (o?: ZippedMetrics, opts: ZippedMetricsOptions = {}, path: string = 'ZippedMetrics::root.'): Error | null => { + if (opts.checkOptionalsAreSet && opts.allOptionalsAreSet) return new Error(path + ': only one of checkOptionalsAreSet or allOptionalNonDefault can be set for each message') + if (typeof o !== 'object' || o === null) return new Error(path + ': object is not an instance of an object or is null') + + if (typeof o.path !== 'string') return new Error(`${path}.path: is not a string`) + if (opts.path_CustomCheck && !opts.path_CustomCheck(o.path)) return new Error(`${path}.path: custom check failed`) + + return null +} + export enum DebitResponse_response_type { DENIED = 'denied', INVOICE = 'invoice', diff --git a/proto/service/methods.proto b/proto/service/methods.proto index 3d6856dd..ee287fb8 100644 --- a/proto/service/methods.proto +++ b/proto/service/methods.proto @@ -234,6 +234,20 @@ service LightningPub { option (nostr) = true; } + rpc ZipMetricsStorages(structs.Empty) returns (structs.ZippedMetrics) { + option (auth_type) = "Admin"; + option (http_method) = "post"; + option (http_route) = "/api/admin/metrics/zip"; + option (nostr) = true; + } + + rpc ResetMetricsStorages(structs.Empty) returns (structs.Empty) { + option (auth_type) = "Admin"; + option (http_method) = "post"; + option (http_route) = "/api/admin/metrics/reset"; + option (nostr) = true; + } + rpc CreateOneTimeInviteLink(structs.CreateOneTimeInviteLinkRequest) returns (structs.CreateOneTimeInviteLinkResponse) { option (auth_type) = "Admin"; option (http_method) = "post"; diff --git a/proto/service/structs.proto b/proto/service/structs.proto index c8514de0..f7693ce7 100644 --- a/proto/service/structs.proto +++ b/proto/service/structs.proto @@ -10,6 +10,10 @@ message LndSeed { repeated string seed = 1; } +message ZippedMetrics { + string path = 1; +} + message EncryptionExchangeRequest { string publicKey = 1; string deviceId = 2; diff --git a/src/services/helpers/utilsWrapper.ts b/src/services/helpers/utilsWrapper.ts index 71a02bdd..d14b5e64 100644 --- a/src/services/helpers/utilsWrapper.ts +++ b/src/services/helpers/utilsWrapper.ts @@ -5,15 +5,15 @@ import { NostrSend } from "../nostr/handler.js"; import { ProcessMetricsCollector } from "../storage/tlv/processMetricsCollector.js"; type UtilsSettings = { noCollector?: boolean - dataDir: string + dataDir: string, + allowResetMetricsStorages: boolean } export class Utils { tlvStorageFactory: TlvStorageFactory stateBundler: StateBundler - settings: MainSettings _nostrSend: NostrSend = () => { throw new Error('nostr send not initialized yet') } - constructor({ noCollector, dataDir }: UtilsSettings) { - this.tlvStorageFactory = new TlvStorageFactory() + constructor({ noCollector, dataDir, allowResetMetricsStorages }: UtilsSettings) { + this.tlvStorageFactory = new TlvStorageFactory(allowResetMetricsStorages) this.stateBundler = new StateBundler(dataDir, this.tlvStorageFactory) if (!noCollector) { new ProcessMetricsCollector((metrics) => { diff --git a/src/services/main/init.ts b/src/services/main/init.ts index 35a5e37d..1f754714 100644 --- a/src/services/main/init.ts +++ b/src/services/main/init.ts @@ -17,7 +17,7 @@ export type AppData = { name: string; } export const initMainHandler = async (log: PubLogger, mainSettings: MainSettings) => { - const utils = new Utils({ dataDir: mainSettings.storageSettings.dataDir }) + const utils = new Utils({ dataDir: mainSettings.storageSettings.dataDir, allowResetMetricsStorages: mainSettings.allowResetMetricsStorages }) const storageManager = new Storage(mainSettings.storageSettings, utils) await storageManager.Connect(log) /* const manualMigration = await TypeOrmMigrationRunner(log, storageManager, mainSettings.storageSettings.dbSettings, process.argv[2]) diff --git a/src/services/main/settings.ts b/src/services/main/settings.ts index 420a0e94..d412e658 100644 --- a/src/services/main/settings.ts +++ b/src/services/main/settings.ts @@ -34,7 +34,8 @@ export type MainSettings = { defaultAppName: string pushBackupsToNostr: boolean lnurlMetaText: string, - bridgeUrl: string + bridgeUrl: string, + allowResetMetricsStorages: boolean } export type BitcoinCoreSettings = { @@ -74,7 +75,8 @@ export const LoadMainSettingsFromEnv = (): MainSettings => { defaultAppName: process.env.DEFAULT_APP_NAME || "wallet", pushBackupsToNostr: process.env.PUSH_BACKUPS_TO_NOSTR === 'true' || false, lnurlMetaText: process.env.LNURL_META_TEXT || "LNURL via Lightning.pub", - bridgeUrl: process.env.BRIDGE_URL || "https://shockwallet.app" + bridgeUrl: process.env.BRIDGE_URL || "https://shockwallet.app", + allowResetMetricsStorages: process.env.ALLOW_RESET_METRICS_STORAGES === 'true' || false } } diff --git a/src/services/serverMethods/index.ts b/src/services/serverMethods/index.ts index 678c9cea..d9f767df 100644 --- a/src/services/serverMethods/index.ts +++ b/src/services/serverMethods/index.ts @@ -46,6 +46,13 @@ export default (mainHandler: Main): Types.ServerMethods => { GetLndMetrics: async ({ ctx, req }) => { return mainHandler.metricsManager.GetLndMetrics(req) }, + ResetMetricsStorages: async ({ ctx }) => { + return mainHandler.utils.tlvStorageFactory.ResetStorages() + }, + ZipMetricsStorages: async ({ ctx }) => { + const path = await mainHandler.utils.tlvStorageFactory.ZipStorages() + return { path } + }, ListChannels: async ({ ctx }) => { return mainHandler.adminManager.ListChannels() }, diff --git a/src/services/storage/tlv/processMetricsCollector.ts b/src/services/storage/tlv/processMetricsCollector.ts index 52047a40..62945b11 100644 --- a/src/services/storage/tlv/processMetricsCollector.ts +++ b/src/services/storage/tlv/processMetricsCollector.ts @@ -5,20 +5,28 @@ export type ProcessMetrics = { memory_heap_total_kb?: number memory_heap_used_kb?: number memory_external_kb?: number + cpu_user_ms?: number + cpu_system_ms?: number } export class ProcessMetricsCollector { reportLog = getLogger({ component: 'ProcessMetricsCollector' }) prevValues: Record = {} interval: NodeJS.Timeout + cpuUsage: { user: number, system: number } constructor(cb: (metrics: ProcessMetrics) => void) { + this.cpuUsage = process.cpuUsage() this.interval = setInterval(() => { const mem = process.memoryUsage() + const diff = process.cpuUsage(this.cpuUsage) + this.cpuUsage = process.cpuUsage() const metrics: ProcessMetrics = { memory_rss_kb: this.AddValue('memory_rss_kb', Math.ceil(mem.rss / 1000 || 0), true), memory_buffer_kb: this.AddValue('memory_buffer_kb', Math.ceil(mem.arrayBuffers / 1000 || 0), true), memory_heap_total_kb: this.AddValue('memory_heap_total_kb', Math.ceil(mem.heapTotal / 1000 || 0), true), memory_heap_used_kb: this.AddValue('memory_heap_used_kb', Math.ceil(mem.heapUsed / 1000 || 0), true), memory_external_kb: this.AddValue('memory_external_kb', Math.ceil(mem.external / 1000 || 0), true), + cpu_user_ms: this.AddValue('cpu_user_ms', Math.ceil(diff.user / 1000), true), + cpu_system_ms: this.AddValue('cpu_system_ms', Math.ceil(diff.system / 1000), true), } cb(metrics) diff --git a/src/services/storage/tlv/tlvFilesStorage.ts b/src/services/storage/tlv/tlvFilesStorage.ts index 22b93e14..3e005e0d 100644 --- a/src/services/storage/tlv/tlvFilesStorage.ts +++ b/src/services/storage/tlv/tlvFilesStorage.ts @@ -9,20 +9,35 @@ export class TlvFilesStorage { private meta: Record> = {} private pending: Record> = {} private metaReady = false + private interval: NodeJS.Timeout constructor(storagePath: string) { this.storagePath = storagePath if (!fs.existsSync(this.storagePath)) { fs.mkdirSync(this.storagePath, { recursive: true }); } + this.init() + process.on('exit', () => { + this.persist() + }); + } + + GetStoragePath = () => { + return this.storagePath + } + + init = () => { this.initMeta() - setInterval(() => { + this.interval = setInterval(() => { if (Date.now() - this.lastPersisted > 1000 * 60 * 4) { this.persist() } }, 1000 * 60 * 5) - process.on('exit', () => { - this.persist() - }); + } + + Reset = () => { + clearInterval(this.interval) + fs.rmSync(this.storagePath, { recursive: true, force: true }) + this.init() } LoadFile = (app: string, dataName: string, chunk: number): TlvFile => { diff --git a/src/services/storage/tlv/tlvFilesStorageFactory.ts b/src/services/storage/tlv/tlvFilesStorageFactory.ts index fc8bfc4d..9840c6ad 100644 --- a/src/services/storage/tlv/tlvFilesStorageFactory.ts +++ b/src/services/storage/tlv/tlvFilesStorageFactory.ts @@ -1,6 +1,6 @@ import { ChildProcess, fork } from 'child_process'; import { EventEmitter } from 'events'; -import { AddTlvOperation, ITlvStorageOperation, SuccessTlvOperationResponse, LoadLatestTlvOperation, LoadTlvFileOperation, NewTlvStorageOperation, SerializableLatestData, SerializableTlvFile, TlvOperationResponse, TlvStorageSettings, WebRtcMessageOperation, ProcessMetricsTlvOperation } from './tlvFilesStorageProcessor'; +import { AddTlvOperation, ITlvStorageOperation, SuccessTlvOperationResponse, LoadLatestTlvOperation, LoadTlvFileOperation, NewTlvStorageOperation, SerializableLatestData, SerializableTlvFile, TlvOperationResponse, TlvStorageSettings, WebRtcMessageOperation, ProcessMetricsTlvOperation, ZipStoragesOperation, ResetTlvStorageOperation } from './tlvFilesStorageProcessor'; import { LatestData, TlvFile } from './tlvFilesStorage'; import { NostrSend, SendData, SendInitiator } from '../../nostr/handler'; import { WebRtcUserInfo } from '../../webRTC'; @@ -17,8 +17,10 @@ export class TlvStorageFactory extends EventEmitter { private isConnected: boolean = false; private debug: boolean = false; private _nostrSend: NostrSend = () => { throw new Error('nostr send not initialized yet') } - constructor() { + private allowResetMetricsStorages: boolean + constructor(allowResetMetricsStorages: boolean) { super(); + this.allowResetMetricsStorages = allowResetMetricsStorages this.initializeSubprocess(); } @@ -61,6 +63,21 @@ export class TlvStorageFactory extends EventEmitter { this.isConnected = true; } + ZipStorages(): Promise { + const opId = Math.random().toString() + const op: ZipStoragesOperation = { type: 'zipStorages', opId } + return this.handleOp(op) + } + + ResetStorages(): Promise { + if (!this.allowResetMetricsStorages) { + throw new Error('Resetting metrics storages is not allowed') + } + const opId = Math.random().toString() + const op: ResetTlvStorageOperation = { type: 'resetStorage', opId } + return this.handleOp(op) + } + NewStorage(settings: TlvStorageSettings): TlvStorageInterface { const opId = Math.random().toString() const op: NewTlvStorageOperation = { type: 'newStorage', opId, settings } diff --git a/src/services/storage/tlv/tlvFilesStorageProcessor.ts b/src/services/storage/tlv/tlvFilesStorageProcessor.ts index 7cc97e1e..4849bc1a 100644 --- a/src/services/storage/tlv/tlvFilesStorageProcessor.ts +++ b/src/services/storage/tlv/tlvFilesStorageProcessor.ts @@ -6,6 +6,8 @@ import { SendData } from '../../nostr/handler.js'; import { SendInitiator } from '../../nostr/handler.js'; import { ProcessMetrics, ProcessMetricsCollector } from './processMetricsCollector.js'; import { integerToUint8Array } from '../../helpers/tlv.js'; +import { zip } from 'zip-a-folder' +import crypto from 'crypto' export type SerializableLatestData = Record> export type SerializableTlvFile = { base64fileData: string, chunks: number[] } export const usageStorageName = 'usage' @@ -15,6 +17,18 @@ export type TlvStorageSettings = { name: typeof usageStorageName | typeof bundlerStorageName } +export type ZipStoragesOperation = { + type: 'zipStorages' + opId: string + debug?: boolean +} + +export type ResetTlvStorageOperation = { + type: 'resetStorage' + opId: string + debug?: boolean +} + export type NewTlvStorageOperation = { type: 'newStorage' opId: string @@ -74,7 +88,7 @@ export interface ITlvStorageOperation { debug?: boolean } -export type TlvStorageOperation = NewTlvStorageOperation | AddTlvOperation | LoadLatestTlvOperation | LoadTlvFileOperation | WebRtcMessageOperation | ProcessMetricsTlvOperation +export type TlvStorageOperation = NewTlvStorageOperation | AddTlvOperation | LoadLatestTlvOperation | LoadTlvFileOperation | WebRtcMessageOperation | ProcessMetricsTlvOperation | ResetTlvStorageOperation | ZipStoragesOperation export type SuccessTlvOperationResponse = { success: true, type: string, data: T, opId: string } export type TlvOperationResponse = SuccessTlvOperationResponse | ErrorTlvOperationResponse @@ -132,11 +146,15 @@ class TlvFilesStorageProcessor { console.log('no bundle storage yet') return } - if (pMetrics.memory_rss_kb) this.storages[bundlerStorageName].AddTlv('_root', 'memory_rss_kb' + pName, this.serializeNowTlv(pMetrics.memory_rss_kb)) - if (pMetrics.memory_buffer_kb) this.storages[bundlerStorageName].AddTlv('_root', 'memory_buffer_kb' + pName, this.serializeNowTlv(pMetrics.memory_buffer_kb)) - if (pMetrics.memory_heap_total_kb) this.storages[bundlerStorageName].AddTlv('_root', 'memory_heap_total_kb' + pName, this.serializeNowTlv(pMetrics.memory_heap_total_kb)) - if (pMetrics.memory_heap_used_kb) this.storages[bundlerStorageName].AddTlv('_root', 'memory_heap_used_kb' + pName, this.serializeNowTlv(pMetrics.memory_heap_used_kb)) - if (pMetrics.memory_external_kb) this.storages[bundlerStorageName].AddTlv('_root', 'memory_external_kb' + pName, this.serializeNowTlv(pMetrics.memory_external_kb)) + for (const key in pMetrics) { + const v = pMetrics[key as keyof ProcessMetrics] + if (v) this.storages[bundlerStorageName].AddTlv('_root', key + pName, this.serializeNowTlv(v)) + } + /* if (pMetrics.memory_rss_kb) this.storages[bundlerStorageName].AddTlv('_root', 'memory_rss_kb' + pName, this.serializeNowTlv(pMetrics.memory_rss_kb)) + if (pMetrics.memory_buffer_kb) this.storages[bundlerStorageName].AddTlv('_root', 'memory_buffer_kb' + pName, this.serializeNowTlv(pMetrics.memory_buffer_kb)) + if (pMetrics.memory_heap_total_kb) this.storages[bundlerStorageName].AddTlv('_root', 'memory_heap_total_kb' + pName, this.serializeNowTlv(pMetrics.memory_heap_total_kb)) + if (pMetrics.memory_heap_used_kb) this.storages[bundlerStorageName].AddTlv('_root', 'memory_heap_used_kb' + pName, this.serializeNowTlv(pMetrics.memory_heap_used_kb)) + if (pMetrics.memory_external_kb) this.storages[bundlerStorageName].AddTlv('_root', 'memory_external_kb' + pName, this.serializeNowTlv(pMetrics.memory_external_kb)) */ } private async handleOperation(operation: TlvStorageOperation) { @@ -144,6 +162,9 @@ class TlvFilesStorageProcessor { const opId = operation.opId; if (operation.debug) console.log('handleOperation', operation) switch (operation.type) { + case 'resetStorage': + await this.handleResetStorage(operation); + break; case 'newStorage': await this.handleNewStorage(operation); break; @@ -162,6 +183,9 @@ class TlvFilesStorageProcessor { case 'processMetrics': await this.handleProcessMetrics(operation); break; + case 'zipStorages': + await this.handleZipStorages(operation); + break; default: this.sendResponse({ success: false, @@ -179,6 +203,19 @@ class TlvFilesStorageProcessor { } } + private async handleResetStorage(operation: ResetTlvStorageOperation) { + for (const storageName in this.storages) { + this.storages[storageName].Reset() + } + this.sendResponse({ + success: true, + type: 'resetStorage', + data: null, + opId: operation.opId + }); + } + + private async handleNewStorage(operation: NewTlvStorageOperation) { if (this.storages[operation.settings.name]) { this.sendResponse({ @@ -279,6 +316,40 @@ class TlvFilesStorageProcessor { }); } + private async handleZipStorages(operation: ZipStoragesOperation) { + const paths = [] + for (const storageName in this.storages) { + paths.push(this.storages[storageName].GetStoragePath()) + } + if (paths.length === 0) { + this.sendResponse({ + success: false, + error: 'No storages to zip', + opId: operation.opId + }) + return + } + const name = crypto.randomBytes(16).toString('hex') + '.zip' + const path = paths.join(', ') + const err = await zip(path, name) + if (err) { + this.sendResponse({ + success: false, + error: err.message, + opId: operation.opId + }) + return + } + this.sendResponse({ + success: true, + type: 'zipStorages', + data: name, + opId: operation.opId + }); + + + } + private sendResponse(response: TlvOperationResponse) { if (process.send) { process.send(response); diff --git a/src/tests/networkSetup.ts b/src/tests/networkSetup.ts index 9ebcaa17..b15e810f 100644 --- a/src/tests/networkSetup.ts +++ b/src/tests/networkSetup.ts @@ -3,14 +3,12 @@ import { BitcoinCoreWrapper } from "./bitcoinCore.js" import LND from '../services/lnd/lnd.js' import { LiquidityProvider } from "../services/main/liquidityProvider.js" import { Utils } from "../services/helpers/utilsWrapper.js" -import { TlvStorageFactory } from "../services/storage/tlv/tlvFilesStorageFactory.js" export const setupNetwork = async () => { const settings = LoadTestSettingsFromEnv() const core = new BitcoinCoreWrapper(settings) await core.InitAddress() await core.Mine(1) - const tlvStorageFactory = new TlvStorageFactory() - const setupUtils = new Utils({ dataDir: settings.storageSettings.dataDir }) + const setupUtils = new Utils({ dataDir: settings.storageSettings.dataDir, allowResetMetricsStorages: settings.allowResetMetricsStorages }) const alice = new LND(settings.lndSettings, new LiquidityProvider("", setupUtils, async () => { }, async () => { }), setupUtils, async () => { }, async () => { }, () => { }, () => { }) const bob = new LND({ ...settings.lndSettings, mainNode: settings.lndSettings.otherNode }, new LiquidityProvider("", setupUtils, async () => { }, async () => { }), setupUtils, async () => { }, async () => { }, () => { }, () => { }) await tryUntil(async i => { diff --git a/src/tests/testBase.ts b/src/tests/testBase.ts index cccb4982..62650472 100644 --- a/src/tests/testBase.ts +++ b/src/tests/testBase.ts @@ -44,7 +44,7 @@ export type StorageTestBase = { export const setupStorageTest = async (d: Describe): Promise => { const settings = GetTestStorageSettings() - const utils = new Utils({ dataDir: settings.dataDir }) + const utils = new Utils({ dataDir: settings.dataDir, allowResetMetricsStorages: true }) const storageManager = new Storage(settings, utils) await storageManager.Connect(console.log) return { @@ -71,7 +71,7 @@ export const SetupTest = async (d: Describe): Promise => { const user1 = { userId: u1.info.userId, appUserIdentifier: u1.identifier, appId: app.appId } const user2 = { userId: u2.info.userId, appUserIdentifier: u2.identifier, appId: app.appId } - const extermnalUtils = new Utils({ dataDir: settings.storageSettings.dataDir }) + const extermnalUtils = new Utils({ dataDir: settings.storageSettings.dataDir, allowResetMetricsStorages: settings.allowResetMetricsStorages }) const externalAccessToMainLnd = new LND(settings.lndSettings, new LiquidityProvider("", extermnalUtils, async () => { }, async () => { }), extermnalUtils, async () => { }, async () => { }, () => { }, () => { }) await externalAccessToMainLnd.Warmup() From b18eab9d20f824edb2c12ce171223618f514e128 Mon Sep 17 00:00:00 2001 From: boufni95 Date: Thu, 10 Apr 2025 17:04:58 +0000 Subject: [PATCH 02/19] fix auth type --- proto/autogenerated/client.md | 8 ++++---- proto/autogenerated/go/http_client.go | 4 ++-- proto/autogenerated/ts/express_server.ts | 4 ++-- proto/autogenerated/ts/http_client.ts | 8 ++++---- proto/autogenerated/ts/nostr_client.ts | 8 ++++---- proto/autogenerated/ts/nostr_transport.ts | 4 ++-- proto/autogenerated/ts/types.ts | 12 ++++++------ proto/service/methods.proto | 4 ++-- 8 files changed, 26 insertions(+), 26 deletions(-) diff --git a/proto/autogenerated/client.md b/proto/autogenerated/client.md index 38eb55c5..4cba96b7 100644 --- a/proto/autogenerated/client.md +++ b/proto/autogenerated/client.md @@ -256,7 +256,7 @@ The nostr server will send back a message response, and inside the body there wi - This methods has an __empty__ __response__ body - ResetMetricsStorages - - auth type: __Admin__ + - auth type: __Metrics__ - This methods has an __empty__ __request__ body - This methods has an __empty__ __response__ body @@ -301,7 +301,7 @@ The nostr server will send back a message response, and inside the body there wi - output: [UserHealthState](#UserHealthState) - ZipMetricsStorages - - auth type: __Admin__ + - auth type: __Metrics__ - This methods has an __empty__ __request__ body - output: [ZippedMetrics](#ZippedMetrics) @@ -805,7 +805,7 @@ The nostr server will send back a message response, and inside the body there wi - This methods has an __empty__ __response__ body - ResetMetricsStorages - - auth type: __Admin__ + - auth type: __Metrics__ - http method: __post__ - http route: __/api/admin/metrics/reset__ - This methods has an __empty__ __request__ body @@ -910,7 +910,7 @@ The nostr server will send back a message response, and inside the body there wi - output: [UserHealthState](#UserHealthState) - ZipMetricsStorages - - auth type: __Admin__ + - auth type: __Metrics__ - http method: __post__ - http route: __/api/admin/metrics/zip__ - This methods has an __empty__ __request__ body diff --git a/proto/autogenerated/go/http_client.go b/proto/autogenerated/go/http_client.go index 652a0838..ec489a7a 100644 --- a/proto/autogenerated/go/http_client.go +++ b/proto/autogenerated/go/http_client.go @@ -1755,7 +1755,7 @@ func NewClient(params ClientParams) *Client { return nil }, ResetMetricsStorages: func() error { - auth, err := params.RetrieveAdminAuth() + auth, err := params.RetrieveMetricsAuth() if err != nil { return err } @@ -2106,7 +2106,7 @@ func NewClient(params ClientParams) *Client { return &res, nil }, ZipMetricsStorages: func() (*ZippedMetrics, error) { - auth, err := params.RetrieveAdminAuth() + auth, err := params.RetrieveMetricsAuth() if err != nil { return nil, err } diff --git a/proto/autogenerated/ts/express_server.ts b/proto/autogenerated/ts/express_server.ts index 9bed5af5..37170364 100644 --- a/proto/autogenerated/ts/express_server.ts +++ b/proto/autogenerated/ts/express_server.ts @@ -1639,7 +1639,7 @@ export default (methods: Types.ServerMethods, opts: ServerOptions) => { let authCtx: Types.AuthContext = {} try { if (!methods.ResetMetricsStorages) throw new Error('method: ResetMetricsStorages is not implemented') - const authContext = await opts.AdminAuthGuard(req.headers['authorization']) + const authContext = await opts.MetricsAuthGuard(req.headers['authorization']) authCtx = authContext stats.guard = process.hrtime.bigint() stats.validate = stats.guard @@ -1941,7 +1941,7 @@ export default (methods: Types.ServerMethods, opts: ServerOptions) => { let authCtx: Types.AuthContext = {} try { if (!methods.ZipMetricsStorages) throw new Error('method: ZipMetricsStorages is not implemented') - const authContext = await opts.AdminAuthGuard(req.headers['authorization']) + const authContext = await opts.MetricsAuthGuard(req.headers['authorization']) authCtx = authContext stats.guard = process.hrtime.bigint() stats.validate = stats.guard diff --git a/proto/autogenerated/ts/http_client.ts b/proto/autogenerated/ts/http_client.ts index afe6ea6a..757f9021 100644 --- a/proto/autogenerated/ts/http_client.ts +++ b/proto/autogenerated/ts/http_client.ts @@ -840,8 +840,8 @@ export default (params: ClientParams) => ({ return { status: 'ERROR', reason: 'invalid response' } }, ResetMetricsStorages: async (): Promise => { - const auth = await params.retrieveAdminAuth() - if (auth === null) throw new Error('retrieveAdminAuth() returned null') + const auth = await params.retrieveMetricsAuth() + if (auth === null) throw new Error('retrieveMetricsAuth() returned null') let finalRoute = '/api/admin/metrics/reset' const { data } = await axios.post(params.baseUrl + finalRoute, {}, { headers: { 'authorization': auth } }) if (data.status === 'ERROR' && typeof data.reason === 'string') return data @@ -1007,8 +1007,8 @@ export default (params: ClientParams) => ({ return { status: 'ERROR', reason: 'invalid response' } }, ZipMetricsStorages: async (): Promise => { - const auth = await params.retrieveAdminAuth() - if (auth === null) throw new Error('retrieveAdminAuth() returned null') + const auth = await params.retrieveMetricsAuth() + if (auth === null) throw new Error('retrieveMetricsAuth() returned null') let finalRoute = '/api/admin/metrics/zip' const { data } = await axios.post(params.baseUrl + finalRoute, {}, { headers: { 'authorization': auth } }) if (data.status === 'ERROR' && typeof data.reason === 'string') return data diff --git a/proto/autogenerated/ts/nostr_client.ts b/proto/autogenerated/ts/nostr_client.ts index 2a2cf713..d7dc1d06 100644 --- a/proto/autogenerated/ts/nostr_client.ts +++ b/proto/autogenerated/ts/nostr_client.ts @@ -699,8 +699,8 @@ export default (params: NostrClientParams, send: (to:string, message: NostrRequ return { status: 'ERROR', reason: 'invalid response' } }, ResetMetricsStorages: async (): Promise => { - const auth = await params.retrieveNostrAdminAuth() - if (auth === null) throw new Error('retrieveNostrAdminAuth() returned null') + const auth = await params.retrieveNostrMetricsAuth() + if (auth === null) throw new Error('retrieveNostrMetricsAuth() returned null') const nostrRequest: NostrRequest = {} const data = await send(params.pubDestination, {rpcName:'ResetMetricsStorages',authIdentifier:auth, ...nostrRequest }) if (data.status === 'ERROR' && typeof data.reason === 'string') return data @@ -817,8 +817,8 @@ export default (params: NostrClientParams, send: (to:string, message: NostrRequ return { status: 'ERROR', reason: 'invalid response' } }, ZipMetricsStorages: async (): Promise => { - const auth = await params.retrieveNostrAdminAuth() - if (auth === null) throw new Error('retrieveNostrAdminAuth() returned null') + const auth = await params.retrieveNostrMetricsAuth() + if (auth === null) throw new Error('retrieveNostrMetricsAuth() returned null') const nostrRequest: NostrRequest = {} const data = await send(params.pubDestination, {rpcName:'ZipMetricsStorages',authIdentifier:auth, ...nostrRequest }) if (data.status === 'ERROR' && typeof data.reason === 'string') return data diff --git a/proto/autogenerated/ts/nostr_transport.ts b/proto/autogenerated/ts/nostr_transport.ts index 6603f973..99d7ec86 100644 --- a/proto/autogenerated/ts/nostr_transport.ts +++ b/proto/autogenerated/ts/nostr_transport.ts @@ -1094,7 +1094,7 @@ export default (methods: Types.ServerMethods, opts: NostrOptions) => { case 'ResetMetricsStorages': try { if (!methods.ResetMetricsStorages) throw new Error('method: ResetMetricsStorages is not implemented') - const authContext = await opts.NostrAdminAuthGuard(req.appId, req.authIdentifier) + const authContext = await opts.NostrMetricsAuthGuard(req.appId, req.authIdentifier) stats.guard = process.hrtime.bigint() authCtx = authContext stats.validate = stats.guard @@ -1229,7 +1229,7 @@ export default (methods: Types.ServerMethods, opts: NostrOptions) => { case 'ZipMetricsStorages': try { if (!methods.ZipMetricsStorages) throw new Error('method: ZipMetricsStorages is not implemented') - const authContext = await opts.NostrAdminAuthGuard(req.appId, req.authIdentifier) + const authContext = await opts.NostrMetricsAuthGuard(req.appId, req.authIdentifier) stats.guard = process.hrtime.bigint() authCtx = authContext stats.validate = stats.guard diff --git a/proto/autogenerated/ts/types.ts b/proto/autogenerated/ts/types.ts index 6997ecef..6bdaa118 100644 --- a/proto/autogenerated/ts/types.ts +++ b/proto/autogenerated/ts/types.ts @@ -7,8 +7,8 @@ export type RequestMetric = AuthContext & RequestInfo & RequestStats & { error?: export type AdminContext = { admin_id: string } -export type AdminMethodInputs = AddApp_Input | AddPeer_Input | AuthApp_Input | BanUser_Input | CloseChannel_Input | CreateOneTimeInviteLink_Input | GetInviteLinkState_Input | GetSeed_Input | ListChannels_Input | LndGetInfo_Input | OpenChannel_Input | ResetMetricsStorages_Input | UpdateChannelPolicy_Input | ZipMetricsStorages_Input -export type AdminMethodOutputs = AddApp_Output | AddPeer_Output | AuthApp_Output | BanUser_Output | CloseChannel_Output | CreateOneTimeInviteLink_Output | GetInviteLinkState_Output | GetSeed_Output | ListChannels_Output | LndGetInfo_Output | OpenChannel_Output | ResetMetricsStorages_Output | UpdateChannelPolicy_Output | ZipMetricsStorages_Output +export type AdminMethodInputs = AddApp_Input | AddPeer_Input | AuthApp_Input | BanUser_Input | CloseChannel_Input | CreateOneTimeInviteLink_Input | GetInviteLinkState_Input | GetSeed_Input | ListChannels_Input | LndGetInfo_Input | OpenChannel_Input | UpdateChannelPolicy_Input +export type AdminMethodOutputs = AddApp_Output | AddPeer_Output | AuthApp_Output | BanUser_Output | CloseChannel_Output | CreateOneTimeInviteLink_Output | GetInviteLinkState_Output | GetSeed_Output | ListChannels_Output | LndGetInfo_Output | OpenChannel_Output | UpdateChannelPolicy_Output export type AppContext = { app_id: string } @@ -28,8 +28,8 @@ export type MetricsContext = { app_id: string operator_id: string } -export type MetricsMethodInputs = GetAppsMetrics_Input | GetBundleMetrics_Input | GetErrorStats_Input | GetLndMetrics_Input | GetSingleBundleMetrics_Input | GetSingleUsageMetrics_Input | GetUsageMetrics_Input | SubmitWebRtcMessage_Input -export type MetricsMethodOutputs = GetAppsMetrics_Output | GetBundleMetrics_Output | GetErrorStats_Output | GetLndMetrics_Output | GetSingleBundleMetrics_Output | GetSingleUsageMetrics_Output | GetUsageMetrics_Output | SubmitWebRtcMessage_Output +export type MetricsMethodInputs = GetAppsMetrics_Input | GetBundleMetrics_Input | GetErrorStats_Input | GetLndMetrics_Input | GetSingleBundleMetrics_Input | GetSingleUsageMetrics_Input | GetUsageMetrics_Input | ResetMetricsStorages_Input | SubmitWebRtcMessage_Input | ZipMetricsStorages_Input +export type MetricsMethodOutputs = GetAppsMetrics_Output | GetBundleMetrics_Output | GetErrorStats_Output | GetLndMetrics_Output | GetSingleBundleMetrics_Output | GetSingleUsageMetrics_Output | GetUsageMetrics_Output | ResetMetricsStorages_Output | SubmitWebRtcMessage_Output | ZipMetricsStorages_Output export type UserContext = { app_id: string app_user_id: string @@ -365,7 +365,7 @@ export type ServerMethods = { PayInvoice?: (req: PayInvoice_Input & {ctx: UserContext }) => Promise RequestNPubLinkingToken?: (req: RequestNPubLinkingToken_Input & {ctx: AppContext }) => Promise ResetDebit?: (req: ResetDebit_Input & {ctx: UserContext }) => Promise - ResetMetricsStorages?: (req: ResetMetricsStorages_Input & {ctx: AdminContext }) => Promise + ResetMetricsStorages?: (req: ResetMetricsStorages_Input & {ctx: MetricsContext }) => Promise ResetNPubLinkingToken?: (req: ResetNPubLinkingToken_Input & {ctx: AppContext }) => Promise RespondToDebit?: (req: RespondToDebit_Input & {ctx: UserContext }) => Promise SendAppUserToAppPayment?: (req: SendAppUserToAppPayment_Input & {ctx: AppContext }) => Promise @@ -380,7 +380,7 @@ export type ServerMethods = { UpdateUserOffer?: (req: UpdateUserOffer_Input & {ctx: UserContext }) => Promise UseInviteLink?: (req: UseInviteLink_Input & {ctx: GuestWithPubContext }) => Promise UserHealth?: (req: UserHealth_Input & {ctx: UserContext }) => Promise - ZipMetricsStorages?: (req: ZipMetricsStorages_Input & {ctx: AdminContext }) => Promise + ZipMetricsStorages?: (req: ZipMetricsStorages_Input & {ctx: MetricsContext }) => Promise } export enum AddressType { diff --git a/proto/service/methods.proto b/proto/service/methods.proto index ee287fb8..f1aebe7b 100644 --- a/proto/service/methods.proto +++ b/proto/service/methods.proto @@ -235,14 +235,14 @@ service LightningPub { } rpc ZipMetricsStorages(structs.Empty) returns (structs.ZippedMetrics) { - option (auth_type) = "Admin"; + option (auth_type) = "Metrics"; option (http_method) = "post"; option (http_route) = "/api/admin/metrics/zip"; option (nostr) = true; } rpc ResetMetricsStorages(structs.Empty) returns (structs.Empty) { - option (auth_type) = "Admin"; + option (auth_type) = "Metrics"; option (http_method) = "post"; option (http_route) = "/api/admin/metrics/reset"; option (nostr) = true; From cf9a10ded48c010bf8460b92994db8c4016841ad Mon Sep 17 00:00:00 2001 From: boufni95 Date: Thu, 10 Apr 2025 17:16:48 +0000 Subject: [PATCH 03/19] fix path --- proto/autogenerated/client.md | 4 ++-- proto/autogenerated/go/http_client.go | 4 ++-- proto/autogenerated/ts/express_server.ts | 4 ++-- proto/autogenerated/ts/http_client.ts | 4 ++-- proto/service/methods.proto | 4 ++-- 5 files changed, 10 insertions(+), 10 deletions(-) diff --git a/proto/autogenerated/client.md b/proto/autogenerated/client.md index 4cba96b7..93fb4efc 100644 --- a/proto/autogenerated/client.md +++ b/proto/autogenerated/client.md @@ -807,7 +807,7 @@ The nostr server will send back a message response, and inside the body there wi - ResetMetricsStorages - auth type: __Metrics__ - http method: __post__ - - http route: __/api/admin/metrics/reset__ + - http route: __/api/metrics/reset__ - This methods has an __empty__ __request__ body - This methods has an __empty__ __response__ body @@ -912,7 +912,7 @@ The nostr server will send back a message response, and inside the body there wi - ZipMetricsStorages - auth type: __Metrics__ - http method: __post__ - - http route: __/api/admin/metrics/zip__ + - http route: __/api/metrics/zip__ - This methods has an __empty__ __request__ body - output: [ZippedMetrics](#ZippedMetrics) diff --git a/proto/autogenerated/go/http_client.go b/proto/autogenerated/go/http_client.go index ec489a7a..970a39d0 100644 --- a/proto/autogenerated/go/http_client.go +++ b/proto/autogenerated/go/http_client.go @@ -1759,7 +1759,7 @@ func NewClient(params ClientParams) *Client { if err != nil { return err } - finalRoute := "/api/admin/metrics/reset" + finalRoute := "/api/metrics/reset" body := []byte{} resBody, err := doPostRequest(params.BaseURL+finalRoute, body, auth) if err != nil { @@ -2110,7 +2110,7 @@ func NewClient(params ClientParams) *Client { if err != nil { return nil, err } - finalRoute := "/api/admin/metrics/zip" + finalRoute := "/api/metrics/zip" body := []byte{} resBody, err := doPostRequest(params.BaseURL+finalRoute, body, auth) if err != nil { diff --git a/proto/autogenerated/ts/express_server.ts b/proto/autogenerated/ts/express_server.ts index 37170364..92f54ba2 100644 --- a/proto/autogenerated/ts/express_server.ts +++ b/proto/autogenerated/ts/express_server.ts @@ -1633,7 +1633,7 @@ export default (methods: Types.ServerMethods, opts: ServerOptions) => { } catch (ex) { const e = ex as any; logErrorAndReturnResponse(e, e.message || e, res, logger, { ...info, ...stats, ...authCtx }, opts.metricsCallback); if (opts.throwErrors) throw e } }) if (!opts.allowNotImplementedMethods && !methods.ResetMetricsStorages) throw new Error('method: ResetMetricsStorages is not implemented') - app.post('/api/admin/metrics/reset', async (req, res) => { + app.post('/api/metrics/reset', async (req, res) => { const info: Types.RequestInfo = { rpcName: 'ResetMetricsStorages', batch: false, nostr: false, batchSize: 0} const stats: Types.RequestStats = { startMs:req.startTimeMs || 0, start:req.startTime || 0n, parse: process.hrtime.bigint(), guard: 0n, validate: 0n, handle: 0n } let authCtx: Types.AuthContext = {} @@ -1935,7 +1935,7 @@ export default (methods: Types.ServerMethods, opts: ServerOptions) => { } catch (ex) { const e = ex as any; logErrorAndReturnResponse(e, e.message || e, res, logger, { ...info, ...stats, ...authCtx }, opts.metricsCallback); if (opts.throwErrors) throw e } }) if (!opts.allowNotImplementedMethods && !methods.ZipMetricsStorages) throw new Error('method: ZipMetricsStorages is not implemented') - app.post('/api/admin/metrics/zip', async (req, res) => { + app.post('/api/metrics/zip', async (req, res) => { const info: Types.RequestInfo = { rpcName: 'ZipMetricsStorages', batch: false, nostr: false, batchSize: 0} const stats: Types.RequestStats = { startMs:req.startTimeMs || 0, start:req.startTime || 0n, parse: process.hrtime.bigint(), guard: 0n, validate: 0n, handle: 0n } let authCtx: Types.AuthContext = {} diff --git a/proto/autogenerated/ts/http_client.ts b/proto/autogenerated/ts/http_client.ts index 757f9021..8ba73a0f 100644 --- a/proto/autogenerated/ts/http_client.ts +++ b/proto/autogenerated/ts/http_client.ts @@ -842,7 +842,7 @@ export default (params: ClientParams) => ({ ResetMetricsStorages: async (): Promise => { const auth = await params.retrieveMetricsAuth() if (auth === null) throw new Error('retrieveMetricsAuth() returned null') - let finalRoute = '/api/admin/metrics/reset' + let finalRoute = '/api/metrics/reset' const { data } = await axios.post(params.baseUrl + finalRoute, {}, { headers: { 'authorization': auth } }) if (data.status === 'ERROR' && typeof data.reason === 'string') return data if (data.status === 'OK') { @@ -1009,7 +1009,7 @@ export default (params: ClientParams) => ({ ZipMetricsStorages: async (): Promise => { const auth = await params.retrieveMetricsAuth() if (auth === null) throw new Error('retrieveMetricsAuth() returned null') - let finalRoute = '/api/admin/metrics/zip' + let finalRoute = '/api/metrics/zip' const { data } = await axios.post(params.baseUrl + finalRoute, {}, { headers: { 'authorization': auth } }) if (data.status === 'ERROR' && typeof data.reason === 'string') return data if (data.status === 'OK') { diff --git a/proto/service/methods.proto b/proto/service/methods.proto index f1aebe7b..0d129d29 100644 --- a/proto/service/methods.proto +++ b/proto/service/methods.proto @@ -237,14 +237,14 @@ service LightningPub { rpc ZipMetricsStorages(structs.Empty) returns (structs.ZippedMetrics) { option (auth_type) = "Metrics"; option (http_method) = "post"; - option (http_route) = "/api/admin/metrics/zip"; + option (http_route) = "/api/metrics/zip"; option (nostr) = true; } rpc ResetMetricsStorages(structs.Empty) returns (structs.Empty) { option (auth_type) = "Metrics"; option (http_method) = "post"; - option (http_route) = "/api/admin/metrics/reset"; + option (http_route) = "/api/metrics/reset"; option (nostr) = true; } From cdb742053b432c1873359352356b8d957a5ca482 Mon Sep 17 00:00:00 2001 From: boufni95 Date: Thu, 10 Apr 2025 17:29:08 +0000 Subject: [PATCH 04/19] up --- env.example | 1 + src/services/storage/tlv/tlvFilesStorage.ts | 3 +++ 2 files changed, 4 insertions(+) diff --git a/env.example b/env.example index d35a7c0f..6a03789b 100644 --- a/env.example +++ b/env.example @@ -103,6 +103,7 @@ LSP_MAX_FEE_BPS=100 #METRICS_TOKEN= # Disable outbound payments aka honeypot mode #DISABLE_EXTERNAL_PAYMENTS=false +#ALLOW_RESET_METRICS_STORAGES=false #WATCHDOG SECURITY # A last line of defense against 0-day drainage attacks diff --git a/src/services/storage/tlv/tlvFilesStorage.ts b/src/services/storage/tlv/tlvFilesStorage.ts index 3e005e0d..56d42228 100644 --- a/src/services/storage/tlv/tlvFilesStorage.ts +++ b/src/services/storage/tlv/tlvFilesStorage.ts @@ -35,6 +35,9 @@ export class TlvFilesStorage { } Reset = () => { + if (this.storagePath === "" && this.storagePath.startsWith("/")) { + throw new Error("cannot delete root storage path") + } clearInterval(this.interval) fs.rmSync(this.storagePath, { recursive: true, force: true }) this.init() From 9dd23f70cfe71822bed36570555c49f31ae26691 Mon Sep 17 00:00:00 2001 From: boufni95 Date: Thu, 10 Apr 2025 17:41:12 +0000 Subject: [PATCH 05/19] up --- src/services/storage/tlv/tlvFilesStorage.ts | 4 ++++ src/services/storage/tlv/tlvFilesStorageProcessor.ts | 2 ++ 2 files changed, 6 insertions(+) diff --git a/src/services/storage/tlv/tlvFilesStorage.ts b/src/services/storage/tlv/tlvFilesStorage.ts index 56d42228..3b6ed01e 100644 --- a/src/services/storage/tlv/tlvFilesStorage.ts +++ b/src/services/storage/tlv/tlvFilesStorage.ts @@ -90,6 +90,10 @@ export class TlvFilesStorage { return data } + PersistNow = () => { + this.persist() + } + private persist = () => { if (!this.metaReady) { throw new Error("meta metrics not ready") diff --git a/src/services/storage/tlv/tlvFilesStorageProcessor.ts b/src/services/storage/tlv/tlvFilesStorageProcessor.ts index 4849bc1a..76912d37 100644 --- a/src/services/storage/tlv/tlvFilesStorageProcessor.ts +++ b/src/services/storage/tlv/tlvFilesStorageProcessor.ts @@ -319,7 +319,9 @@ class TlvFilesStorageProcessor { private async handleZipStorages(operation: ZipStoragesOperation) { const paths = [] for (const storageName in this.storages) { + this.storages[storageName].PersistNow() paths.push(this.storages[storageName].GetStoragePath()) + } if (paths.length === 0) { this.sendResponse({ From c2c36c7e8a3f872b5cdceb0014614874ccdf3943 Mon Sep 17 00:00:00 2001 From: boufni95 Date: Thu, 10 Apr 2025 17:45:21 +0000 Subject: [PATCH 06/19] up --- src/services/storage/tlv/tlvFilesStorageProcessor.ts | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/services/storage/tlv/tlvFilesStorageProcessor.ts b/src/services/storage/tlv/tlvFilesStorageProcessor.ts index 76912d37..0d931cde 100644 --- a/src/services/storage/tlv/tlvFilesStorageProcessor.ts +++ b/src/services/storage/tlv/tlvFilesStorageProcessor.ts @@ -318,9 +318,10 @@ class TlvFilesStorageProcessor { private async handleZipStorages(operation: ZipStoragesOperation) { const paths = [] + const cwd = process.cwd() for (const storageName in this.storages) { this.storages[storageName].PersistNow() - paths.push(this.storages[storageName].GetStoragePath()) + paths.push(cwd + '/' + this.storages[storageName].GetStoragePath()) } if (paths.length === 0) { From c846148b2197a6b3ed8ce607976cf3e5d3c7790a Mon Sep 17 00:00:00 2001 From: boufni95 Date: Thu, 10 Apr 2025 17:50:02 +0000 Subject: [PATCH 07/19] up --- .../storage/tlv/tlvFilesStorageProcessor.ts | 27 ++++++++++--------- 1 file changed, 14 insertions(+), 13 deletions(-) diff --git a/src/services/storage/tlv/tlvFilesStorageProcessor.ts b/src/services/storage/tlv/tlvFilesStorageProcessor.ts index 0d931cde..249b31c8 100644 --- a/src/services/storage/tlv/tlvFilesStorageProcessor.ts +++ b/src/services/storage/tlv/tlvFilesStorageProcessor.ts @@ -318,10 +318,21 @@ class TlvFilesStorageProcessor { private async handleZipStorages(operation: ZipStoragesOperation) { const paths = [] - const cwd = process.cwd() + //const cwd = process.cwd() + const name = crypto.randomBytes(16).toString('hex') + '.zip' for (const storageName in this.storages) { this.storages[storageName].PersistNow() - paths.push(cwd + '/' + this.storages[storageName].GetStoragePath()) + //paths.push(cwd + '/' + this.storages[storageName].GetStoragePath()) + const path = this.storages[storageName].GetStoragePath() + const err = await zip(path, name) + if (err) { + this.sendResponse({ + success: false, + error: err.message, + opId: operation.opId + }) + return + } } if (paths.length === 0) { @@ -332,17 +343,7 @@ class TlvFilesStorageProcessor { }) return } - const name = crypto.randomBytes(16).toString('hex') + '.zip' - const path = paths.join(', ') - const err = await zip(path, name) - if (err) { - this.sendResponse({ - success: false, - error: err.message, - opId: operation.opId - }) - return - } + this.sendResponse({ success: true, type: 'zipStorages', From 75afb82966d64fc27c6122dc138a1afcb8599ab1 Mon Sep 17 00:00:00 2001 From: boufni95 Date: Thu, 10 Apr 2025 17:51:35 +0000 Subject: [PATCH 08/19] fix --- src/services/storage/tlv/tlvFilesStorageProcessor.ts | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/services/storage/tlv/tlvFilesStorageProcessor.ts b/src/services/storage/tlv/tlvFilesStorageProcessor.ts index 249b31c8..39e071a8 100644 --- a/src/services/storage/tlv/tlvFilesStorageProcessor.ts +++ b/src/services/storage/tlv/tlvFilesStorageProcessor.ts @@ -317,10 +317,11 @@ class TlvFilesStorageProcessor { } private async handleZipStorages(operation: ZipStoragesOperation) { - const paths = [] + let noAction = true //const cwd = process.cwd() const name = crypto.randomBytes(16).toString('hex') + '.zip' for (const storageName in this.storages) { + noAction = false this.storages[storageName].PersistNow() //paths.push(cwd + '/' + this.storages[storageName].GetStoragePath()) const path = this.storages[storageName].GetStoragePath() @@ -335,7 +336,7 @@ class TlvFilesStorageProcessor { } } - if (paths.length === 0) { + if (noAction) { this.sendResponse({ success: false, error: 'No storages to zip', From 2367250def302f06edab4e8360e124ea62a182bf Mon Sep 17 00:00:00 2001 From: boufni95 Date: Wed, 16 Apr 2025 16:50:32 +0000 Subject: [PATCH 09/19] chain test --- src/tests/externalPayment.spec.ts | 24 +++++++++++++++++++++++- 1 file changed, 23 insertions(+), 1 deletion(-) diff --git a/src/tests/externalPayment.spec.ts b/src/tests/externalPayment.spec.ts index f5524ed1..f3cfef52 100644 --- a/src/tests/externalPayment.spec.ts +++ b/src/tests/externalPayment.spec.ts @@ -1,11 +1,13 @@ import { defaultInvoiceExpiry } from '../services/storage/paymentStorage.js' import { Describe, expect, expectThrowsAsync, runSanityCheck, safelySetUserBalance, SetupTest, TestBase } from './testBase.js' +import * as Types from '../../proto/autogenerated/ts/types.js' export const ignore = false export const dev = false export default async (T: TestBase) => { await safelySetUserBalance(T, T.user1, 2000) await testSuccessfulExternalPayment(T) await testFailedExternalPayment(T) + await testSuccesfulReceivedExternalChainPayment(T) await runSanityCheck(T) } @@ -44,4 +46,24 @@ const testFailedExternalPayment = async (T: TestBase) => { const owner = await T.main.storage.userStorage.GetUser(application.owner.user_id) expect(owner.balance_sats).to.be.equal(3) T.d("app balance is still 3 sats") -} \ No newline at end of file +} + +const testSuccesfulReceivedExternalChainPayment = async (T: TestBase) => { + T.d("starting testSuccesfulReceivedExternalChainPayment") + const balanceBefore = await T.main.storage.userStorage.GetUser(T.user1.userId) + expect(balanceBefore.balance_sats).to.be.equal(0) + const user2Address = await T.main.paymentManager.NewAddress({ app_id: T.app.appId, app_user_id: T.user2.appUserIdentifier, user_id: T.user2.userId }, { addressType: Types.AddressType.WITNESS_PUBKEY_HASH }) + expect(user2Address.address).to.startWith("bc1") + T.d("generated external chain address for user2") + const payment = await T.externalAccessToOtherLnd.PayAddress(user2Address.address, 1000, 3, "test", { from: 'system', useProvider: false }) + expect(payment.txid).to.not.be.undefined + T.d("paid 1000 sats to user2's external chain address") + const u2 = await T.main.storage.userStorage.GetUser(T.user2.userId) + expect(u2.balance_sats).to.be.equal(1000) + T.d("user2 balance is now 1000") + + + + + +} From 049af2c81f05fc11d5bbc231080381941bf30b5b Mon Sep 17 00:00:00 2001 From: boufni95 Date: Wed, 16 Apr 2025 16:53:24 +0000 Subject: [PATCH 10/19] up --- src/tests/externalPayment.spec.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/tests/externalPayment.spec.ts b/src/tests/externalPayment.spec.ts index f3cfef52..1c82e601 100644 --- a/src/tests/externalPayment.spec.ts +++ b/src/tests/externalPayment.spec.ts @@ -50,8 +50,8 @@ const testFailedExternalPayment = async (T: TestBase) => { const testSuccesfulReceivedExternalChainPayment = async (T: TestBase) => { T.d("starting testSuccesfulReceivedExternalChainPayment") - const balanceBefore = await T.main.storage.userStorage.GetUser(T.user1.userId) - expect(balanceBefore.balance_sats).to.be.equal(0) + /* const balanceBefore = await T.main.storage.userStorage.GetUser(T.user1.userId) + expect(balanceBefore.balance_sats).to.be.equal(0) */ const user2Address = await T.main.paymentManager.NewAddress({ app_id: T.app.appId, app_user_id: T.user2.appUserIdentifier, user_id: T.user2.userId }, { addressType: Types.AddressType.WITNESS_PUBKEY_HASH }) expect(user2Address.address).to.startWith("bc1") T.d("generated external chain address for user2") From 97be3089e8be215c66f790d2f7cd0d45d1da92f2 Mon Sep 17 00:00:00 2001 From: boufni95 Date: Wed, 16 Apr 2025 16:56:03 +0000 Subject: [PATCH 11/19] up --- src/tests/externalPayment.spec.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/tests/externalPayment.spec.ts b/src/tests/externalPayment.spec.ts index 1c82e601..32e309d9 100644 --- a/src/tests/externalPayment.spec.ts +++ b/src/tests/externalPayment.spec.ts @@ -53,7 +53,7 @@ const testSuccesfulReceivedExternalChainPayment = async (T: TestBase) => { /* const balanceBefore = await T.main.storage.userStorage.GetUser(T.user1.userId) expect(balanceBefore.balance_sats).to.be.equal(0) */ const user2Address = await T.main.paymentManager.NewAddress({ app_id: T.app.appId, app_user_id: T.user2.appUserIdentifier, user_id: T.user2.userId }, { addressType: Types.AddressType.WITNESS_PUBKEY_HASH }) - expect(user2Address.address).to.startWith("bc1") + expect(user2Address.address).to.startWith("bcrt1") T.d("generated external chain address for user2") const payment = await T.externalAccessToOtherLnd.PayAddress(user2Address.address, 1000, 3, "test", { from: 'system', useProvider: false }) expect(payment.txid).to.not.be.undefined From 98d856196398504ea0772ebca42fd9dedf4bd28e Mon Sep 17 00:00:00 2001 From: boufni95 Date: Wed, 16 Apr 2025 17:07:35 +0000 Subject: [PATCH 12/19] up --- src/tests/externalPayment.spec.ts | 4 +++- src/tests/internalPayment.spec.ts | 2 +- src/tests/networkSetup.ts | 8 +++++++- src/tests/spamExternalPayments.spec.ts | 2 +- src/tests/spamMixedPayments.spec.ts | 2 +- src/tests/testBase.ts | 7 +++++-- src/tests/testRunner.ts | 18 +++++++++++------- src/tests/userToUserPayment.spec.ts | 2 +- 8 files changed, 30 insertions(+), 15 deletions(-) diff --git a/src/tests/externalPayment.spec.ts b/src/tests/externalPayment.spec.ts index 32e309d9..0bb2f181 100644 --- a/src/tests/externalPayment.spec.ts +++ b/src/tests/externalPayment.spec.ts @@ -1,5 +1,5 @@ import { defaultInvoiceExpiry } from '../services/storage/paymentStorage.js' -import { Describe, expect, expectThrowsAsync, runSanityCheck, safelySetUserBalance, SetupTest, TestBase } from './testBase.js' +import { Describe, expect, expectThrowsAsync, runSanityCheck, safelySetUserBalance, TestBase } from './testBase.js' import * as Types from '../../proto/autogenerated/ts/types.js' export const ignore = false export const dev = false @@ -58,6 +58,8 @@ const testSuccesfulReceivedExternalChainPayment = async (T: TestBase) => { const payment = await T.externalAccessToOtherLnd.PayAddress(user2Address.address, 1000, 3, "test", { from: 'system', useProvider: false }) expect(payment.txid).to.not.be.undefined T.d("paid 1000 sats to user2's external chain address") + await T.chainTools.mine(2) + T.d("mined 2 blocks to confirm the payment") const u2 = await T.main.storage.userStorage.GetUser(T.user2.userId) expect(u2.balance_sats).to.be.equal(1000) T.d("user2 balance is now 1000") diff --git a/src/tests/internalPayment.spec.ts b/src/tests/internalPayment.spec.ts index 7a4b3055..f94bb994 100644 --- a/src/tests/internalPayment.spec.ts +++ b/src/tests/internalPayment.spec.ts @@ -1,5 +1,5 @@ import { defaultInvoiceExpiry } from '../services/storage/paymentStorage.js' -import { Describe, expect, expectThrowsAsync, runSanityCheck, safelySetUserBalance, SetupTest, TestBase } from './testBase.js' +import { Describe, expect, expectThrowsAsync, runSanityCheck, safelySetUserBalance, TestBase } from './testBase.js' export const ignore = false export default async (T: TestBase) => { diff --git a/src/tests/networkSetup.ts b/src/tests/networkSetup.ts index b15e810f..b169c7c7 100644 --- a/src/tests/networkSetup.ts +++ b/src/tests/networkSetup.ts @@ -3,7 +3,12 @@ import { BitcoinCoreWrapper } from "./bitcoinCore.js" import LND from '../services/lnd/lnd.js' import { LiquidityProvider } from "../services/main/liquidityProvider.js" import { Utils } from "../services/helpers/utilsWrapper.js" -export const setupNetwork = async () => { + +export type ChainTools = { + mine: (amount: number) => Promise +} + +export const setupNetwork = async (): Promise => { const settings = LoadTestSettingsFromEnv() const core = new BitcoinCoreWrapper(settings) await core.InitAddress() @@ -51,6 +56,7 @@ export const setupNetwork = async () => { alice.Stop() bob.Stop() + return { mine: (amount: number) => core.Mine(amount) } } const tryUntil = async (fn: (attempt: number) => Promise, maxTries: number, interval: number) => { diff --git a/src/tests/spamExternalPayments.spec.ts b/src/tests/spamExternalPayments.spec.ts index e5fc9467..e7956632 100644 --- a/src/tests/spamExternalPayments.spec.ts +++ b/src/tests/spamExternalPayments.spec.ts @@ -1,6 +1,6 @@ import { disableLoggers } from '../services/helpers/logger.js' import { defaultInvoiceExpiry } from '../services/storage/paymentStorage.js' -import { Describe, expect, expectThrowsAsync, runSanityCheck, safelySetUserBalance, SetupTest, TestBase } from './testBase.js' +import { Describe, expect, expectThrowsAsync, runSanityCheck, safelySetUserBalance, TestBase } from './testBase.js' export const ignore = false export default async (T: TestBase) => { diff --git a/src/tests/spamMixedPayments.spec.ts b/src/tests/spamMixedPayments.spec.ts index 4851ab2c..9f85db64 100644 --- a/src/tests/spamMixedPayments.spec.ts +++ b/src/tests/spamMixedPayments.spec.ts @@ -1,6 +1,6 @@ import { disableLoggers } from '../services/helpers/logger.js' import { defaultInvoiceExpiry } from '../services/storage/paymentStorage.js' -import { Describe, expect, expectThrowsAsync, runSanityCheck, safelySetUserBalance, SetupTest, TestBase } from './testBase.js' +import { Describe, expect, expectThrowsAsync, runSanityCheck, safelySetUserBalance, TestBase } from './testBase.js' import * as Types from '../../proto/autogenerated/ts/types.js' export const ignore = false diff --git a/src/tests/testBase.ts b/src/tests/testBase.ts index 62650472..aa4d3714 100644 --- a/src/tests/testBase.ts +++ b/src/tests/testBase.ts @@ -14,6 +14,7 @@ import { LiquidityProvider } from '../services/main/liquidityProvider.js' import { Utils } from '../services/helpers/utilsWrapper.js' import { AdminManager } from '../services/main/adminManager.js' import { TlvStorageFactory } from '../services/storage/tlv/tlvFilesStorageFactory.js' +import { ChainTools } from './networkSetup.js' chai.use(chaiString) export const expect = chai.expect export type Describe = (message: string, failure?: boolean) => void @@ -34,6 +35,7 @@ export type TestBase = { externalAccessToThirdLnd: LND adminManager: AdminManager d: Describe + chainTools: ChainTools } export type StorageTestBase = { @@ -58,7 +60,7 @@ export const teardownStorageTest = async (T: StorageTestBase) => { T.storage.Stop() } -export const SetupTest = async (d: Describe): Promise => { +export const SetupTest = async (d: Describe, chainTools: ChainTools): Promise => { const settings = LoadTestSettingsFromEnv() const initialized = await initMainHandler(getLogger({ component: "mainForTest" }), settings) if (!initialized) { @@ -89,7 +91,8 @@ export const SetupTest = async (d: Describe): Promise => { user1, user2, externalAccessToMainLnd, externalAccessToOtherLnd, externalAccessToThirdLnd, d, - adminManager: initialized.adminManager + adminManager: initialized.adminManager, + chainTools } } diff --git a/src/tests/testRunner.ts b/src/tests/testRunner.ts index d283cc43..242fed46 100644 --- a/src/tests/testRunner.ts +++ b/src/tests/testRunner.ts @@ -1,6 +1,6 @@ //import whyIsNodeRunning from 'why-is-node-running' import { globby } from 'globby' -import { setupNetwork } from './networkSetup.js' +import { ChainTools, setupNetwork } from './networkSetup.js' import { Describe, SetupTest, teardown, TestBase, StorageTestBase, setupStorageTest, teardownStorageTest } from './testBase.js' type TestModule = { ignore?: boolean @@ -38,17 +38,18 @@ const start = async () => { if (devModule !== -1) { console.log("running dev module") const { file, module } = modules[devModule] + let chainTools: ChainTools | undefined if (module.requires === 'storage') { console.log("dev module requires only storage, skipping network setup") } else { - await setupNetwork() + chainTools = await setupNetwork() } - await runTestFile(file, module) + await runTestFile(file, module, chainTools) } else { console.log("running all tests") - await setupNetwork() + const chainTools = await setupNetwork() for (const { file, module } of modules) { - await runTestFile(file, module) + await runTestFile(file, module, chainTools) } } console.log(failures) @@ -62,7 +63,7 @@ const start = async () => { } -const runTestFile = async (fileName: string, mod: TestModule) => { +const runTestFile = async (fileName: string, mod: TestModule, chainTools?: ChainTools) => { console.log(fileName) const d = getDescribe(fileName) if (mod.ignore) { @@ -78,7 +79,10 @@ const runTestFile = async (fileName: string, mod: TestModule) => { T = await setupStorageTest(d) } else { d("-----requires all-----") - T = await SetupTest(d) + if (!chainTools) { + throw new Error("chainTools are required for this test") + } + T = await SetupTest(d, chainTools) } try { d("test starting") diff --git a/src/tests/userToUserPayment.spec.ts b/src/tests/userToUserPayment.spec.ts index 808cc62c..e93f0604 100644 --- a/src/tests/userToUserPayment.spec.ts +++ b/src/tests/userToUserPayment.spec.ts @@ -1,5 +1,5 @@ import { defaultInvoiceExpiry } from '../services/storage/paymentStorage.js' -import { Describe, expect, expectThrowsAsync, runSanityCheck, safelySetUserBalance, SetupTest, TestBase } from './testBase.js' +import { Describe, expect, expectThrowsAsync, runSanityCheck, safelySetUserBalance, TestBase } from './testBase.js' export const ignore = false export const dev = false export default async (T: TestBase) => { From 1e27823ce98e66284a9c1251764dd6ee251984f1 Mon Sep 17 00:00:00 2001 From: boufni95 Date: Wed, 16 Apr 2025 17:28:06 +0000 Subject: [PATCH 13/19] deb --- src/services/main/index.ts | 1 + src/services/main/paymentManager.ts | 2 ++ 2 files changed, 3 insertions(+) diff --git a/src/services/main/index.ts b/src/services/main/index.ts index 0aaf5c4e..a4691246 100644 --- a/src/services/main/index.ts +++ b/src/services/main/index.ts @@ -119,6 +119,7 @@ export default class { const balanceEvents = await this.paymentManager.GetLndBalance() await this.metricsManager.NewBlockCb(height, balanceEvents) confirmed = await this.paymentManager.CheckNewlyConfirmedTxs(height) + log("checked newly confirmed transactions", confirmed) await this.liquidityManager.onNewBlock() } catch (err: any) { log(ERROR, "failed to check transactions after new block", err.message || err) diff --git a/src/services/main/paymentManager.ts b/src/services/main/paymentManager.ts index 2a1c1c34..b769adef 100644 --- a/src/services/main/paymentManager.ts +++ b/src/services/main/paymentManager.ts @@ -211,9 +211,11 @@ export default class { } const existingAddress = await this.storage.paymentStorage.GetExistingUserAddress(ctx.user_id, app) if (existingAddress) { + this.log("returning existing address", existingAddress.address) return { address: existingAddress.address } } const res = await this.lnd.NewAddress(req.addressType, { useProvider: false, from: 'user' }) + this.log("generated new address", res.address) const userAddress = await this.storage.paymentStorage.AddUserAddress(user, res.address, { linkedApplication: app }) this.storage.eventsLog.LogEvent({ type: 'new_address', userId: user.user_id, appUserId: "", appId: app.app_id, balance: user.balance_sats, data: res.address, amount: 0 }) return { address: userAddress.address } From b42af40a7fe5875991459229645121f2c24e42e3 Mon Sep 17 00:00:00 2001 From: boufni95 Date: Wed, 16 Apr 2025 17:34:50 +0000 Subject: [PATCH 14/19] up --- src/services/main/index.ts | 2 +- src/tests/externalPayment.spec.ts | 6 +++--- src/tests/testBase.ts | 10 +++++----- 3 files changed, 9 insertions(+), 9 deletions(-) diff --git a/src/services/main/index.ts b/src/services/main/index.ts index a4691246..e9114445 100644 --- a/src/services/main/index.ts +++ b/src/services/main/index.ts @@ -114,7 +114,7 @@ export default class { NewBlockHandler = async (height: number) => { let confirmed: (PendingTx & { confs: number; })[] let log = getLogger({}) - + log("checking new block", height) try { const balanceEvents = await this.paymentManager.GetLndBalance() await this.metricsManager.NewBlockCb(height, balanceEvents) diff --git a/src/tests/externalPayment.spec.ts b/src/tests/externalPayment.spec.ts index 0bb2f181..10d31fa6 100644 --- a/src/tests/externalPayment.spec.ts +++ b/src/tests/externalPayment.spec.ts @@ -2,7 +2,7 @@ import { defaultInvoiceExpiry } from '../services/storage/paymentStorage.js' import { Describe, expect, expectThrowsAsync, runSanityCheck, safelySetUserBalance, TestBase } from './testBase.js' import * as Types from '../../proto/autogenerated/ts/types.js' export const ignore = false -export const dev = false +export const dev = true export default async (T: TestBase) => { await safelySetUserBalance(T, T.user1, 2000) await testSuccessfulExternalPayment(T) @@ -58,8 +58,8 @@ const testSuccesfulReceivedExternalChainPayment = async (T: TestBase) => { const payment = await T.externalAccessToOtherLnd.PayAddress(user2Address.address, 1000, 3, "test", { from: 'system', useProvider: false }) expect(payment.txid).to.not.be.undefined T.d("paid 1000 sats to user2's external chain address") - await T.chainTools.mine(2) - T.d("mined 2 blocks to confirm the payment") + await T.chainTools.mine(1) + T.d("mined 1 blocks to confirm the payment") const u2 = await T.main.storage.userStorage.GetUser(T.user2.userId) expect(u2.balance_sats).to.be.equal(1000) T.d("user2 balance is now 1000") diff --git a/src/tests/testBase.ts b/src/tests/testBase.ts index aa4d3714..939ce5fb 100644 --- a/src/tests/testBase.ts +++ b/src/tests/testBase.ts @@ -30,7 +30,7 @@ export type TestBase = { app: AppData user1: TestUserData user2: TestUserData - externalAccessToMainLnd: LND + //externalAccessToMainLnd: LND externalAccessToOtherLnd: LND externalAccessToThirdLnd: LND adminManager: AdminManager @@ -74,8 +74,8 @@ export const SetupTest = async (d: Describe, chainTools: ChainTools): Promise { }, async () => { }), extermnalUtils, async () => { }, async () => { }, () => { }, () => { }) - await externalAccessToMainLnd.Warmup() + /* const externalAccessToMainLnd = new LND(settings.lndSettings, new LiquidityProvider("", extermnalUtils, async () => { }, async () => { }), extermnalUtils, async () => { }, async () => { }, () => { }, () => { }) + await externalAccessToMainLnd.Warmup() */ const otherLndSetting = { ...settings.lndSettings, mainNode: settings.lndSettings.otherNode } const externalAccessToOtherLnd = new LND(otherLndSetting, new LiquidityProvider("", extermnalUtils, async () => { }, async () => { }), extermnalUtils, async () => { }, async () => { }, () => { }, () => { }) @@ -89,7 +89,7 @@ export const SetupTest = async (d: Describe, chainTools: ChainTools): Promise { T.main.Stop() - T.externalAccessToMainLnd.Stop() + /* T.externalAccessToMainLnd.Stop() */ T.externalAccessToOtherLnd.Stop() T.externalAccessToThirdLnd.Stop() T.adminManager.Stop() From 2879517c9291faaf93167c3558f649ddcf95ceb1 Mon Sep 17 00:00:00 2001 From: boufni95 Date: Wed, 16 Apr 2025 17:38:10 +0000 Subject: [PATCH 15/19] deb --- src/tests/externalPayment.spec.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/src/tests/externalPayment.spec.ts b/src/tests/externalPayment.spec.ts index 10d31fa6..dd19eb7e 100644 --- a/src/tests/externalPayment.spec.ts +++ b/src/tests/externalPayment.spec.ts @@ -59,6 +59,7 @@ const testSuccesfulReceivedExternalChainPayment = async (T: TestBase) => { expect(payment.txid).to.not.be.undefined T.d("paid 1000 sats to user2's external chain address") await T.chainTools.mine(1) + await new Promise(resolve => setTimeout(resolve, 100)) T.d("mined 1 blocks to confirm the payment") const u2 = await T.main.storage.userStorage.GetUser(T.user2.userId) expect(u2.balance_sats).to.be.equal(1000) From a6aef23df198b471929cb356fde81c2fa7de36c0 Mon Sep 17 00:00:00 2001 From: boufni95 Date: Wed, 16 Apr 2025 17:41:40 +0000 Subject: [PATCH 16/19] fixed? --- src/services/main/index.ts | 3 +-- src/services/main/paymentManager.ts | 2 -- 2 files changed, 1 insertion(+), 4 deletions(-) diff --git a/src/services/main/index.ts b/src/services/main/index.ts index e9114445..0aaf5c4e 100644 --- a/src/services/main/index.ts +++ b/src/services/main/index.ts @@ -114,12 +114,11 @@ export default class { NewBlockHandler = async (height: number) => { let confirmed: (PendingTx & { confs: number; })[] let log = getLogger({}) - log("checking new block", height) + try { const balanceEvents = await this.paymentManager.GetLndBalance() await this.metricsManager.NewBlockCb(height, balanceEvents) confirmed = await this.paymentManager.CheckNewlyConfirmedTxs(height) - log("checked newly confirmed transactions", confirmed) await this.liquidityManager.onNewBlock() } catch (err: any) { log(ERROR, "failed to check transactions after new block", err.message || err) diff --git a/src/services/main/paymentManager.ts b/src/services/main/paymentManager.ts index b769adef..2a1c1c34 100644 --- a/src/services/main/paymentManager.ts +++ b/src/services/main/paymentManager.ts @@ -211,11 +211,9 @@ export default class { } const existingAddress = await this.storage.paymentStorage.GetExistingUserAddress(ctx.user_id, app) if (existingAddress) { - this.log("returning existing address", existingAddress.address) return { address: existingAddress.address } } const res = await this.lnd.NewAddress(req.addressType, { useProvider: false, from: 'user' }) - this.log("generated new address", res.address) const userAddress = await this.storage.paymentStorage.AddUserAddress(user, res.address, { linkedApplication: app }) this.storage.eventsLog.LogEvent({ type: 'new_address', userId: user.user_id, appUserId: "", appId: app.app_id, balance: user.balance_sats, data: res.address, amount: 0 }) return { address: userAddress.address } From 94024c069367c77e0f25eded73475b05a51420e6 Mon Sep 17 00:00:00 2001 From: boufni95 Date: Wed, 16 Apr 2025 17:46:42 +0000 Subject: [PATCH 17/19] deb --- src/services/main/sanityChecker.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/src/services/main/sanityChecker.ts b/src/services/main/sanityChecker.ts index 876c122e..69251f75 100644 --- a/src/services/main/sanityChecker.ts +++ b/src/services/main/sanityChecker.ts @@ -236,6 +236,7 @@ export default class SanityChecker { this.decrementEvents = {} for (let i = 0; i < this.events.length; i++) { const e = this.events[i] + this.log("checking event", e.type, e.data) if (e.type === 'balance_decrement') { await this.verifyDecrementEvent(e) } else if (e.type === 'balance_increment') { From 8f60e81f8edc30be8383b4da4af9bd1880a440a7 Mon Sep 17 00:00:00 2001 From: boufni95 Date: Wed, 16 Apr 2025 17:53:00 +0000 Subject: [PATCH 18/19] fix --- src/services/main/sanityChecker.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/services/main/sanityChecker.ts b/src/services/main/sanityChecker.ts index 69251f75..6ef7419a 100644 --- a/src/services/main/sanityChecker.ts +++ b/src/services/main/sanityChecker.ts @@ -36,7 +36,7 @@ export default class SanityChecker { return { type: fullData, data: fullData } } else if (LN_INVOICE_REGEX.test(fullData)) { return { type: 'invoice', data: fullData } - } else if (BITCOIN_ADDRESS_REGEX.test(fullData)) { + } else if (BITCOIN_ADDRESS_REGEX.test(fullData) || fullData.startsWith("bcrt1")) { return { type: 'address', data: fullData } } else { return { type: 'u2u', data: fullData } @@ -45,7 +45,7 @@ export default class SanityChecker { const [prefix, data] = parts if (prefix === 'routing_fee_refund' || prefix === 'payment_refund') { return { type: prefix, data } - } else if (BITCOIN_ADDRESS_REGEX.test(prefix)) { + } else if (BITCOIN_ADDRESS_REGEX.test(prefix) || prefix.startsWith("bcrt1")) { return { type: 'address', data: prefix, txHash: data } } else { return { type: 'u2u', data: prefix, serialId: +data } From bfbbcfbb83dc4ca502c300d435c9a7305e1dfdbf Mon Sep 17 00:00:00 2001 From: boufni95 Date: Wed, 16 Apr 2025 17:55:25 +0000 Subject: [PATCH 19/19] clean --- src/tests/externalPayment.spec.ts | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/src/tests/externalPayment.spec.ts b/src/tests/externalPayment.spec.ts index dd19eb7e..a92d0054 100644 --- a/src/tests/externalPayment.spec.ts +++ b/src/tests/externalPayment.spec.ts @@ -2,7 +2,7 @@ import { defaultInvoiceExpiry } from '../services/storage/paymentStorage.js' import { Describe, expect, expectThrowsAsync, runSanityCheck, safelySetUserBalance, TestBase } from './testBase.js' import * as Types from '../../proto/autogenerated/ts/types.js' export const ignore = false -export const dev = true +export const dev = false export default async (T: TestBase) => { await safelySetUserBalance(T, T.user1, 2000) await testSuccessfulExternalPayment(T) @@ -64,9 +64,13 @@ const testSuccesfulReceivedExternalChainPayment = async (T: TestBase) => { const u2 = await T.main.storage.userStorage.GetUser(T.user2.userId) expect(u2.balance_sats).to.be.equal(1000) T.d("user2 balance is now 1000") - - - - - + const payment2 = await T.externalAccessToOtherLnd.PayAddress(user2Address.address, 1000, 3, "test", { from: 'system', useProvider: false }) + expect(payment2.txid).to.not.be.undefined + T.d("paid 1000 sats to user2's external chain address again") + await T.chainTools.mine(1) + await new Promise(resolve => setTimeout(resolve, 100)) + T.d("mined 1 blocks to confirm the payment") + const u2_2 = await T.main.storage.userStorage.GetUser(T.user2.userId) + expect(u2_2.balance_sats).to.be.equal(2000) + T.d("user2 balance is now 2000") }