From ff76ef9947cd3c0e12012ec5dcf7d1172a80e175 Mon Sep 17 00:00:00 2001 From: boufni95 Date: Wed, 4 Jun 2025 18:44:23 +0000 Subject: [PATCH] events sharding --- env.example | 2 + package.json | 3 +- src/services/nostr/handler.ts | 89 ++++++++++++++++++++++------------- src/services/nostr/index.ts | 6 ++- src/tests/testRelay.ts | 29 ++++++++++++ 5 files changed, 93 insertions(+), 36 deletions(-) create mode 100644 src/tests/testRelay.ts diff --git a/env.example b/env.example index 319585fb..f85faf5d 100644 --- a/env.example +++ b/env.example @@ -73,6 +73,8 @@ LSP_MAX_FEE_BPS=100 #NOSTR # Default relay may become rate-limited without a paid subscription #NOSTR_RELAYS=wss://relay.lightning.pub +# Max content lengh of single nostr event content, events will be sharded above this size +#NOSTR_MAX_EVENT_CONTENT_LENGTH=45000 #LNURL # Optional diff --git a/package.json b/package.json index cf1bafbb..ed6d727b 100644 --- a/package.json +++ b/package.json @@ -12,7 +12,8 @@ "build_autogenerated": "cd proto && rimraf autogenerated && protoc -I ./service --pub_out=. service/*", "build_lnd_client_1": "cd proto && protoc -I ./others --plugin=.\\node_modules\\.bin\\protoc-gen-ts_proto.cmd --ts_proto_out=./lnd --ts_proto_opt=esModuleInterop=true others/* ", "build_lnd_client": "cd proto && rimraf lnd/* && npx protoc --ts_out ./lnd --ts_opt long_type_string --proto_path others others/* ", - "typeorm": "typeorm-ts-node-commonjs" + "typeorm": "typeorm-ts-node-commonjs", + "test_relay": "npm run clean && tsc && node build/src/tests/testRelay.js" }, "repository": { "type": "git", diff --git a/src/services/nostr/handler.ts b/src/services/nostr/handler.ts index c4fad0bd..32faddfa 100644 --- a/src/services/nostr/handler.ts +++ b/src/services/nostr/handler.ts @@ -1,6 +1,7 @@ //import { SimplePool, Sub, Event, UnsignedEvent, getEventHash, signEvent } from 'nostr-tools' import WebSocket from 'ws' Object.assign(global, { WebSocket: WebSocket }); +import crypto from 'crypto' import { SimplePool, Event, UnsignedEvent, finalizeEvent, Relay, nip44 } from 'nostr-tools' import { ERROR, getLogger } from '../helpers/logger.js' import { nip19 } from 'nostr-tools' @@ -13,7 +14,9 @@ const { getConversationKey: getConversationKeyV2 } = utils const handledEvents: string[] = [] // TODO: - big memory leak here, add TTL type AppInfo = { appId: string, publicKey: string, privateKey: string, name: string } type ClientInfo = { clientId: string, publicKey: string, privateKey: string, name: string } -export type SendData = { type: "content", content: string, pub: string } | { type: "event", event: UnsignedEvent, encrypt?: { toPub: string } } +type SendDataContent = { type: "content", content: string, pub: string, index?: number, totalShards?: number, shardsId?: string } +type SendDataEvent = { type: "event", event: UnsignedEvent, encrypt?: { toPub: string } } +export type SendData = SendDataContent | SendDataEvent export type SendInitiator = { type: 'app', appId: string } | { type: 'client', clientId: string } export type NostrSend = (initiator: SendInitiator, data: SendData, relays?: string[] | undefined) => void @@ -21,6 +24,7 @@ export type NostrSettings = { apps: AppInfo[] relays: string[] clients: ClientInfo[] + maxEventContentLength: number } export type NostrEvent = { id: string @@ -218,38 +222,49 @@ export default class Handler { async Send(initiator: SendInitiator, data: SendData, relays?: string[]) { const keys = this.GetSendKeys(initiator) const privateKey = Buffer.from(keys.privateKey, 'hex') - let toSign: UnsignedEvent - if (data.type === 'content') { - let content: string - try { - content = encryptV1(data.content, getConversationKeyV1(keys.privateKey, data.pub)) - } catch (e: any) { - this.log(ERROR, "failed to encrypt content", e.message, data.content) - return - } - toSign = { - content, - created_at: Math.floor(Date.now() / 1000), - kind: 21000, - pubkey: keys.publicKey, - tags: [['p', data.pub]], - } - } else { - toSign = data.event - if (data.encrypt) { - try { - toSign.content = encryptV2(data.event.content, getConversationKeyV2(Buffer.from(keys.privateKey, 'hex'), data.encrypt.toPub)) - } catch (e: any) { - this.log(ERROR, "failed to encrypt content", e.message) - return - } - } - if (!toSign.pubkey) { - toSign.pubkey = keys.publicKey - } - } + const toSign = await this.handleSend(data, keys) + await Promise.all(toSign.map(ue => this.sendEvent(ue, keys, relays))) + } - const signed = finalizeEvent(toSign, Buffer.from(keys.privateKey, 'hex')) + async handleSend(data: SendData, keys: { name: string, privateKey: string, publicKey: string }): Promise { + if (data.type === 'content') { + const parts = splitContent(data.content, this.settings.maxEventContentLength) + if (parts.length > 1) { + const shardsId = crypto.randomBytes(16).toString('hex') + const totalShards = parts.length + const ues = await Promise.all(parts.map((part, index) => this.handleSendDataContent({ ...data, content: part, index, totalShards, shardsId }, keys))) + return ues + } + return [await this.handleSendDataContent(data, keys)] + } + const ue = await this.handleSendDataEvent(data, keys) + return [ue] + } + + async handleSendDataContent(data: SendDataContent, keys: { name: string, privateKey: string, publicKey: string }): Promise { + let content = encryptV1(data.content, getConversationKeyV1(keys.privateKey, data.pub)) + return { + content, + created_at: Math.floor(Date.now() / 1000), + kind: 21000, + pubkey: keys.publicKey, + tags: [['p', data.pub]], + } + } + + async handleSendDataEvent(data: SendDataEvent, keys: { name: string, privateKey: string, publicKey: string }): Promise { + const toSign = data.event + if (data.encrypt) { + toSign.content = encryptV2(data.event.content, getConversationKeyV2(Buffer.from(keys.privateKey, 'hex'), data.encrypt.toPub)) + } + if (!toSign.pubkey) { + toSign.pubkey = keys.publicKey + } + return toSign + } + + async sendEvent(event: UnsignedEvent, keys: { name: string, privateKey: string }, relays?: string[]) { + const signed = finalizeEvent(event, Buffer.from(keys.privateKey, 'hex')) let sent = false const log = getLogger({ appName: keys.name }) await Promise.all(this.pool.publish(relays || this.settings.relays, signed).map(async p => { @@ -264,7 +279,7 @@ export default class Handler { if (!sent) { log("failed to send event") } else { - log("sent event") + //log("sent event") } } @@ -286,4 +301,12 @@ export default class Handler { } throw new Error("unkown initiator type") } +} + +const splitContent = (content: string, maxLength: number) => { + const parts = [] + for (let i = 0; i < content.length; i += maxLength) { + parts.push(content.slice(i, i + maxLength)) + } + return parts } \ No newline at end of file diff --git a/src/services/nostr/index.ts b/src/services/nostr/index.ts index 9b9745e3..6d7bb7ce 100644 --- a/src/services/nostr/index.ts +++ b/src/services/nostr/index.ts @@ -1,5 +1,5 @@ import { ChildProcess, fork } from 'child_process' -import { EnvMustBeNonEmptyString } from "../helpers/envParser.js" +import { EnvCanBeInteger, EnvMustBeNonEmptyString } from "../helpers/envParser.js" import { NostrSettings, NostrEvent, ChildProcessRequest, ChildProcessResponse, SendData, SendInitiator } from "./handler.js" import { Utils } from '../helpers/utilsWrapper.js' type EventCallback = (event: NostrEvent) => void @@ -10,8 +10,10 @@ const getEnvOrDefault = (name: string, defaultValue: string): string => { export const LoadNosrtSettingsFromEnv = (test = false) => { const relaysEnv = getEnvOrDefault("NOSTR_RELAYS", "wss://relay.lightning.pub"); + const maxEventContentLength = EnvCanBeInteger("NOSTR_MAX_EVENT_CONTENT_LENGTH", 45000) return { - relays: relaysEnv.split(' ') + relays: relaysEnv.split(' '), + maxEventContentLength } } diff --git a/src/tests/testRelay.ts b/src/tests/testRelay.ts new file mode 100644 index 00000000..cc662bff --- /dev/null +++ b/src/tests/testRelay.ts @@ -0,0 +1,29 @@ +import { SimplePool, UnsignedEvent, finalizeEvent, generateSecretKey, getPublicKey } from "nostr-tools" +import { encrypt as encryptV1, decrypt as decryptV1, getSharedSecret as getConversationKeyV1 } from '../services/nostr/nip44v1.js' +import WebSocket from 'ws' +Object.assign(global, { WebSocket: WebSocket }); +const pool = new SimplePool() + +const relays = [ + "wss://strfry.shock.network" +] +const secretKey = generateSecretKey() +const publicKey = getPublicKey(secretKey) +const content = Array(1000).fill("A").join("") +console.log("content length", content.length) +const encrypted = encryptV1(content, getConversationKeyV1(Buffer.from(secretKey).toString('hex'), publicKey)) +console.log("encrypted length", encrypted.length) +console.log("encrypted", encrypted) +const e: UnsignedEvent = { + content: encrypted, + created_at: Math.floor(Date.now() / 1000), + kind: 21000, + pubkey: publicKey, + tags: [["p", publicKey]], +} +const signed = finalizeEvent(e, Buffer.from(secretKey)) +Promise.all(pool.publish(relays, signed)).then(r => { + console.log("sent", r) +}).catch(e => { + console.error("error", e) +}) \ No newline at end of file