events sharding

env.example

@@ -73,6 +73,8 @@ LSP_MAX_FEE_BPS=100
 #NOSTR
 # Default relay may become rate-limited without a paid subscription
 #NOSTR_RELAYS=wss://relay.lightning.pub
+# Max content lengh of single nostr event content, events will be sharded above this size
+#NOSTR_MAX_EVENT_CONTENT_LENGTH=45000
 
 #LNURL
 # Optional

package.json

@@ -12,7 +12,8 @@
     "build_autogenerated": "cd proto && rimraf autogenerated && protoc -I ./service --pub_out=. service/*",
     "build_lnd_client_1": "cd proto && protoc -I ./others --plugin=.\\node_modules\\.bin\\protoc-gen-ts_proto.cmd --ts_proto_out=./lnd --ts_proto_opt=esModuleInterop=true others/* ",
     "build_lnd_client": "cd proto && rimraf lnd/*  && npx protoc --ts_out ./lnd --ts_opt long_type_string  --proto_path others others/* ",
-    "typeorm": "typeorm-ts-node-commonjs"
+    "typeorm": "typeorm-ts-node-commonjs",
+    "test_relay": "npm run clean && tsc && node build/src/tests/testRelay.js"
   },
   "repository": {
     "type": "git",

src/services/nostr/handler.ts

@@ -1,6 +1,7 @@
 //import { SimplePool, Sub, Event, UnsignedEvent, getEventHash, signEvent } from 'nostr-tools'
 import WebSocket from 'ws'
 Object.assign(global, { WebSocket: WebSocket });
+import crypto from 'crypto'
 import { SimplePool, Event, UnsignedEvent, finalizeEvent, Relay, nip44 } from 'nostr-tools'
 import { ERROR, getLogger } from '../helpers/logger.js'
 import { nip19 } from 'nostr-tools'
@@ -13,7 +14,9 @@ const { getConversationKey: getConversationKeyV2 } = utils
 const handledEvents: string[] = [] // TODO: - big memory leak here, add TTL
 type AppInfo = { appId: string, publicKey: string, privateKey: string, name: string }
 type ClientInfo = { clientId: string, publicKey: string, privateKey: string, name: string }
-export type SendData = { type: "content", content: string, pub: string } | { type: "event", event: UnsignedEvent, encrypt?: { toPub: string } }
+type SendDataContent = { type: "content", content: string, pub: string, index?: number, totalShards?: number, shardsId?: string }
+type SendDataEvent = { type: "event", event: UnsignedEvent, encrypt?: { toPub: string } }
+export type SendData = SendDataContent | SendDataEvent
 export type SendInitiator = { type: 'app', appId: string } | { type: 'client', clientId: string }
 export type NostrSend = (initiator: SendInitiator, data: SendData, relays?: string[] | undefined) => void
 
@@ -21,6 +24,7 @@ export type NostrSettings = {
     apps: AppInfo[]
     relays: string[]
     clients: ClientInfo[]
+    maxEventContentLength: number
 }
 export type NostrEvent = {
     id: string
@@ -218,38 +222,49 @@ export default class Handler {
     async Send(initiator: SendInitiator, data: SendData, relays?: string[]) {
         const keys = this.GetSendKeys(initiator)
         const privateKey = Buffer.from(keys.privateKey, 'hex')
-        let toSign: UnsignedEvent
+        const toSign = await this.handleSend(data, keys)
-        if (data.type === 'content') {
+        await Promise.all(toSign.map(ue => this.sendEvent(ue, keys, relays)))
-            let content: string
+    }
-            try {
-                content = encryptV1(data.content, getConversationKeyV1(keys.privateKey, data.pub))
-            } catch (e: any) {
-                this.log(ERROR, "failed to encrypt content", e.message, data.content)
-                return
-            }
-            toSign = {
-                content,
-                created_at: Math.floor(Date.now() / 1000),
-                kind: 21000,
-                pubkey: keys.publicKey,
-                tags: [['p', data.pub]],
-            }
-        } else {
-            toSign = data.event
-            if (data.encrypt) {
-                try {
-                    toSign.content = encryptV2(data.event.content, getConversationKeyV2(Buffer.from(keys.privateKey, 'hex'), data.encrypt.toPub))
-                } catch (e: any) {
-                    this.log(ERROR, "failed to encrypt content", e.message)
-                    return
-                }
-            }
-            if (!toSign.pubkey) {
-                toSign.pubkey = keys.publicKey
-            }
-        }
 
-        const signed = finalizeEvent(toSign, Buffer.from(keys.privateKey, 'hex'))
+    async handleSend(data: SendData, keys: { name: string, privateKey: string, publicKey: string }): Promise<UnsignedEvent[]> {
+        if (data.type === 'content') {
+            const parts = splitContent(data.content, this.settings.maxEventContentLength)
+            if (parts.length > 1) {
+                const shardsId = crypto.randomBytes(16).toString('hex')
+                const totalShards = parts.length
+                const ues = await Promise.all(parts.map((part, index) => this.handleSendDataContent({ ...data, content: part, index, totalShards, shardsId }, keys)))
+                return ues
+            }
+            return [await this.handleSendDataContent(data, keys)]
+        }
+        const ue = await this.handleSendDataEvent(data, keys)
+        return [ue]
+    }
+
+    async handleSendDataContent(data: SendDataContent, keys: { name: string, privateKey: string, publicKey: string }): Promise<UnsignedEvent> {
+        let content = encryptV1(data.content, getConversationKeyV1(keys.privateKey, data.pub))
+        return {
+            content,
+            created_at: Math.floor(Date.now() / 1000),
+            kind: 21000,
+            pubkey: keys.publicKey,
+            tags: [['p', data.pub]],
+        }
+    }
+
+    async handleSendDataEvent(data: SendDataEvent, keys: { name: string, privateKey: string, publicKey: string }): Promise<UnsignedEvent> {
+        const toSign = data.event
+        if (data.encrypt) {
+            toSign.content = encryptV2(data.event.content, getConversationKeyV2(Buffer.from(keys.privateKey, 'hex'), data.encrypt.toPub))
+        }
+        if (!toSign.pubkey) {
+            toSign.pubkey = keys.publicKey
+        }
+        return toSign
+    }
+
+    async sendEvent(event: UnsignedEvent, keys: { name: string, privateKey: string }, relays?: string[]) {
+        const signed = finalizeEvent(event, Buffer.from(keys.privateKey, 'hex'))
         let sent = false
         const log = getLogger({ appName: keys.name })
         await Promise.all(this.pool.publish(relays || this.settings.relays, signed).map(async p => {
@@ -264,7 +279,7 @@ export default class Handler {
         if (!sent) {
             log("failed to send event")
         } else {
-            log("sent event")
+            //log("sent event")
         }
     }
 
@@ -287,3 +302,11 @@ export default class Handler {
         throw new Error("unkown initiator type")
     }
 }
+
+const splitContent = (content: string, maxLength: number) => {
+    const parts = []
+    for (let i = 0; i < content.length; i += maxLength) {
+        parts.push(content.slice(i, i + maxLength))
+    }
+    return parts
+}

src/services/nostr/index.ts

@@ -1,5 +1,5 @@
 import { ChildProcess, fork } from 'child_process'
-import { EnvMustBeNonEmptyString } from "../helpers/envParser.js"
+import { EnvCanBeInteger, EnvMustBeNonEmptyString } from "../helpers/envParser.js"
 import { NostrSettings, NostrEvent, ChildProcessRequest, ChildProcessResponse, SendData, SendInitiator } from "./handler.js"
 import { Utils } from '../helpers/utilsWrapper.js'
 type EventCallback = (event: NostrEvent) => void
@@ -10,8 +10,10 @@ const getEnvOrDefault = (name: string, defaultValue: string): string => {
 
 export const LoadNosrtSettingsFromEnv = (test = false) => {
     const relaysEnv = getEnvOrDefault("NOSTR_RELAYS", "wss://relay.lightning.pub");
+    const maxEventContentLength = EnvCanBeInteger("NOSTR_MAX_EVENT_CONTENT_LENGTH", 45000)
     return {
-        relays: relaysEnv.split(' ')
+        relays: relaysEnv.split(' '),
+        maxEventContentLength
     }
 }
 

src/tests/testRelay.ts

@@ -0,0 +1,29 @@
+import { SimplePool, UnsignedEvent, finalizeEvent, generateSecretKey, getPublicKey } from "nostr-tools"
+import { encrypt as encryptV1, decrypt as decryptV1, getSharedSecret as getConversationKeyV1 } from '../services/nostr/nip44v1.js'
+import WebSocket from 'ws'
+Object.assign(global, { WebSocket: WebSocket });
+const pool = new SimplePool()
+
+const relays = [
+    "wss://strfry.shock.network"
+]
+const secretKey = generateSecretKey()
+const publicKey = getPublicKey(secretKey)
+const content = Array(1000).fill("A").join("")
+console.log("content length", content.length)
+const encrypted = encryptV1(content, getConversationKeyV1(Buffer.from(secretKey).toString('hex'), publicKey))
+console.log("encrypted length", encrypted.length)
+console.log("encrypted", encrypted)
+const e: UnsignedEvent = {
+    content: encrypted,
+    created_at: Math.floor(Date.now() / 1000),
+    kind: 21000,
+    pubkey: publicKey,
+    tags: [["p", publicKey]],
+}
+const signed = finalizeEvent(e, Buffer.from(secretKey))
+Promise.all(pool.publish(relays, signed)).then(r => {
+    console.log("sent", r)
+}).catch(e => {
+    console.error("error", e)
+})