Merge pull request #813 from shocknet/nostr-sharding

Nostr sharding

env.example

@@ -73,6 +73,8 @@ LSP_MAX_FEE_BPS=100
 #NOSTR
 # Default relay may become rate-limited without a paid subscription
 #NOSTR_RELAYS=wss://relay.lightning.pub
+# Max content lengh of single nostr event content, events will be sharded above this size
+#NOSTR_MAX_EVENT_CONTENT_LENGTH=45000
 
 #LNURL
 # Optional

package.json

@@ -7,12 +7,13 @@
     "clean": "rimraf build",
     "test": "npm run clean && tsc  && node build/src/tests/testRunner.js",
     "start": "npm run clean && tsc && node build/src/index.js",
-    "start:ci": "git reset --hard && git pull &&  npm run start",
+    "start:ci": "git reset --hard && git pull && export NODE_TLS_REJECT_UNAUTHORIZED='0' && npm run start",
     "gen": "cd proto && rimraf autogenerated && export PATH=$PATH:~/Lightning.Pub/proto && protoc -I ./service --pub_out=. service/*",
     "build_autogenerated": "cd proto && rimraf autogenerated && protoc -I ./service --pub_out=. service/*",
     "build_lnd_client_1": "cd proto && protoc -I ./others --plugin=.\\node_modules\\.bin\\protoc-gen-ts_proto.cmd --ts_proto_out=./lnd --ts_proto_opt=esModuleInterop=true others/* ",
     "build_lnd_client": "cd proto && rimraf lnd/*  && npx protoc --ts_out ./lnd --ts_opt long_type_string  --proto_path others others/* ",
-    "typeorm": "typeorm-ts-node-commonjs"
+    "typeorm": "typeorm-ts-node-commonjs",
+    "test_relay": "npm run clean && tsc && node build/src/tests/testRelay.js"
   },
   "repository": {
     "type": "git",
@@ -30,6 +31,7 @@
     "@protobuf-ts/grpc-transport": "^2.9.4",
     "@protobuf-ts/plugin": "^2.5.0",
     "@protobuf-ts/runtime": "^2.5.0",
+    "@shocknet/clink-sdk": "^1.0.4",
     "@stablelib/xchacha20": "^1.0.1",
     "@types/express": "^4.17.21",
     "@types/node": "^17.0.31",
@@ -49,7 +51,7 @@
     "grpc-tools": "^1.12.4",
     "jsonwebtoken": "^9.0.2",
     "lodash": "^4.17.21",
-    "nostr-tools": "github:shocknet/nostr-tools#27575ffb69d615691242df433a0ccc063f6b8346",
+    "nostr-tools": "^2.13.0",
     "pg": "^8.4.0",
     "reflect-metadata": "^0.2.2",
     "rimraf": "^3.0.2",
@@ -84,4 +86,4 @@
     "ts-node": "10.7.0",
     "typescript": "5.5.4"
   }
-}
+}

src/nostrMiddleware.ts

@@ -4,8 +4,7 @@ import { NostrEvent, NostrSend, NostrSettings } from "./services/nostr/handler.j
 import * as Types from '../proto/autogenerated/ts/types.js'
 import NewNostrTransport, { NostrRequest } from '../proto/autogenerated/ts/nostr_transport.js';
 import { ERROR, getLogger } from "./services/helpers/logger.js";
-import { NdebitData } from "nostr-tools/lib/types/nip68.js";
-import { NofferData } from "nostr-tools/lib/types/nip69.js";
+import { NdebitData, NofferData } from "@shocknet/clink-sdk";
 
 export default (serverMethods: Types.ServerMethods, mainHandler: Main, nostrSettings: NostrSettings, onClientEvent: (e: { requestId: string }, fromPub: string) => void): { Stop: () => void, Send: NostrSend, Ping: () => Promise<void> } => {
     const log = getLogger({})

src/services/main/appUserManager.ts

@@ -4,9 +4,8 @@ import * as Types from '../../../proto/autogenerated/ts/types.js'
 
 import { MainSettings } from './settings.js'
 import ApplicationManager from './applicationManager.js'
-import { nip19 } from 'nostr-tools'
 import { LoadNosrtSettingsFromEnv } from '../nostr/index.js'
-const { ndebitEncode, nofferEncode, OfferPriceType } = nip19
+import { OfferPriceType, ndebitEncode, nofferEncode } from '@shocknet/clink-sdk'
 export default class {
 
     storage: Storage

src/services/main/applicationManager.ts

@@ -8,11 +8,9 @@ import { ApplicationUser } from '../storage/entity/ApplicationUser.js'
 import { PubLogger, getLogger } from '../helpers/logger.js'
 import crypto from 'crypto'
 import { Application } from '../storage/entity/Application.js'
-import { nip69, nip19 } from 'nostr-tools'
 import { LoadNosrtSettingsFromEnv } from '../nostr/index.js'
 import { ZapInfo } from '../storage/entity/UserReceivingInvoice.js'
-const { SendNofferRequest } = nip69
-const { nofferEncode, ndebitEncode, OfferPriceType } = nip19
+import { nofferEncode, ndebitEncode, OfferPriceType } from '@shocknet/clink-sdk'
 const TOKEN_EXPIRY_TIME = 2 * 60 * 1000 // 2 minutes, in milliseconds
 
 type NsecLinkingData = {

src/services/main/debitManager.ts

@@ -1,16 +1,15 @@
-import crypto from 'crypto';
 import * as Types from "../../../proto/autogenerated/ts/types.js";
 import ApplicationManager from "./applicationManager.js";
 import Storage from '../storage/index.js'
 import LND from "../lnd/lnd.js"
 import { ERROR, getLogger } from "../helpers/logger.js";
 import { DebitAccess, DebitAccessRules } from '../storage/entity/DebitAccess.js';
-import paymentManager from './paymentManager.js';
 import { Application } from '../storage/entity/Application.js';
 import { ApplicationUser } from '../storage/entity/ApplicationUser.js';
 import { NostrEvent, NostrSend, SendData, SendInitiator } from '../nostr/handler.js';
 import { UnsignedEvent } from 'nostr-tools';
-import { BudgetFrequency, NdebitData, NdebitFailure, NdebitSuccess, NdebitSuccessPayment, RecurringDebitTimeUnit } from 'nostr-tools/lib/types/nip68.js';
+import { NdebitData, NdebitFailure, NdebitSuccess, NdebitSuccessPayment, RecurringDebitTimeUnit } from "@shocknet/clink-sdk";
+
 export const expirationRuleName = 'expiration'
 export const frequencyRuleName = 'frequency'
 const unitToIntervalType = (unit: RecurringDebitTimeUnit) => {

src/services/main/index.ts

@@ -23,7 +23,6 @@ import { AdminManager } from "./adminManager.js"
 import { Unlocker } from "./unlocker.js"
 import { defaultInvoiceExpiry } from "../storage/paymentStorage.js"
 import { DebitManager } from "./debitManager.js"
-import { NofferData } from "nostr-tools/lib/types/nip69.js"
 import { OfferManager } from "./offerManager.js"
 import webRTC from "../webRTC/index.js"
 

src/services/main/offerManager.ts

@@ -1,23 +1,15 @@
-import crypto from 'crypto';
 import * as Types from "../../../proto/autogenerated/ts/types.js";
 import ApplicationManager from "./applicationManager.js";
 import ProductManager from "./productManager.js";
 import Storage from '../storage/index.js'
 import LND from "../lnd/lnd.js"
 import { ERROR, getLogger } from "../helpers/logger.js";
-import { DebitAccess, DebitAccessRules } from '../storage/entity/DebitAccess.js';
-import paymentManager from './paymentManager.js';
-import { Application } from '../storage/entity/Application.js';
-import { ApplicationUser } from '../storage/entity/ApplicationUser.js';
 import { NostrEvent, NostrSend, SendData, SendInitiator } from '../nostr/handler.js';
 import { UnsignedEvent } from 'nostr-tools';
-import { BudgetFrequency, NdebitData, NdebitFailure, NdebitSuccess, NdebitSuccessPayment, RecurringDebitTimeUnit } from 'nostr-tools/lib/types/nip68.js';
-import { NofferData } from "nostr-tools/lib/types/nip69.js"
 import { UserOffer } from '../storage/entity/UserOffer.js';
-import { DeepPartial } from 'typeorm';
-import { nip19 } from 'nostr-tools';
 import { LoadNosrtSettingsFromEnv } from '../nostr/index.js';
 import { LiquidityManager } from "./liquidityManager.js"
+import { NofferData, OfferPriceType, nofferEncode } from '@shocknet/clink-sdk';
 
 const mapToOfferConfig = (appUserId: string, offer: UserOffer, { pubkey, relay }: { pubkey: string, relay: string }): Types.OfferConfig => {
     if (offer.expected_data) {
@@ -30,8 +22,8 @@ const mapToOfferConfig = (appUserId: string, offer: UserOffer, { pubkey, relay }
         }
     }
     const offerStr = offer.offer_id
-    const priceType: nip19.OfferPriceType = offer.price_sats === 0 ? nip19.OfferPriceType.Spontaneous : nip19.OfferPriceType.Fixed
-    const noffer = nip19.nofferEncode({ pubkey, offer: offerStr, priceType, relay, price: offer.price_sats || undefined })
+    const priceType: OfferPriceType = offer.price_sats === 0 ? OfferPriceType.Spontaneous : OfferPriceType.Fixed
+    const noffer = nofferEncode({ pubkey, offer: offerStr, priceType, relay, price: offer.price_sats || undefined })
     return {
         label: offer.label,
         price_sats: offer.price_sats,

src/services/main/paymentManager.ts

@@ -7,15 +7,13 @@ import { InboundOptionals, defaultInvoiceExpiry } from '../storage/paymentStorag
 import LND from '../lnd/lnd.js'
 import { Application } from '../storage/entity/Application.js'
 import { ERROR, getLogger, PubLogger } from '../helpers/logger.js'
-import { UserReceivingAddress } from '../storage/entity/UserReceivingAddress.js'
-import { AddressPaidCb, InvoicePaidCb, PaidInvoice } from '../lnd/settings.js'
+import { AddressPaidCb, InvoicePaidCb } from '../lnd/settings.js'
 import { UserReceivingInvoice, ZapInfo } from '../storage/entity/UserReceivingInvoice.js'
-import { Payment_PaymentStatus, SendCoinsResponse } from '../../../proto/lnd/lightning.js'
+import { Payment_PaymentStatus } from '../../../proto/lnd/lightning.js'
 import { Event, verifiedSymbol, verifyEvent } from 'nostr-tools'
 import { AddressReceivingTransaction } from '../storage/entity/AddressReceivingTransaction.js'
 import { UserTransactionPayment } from '../storage/entity/UserTransactionPayment.js'
 import { Watchdog } from './watchdog.js'
-import { LiquidityProvider } from './liquidityProvider.js'
 import { LiquidityManager } from './liquidityManager.js'
 import { Utils } from '../helpers/utilsWrapper.js'
 import { UserInvoicePayment } from '../storage/entity/UserInvoicePayment.js'

src/services/main/productManager.ts

@@ -5,8 +5,7 @@ import * as Types from '../../../proto/autogenerated/ts/types.js'
 import { MainSettings } from './settings.js'
 import PaymentManager from './paymentManager.js'
 import { defaultInvoiceExpiry } from '../storage/paymentStorage.js'
-import { nip19 } from 'nostr-tools'
-const { nofferEncode, OfferPriceType } = nip19
+import { nofferEncode, OfferPriceType } from '@shocknet/clink-sdk'
 
 export default class {
     storage: Storage

src/services/nostr/handler.ts

@@ -1,8 +1,8 @@
 //import { SimplePool, Sub, Event, UnsignedEvent, getEventHash, signEvent } from 'nostr-tools'
 import WebSocket from 'ws'
 Object.assign(global, { WebSocket: WebSocket });
-import { SimplePool, Event, UnsignedEvent, getEventHash, finalizeEvent, Relay, nip44 } from 'nostr-tools'
-//import { encryptData, decryptData, getSharedSecret, decodePayload, encodePayload, EncryptedData, nip44 } from 'nostr-tools'
+import crypto from 'crypto'
+import { SimplePool, Event, UnsignedEvent, finalizeEvent, Relay, nip44 } from 'nostr-tools'
 import { ERROR, getLogger } from '../helpers/logger.js'
 import { nip19 } from 'nostr-tools'
 import { encrypt as encryptV1, decrypt as decryptV1, getSharedSecret as getConversationKeyV1 } from './nip44v1.js'
@@ -14,7 +14,9 @@ const { getConversationKey: getConversationKeyV2 } = utils
 const handledEvents: string[] = [] // TODO: - big memory leak here, add TTL
 type AppInfo = { appId: string, publicKey: string, privateKey: string, name: string }
 type ClientInfo = { clientId: string, publicKey: string, privateKey: string, name: string }
-export type SendData = { type: "content", content: string, pub: string } | { type: "event", event: UnsignedEvent, encrypt?: { toPub: string } }
+type SendDataContent = { type: "content", content: string, pub: string, index?: number, totalShards?: number, shardsId?: string }
+type SendDataEvent = { type: "event", event: UnsignedEvent, encrypt?: { toPub: string } }
+export type SendData = SendDataContent | SendDataEvent
 export type SendInitiator = { type: 'app', appId: string } | { type: 'client', clientId: string }
 export type NostrSend = (initiator: SendInitiator, data: SendData, relays?: string[] | undefined) => void
 
@@ -22,6 +24,7 @@ export type NostrSettings = {
     apps: AppInfo[]
     relays: string[]
     clients: ClientInfo[]
+    maxEventContentLength: number
 }
 export type NostrEvent = {
     id: string
@@ -219,38 +222,49 @@ export default class Handler {
     async Send(initiator: SendInitiator, data: SendData, relays?: string[]) {
         const keys = this.GetSendKeys(initiator)
         const privateKey = Buffer.from(keys.privateKey, 'hex')
-        let toSign: UnsignedEvent
-        if (data.type === 'content') {
-            let content: string
-            try {
-                content = encryptV1(data.content, getConversationKeyV1(keys.privateKey, data.pub))
-            } catch (e: any) {
-                this.log(ERROR, "failed to encrypt content", e.message, data.content)
-                return
-            }
-            toSign = {
-                content,
-                created_at: Math.floor(Date.now() / 1000),
-                kind: 21000,
-                pubkey: keys.publicKey,
-                tags: [['p', data.pub]],
-            }
-        } else {
-            toSign = data.event
-            if (data.encrypt) {
-                try {
-                    toSign.content = encryptV2(data.event.content, getConversationKeyV2(Buffer.from(keys.privateKey, 'hex'), data.encrypt.toPub))
-                } catch (e: any) {
-                    this.log(ERROR, "failed to encrypt content", e.message)
-                    return
-                }
-            }
-            if (!toSign.pubkey) {
-                toSign.pubkey = keys.publicKey
-            }
-        }
+        const toSign = await this.handleSend(data, keys)
+        await Promise.all(toSign.map(ue => this.sendEvent(ue, keys, relays)))
+    }
 
-        const signed = finalizeEvent(toSign, Buffer.from(keys.privateKey, 'hex'))
+    async handleSend(data: SendData, keys: { name: string, privateKey: string, publicKey: string }): Promise<UnsignedEvent[]> {
+        if (data.type === 'content') {
+            const parts = splitContent(data.content, this.settings.maxEventContentLength)
+            if (parts.length > 1) {
+                const shardsId = crypto.randomBytes(16).toString('hex')
+                const totalShards = parts.length
+                const ues = await Promise.all(parts.map((part, index) => this.handleSendDataContent({ ...data, content: part, index, totalShards, shardsId }, keys)))
+                return ues
+            }
+            return [await this.handleSendDataContent(data, keys)]
+        }
+        const ue = await this.handleSendDataEvent(data, keys)
+        return [ue]
+    }
+
+    async handleSendDataContent(data: SendDataContent, keys: { name: string, privateKey: string, publicKey: string }): Promise<UnsignedEvent> {
+        let content = encryptV1(data.content, getConversationKeyV1(keys.privateKey, data.pub))
+        return {
+            content,
+            created_at: Math.floor(Date.now() / 1000),
+            kind: 21000,
+            pubkey: keys.publicKey,
+            tags: [['p', data.pub]],
+        }
+    }
+
+    async handleSendDataEvent(data: SendDataEvent, keys: { name: string, privateKey: string, publicKey: string }): Promise<UnsignedEvent> {
+        const toSign = data.event
+        if (data.encrypt) {
+            toSign.content = encryptV2(data.event.content, getConversationKeyV2(Buffer.from(keys.privateKey, 'hex'), data.encrypt.toPub))
+        }
+        if (!toSign.pubkey) {
+            toSign.pubkey = keys.publicKey
+        }
+        return toSign
+    }
+
+    async sendEvent(event: UnsignedEvent, keys: { name: string, privateKey: string }, relays?: string[]) {
+        const signed = finalizeEvent(event, Buffer.from(keys.privateKey, 'hex'))
         let sent = false
         const log = getLogger({ appName: keys.name })
         await Promise.all(this.pool.publish(relays || this.settings.relays, signed).map(async p => {
@@ -265,7 +279,7 @@ export default class Handler {
         if (!sent) {
             log("failed to send event")
         } else {
-            log("sent event")
+            //log("sent event")
         }
     }
 
@@ -287,4 +301,12 @@ export default class Handler {
         }
         throw new Error("unkown initiator type")
     }
+}
+
+const splitContent = (content: string, maxLength: number) => {
+    const parts = []
+    for (let i = 0; i < content.length; i += maxLength) {
+        parts.push(content.slice(i, i + maxLength))
+    }
+    return parts
 }

src/services/nostr/index.ts

@@ -1,5 +1,5 @@
 import { ChildProcess, fork } from 'child_process'
-import { EnvMustBeNonEmptyString } from "../helpers/envParser.js"
+import { EnvCanBeInteger, EnvMustBeNonEmptyString } from "../helpers/envParser.js"
 import { NostrSettings, NostrEvent, ChildProcessRequest, ChildProcessResponse, SendData, SendInitiator } from "./handler.js"
 import { Utils } from '../helpers/utilsWrapper.js'
 type EventCallback = (event: NostrEvent) => void
@@ -10,8 +10,10 @@ const getEnvOrDefault = (name: string, defaultValue: string): string => {
 
 export const LoadNosrtSettingsFromEnv = (test = false) => {
     const relaysEnv = getEnvOrDefault("NOSTR_RELAYS", "wss://relay.lightning.pub");
+    const maxEventContentLength = EnvCanBeInteger("NOSTR_MAX_EVENT_CONTENT_LENGTH", 45000)
     return {
-        relays: relaysEnv.split(' ')
+        relays: relaysEnv.split(' '),
+        maxEventContentLength
     }
 }
 

src/tests/testRelay.ts

@@ -0,0 +1,29 @@
+import { SimplePool, UnsignedEvent, finalizeEvent, generateSecretKey, getPublicKey } from "nostr-tools"
+import { encrypt as encryptV1, decrypt as decryptV1, getSharedSecret as getConversationKeyV1 } from '../services/nostr/nip44v1.js'
+import WebSocket from 'ws'
+Object.assign(global, { WebSocket: WebSocket });
+const pool = new SimplePool()
+
+const relays = [
+    "wss://strfry.shock.network"
+]
+const secretKey = generateSecretKey()
+const publicKey = getPublicKey(secretKey)
+const content = Array(1000).fill("A").join("")
+console.log("content length", content.length)
+const encrypted = encryptV1(content, getConversationKeyV1(Buffer.from(secretKey).toString('hex'), publicKey))
+console.log("encrypted length", encrypted.length)
+console.log("encrypted", encrypted)
+const e: UnsignedEvent = {
+    content: encrypted,
+    created_at: Math.floor(Date.now() / 1000),
+    kind: 21000,
+    pubkey: publicKey,
+    tags: [["p", publicKey]],
+}
+const signed = finalizeEvent(e, Buffer.from(secretKey))
+Promise.all(pool.publish(relays, signed)).then(r => {
+    console.log("sent", r)
+}).catch(e => {
+    console.error("error", e)
+})